From python-checkins at python.org  Thu Oct  1 00:54:53 2015
From: python-checkins at python.org (victor.stinner)
Date: Wed, 30 Sep 2015 22:54:53 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Fix_regrtest_--coverage_on?=
	=?utf-8?q?_Windows?=
Message-ID: <20150930225451.115507.12471@psf.io>

https://hg.python.org/cpython/rev/0f10e0b3e76d
changeset:   98443:0f10e0b3e76d
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Oct 01 00:53:09 2015 +0200
summary:
  Fix regrtest --coverage on Windows

Issue #25260: Fix ``python -m test --coverage`` on Windows. Remove the list of
ignored directories.

files:
  Lib/test/libregrtest/main.py |  5 +----
  Lib/test/test_regrtest.py    |  2 --
  Misc/NEWS                    |  3 +++
  3 files changed, 4 insertions(+), 6 deletions(-)


diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -272,10 +272,7 @@
     def run_tests_sequential(self):
         if self.ns.trace:
             import trace
-            self.tracer = trace.Trace(ignoredirs=[sys.base_prefix,
-                                                  sys.base_exec_prefix,
-                                                  tempfile.gettempdir()],
-                                      trace=False, count=True)
+            self.tracer = trace.Trace(trace=False, count=True)
 
         save_modules = sys.modules.keys()
 
diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -605,8 +605,6 @@
                  % (self.TESTNAME_REGEX, len(tests)))
         self.check_line(output, regex)
 
-    @unittest.skipIf(sys.platform == 'win32',
-                     "FIXME: coverage doesn't work on Windows")
     def test_coverage(self):
         # test --coverage
         test = self.create_test()
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -157,6 +157,9 @@
 Tests
 -----
 
+- Issue #25260: Fix ``python -m test --coverage`` on Windows. Remove the
+  list of ignored directories.
+
 - PCbuild\rt.bat now accepts an unlimited number of arguments to pass along
   to regrtest.py.  Previously there was a limit of 9.
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  1 08:15:08 2015
From: python-checkins at python.org (raymond.hettinger)
Date: Thu, 01 Oct 2015 06:15:08 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Add_fast_paths_to_deque=5F?=
 =?utf-8?q?init=28=29_for_the_common_cases?=
Message-ID: <20151001061507.115366.64922@psf.io>

https://hg.python.org/cpython/rev/5352badd200e
changeset:   98444:5352badd200e
user:        Raymond Hettinger <python at rcn.com>
date:        Wed Sep 30 23:15:02 2015 -0700
summary:
  Add fast paths to deque_init() for the common cases

files:
  Modules/_collectionsmodule.c |  13 ++++++++++---
  1 files changed, 10 insertions(+), 3 deletions(-)


diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c
--- a/Modules/_collectionsmodule.c
+++ b/Modules/_collectionsmodule.c
@@ -1456,8 +1456,14 @@
     Py_ssize_t maxlen = -1;
     char *kwlist[] = {"iterable", "maxlen", 0};
 
-    if (!PyArg_ParseTupleAndKeywords(args, kwdargs, "|OO:deque", kwlist, &iterable, &maxlenobj))
-        return -1;
+    if (kwdargs == NULL) {
+        if (!PyArg_UnpackTuple(args, "deque()", 0, 2, &iterable, &maxlenobj))
+            return -1;
+    } else {
+        if (!PyArg_ParseTupleAndKeywords(args, kwdargs, "|OO:deque", kwlist,
+                                         &iterable, &maxlenobj))
+            return -1;
+    }
     if (maxlenobj != NULL && maxlenobj != Py_None) {
         maxlen = PyLong_AsSsize_t(maxlenobj);
         if (maxlen == -1 && PyErr_Occurred())
@@ -1468,7 +1474,8 @@
         }
     }
     deque->maxlen = maxlen;
-    deque_clear(deque);
+    if (Py_SIZE(deque) > 0)
+        deque_clear(deque);
     if (iterable != NULL) {
         PyObject *rv = deque_extend(deque, iterable);
         if (rv == NULL)

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  1 08:44:41 2015
From: python-checkins at python.org (victor.stinner)
Date: Thu, 01 Oct 2015 06:44:41 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Fix_=5FPyTime=5FAsTimevalS?=
 =?utf-8?q?truct=5Fimpl=28=29_on_OpenBSD?=
Message-ID: <20151001064441.82658.27044@psf.io>

https://hg.python.org/cpython/rev/33a5ab6c578a
changeset:   98445:33a5ab6c578a
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Oct 01 08:44:03 2015 +0200
summary:
  Fix _PyTime_AsTimevalStruct_impl() on OpenBSD

On the x86 OpenBSD 5.8 buildbot, the integer overflow check is ignored. Copy
the tv_sec variable into a Py_time_t variable instead of "simply" casting it to
Py_time_t, to fix the integer overflow check.

files:
  Python/pytime.c |  5 +++--
  1 files changed, 3 insertions(+), 2 deletions(-)


diff --git a/Python/pytime.c b/Python/pytime.c
--- a/Python/pytime.c
+++ b/Python/pytime.c
@@ -454,7 +454,7 @@
 _PyTime_AsTimevalStruct_impl(_PyTime_t t, struct timeval *tv,
                              _PyTime_round_t round, int raise)
 {
-    _PyTime_t secs;
+    _PyTime_t secs, secs2;
     int us;
     int res;
 
@@ -467,7 +467,8 @@
 #endif
     tv->tv_usec = us;
 
-    if (res < 0 || (_PyTime_t)tv->tv_sec != secs) {
+    secs2 = (_PyTime_t)tv->tv_sec;
+    if (res < 0 || secs2 != secs) {
         if (raise)
             error_time_t_overflow();
         return -1;

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  1 08:49:12 2015
From: python-checkins at python.org (andrew.svetlov)
Date: Thu, 01 Oct 2015 06:49:12 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
	=?utf-8?q?_Merge_3=2E4_-=3E_3=2E5?=
Message-ID: <20151001064912.94107.11455@psf.io>

https://hg.python.org/cpython/rev/d7d18ef3e05c
changeset:   98447:d7d18ef3e05c
branch:      3.5
parent:      98439:0eb26a4d5ffa
parent:      98446:9a10055e12fa
user:        Andrew Svetlov <andrew.svetlov at gmail.com>
date:        Thu Oct 01 09:48:36 2015 +0300
summary:
  Merge 3.4 -> 3.5

files:
  Doc/library/asyncio-eventloop.rst |  6 +++---
  1 files changed, 3 insertions(+), 3 deletions(-)


diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst
--- a/Doc/library/asyncio-eventloop.rst
+++ b/Doc/library/asyncio-eventloop.rst
@@ -586,14 +586,14 @@
 pool of processes). By default, an event loop uses a thread pool executor
 (:class:`~concurrent.futures.ThreadPoolExecutor`).
 
-.. coroutinemethod:: BaseEventLoop.run_in_executor(executor, callback, \*args)
+.. coroutinemethod:: BaseEventLoop.run_in_executor(executor, func, \*args)
 
-   Arrange for a callback to be called in the specified executor.
+   Arrange for a *func* to be called in the specified executor.
 
    The *executor* argument should be an :class:`~concurrent.futures.Executor`
    instance. The default executor is used if *executor* is ``None``.
 
-   :ref:`Use functools.partial to pass keywords to the callback
+   :ref:`Use functools.partial to pass keywords to the *func*
    <asyncio-pass-keywords>`.
 
    This method is a :ref:`coroutine <coroutine>`.

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  1 08:49:12 2015
From: python-checkins at python.org (andrew.svetlov)
Date: Thu, 01 Oct 2015 06:49:12 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E4=29=3A_Reflect_parame?=
	=?utf-8?q?ter_name_change_in_the_doc?=
Message-ID: <20151001064912.11694.9265@psf.io>

https://hg.python.org/cpython/rev/9a10055e12fa
changeset:   98446:9a10055e12fa
branch:      3.4
parent:      98438:2652c1798f7d
user:        Andrew Svetlov <andrew.svetlov at gmail.com>
date:        Thu Oct 01 09:48:08 2015 +0300
summary:
  Reflect parameter name change in the doc

files:
  Doc/library/asyncio-eventloop.rst |  6 +++---
  1 files changed, 3 insertions(+), 3 deletions(-)


diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst
--- a/Doc/library/asyncio-eventloop.rst
+++ b/Doc/library/asyncio-eventloop.rst
@@ -582,14 +582,14 @@
 pool of processes). By default, an event loop uses a thread pool executor
 (:class:`~concurrent.futures.ThreadPoolExecutor`).
 
-.. coroutinemethod:: BaseEventLoop.run_in_executor(executor, callback, \*args)
+.. coroutinemethod:: BaseEventLoop.run_in_executor(executor, func, \*args)
 
-   Arrange for a callback to be called in the specified executor.
+   Arrange for a *func* to be called in the specified executor.
 
    The *executor* argument should be an :class:`~concurrent.futures.Executor`
    instance. The default executor is used if *executor* is ``None``.
 
-   :ref:`Use functools.partial to pass keywords to the callback
+   :ref:`Use functools.partial to pass keywords to the *func*
    <asyncio-pass-keywords>`.
 
    This method is a :ref:`coroutine <coroutine>`.

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  1 08:49:13 2015
From: python-checkins at python.org (andrew.svetlov)
Date: Thu, 01 Oct 2015 06:49:13 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
 =?utf-8?q?=29=3A_Merge_3=2E5_-=3E_default?=
Message-ID: <20151001064912.3648.54297@psf.io>

https://hg.python.org/cpython/rev/1465b18ef4fc
changeset:   98448:1465b18ef4fc
parent:      98445:33a5ab6c578a
parent:      98447:d7d18ef3e05c
user:        Andrew Svetlov <andrew.svetlov at gmail.com>
date:        Thu Oct 01 09:49:03 2015 +0300
summary:
  Merge 3.5 -> default

files:
  Doc/library/asyncio-eventloop.rst |  6 +++---
  1 files changed, 3 insertions(+), 3 deletions(-)


diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst
--- a/Doc/library/asyncio-eventloop.rst
+++ b/Doc/library/asyncio-eventloop.rst
@@ -586,14 +586,14 @@
 pool of processes). By default, an event loop uses a thread pool executor
 (:class:`~concurrent.futures.ThreadPoolExecutor`).
 
-.. coroutinemethod:: BaseEventLoop.run_in_executor(executor, callback, \*args)
+.. coroutinemethod:: BaseEventLoop.run_in_executor(executor, func, \*args)
 
-   Arrange for a callback to be called in the specified executor.
+   Arrange for a *func* to be called in the specified executor.
 
    The *executor* argument should be an :class:`~concurrent.futures.Executor`
    instance. The default executor is used if *executor* is ``None``.
 
-   :ref:`Use functools.partial to pass keywords to the callback
+   :ref:`Use functools.partial to pass keywords to the *func*
    <asyncio-pass-keywords>`.
 
    This method is a :ref:`coroutine <coroutine>`.

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  1 08:57:34 2015
From: python-checkins at python.org (victor.stinner)
Date: Thu, 01 Oct 2015 06:57:34 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
 =?utf-8?q?=29=3A_=28Merge_3=2E5=29_Issue_=2325274=3A_test=5Frecursionlimi?=
 =?utf-8?q?t=5Frecovery=28=29_of_test=5Fsys_now_checks?=
Message-ID: <20151001065734.98358.78229@psf.io>

https://hg.python.org/cpython/rev/bae0912dd160
changeset:   98451:bae0912dd160
parent:      98448:1465b18ef4fc
parent:      98450:898a9a959927
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Oct 01 08:56:54 2015 +0200
summary:
  (Merge 3.5) Issue #25274: test_recursionlimit_recovery() of test_sys now checks
sys.gettrace() when the test is executed, not when the module is loaded.

sys.settrace() may be after after the test is loaded.

files:
  Lib/test/test_sys.py |  5 +++--
  1 files changed, 3 insertions(+), 2 deletions(-)


diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
--- a/Lib/test/test_sys.py
+++ b/Lib/test/test_sys.py
@@ -197,9 +197,10 @@
         self.assertEqual(sys.getrecursionlimit(), 10000)
         sys.setrecursionlimit(oldlimit)
 
-    @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
-                     'fatal error if run with a trace function')
     def test_recursionlimit_recovery(self):
+        if hasattr(sys, 'gettrace') and sys.gettrace():
+            self.skipTest('fatal error if run with a trace function')
+
         # NOTE: this test is slightly fragile in that it depends on the current
         # recursion count when executing the test being low enough so as to
         # trigger the recursion recovery detection in the _Py_MakeEndRecCheck

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  1 08:57:34 2015
From: python-checkins at python.org (victor.stinner)
Date: Thu, 01 Oct 2015 06:57:34 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
 =?utf-8?q?_=28Merge_3=2E4=29_Issue_=2325274=3A_test=5Frecursionlimit=5Fre?=
 =?utf-8?q?covery=28=29_of_test=5Fsys_now_checks?=
Message-ID: <20151001065734.3664.80792@psf.io>

https://hg.python.org/cpython/rev/898a9a959927
changeset:   98450:898a9a959927
branch:      3.5
parent:      98447:d7d18ef3e05c
parent:      98449:60c4fd84ef92
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Oct 01 08:56:27 2015 +0200
summary:
  (Merge 3.4) Issue #25274: test_recursionlimit_recovery() of test_sys now checks
sys.gettrace() when the test is executed, not when the module is loaded.

sys.settrace() may be after after the test is loaded.

files:
  Lib/test/test_sys.py |  5 +++--
  1 files changed, 3 insertions(+), 2 deletions(-)


diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
--- a/Lib/test/test_sys.py
+++ b/Lib/test/test_sys.py
@@ -197,9 +197,10 @@
         self.assertEqual(sys.getrecursionlimit(), 10000)
         sys.setrecursionlimit(oldlimit)
 
-    @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
-                     'fatal error if run with a trace function')
     def test_recursionlimit_recovery(self):
+        if hasattr(sys, 'gettrace') and sys.gettrace():
+            self.skipTest('fatal error if run with a trace function')
+
         # NOTE: this test is slightly fragile in that it depends on the current
         # recursion count when executing the test being low enough so as to
         # trigger the recursion recovery detection in the _Py_MakeEndRecCheck

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  1 08:57:34 2015
From: python-checkins at python.org (victor.stinner)
Date: Thu, 01 Oct 2015 06:57:34 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzI1Mjc0?=
 =?utf-8?q?=3A_test=5Frecursionlimit=5Frecovery=28=29_of_test=5Fsys_now_ch?=
 =?utf-8?q?ecks?=
Message-ID: <20151001065734.9957.26964@psf.io>

https://hg.python.org/cpython/rev/60c4fd84ef92
changeset:   98449:60c4fd84ef92
branch:      3.4
parent:      98446:9a10055e12fa
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Oct 01 08:55:33 2015 +0200
summary:
  Issue #25274: test_recursionlimit_recovery() of test_sys now checks
sys.gettrace() when the test is executed, not when the module is loaded.

sys.settrace() may be after after the test is loaded.

files:
  Lib/test/test_sys.py |  5 +++--
  1 files changed, 3 insertions(+), 2 deletions(-)


diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
--- a/Lib/test/test_sys.py
+++ b/Lib/test/test_sys.py
@@ -197,9 +197,10 @@
         self.assertEqual(sys.getrecursionlimit(), 10000)
         sys.setrecursionlimit(oldlimit)
 
-    @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
-                     'fatal error if run with a trace function')
     def test_recursionlimit_recovery(self):
+        if hasattr(sys, 'gettrace') and sys.gettrace():
+            self.skipTest('fatal error if run with a trace function')
+
         # NOTE: this test is slightly fragile in that it depends on the current
         # recursion count when executing the test being low enough so as to
         # trigger the recursion recovery detection in the _Py_MakeEndRecCheck

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  1 09:51:52 2015
From: python-checkins at python.org (victor.stinner)
Date: Thu, 01 Oct 2015 07:51:52 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
	=?utf-8?b?KTogTWVyZ2UgMy41?=
Message-ID: <20151001075152.115214.35513@psf.io>

https://hg.python.org/cpython/rev/710ef035ee44
changeset:   98453:710ef035ee44
parent:      98451:bae0912dd160
parent:      98452:835085cc28cd
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Oct 01 09:51:02 2015 +0200
summary:
  Merge 3.5

files:
  Python/random.c |  4 ++--
  1 files changed, 2 insertions(+), 2 deletions(-)


diff --git a/Python/random.c b/Python/random.c
--- a/Python/random.c
+++ b/Python/random.c
@@ -73,7 +73,7 @@
 }
 
 #elif defined(HAVE_GETENTROPY) && !defined(sun)
-#define PY_GETENTROPY
+#define PY_GETENTROPY 1
 
 /* Fill buffer with size pseudo-random bytes generated by getentropy().
    Return 0 on success, or raise an exception and return -1 on error.
@@ -112,7 +112,7 @@
 #else
 
 #if defined(HAVE_GETRANDOM) || defined(HAVE_GETRANDOM_SYSCALL)
-#define PY_GETRANDOM
+#define PY_GETRANDOM 1
 
 static int
 py_getrandom(void *buffer, Py_ssize_t size, int raise)

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  1 09:51:52 2015
From: python-checkins at python.org (victor.stinner)
Date: Thu, 01 Oct 2015 07:51:52 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy41KTogSXNzdWUgIzI1MDAz?=
 =?utf-8?q?=3A_On_Solaris_11=2E3_or_newer=2C_os=2Eurandom=28=29_now_uses_t?=
 =?utf-8?q?he_getrandom=28=29?=
Message-ID: <20151001075152.115050.90181@psf.io>

https://hg.python.org/cpython/rev/835085cc28cd
changeset:   98452:835085cc28cd
branch:      3.5
parent:      98450:898a9a959927
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Oct 01 09:47:30 2015 +0200
summary:
  Issue #25003: On Solaris 11.3 or newer, os.urandom() now uses the getrandom()
function instead of the getentropy() function. The getentropy() function is
blocking to generate very good quality entropy, os.urandom() doesn't need such
high-quality entropy.

files:
  Lib/test/test_os.py |  16 ++++++----
  Misc/NEWS           |   5 +++
  Python/random.c     |  49 ++++++++++++++++++++++----------
  configure           |  43 ++++++++++++++++++++++++++--
  configure.ac        |  31 ++++++++++++++++++--
  pyconfig.h.in       |   3 ++
  6 files changed, 119 insertions(+), 28 deletions(-)


diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -1226,13 +1226,15 @@
         self.assertNotEqual(data1, data2)
 
 
-HAVE_GETENTROPY = (sysconfig.get_config_var('HAVE_GETENTROPY') == 1)
-HAVE_GETRANDOM = (sysconfig.get_config_var('HAVE_GETRANDOM_SYSCALL') == 1)
-
- at unittest.skipIf(HAVE_GETENTROPY,
-                 "getentropy() does not use a file descriptor")
- at unittest.skipIf(HAVE_GETRANDOM,
-                 "getrandom() does not use a file descriptor")
+# os.urandom() doesn't use a file descriptor when it is implemented with the
+# getentropy() function, the getrandom() function or the getrandom() syscall
+OS_URANDOM_DONT_USE_FD = (
+    sysconfig.get_config_var('HAVE_GETENTROPY') == 1
+    or sysconfig.get_config_var('HAVE_GETRANDOM') == 1
+    or sysconfig.get_config_var('HAVE_GETRANDOM_SYSCALL') == 1)
+
+ at unittest.skipIf(OS_URANDOM_DONT_USE_FD ,
+                 "os.random() does not use a file descriptor")
 class URandomFDTests(unittest.TestCase):
     @unittest.skipUnless(resource, "test requires the resource module")
     def test_urandom_failure(self):
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -11,6 +11,11 @@
 Core and Builtins
 -----------------
 
+- Issue #25003: On Solaris 11.3 or newer, os.urandom() now uses the
+  getrandom() function instead of the getentropy() function. The getentropy()
+  function is blocking to generate very good quality entropy, os.urandom()
+  doesn't need such high-quality entropy.
+
 - Issue #25182: The stdprinter (used as sys.stderr before the io module is
   imported at startup) now uses the backslashreplace error handler.
 
diff --git a/Python/random.c b/Python/random.c
--- a/Python/random.c
+++ b/Python/random.c
@@ -6,7 +6,9 @@
 #  ifdef HAVE_SYS_STAT_H
 #    include <sys/stat.h>
 #  endif
-#  ifdef HAVE_GETRANDOM_SYSCALL
+#  ifdef HAVE_GETRANDOM
+#    include <sys/random.h>
+#  elif defined(HAVE_GETRANDOM_SYSCALL)
 #    include <sys/syscall.h>
 #  endif
 #endif
@@ -70,7 +72,9 @@
     return 0;
 }
 
-#elif HAVE_GETENTROPY
+#elif defined(HAVE_GETENTROPY) && !defined(sun)
+#define PY_GETENTROPY 1
+
 /* Fill buffer with size pseudo-random bytes generated by getentropy().
    Return 0 on success, or raise an exception and return -1 on error.
 
@@ -105,16 +109,19 @@
     return 0;
 }
 
-#else   /* !HAVE_GETENTROPY */
+#else
 
-#ifdef HAVE_GETRANDOM_SYSCALL
+#if defined(HAVE_GETRANDOM) || defined(HAVE_GETRANDOM_SYSCALL)
+#define PY_GETRANDOM 1
+
 static int
 py_getrandom(void *buffer, Py_ssize_t size, int raise)
 {
-    /* is getrandom() supported by the running kernel?
-     * need Linux kernel 3.17 or later */
+    /* Is getrandom() supported by the running kernel?
+     * Need Linux kernel 3.17 or newer, or Solaris 11.3 or newer */
     static int getrandom_works = 1;
-    /* Use /dev/urandom, block if the kernel has no entropy */
+    /* Use non-blocking /dev/urandom device. On Linux at boot, the getrandom()
+     * syscall blocks until /dev/urandom is initialized with enough entropy. */
     const int flags = 0;
     int n;
 
@@ -124,7 +131,18 @@
     while (0 < size) {
         errno = 0;
 
-        /* Use syscall() because the libc doesn't expose getrandom() yet, see:
+#ifdef HAVE_GETRANDOM
+        if (raise) {
+            Py_BEGIN_ALLOW_THREADS
+            n = getrandom(buffer, size, flags);
+            Py_END_ALLOW_THREADS
+        }
+        else {
+            n = getrandom(buffer, size, flags);
+        }
+#else
+        /* On Linux, use the syscall() function because the GNU libc doesn't
+         * expose the Linux getrandom() syscall yet. See:
          * https://sourceware.org/bugzilla/show_bug.cgi?id=17252 */
         if (raise) {
             Py_BEGIN_ALLOW_THREADS
@@ -134,6 +152,7 @@
         else {
             n = syscall(SYS_getrandom, buffer, size, flags);
         }
+#endif
 
         if (n < 0) {
             if (errno == ENOSYS) {
@@ -182,7 +201,7 @@
 
     assert (0 < size);
 
-#ifdef HAVE_GETRANDOM_SYSCALL
+#ifdef PY_GETRANDOM
     if (py_getrandom(buffer, size, 0) == 1)
         return;
     /* getrandom() is not supported by the running kernel, fall back
@@ -218,14 +237,14 @@
     int fd;
     Py_ssize_t n;
     struct _Py_stat_struct st;
-#ifdef HAVE_GETRANDOM_SYSCALL
+#ifdef PY_GETRANDOM
     int res;
 #endif
 
     if (size <= 0)
         return 0;
 
-#ifdef HAVE_GETRANDOM_SYSCALL
+#ifdef PY_GETRANDOM
     res = py_getrandom(buffer, size, 1);
     if (res < 0)
         return -1;
@@ -304,7 +323,7 @@
     }
 }
 
-#endif /* HAVE_GETENTROPY */
+#endif
 
 /* Fill buffer with pseudo-random bytes generated by a linear congruent
    generator (LCG):
@@ -345,7 +364,7 @@
 
 #ifdef MS_WINDOWS
     return win32_urandom((unsigned char *)buffer, size, 1);
-#elif HAVE_GETENTROPY
+#elif defined(PY_GETENTROPY)
     return py_getentropy(buffer, size, 0);
 #else
     return dev_urandom_python((char*)buffer, size);
@@ -392,7 +411,7 @@
     else {
 #ifdef MS_WINDOWS
         (void)win32_urandom(secret, secret_size, 0);
-#elif HAVE_GETENTROPY
+#elif defined(PY_GETENTROPY)
         (void)py_getentropy(secret, secret_size, 1);
 #else
         dev_urandom_noraise(secret, secret_size);
@@ -408,7 +427,7 @@
         CryptReleaseContext(hCryptProv, 0);
         hCryptProv = 0;
     }
-#elif HAVE_GETENTROPY
+#elif defined(PY_GETENTROPY)
     /* nothing to clean */
 #else
     dev_urandom_close();
diff --git a/configure b/configure
--- a/configure
+++ b/configure
@@ -16085,11 +16085,11 @@
     #include <sys/syscall.h>
 
     int main() {
+        char buffer[1];
+        const size_t buflen = sizeof(buffer);
         const int flags = 0;
-        char buffer[1];
-        int n;
         /* ignore the result, Python checks for ENOSYS at runtime */
-        (void)syscall(SYS_getrandom, buffer, sizeof(buffer), flags);
+        (void)syscall(SYS_getrandom, buffer, buflen, flags);
         return 0;
     }
 
@@ -16111,6 +16111,43 @@
 
 fi
 
+# check if the getrandom() function is available
+# the test was written for the Solaris function of <sys/random.h>
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for the getrandom() function" >&5
+$as_echo_n "checking for the getrandom() function... " >&6; }
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+
+    #include <sys/random.h>
+
+    int main() {
+        char buffer[1];
+        const size_t buflen = sizeof(buffer);
+        const int flags = 0;
+        /* ignore the result, Python checks for ENOSYS at runtime */
+        (void)getrandom(buffer, buflen, flags);
+        return 0;
+    }
+
+
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+  have_getrandom=yes
+else
+  have_getrandom=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $have_getrandom" >&5
+$as_echo "$have_getrandom" >&6; }
+
+if test "$have_getrandom" = yes; then
+
+$as_echo "#define HAVE_GETRANDOM 1" >>confdefs.h
+
+fi
+
 # generate output files
 ac_config_files="$ac_config_files Makefile.pre Modules/Setup.config Misc/python.pc Misc/python-config.sh"
 
diff --git a/configure.ac b/configure.ac
--- a/configure.ac
+++ b/configure.ac
@@ -5154,11 +5154,11 @@
     #include <sys/syscall.h>
 
     int main() {
+        char buffer[1];
+        const size_t buflen = sizeof(buffer);
         const int flags = 0;
-        char buffer[1];
-        int n;
         /* ignore the result, Python checks for ENOSYS at runtime */
-        (void)syscall(SYS_getrandom, buffer, sizeof(buffer), flags);
+        (void)syscall(SYS_getrandom, buffer, buflen, flags);
         return 0;
     }
   ]])
@@ -5170,6 +5170,31 @@
               [Define to 1 if the Linux getrandom() syscall is available])
 fi
 
+# check if the getrandom() function is available
+# the test was written for the Solaris function of <sys/random.h>
+AC_MSG_CHECKING(for the getrandom() function)
+AC_LINK_IFELSE(
+[
+  AC_LANG_SOURCE([[
+    #include <sys/random.h>
+
+    int main() {
+        char buffer[1];
+        const size_t buflen = sizeof(buffer);
+        const int flags = 0;
+        /* ignore the result, Python checks for ENOSYS at runtime */
+        (void)getrandom(buffer, buflen, flags);
+        return 0;
+    }
+  ]])
+],[have_getrandom=yes],[have_getrandom=no])
+AC_MSG_RESULT($have_getrandom)
+
+if test "$have_getrandom" = yes; then
+    AC_DEFINE(HAVE_GETRANDOM, 1,
+              [Define to 1 if the getrandom() function is available])
+fi
+
 # generate output files
 AC_CONFIG_FILES(Makefile.pre Modules/Setup.config Misc/python.pc Misc/python-config.sh)
 AC_CONFIG_FILES([Modules/ld_so_aix], [chmod +x Modules/ld_so_aix])
diff --git a/pyconfig.h.in b/pyconfig.h.in
--- a/pyconfig.h.in
+++ b/pyconfig.h.in
@@ -395,6 +395,9 @@
 /* Define to 1 if you have the `getpwent' function. */
 #undef HAVE_GETPWENT
 
+/* Define to 1 if the getrandom() function is available */
+#undef HAVE_GETRANDOM
+
 /* Define to 1 if the Linux getrandom() syscall is available */
 #undef HAVE_GETRANDOM_SYSCALL
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  1 10:02:00 2015
From: python-checkins at python.org (victor.stinner)
Date: Thu, 01 Oct 2015 08:02:00 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzI1MDAz?=
 =?utf-8?q?=3A_os=2Eurandom=28=29_doesn=27t_use_getentropy=28=29_on_Solari?=
 =?utf-8?q?s_because?=
Message-ID: <20151001080200.3652.15402@psf.io>

https://hg.python.org/cpython/rev/202c827f86df
changeset:   98454:202c827f86df
branch:      2.7
parent:      98411:8274fc521e69
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Oct 01 09:57:26 2015 +0200
summary:
  Issue #25003: os.urandom() doesn't use getentropy() on Solaris because
getentropy() is blocking, whereas os.urandom() should not block. getentropy()
is supported since Solaris 11.3.

files:
  Misc/NEWS       |   4 ++++
  Python/random.c |  12 ++++++++----
  2 files changed, 12 insertions(+), 4 deletions(-)


diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,10 @@
 Core and Builtins
 -----------------
 
+- Issue #25003: os.urandom() doesn't use getentropy() on Solaris because
+  getentropy() is blocking, whereas os.urandom() should not block. getentropy()
+  is supported since Solaris 11.3.
+
 - Issue #21167: NAN operations are now handled correctly when python is
   compiled with ICC even if -fp-model strict is not specified.
 
diff --git a/Python/random.c b/Python/random.c
--- a/Python/random.c
+++ b/Python/random.c
@@ -93,7 +93,11 @@
     return 0;
 }
 
-#elif HAVE_GETENTROPY
+/* Issue #25003: Don' use getentropy() on Solaris (available since
+ * Solaris 11.3), it is blocking whereas os.urandom() should not block. */
+#elif defined(HAVE_GETENTROPY) && !defined(sun)
+#define PY_GETENTROPY 1
+
 /* Fill buffer with size pseudo-random bytes generated by getentropy().
    Return 0 on success, or raise an exception and return -1 on error.
    If fatal is nonzero, call Py_FatalError() instead of raising an exception
@@ -333,7 +337,7 @@
 
 #ifdef MS_WINDOWS
     return win32_urandom((unsigned char *)buffer, size, 1);
-#elif HAVE_GETENTROPY
+#elif defined(PY_GETENTROPY)
     return py_getentropy(buffer, size, 0);
 #else
 # ifdef __VMS
@@ -396,7 +400,7 @@
         (void)win32_urandom((unsigned char *)secret, secret_size, 0);
 #elif __VMS
         vms_urandom((unsigned char *)secret, secret_size, 0);
-#elif HAVE_GETENTROPY
+#elif defined(PY_GETENTROPY)
         (void)py_getentropy(secret, secret_size, 1);
 #else
         dev_urandom_noraise(secret, secret_size);
@@ -412,7 +416,7 @@
         CryptReleaseContext(hCryptProv, 0);
         hCryptProv = 0;
     }
-#elif HAVE_GETENTROPY
+#elif defined(PY_GETENTROPY)
     /* nothing to clean */
 #else
     dev_urandom_close();

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  1 10:02:01 2015
From: python-checkins at python.org (victor.stinner)
Date: Thu, 01 Oct 2015 08:02:01 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
	=?utf-8?b?KTogTWVyZ2UgMy41?=
Message-ID: <20151001080201.115149.13299@psf.io>

https://hg.python.org/cpython/rev/e885f3f00256
changeset:   98457:e885f3f00256
parent:      98453:710ef035ee44
parent:      98456:8165c7460596
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Oct 01 10:01:31 2015 +0200
summary:
  Merge 3.5

files:
  Python/random.c |  2 ++
  1 files changed, 2 insertions(+), 0 deletions(-)


diff --git a/Python/random.c b/Python/random.c
--- a/Python/random.c
+++ b/Python/random.c
@@ -111,6 +111,8 @@
 
 #else
 
+/* Issue #25003: Don' use getentropy() on Solaris (available since
+ * Solaris 11.3), it is blocking whereas os.urandom() should not block. */
 #if defined(HAVE_GETRANDOM) || defined(HAVE_GETRANDOM_SYSCALL)
 #define PY_GETRANDOM 1
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  1 10:02:01 2015
From: python-checkins at python.org (victor.stinner)
Date: Thu, 01 Oct 2015 08:02:01 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzI1MDAz?=
 =?utf-8?q?=3A_os=2Eurandom=28=29_doesn=27t_use_getentropy=28=29_on_Solari?=
 =?utf-8?q?s_because?=
Message-ID: <20151001080200.82662.7817@psf.io>

https://hg.python.org/cpython/rev/83dc79eeaf7f
changeset:   98455:83dc79eeaf7f
branch:      3.4
parent:      98449:60c4fd84ef92
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Oct 01 09:59:32 2015 +0200
summary:
  Issue #25003: os.urandom() doesn't use getentropy() on Solaris because
getentropy() is blocking, whereas os.urandom() should not block. getentropy()
is supported since Solaris 11.3.

files:
  Misc/NEWS       |   4 ++++
  Python/random.c |  12 ++++++++----
  2 files changed, 12 insertions(+), 4 deletions(-)


diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,10 @@
 Core and Builtins
 -----------------
 
+- Issue #25003: os.urandom() doesn't use getentropy() on Solaris because
+  getentropy() is blocking, whereas os.urandom() should not block. getentropy()
+  is supported since Solaris 11.3.
+
 - Issue #25182: The stdprinter (used as sys.stderr before the io module is
   imported at startup) now uses the backslashreplace error handler.
 
diff --git a/Python/random.c b/Python/random.c
--- a/Python/random.c
+++ b/Python/random.c
@@ -67,7 +67,11 @@
     return 0;
 }
 
-#elif HAVE_GETENTROPY
+/* Issue #25003: Don' use getentropy() on Solaris (available since
+ * Solaris 11.3), it is blocking whereas os.urandom() should not block. */
+#elif defined(HAVE_GETENTROPY) && !defined(sun)
+#define PY_GETENTROPY 1
+
 /* Fill buffer with size pseudo-random bytes generated by getentropy().
    Return 0 on success, or raise an exception and return -1 on error.
 
@@ -275,7 +279,7 @@
 
 #ifdef MS_WINDOWS
     return win32_urandom((unsigned char *)buffer, size, 1);
-#elif HAVE_GETENTROPY
+#elif defined(PY_GETENTROPY)
     return py_getentropy(buffer, size, 0);
 #else
     return dev_urandom_python((char*)buffer, size);
@@ -322,7 +326,7 @@
     else {
 #ifdef MS_WINDOWS
         (void)win32_urandom(secret, secret_size, 0);
-#elif HAVE_GETENTROPY
+#elif defined(PY_GETENTROPY)
         (void)py_getentropy(secret, secret_size, 1);
 #else
         dev_urandom_noraise(secret, secret_size);
@@ -338,7 +342,7 @@
         CryptReleaseContext(hCryptProv, 0);
         hCryptProv = 0;
     }
-#elif HAVE_GETENTROPY
+#elif defined(PY_GETENTROPY)
     /* nothing to clean */
 #else
     dev_urandom_close();

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  1 10:02:01 2015
From: python-checkins at python.org (victor.stinner)
Date: Thu, 01 Oct 2015 08:02:01 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
 =?utf-8?q?_Merge_3=2E4_=28os=2Eurandom=29?=
Message-ID: <20151001080200.81627.19030@psf.io>

https://hg.python.org/cpython/rev/8165c7460596
changeset:   98456:8165c7460596
branch:      3.5
parent:      98452:835085cc28cd
parent:      98455:83dc79eeaf7f
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Oct 01 10:00:23 2015 +0200
summary:
  Merge 3.4 (os.urandom)

files:
  Python/random.c |  2 ++
  1 files changed, 2 insertions(+), 0 deletions(-)


diff --git a/Python/random.c b/Python/random.c
--- a/Python/random.c
+++ b/Python/random.c
@@ -111,6 +111,8 @@
 
 #else
 
+/* Issue #25003: Don' use getentropy() on Solaris (available since
+ * Solaris 11.3), it is blocking whereas os.urandom() should not block. */
 #if defined(HAVE_GETRANDOM) || defined(HAVE_GETRANDOM_SYSCALL)
 #define PY_GETRANDOM 1
 

-- 
Repository URL: https://hg.python.org/cpython

From solipsis at pitrou.net  Thu Oct  1 10:45:15 2015
From: solipsis at pitrou.net (solipsis at pitrou.net)
Date: Thu, 01 Oct 2015 08:45:15 +0000
Subject: [Python-checkins] Daily reference leaks (0f10e0b3e76d): sum=17877
Message-ID: <20151001084515.94137.1702@psf.io>

results for 0f10e0b3e76d on branch "default"
--------------------------------------------

test_capi leaked [1598, 1598, 1598] references, sum=4794
test_capi leaked [387, 389, 389] memory blocks, sum=1165
test_functools leaked [0, 2, 2] memory blocks, sum=4
test_threading leaked [3196, 3196, 3196] references, sum=9588
test_threading leaked [774, 776, 776] memory blocks, sum=2326


Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogZKhs_L', '--timeout', '7200']

From python-checkins at python.org  Thu Oct  1 10:57:04 2015
From: python-checkins at python.org (serhiy.storchaka)
Date: Thu, 01 Oct 2015 08:57:04 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2325280=3A_Import_trace_messages_emitted_in_verbo?=
 =?utf-8?q?se_=28-v=29_mode_are_no?=
Message-ID: <20151001085704.94127.94122@psf.io>

https://hg.python.org/cpython/rev/e377d568928b
changeset:   98460:e377d568928b
parent:      98457:e885f3f00256
parent:      98459:10c13441bf8d
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Thu Oct 01 11:55:52 2015 +0300
summary:
  Issue #25280: Import trace messages emitted in verbose (-v) mode are no
longer formatted twice.

files:
  Lib/importlib/_bootstrap_external.py |  8 ++++----
  Misc/NEWS                            |  3 +++
  2 files changed, 7 insertions(+), 4 deletions(-)


diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
--- a/Lib/importlib/_bootstrap_external.py
+++ b/Lib/importlib/_bootstrap_external.py
@@ -429,15 +429,15 @@
     raw_size = data[8:12]
     if magic != MAGIC_NUMBER:
         message = 'bad magic number in {!r}: {!r}'.format(name, magic)
-        _bootstrap._verbose_message(message)
+        _bootstrap._verbose_message('{}', message)
         raise ImportError(message, **exc_details)
     elif len(raw_timestamp) != 4:
         message = 'reached EOF while reading timestamp in {!r}'.format(name)
-        _bootstrap._verbose_message(message)
+        _bootstrap._verbose_message('{}', message)
         raise EOFError(message)
     elif len(raw_size) != 4:
         message = 'reached EOF while reading size of source in {!r}'.format(name)
-        _bootstrap._verbose_message(message)
+        _bootstrap._verbose_message('{}', message)
         raise EOFError(message)
     if source_stats is not None:
         try:
@@ -447,7 +447,7 @@
         else:
             if _r_long(raw_timestamp) != source_mtime:
                 message = 'bytecode is stale for {!r}'.format(name)
-                _bootstrap._verbose_message(message)
+                _bootstrap._verbose_message('{}', message)
                 raise ImportError(message, **exc_details)
         try:
             source_size = source_stats['size'] & 0xFFFFFFFF
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,9 @@
 Core and Builtins
 -----------------
 
+- Issue #25280: Import trace messages emitted in verbose (-v) mode are no
+  longer formatted twice.
+
 - Issue #25227: Optimize ASCII and latin1 encoders with the ``surrogateescape``
   error handler: the encoders are now up to 3 times as fast. Initial patch
   written by Serhiy Storchaka.

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  1 10:57:04 2015
From: python-checkins at python.org (serhiy.storchaka)
Date: Thu, 01 Oct 2015 08:57:04 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
 =?utf-8?q?_Issue_=2325280=3A_Import_trace_messages_emitted_in_verbose_=28?=
 =?utf-8?q?-v=29_mode_are_no?=
Message-ID: <20151001085704.11706.18971@psf.io>

https://hg.python.org/cpython/rev/10c13441bf8d
changeset:   98459:10c13441bf8d
branch:      3.5
parent:      98456:8165c7460596
parent:      98458:da42b38f7470
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Thu Oct 01 11:40:22 2015 +0300
summary:
  Issue #25280: Import trace messages emitted in verbose (-v) mode are no
longer formatted twice.

files:
  Lib/importlib/_bootstrap_external.py |    8 +-
  Misc/NEWS                            |    3 +
  Python/importlib_external.h          |  113 +++++++-------
  3 files changed, 64 insertions(+), 60 deletions(-)


diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
--- a/Lib/importlib/_bootstrap_external.py
+++ b/Lib/importlib/_bootstrap_external.py
@@ -437,15 +437,15 @@
     raw_size = data[8:12]
     if magic != MAGIC_NUMBER:
         message = 'bad magic number in {!r}: {!r}'.format(name, magic)
-        _verbose_message(message)
+        _verbose_message('{}', message)
         raise ImportError(message, **exc_details)
     elif len(raw_timestamp) != 4:
         message = 'reached EOF while reading timestamp in {!r}'.format(name)
-        _verbose_message(message)
+        _verbose_message('{}', message)
         raise EOFError(message)
     elif len(raw_size) != 4:
         message = 'reached EOF while reading size of source in {!r}'.format(name)
-        _verbose_message(message)
+        _verbose_message('{}', message)
         raise EOFError(message)
     if source_stats is not None:
         try:
@@ -455,7 +455,7 @@
         else:
             if _r_long(raw_timestamp) != source_mtime:
                 message = 'bytecode is stale for {!r}'.format(name)
-                _verbose_message(message)
+                _verbose_message('{}', message)
                 raise ImportError(message, **exc_details)
         try:
             source_size = source_stats['size'] & 0xFFFFFFFF
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -11,6 +11,9 @@
 Core and Builtins
 -----------------
 
+- Issue #25280: Import trace messages emitted in verbose (-v) mode are no
+  longer formatted twice.
+
 - Issue #25003: On Solaris 11.3 or newer, os.urandom() now uses the
   getrandom() function instead of the getentropy() function. The getentropy()
   function is blocking to generate very good quality entropy, os.urandom()
diff --git a/Python/importlib_external.h b/Python/importlib_external.h
--- a/Python/importlib_external.h
+++ b/Python/importlib_external.h
@@ -635,7 +635,7 @@
     95,102,105,110,100,95,109,111,100,117,108,101,95,115,104,105,
     109,143,1,0,0,115,10,0,0,0,0,10,21,1,24,1,
     6,1,29,1,114,130,0,0,0,99,4,0,0,0,0,0,
-    0,0,11,0,0,0,19,0,0,0,67,0,0,0,115,228,
+    0,0,11,0,0,0,19,0,0,0,67,0,0,0,115,240,
     1,0,0,105,0,0,125,4,0,124,2,0,100,1,0,107,
     9,0,114,31,0,124,2,0,124,4,0,100,2,0,60,110,
     6,0,100,3,0,125,2,0,124,3,0,100,1,0,107,9,
@@ -643,58 +643,59 @@
     0,100,1,0,100,5,0,133,2,0,25,125,5,0,124,0,
     0,100,5,0,100,6,0,133,2,0,25,125,6,0,124,0,
     0,100,6,0,100,7,0,133,2,0,25,125,7,0,124,5,
-    0,116,0,0,107,3,0,114,165,0,100,8,0,106,1,0,
-    124,2,0,124,5,0,131,2,0,125,8,0,116,2,0,124,
-    8,0,131,1,0,1,116,3,0,124,8,0,124,4,0,141,
-    1,0,130,1,0,110,113,0,116,4,0,124,6,0,131,1,
-    0,100,5,0,107,3,0,114,223,0,100,9,0,106,1,0,
-    124,2,0,131,1,0,125,8,0,116,2,0,124,8,0,131,
-    1,0,1,116,5,0,124,8,0,131,1,0,130,1,0,110,
-    55,0,116,4,0,124,7,0,131,1,0,100,5,0,107,3,
-    0,114,22,1,100,10,0,106,1,0,124,2,0,131,1,0,
-    125,8,0,116,2,0,124,8,0,131,1,0,1,116,5,0,
-    124,8,0,131,1,0,130,1,0,124,1,0,100,1,0,107,
-    9,0,114,214,1,121,20,0,116,6,0,124,1,0,100,11,
-    0,25,131,1,0,125,9,0,87,110,18,0,4,116,7,0,
-    107,10,0,114,74,1,1,1,1,89,110,59,0,88,116,8,
-    0,124,6,0,131,1,0,124,9,0,107,3,0,114,133,1,
-    100,12,0,106,1,0,124,2,0,131,1,0,125,8,0,116,
-    2,0,124,8,0,131,1,0,1,116,3,0,124,8,0,124,
-    4,0,141,1,0,130,1,0,121,18,0,124,1,0,100,13,
-    0,25,100,14,0,64,125,10,0,87,110,18,0,4,116,7,
-    0,107,10,0,114,171,1,1,1,1,89,110,43,0,88,116,
-    8,0,124,7,0,131,1,0,124,10,0,107,3,0,114,214,
-    1,116,3,0,100,12,0,106,1,0,124,2,0,131,1,0,
-    124,4,0,141,1,0,130,1,0,124,0,0,100,7,0,100,
-    1,0,133,2,0,25,83,41,15,97,122,1,0,0,86,97,
-    108,105,100,97,116,101,32,116,104,101,32,104,101,97,100,101,
-    114,32,111,102,32,116,104,101,32,112,97,115,115,101,100,45,
-    105,110,32,98,121,116,101,99,111,100,101,32,97,103,97,105,
-    110,115,116,32,115,111,117,114,99,101,95,115,116,97,116,115,
-    32,40,105,102,10,32,32,32,32,103,105,118,101,110,41,32,
-    97,110,100,32,114,101,116,117,114,110,105,110,103,32,116,104,
-    101,32,98,121,116,101,99,111,100,101,32,116,104,97,116,32,
-    99,97,110,32,98,101,32,99,111,109,112,105,108,101,100,32,
-    98,121,32,99,111,109,112,105,108,101,40,41,46,10,10,32,
-    32,32,32,65,108,108,32,111,116,104,101,114,32,97,114,103,
-    117,109,101,110,116,115,32,97,114,101,32,117,115,101,100,32,
-    116,111,32,101,110,104,97,110,99,101,32,101,114,114,111,114,
-    32,114,101,112,111,114,116,105,110,103,46,10,10,32,32,32,
-    32,73,109,112,111,114,116,69,114,114,111,114,32,105,115,32,
-    114,97,105,115,101,100,32,119,104,101,110,32,116,104,101,32,
-    109,97,103,105,99,32,110,117,109,98,101,114,32,105,115,32,
-    105,110,99,111,114,114,101,99,116,32,111,114,32,116,104,101,
-    32,98,121,116,101,99,111,100,101,32,105,115,10,32,32,32,
-    32,102,111,117,110,100,32,116,111,32,98,101,32,115,116,97,
-    108,101,46,32,69,79,70,69,114,114,111,114,32,105,115,32,
-    114,97,105,115,101,100,32,119,104,101,110,32,116,104,101,32,
-    100,97,116,97,32,105,115,32,102,111,117,110,100,32,116,111,
-    32,98,101,10,32,32,32,32,116,114,117,110,99,97,116,101,
-    100,46,10,10,32,32,32,32,78,114,106,0,0,0,122,10,
-    60,98,121,116,101,99,111,100,101,62,114,35,0,0,0,114,
-    12,0,0,0,233,8,0,0,0,233,12,0,0,0,122,30,
-    98,97,100,32,109,97,103,105,99,32,110,117,109,98,101,114,
-    32,105,110,32,123,33,114,125,58,32,123,33,114,125,122,43,
+    0,116,0,0,107,3,0,114,168,0,100,8,0,106,1,0,
+    124,2,0,124,5,0,131,2,0,125,8,0,116,2,0,100,
+    9,0,124,8,0,131,2,0,1,116,3,0,124,8,0,124,
+    4,0,141,1,0,130,1,0,110,119,0,116,4,0,124,6,
+    0,131,1,0,100,5,0,107,3,0,114,229,0,100,10,0,
+    106,1,0,124,2,0,131,1,0,125,8,0,116,2,0,100,
+    9,0,124,8,0,131,2,0,1,116,5,0,124,8,0,131,
+    1,0,130,1,0,110,58,0,116,4,0,124,7,0,131,1,
+    0,100,5,0,107,3,0,114,31,1,100,11,0,106,1,0,
+    124,2,0,131,1,0,125,8,0,116,2,0,100,9,0,124,
+    8,0,131,2,0,1,116,5,0,124,8,0,131,1,0,130,
+    1,0,124,1,0,100,1,0,107,9,0,114,226,1,121,20,
+    0,116,6,0,124,1,0,100,12,0,25,131,1,0,125,9,
+    0,87,110,18,0,4,116,7,0,107,10,0,114,83,1,1,
+    1,1,89,110,62,0,88,116,8,0,124,6,0,131,1,0,
+    124,9,0,107,3,0,114,145,1,100,13,0,106,1,0,124,
+    2,0,131,1,0,125,8,0,116,2,0,100,9,0,124,8,
+    0,131,2,0,1,116,3,0,124,8,0,124,4,0,141,1,
+    0,130,1,0,121,18,0,124,1,0,100,14,0,25,100,15,
+    0,64,125,10,0,87,110,18,0,4,116,7,0,107,10,0,
+    114,183,1,1,1,1,89,110,43,0,88,116,8,0,124,7,
+    0,131,1,0,124,10,0,107,3,0,114,226,1,116,3,0,
+    100,13,0,106,1,0,124,2,0,131,1,0,124,4,0,141,
+    1,0,130,1,0,124,0,0,100,7,0,100,1,0,133,2,
+    0,25,83,41,16,97,122,1,0,0,86,97,108,105,100,97,
+    116,101,32,116,104,101,32,104,101,97,100,101,114,32,111,102,
+    32,116,104,101,32,112,97,115,115,101,100,45,105,110,32,98,
+    121,116,101,99,111,100,101,32,97,103,97,105,110,115,116,32,
+    115,111,117,114,99,101,95,115,116,97,116,115,32,40,105,102,
+    10,32,32,32,32,103,105,118,101,110,41,32,97,110,100,32,
+    114,101,116,117,114,110,105,110,103,32,116,104,101,32,98,121,
+    116,101,99,111,100,101,32,116,104,97,116,32,99,97,110,32,
+    98,101,32,99,111,109,112,105,108,101,100,32,98,121,32,99,
+    111,109,112,105,108,101,40,41,46,10,10,32,32,32,32,65,
+    108,108,32,111,116,104,101,114,32,97,114,103,117,109,101,110,
+    116,115,32,97,114,101,32,117,115,101,100,32,116,111,32,101,
+    110,104,97,110,99,101,32,101,114,114,111,114,32,114,101,112,
+    111,114,116,105,110,103,46,10,10,32,32,32,32,73,109,112,
+    111,114,116,69,114,114,111,114,32,105,115,32,114,97,105,115,
+    101,100,32,119,104,101,110,32,116,104,101,32,109,97,103,105,
+    99,32,110,117,109,98,101,114,32,105,115,32,105,110,99,111,
+    114,114,101,99,116,32,111,114,32,116,104,101,32,98,121,116,
+    101,99,111,100,101,32,105,115,10,32,32,32,32,102,111,117,
+    110,100,32,116,111,32,98,101,32,115,116,97,108,101,46,32,
+    69,79,70,69,114,114,111,114,32,105,115,32,114,97,105,115,
+    101,100,32,119,104,101,110,32,116,104,101,32,100,97,116,97,
+    32,105,115,32,102,111,117,110,100,32,116,111,32,98,101,10,
+    32,32,32,32,116,114,117,110,99,97,116,101,100,46,10,10,
+    32,32,32,32,78,114,106,0,0,0,122,10,60,98,121,116,
+    101,99,111,100,101,62,114,35,0,0,0,114,12,0,0,0,
+    233,8,0,0,0,233,12,0,0,0,122,30,98,97,100,32,
+    109,97,103,105,99,32,110,117,109,98,101,114,32,105,110,32,
+    123,33,114,125,58,32,123,33,114,125,122,2,123,125,122,43,
     114,101,97,99,104,101,100,32,69,79,70,32,119,104,105,108,
     101,32,114,101,97,100,105,110,103,32,116,105,109,101,115,116,
     97,109,112,32,105,110,32,123,33,114,125,122,48,114,101,97,
@@ -719,9 +720,9 @@
     95,118,97,108,105,100,97,116,101,95,98,121,116,101,99,111,
     100,101,95,104,101,97,100,101,114,160,1,0,0,115,76,0,
     0,0,0,11,6,1,12,1,13,3,6,1,12,1,10,1,
-    16,1,16,1,16,1,12,1,18,1,10,1,18,1,18,1,
-    15,1,10,1,15,1,18,1,15,1,10,1,12,1,12,1,
-    3,1,20,1,13,1,5,2,18,1,15,1,10,1,15,1,
+    16,1,16,1,16,1,12,1,18,1,13,1,18,1,18,1,
+    15,1,13,1,15,1,18,1,15,1,13,1,12,1,12,1,
+    3,1,20,1,13,1,5,2,18,1,15,1,13,1,15,1,
     3,1,18,1,13,1,5,2,18,1,15,1,9,1,114,141,
     0,0,0,99,4,0,0,0,0,0,0,0,5,0,0,0,
     6,0,0,0,67,0,0,0,115,112,0,0,0,116,0,0,

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  1 10:57:05 2015
From: python-checkins at python.org (serhiy.storchaka)
Date: Thu, 01 Oct 2015 08:57:05 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzI1Mjgw?=
 =?utf-8?q?=3A_Import_trace_messages_emitted_in_verbose_=28-v=29_mode_are_?=
 =?utf-8?q?no?=
Message-ID: <20151001085703.11710.40499@psf.io>

https://hg.python.org/cpython/rev/da42b38f7470
changeset:   98458:da42b38f7470
branch:      3.4
parent:      98455:83dc79eeaf7f
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Thu Oct 01 11:08:50 2015 +0300
summary:
  Issue #25280: Import trace messages emitted in verbose (-v) mode are no
longer formatted twice.

files:
  Lib/importlib/_bootstrap.py |    8 +-
  Misc/NEWS                   |    3 +
  Python/importlib.h          |  115 ++++++++++++-----------
  3 files changed, 65 insertions(+), 61 deletions(-)


diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py
--- a/Lib/importlib/_bootstrap.py
+++ b/Lib/importlib/_bootstrap.py
@@ -620,15 +620,15 @@
     raw_size = data[8:12]
     if magic != MAGIC_NUMBER:
         message = 'bad magic number in {!r}: {!r}'.format(name, magic)
-        _verbose_message(message)
+        _verbose_message('{}', message)
         raise ImportError(message, **exc_details)
     elif len(raw_timestamp) != 4:
         message = 'reached EOF while reading timestamp in {!r}'.format(name)
-        _verbose_message(message)
+        _verbose_message('{}', message)
         raise EOFError(message)
     elif len(raw_size) != 4:
         message = 'reached EOF while reading size of source in {!r}'.format(name)
-        _verbose_message(message)
+        _verbose_message('{}', message)
         raise EOFError(message)
     if source_stats is not None:
         try:
@@ -638,7 +638,7 @@
         else:
             if _r_long(raw_timestamp) != source_mtime:
                 message = 'bytecode is stale for {!r}'.format(name)
-                _verbose_message(message)
+                _verbose_message('{}', message)
                 raise ImportError(message, **exc_details)
         try:
             source_size = source_stats['size'] & 0xFFFFFFFF
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,9 @@
 Core and Builtins
 -----------------
 
+- Issue #25280: Import trace messages emitted in verbose (-v) mode are no
+  longer formatted twice.
+
 - Issue #25003: os.urandom() doesn't use getentropy() on Solaris because
   getentropy() is blocking, whereas os.urandom() should not block. getentropy()
   is supported since Solaris 11.3.
diff --git a/Python/importlib.h b/Python/importlib.h
--- a/Python/importlib.h
+++ b/Python/importlib.h
[stripped]

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  1 12:28:49 2015
From: python-checkins at python.org (vinay.sajip)
Date: Thu, 01 Oct 2015 10:28:49 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogQ2xvc2VzICMyNTE4?=
 =?utf-8?q?5=3A_Use_UTF-8_encoding_when_reading_pyvenv=2Ecfg=2E?=
Message-ID: <20151001102849.98350.41256@psf.io>

https://hg.python.org/cpython/rev/d927c6cae05f
changeset:   98461:d927c6cae05f
branch:      3.4
parent:      98458:da42b38f7470
user:        Vinay Sajip <vinay_sajip at yahoo.co.uk>
date:        Thu Oct 01 11:27:00 2015 +0100
summary:
  Closes #25185: Use UTF-8 encoding when reading pyvenv.cfg.

files:
  Lib/site.py |  4 +++-
  1 files changed, 3 insertions(+), 1 deletions(-)


diff --git a/Lib/site.py b/Lib/site.py
--- a/Lib/site.py
+++ b/Lib/site.py
@@ -472,7 +472,9 @@
         config_line = re.compile(CONFIG_LINE)
         virtual_conf = candidate_confs[0]
         system_site = "true"
-        with open(virtual_conf) as f:
+        # Issue 25185: Use UTF-8, as that's what the venv module uses when
+        # writing the file.
+        with open(virtual_conf, encoding='utf-8') as f:
             for line in f:
                 line = line.strip()
                 m = config_line.match(line)

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  1 12:28:51 2015
From: python-checkins at python.org (vinay.sajip)
Date: Thu, 01 Oct 2015 10:28:51 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
 =?utf-8?q?=29=3A_Closes_=2325185=3A_merged_fix_from_3=2E5=2E?=
Message-ID: <20151001102849.98376.95963@psf.io>

https://hg.python.org/cpython/rev/69dd42cef190
changeset:   98463:69dd42cef190
parent:      98460:e377d568928b
parent:      98462:eaf9220bdee3
user:        Vinay Sajip <vinay_sajip at yahoo.co.uk>
date:        Thu Oct 01 11:28:34 2015 +0100
summary:
  Closes #25185: merged fix from 3.5.

files:
  Lib/site.py |  4 +++-
  1 files changed, 3 insertions(+), 1 deletions(-)


diff --git a/Lib/site.py b/Lib/site.py
--- a/Lib/site.py
+++ b/Lib/site.py
@@ -465,7 +465,9 @@
         config_line = re.compile(CONFIG_LINE)
         virtual_conf = candidate_confs[0]
         system_site = "true"
-        with open(virtual_conf) as f:
+        # Issue 25185: Use UTF-8, as that's what the venv module uses when
+        # writing the file.
+        with open(virtual_conf, encoding='utf-8') as f:
             for line in f:
                 line = line.strip()
                 m = config_line.match(line)

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  1 12:28:51 2015
From: python-checkins at python.org (vinay.sajip)
Date: Thu, 01 Oct 2015 10:28:51 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
 =?utf-8?q?_Closes_=2325185=3A_merged_fix_from_3=2E4=2E?=
Message-ID: <20151001102849.81633.31210@psf.io>

https://hg.python.org/cpython/rev/eaf9220bdee3
changeset:   98462:eaf9220bdee3
branch:      3.5
parent:      98459:10c13441bf8d
parent:      98461:d927c6cae05f
user:        Vinay Sajip <vinay_sajip at yahoo.co.uk>
date:        Thu Oct 01 11:27:57 2015 +0100
summary:
  Closes #25185: merged fix from 3.4.

files:
  Lib/site.py |  4 +++-
  1 files changed, 3 insertions(+), 1 deletions(-)


diff --git a/Lib/site.py b/Lib/site.py
--- a/Lib/site.py
+++ b/Lib/site.py
@@ -465,7 +465,9 @@
         config_line = re.compile(CONFIG_LINE)
         virtual_conf = candidate_confs[0]
         system_site = "true"
-        with open(virtual_conf) as f:
+        # Issue 25185: Use UTF-8, as that's what the venv module uses when
+        # writing the file.
+        with open(virtual_conf, encoding='utf-8') as f:
             for line in f:
                 line = line.strip()
                 m = config_line.match(line)

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  1 13:17:14 2015
From: python-checkins at python.org (victor.stinner)
Date: Thu, 01 Oct 2015 11:17:14 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2325277=3A_Set_a_ti?=
 =?utf-8?q?meout_of_10_minutes_in_test=5Feintr_using_faulthandler_to?=
Message-ID: <20151001111713.98374.93926@psf.io>

https://hg.python.org/cpython/rev/10efb1797e7b
changeset:   98464:10efb1797e7b
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Oct 01 13:16:43 2015 +0200
summary:
  Issue #25277: Set a timeout of 10 minutes in test_eintr using faulthandler to
try to debug a hang on the FreeBSD 9 buildbot.

Run also eintr_tester.py with python "-u" command line option to try to get the
full output on hang/crash.

files:
  Lib/test/eintrdata/eintr_tester.py |  5 +++++
  Lib/test/test_eintr.py             |  3 ++-
  2 files changed, 7 insertions(+), 1 deletions(-)


diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py
--- a/Lib/test/eintrdata/eintr_tester.py
+++ b/Lib/test/eintrdata/eintr_tester.py
@@ -9,6 +9,7 @@
 """
 
 import contextlib
+import faulthandler
 import io
 import os
 import select
@@ -50,6 +51,9 @@
         signal.setitimer(signal.ITIMER_REAL, cls.signal_delay,
                          cls.signal_period)
 
+        # Issue #25277: Use faulthandler to try to debug a hang on FreeBSD
+        faulthandler.dump_traceback_later(10 * 60, exit=True)
+
     @classmethod
     def stop_alarm(cls):
         signal.setitimer(signal.ITIMER_REAL, 0, 0)
@@ -58,6 +62,7 @@
     def tearDownClass(cls):
         cls.stop_alarm()
         signal.signal(signal.SIGALRM, cls.orig_handler)
+        faulthandler.cancel_dump_traceback_later()
 
     @classmethod
     def _sleep(cls):
diff --git a/Lib/test/test_eintr.py b/Lib/test/test_eintr.py
--- a/Lib/test/test_eintr.py
+++ b/Lib/test/test_eintr.py
@@ -16,7 +16,8 @@
         # Run the tester in a sub-process, to make sure there is only one
         # thread (for reliable signal delivery).
         tester = support.findfile("eintr_tester.py", subdir="eintrdata")
-        script_helper.assert_python_ok(tester)
+        # use -u to try to get the full output if the test hangs or crash
+        script_helper.assert_python_ok("-u", tester)
 
 
 if __name__ == "__main__":

-- 
Repository URL: https://hg.python.org/cpython

From lp_benchmark_robot at intel.com  Thu Oct  1 16:47:42 2015
From: lp_benchmark_robot at intel.com (lp_benchmark_robot at intel.com)
Date: Thu, 1 Oct 2015 15:47:42 +0100
Subject: [Python-checkins] Benchmark Results for Python Default 2015-10-01
Message-ID: <284e6abe-30cf-455f-8d9e-0581dcb6180b@irsmsx153.ger.corp.intel.com>

Results for project python_default-nightly, build date 2015-10-01 03:02:02 
commit:		0f10e0b3e76d2ec4958d1260eef91055a096d990
revision date:	2015-09-30 22:53:09 +0000
environment:	Haswell-EP
	cpu:	Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz 2x18 cores, stepping 2, LLC 45 MB
	mem:	128 GB
	os:	CentOS 7.1
	kernel:	Linux 3.10.0-229.4.2.el7.x86_64

Baseline results were generated using release v3.4.3, with hash
b4cbecbc0781e89a309d03b60a1f75f8499250e6 from 2015-02-25 12:15:33+00:00

------------------------------------------------------------------------------------------
               benchmark    relative      change since      change since  current rev with
                            std_dev*          last run            v3.4.3      regrtest PGO
------------------------------------------------------------------------------------------
:-)            django_v2    0.23373%          1.27877%          8.62931%         16.74727%
:-|              pybench    0.14534%         -0.15742%         -1.72834%          8.99633%
:-(             regex_v8    2.70946%          1.05918%         -3.98469%          3.95760%
:-|                nbody    0.13576%          0.92277%          0.13154%          7.82985%
:-|         json_dump_v2    0.20029%          0.43084%         -1.07688%          9.96645%
:-|       normal_startup    0.73680%         -0.17347%          0.43557%          4.99884%
------------------------------------------------------------------------------------------
Note: Benchmark results are measured in seconds.
* Relative Standard Deviation (Standard Deviation/Average)

Our lab does a nightly source pull and build of the Python project and measures
performance changes against the previous stable version and the previous nightly
measurement. This is provided as a service to the community so that quality
issues with current hardware can be identified quickly.

Intel technologies' features and benefits depend on system configuration and may
require enabled hardware, software or service activation. Performance varies
depending on system configuration.


From lp_benchmark_robot at intel.com  Thu Oct  1 16:48:09 2015
From: lp_benchmark_robot at intel.com (lp_benchmark_robot at intel.com)
Date: Thu, 1 Oct 2015 15:48:09 +0100
Subject: [Python-checkins] Benchmark Results for Python 2.7 2015-10-01
Message-ID: <1da2a71b-25ce-47da-b41e-10cbe2cdb2d9@irsmsx153.ger.corp.intel.com>

No new revisions. Here are the previous results:

Results for project python_2.7-nightly, build date 2015-10-01 03:44:50 
commit:		8274fc521e69cb6baf2932ffd3d81bef4a45b5e8
revision date:	2015-09-29 20:51:27 +0000
environment:	Haswell-EP
	cpu:	Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz 2x18 cores, stepping 2, LLC 45 MB
	mem:	128 GB
	os:	CentOS 7.1
	kernel:	Linux 3.10.0-229.4.2.el7.x86_64

Baseline results were generated using release v2.7.10, with hash
15c95b7d81dcf821daade360741e00714667653f from 2015-05-23 16:02:14+00:00

------------------------------------------------------------------------------------------
               benchmark    relative      change since      change since  current rev with
                            std_dev*          last run           v2.7.10      regrtest PGO
------------------------------------------------------------------------------------------
:-)            django_v2    0.37820%         -0.19226%          4.02348%          9.24847%
:-)              pybench    0.18635%         -0.18920%          6.61514%          7.24446%
:-|             regex_v8    1.06512%         -0.76123%         -1.89789%          8.19914%
:-)                nbody    0.11606%         -0.51994%          8.62419%          3.90676%
:-)         json_dump_v2    0.28493%         -0.97989%          3.44541%         12.26374%
:-|       normal_startup    1.57922%          0.15100%         -1.28399%          2.33192%
:-|              ssbench    0.43283%          0.56654%          1.58202%          2.52947%
------------------------------------------------------------------------------------------
Note: Benchmark results for ssbench are measured in requests/second while all
other are measured in seconds.
* Relative Standard Deviation (Standard Deviation/Average)

Our lab does a nightly source pull and build of the Python project and measures
performance changes against the previous stable version and the previous nightly
measurement. This is provided as a service to the community so that quality
issues with current hardware can be identified quickly.

Intel technologies' features and benefits depend on system configuration and may
require enabled hardware, software or service activation. Performance varies
depending on system configuration.


From python-checkins at python.org  Thu Oct  1 21:39:51 2015
From: python-checkins at python.org (vinay.sajip)
Date: Thu, 01 Oct 2015 19:39:51 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogRml4ZXMgIzI1MDk3?=
 =?utf-8?q?=3A_Windows_test_is_skipped_if_there_are_insufficient_privilege?=
 =?utf-8?q?s=2C?=
Message-ID: <20151001193949.81647.28965@psf.io>

https://hg.python.org/cpython/rev/72c57c120c19
changeset:   98465:72c57c120c19
branch:      3.4
parent:      98461:d927c6cae05f
user:        Vinay Sajip <vinay_sajip at yahoo.co.uk>
date:        Thu Oct 01 20:37:54 2015 +0100
summary:
  Fixes #25097: Windows test is skipped if there are insufficient privileges, rather than failing.

files:
  Lib/test/test_logging.py |  20 +++++++++++---------
  1 files changed, 11 insertions(+), 9 deletions(-)


diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py
--- a/Lib/test/test_logging.py
+++ b/Lib/test/test_logging.py
@@ -65,14 +65,10 @@
 except ImportError:
     threading = None
 try:
-    import win32evtlog
+    import win32evtlog, win32evtlogutil, pywintypes
 except ImportError:
-    win32evtlog = None
-try:
-    import win32evtlogutil
-except ImportError:
-    win32evtlogutil = None
-    win32evtlog = None
+    win32evtlog = win32evtlogutil = pywintypes = None
+
 try:
     import zlib
 except ImportError:
@@ -4098,13 +4094,19 @@
     setattr(TimedRotatingFileHandlerTest, "test_compute_rollover_%s" % when, test_compute_rollover)
 
 
- at unittest.skipUnless(win32evtlog, 'win32evtlog/win32evtlogutil required for this test.')
+ at unittest.skipUnless(win32evtlog, 'win32evtlog/win32evtlogutil/pywintypes required for this test.')
 class NTEventLogHandlerTest(BaseTest):
     def test_basic(self):
         logtype = 'Application'
         elh = win32evtlog.OpenEventLog(None, logtype)
         num_recs = win32evtlog.GetNumberOfEventLogRecords(elh)
-        h = logging.handlers.NTEventLogHandler('test_logging')
+
+        try:
+            h = logging.handlers.NTEventLogHandler('test_logging')
+        except pywintypes.error as e:
+            if e[0] == 5:  # access denied
+                raise unittest.SkipTest('Insufficient privileges to run test')
+
         r = logging.makeLogRecord({'msg': 'Test Log Message'})
         h.handle(r)
         h.close()

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  1 21:39:51 2015
From: python-checkins at python.org (vinay.sajip)
Date: Thu, 01 Oct 2015 19:39:51 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
 =?utf-8?q?_Fixes_=2325097=3A_Merged_fi_from_3=2E4=2E?=
Message-ID: <20151001193949.82638.92770@psf.io>

https://hg.python.org/cpython/rev/b54528d8d8c3
changeset:   98466:b54528d8d8c3
branch:      3.5
parent:      98462:eaf9220bdee3
parent:      98465:72c57c120c19
user:        Vinay Sajip <vinay_sajip at yahoo.co.uk>
date:        Thu Oct 01 20:38:53 2015 +0100
summary:
  Fixes #25097: Merged fi from 3.4.

files:
  Lib/test/test_logging.py |  20 +++++++++++---------
  1 files changed, 11 insertions(+), 9 deletions(-)


diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py
--- a/Lib/test/test_logging.py
+++ b/Lib/test/test_logging.py
@@ -58,14 +58,10 @@
 except ImportError:
     threading = None
 try:
-    import win32evtlog
+    import win32evtlog, win32evtlogutil, pywintypes
 except ImportError:
-    win32evtlog = None
-try:
-    import win32evtlogutil
-except ImportError:
-    win32evtlogutil = None
-    win32evtlog = None
+    win32evtlog = win32evtlogutil = pywintypes = None
+
 try:
     import zlib
 except ImportError:
@@ -4128,13 +4124,19 @@
     setattr(TimedRotatingFileHandlerTest, "test_compute_rollover_%s" % when, test_compute_rollover)
 
 
- at unittest.skipUnless(win32evtlog, 'win32evtlog/win32evtlogutil required for this test.')
+ at unittest.skipUnless(win32evtlog, 'win32evtlog/win32evtlogutil/pywintypes required for this test.')
 class NTEventLogHandlerTest(BaseTest):
     def test_basic(self):
         logtype = 'Application'
         elh = win32evtlog.OpenEventLog(None, logtype)
         num_recs = win32evtlog.GetNumberOfEventLogRecords(elh)
-        h = logging.handlers.NTEventLogHandler('test_logging')
+
+        try:
+            h = logging.handlers.NTEventLogHandler('test_logging')
+        except pywintypes.error as e:
+            if e[0] == 5:  # access denied
+                raise unittest.SkipTest('Insufficient privileges to run test')
+
         r = logging.makeLogRecord({'msg': 'Test Log Message'})
         h.handle(r)
         h.close()

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  1 21:39:52 2015
From: python-checkins at python.org (vinay.sajip)
Date: Thu, 01 Oct 2015 19:39:52 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
 =?utf-8?q?=29=3A_Fixes_=2325097=3A_Merged_fix_from_3=2E5=2E?=
Message-ID: <20151001193949.9951.91082@psf.io>

https://hg.python.org/cpython/rev/757baaedc043
changeset:   98467:757baaedc043
parent:      98464:10efb1797e7b
parent:      98466:b54528d8d8c3
user:        Vinay Sajip <vinay_sajip at yahoo.co.uk>
date:        Thu Oct 01 20:39:30 2015 +0100
summary:
  Fixes #25097: Merged fix from 3.5.

files:
  Lib/test/test_logging.py |  20 +++++++++++---------
  1 files changed, 11 insertions(+), 9 deletions(-)


diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py
--- a/Lib/test/test_logging.py
+++ b/Lib/test/test_logging.py
@@ -58,14 +58,10 @@
 except ImportError:
     threading = None
 try:
-    import win32evtlog
+    import win32evtlog, win32evtlogutil, pywintypes
 except ImportError:
-    win32evtlog = None
-try:
-    import win32evtlogutil
-except ImportError:
-    win32evtlogutil = None
-    win32evtlog = None
+    win32evtlog = win32evtlogutil = pywintypes = None
+
 try:
     import zlib
 except ImportError:
@@ -4128,13 +4124,19 @@
     setattr(TimedRotatingFileHandlerTest, "test_compute_rollover_%s" % when, test_compute_rollover)
 
 
- at unittest.skipUnless(win32evtlog, 'win32evtlog/win32evtlogutil required for this test.')
+ at unittest.skipUnless(win32evtlog, 'win32evtlog/win32evtlogutil/pywintypes required for this test.')
 class NTEventLogHandlerTest(BaseTest):
     def test_basic(self):
         logtype = 'Application'
         elh = win32evtlog.OpenEventLog(None, logtype)
         num_recs = win32evtlog.GetNumberOfEventLogRecords(elh)
-        h = logging.handlers.NTEventLogHandler('test_logging')
+
+        try:
+            h = logging.handlers.NTEventLogHandler('test_logging')
+        except pywintypes.error as e:
+            if e[0] == 5:  # access denied
+                raise unittest.SkipTest('Insufficient privileges to run test')
+
         r = logging.makeLogRecord({'msg': 'Test Log Message'})
         h.handle(r)
         h.close()

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  1 21:55:18 2015
From: python-checkins at python.org (vinay.sajip)
Date: Thu, 01 Oct 2015 19:55:18 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Closes_=2324884=3A_refacto?=
 =?utf-8?q?red_WatchedFileHandler_file_reopening_into_a_separate?=
Message-ID: <20151001195501.94115.96069@psf.io>

https://hg.python.org/cpython/rev/6d61b057c375
changeset:   98468:6d61b057c375
user:        Vinay Sajip <vinay_sajip at yahoo.co.uk>
date:        Thu Oct 01 20:54:41 2015 +0100
summary:
  Closes #24884: refactored WatchedFileHandler file reopening into a separate method, based on a suggestion and patch by Marian Horban.

files:
  Doc/library/logging.handlers.rst |  12 +++++++++---
  Lib/logging/handlers.py          |  15 ++++++++++++---
  2 files changed, 21 insertions(+), 6 deletions(-)


diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst
--- a/Doc/library/logging.handlers.rst
+++ b/Doc/library/logging.handlers.rst
@@ -162,11 +162,17 @@
    first call to :meth:`emit`.  By default, the file grows indefinitely.
 
 
+   .. method:: reopenIfNeeded()
+
+      Checks to see if the file has changed.  If it has, the existing stream is
+      flushed and closed and the file opened again, typically as a precursor to
+      outputting the record to the file.
+
+
    .. method:: emit(record)
 
-      Outputs the record to the file, but first checks to see if the file has
-      changed.  If it has, the existing stream is flushed and closed and the
-      file opened again, before outputting the record to the file.
+      Outputs the record to the file, but first calls :meth:`reopenIfNeeded` to
+      reopen the file if it has changed.
 
 .. _base-rotating-handler:
 
diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py
--- a/Lib/logging/handlers.py
+++ b/Lib/logging/handlers.py
@@ -440,11 +440,11 @@
             sres = os.fstat(self.stream.fileno())
             self.dev, self.ino = sres[ST_DEV], sres[ST_INO]
 
-    def emit(self, record):
+    def reopenIfNeeded(self):
         """
-        Emit a record.
+        Reopen log file if needed.
 
-        First check if the underlying file has changed, and if it
+        Checks if the underlying file has changed, and if it
         has, close the old stream and reopen the file to get the
         current stream.
         """
@@ -467,6 +467,15 @@
                 # open a new file handle and get new stat info from that fd
                 self.stream = self._open()
                 self._statstream()
+
+    def emit(self, record):
+        """
+        Emit a record.
+
+        If underlying file has changed, reopen the file before emitting the
+        record to it.
+        """
+        self.reopenIfNeeded()
         logging.FileHandler.emit(self, record)
 
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  1 23:20:08 2015
From: python-checkins at python.org (victor.stinner)
Date: Thu, 01 Oct 2015 21:20:08 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Make_=5FPyUnicode=5FTransl?=
 =?utf-8?q?ateCharmap=28=29_symbol_private?=
Message-ID: <20151001212007.31179.19998@psf.io>

https://hg.python.org/cpython/rev/3bcf60b12094
changeset:   98471:3bcf60b12094
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Oct 01 22:07:32 2015 +0200
summary:
  Make _PyUnicode_TranslateCharmap() symbol private

unicodeobject.h exposes PyUnicode_TranslateCharmap() and PyUnicode_Translate().

files:
  Objects/unicodeobject.c |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -8683,7 +8683,7 @@
     return res;
 }
 
-PyObject *
+static PyObject *
 _PyUnicode_TranslateCharmap(PyObject *input,
                             PyObject *mapping,
                             const char *errors)

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  1 23:20:08 2015
From: python-checkins at python.org (victor.stinner)
Date: Thu, 01 Oct 2015 21:20:08 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Update_importlib=5Fexterna?=
	=?utf-8?b?bC5o?=
Message-ID: <20151001212007.16569.68394@psf.io>

https://hg.python.org/cpython/rev/d3c6c1ff35aa
changeset:   98470:d3c6c1ff35aa
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Oct 01 22:06:54 2015 +0200
summary:
  Update importlib_external.h

files:
  Python/importlib_external.h |  113 ++++++++++++-----------
  1 files changed, 57 insertions(+), 56 deletions(-)


diff --git a/Python/importlib_external.h b/Python/importlib_external.h
--- a/Python/importlib_external.h
+++ b/Python/importlib_external.h
@@ -613,7 +613,7 @@
     95,102,105,110,100,95,109,111,100,117,108,101,95,115,104,105,
     109,135,1,0,0,115,10,0,0,0,0,10,21,1,24,1,
     6,1,29,1,114,123,0,0,0,99,4,0,0,0,0,0,
-    0,0,11,0,0,0,19,0,0,0,67,0,0,0,115,240,
+    0,0,11,0,0,0,19,0,0,0,67,0,0,0,115,252,
     1,0,0,105,0,0,125,4,0,124,2,0,100,1,0,107,
     9,0,114,31,0,124,2,0,124,4,0,100,2,0,60,110,
     6,0,100,3,0,125,2,0,124,3,0,100,1,0,107,9,
@@ -621,59 +621,60 @@
     0,100,1,0,100,5,0,133,2,0,25,125,5,0,124,0,
     0,100,5,0,100,6,0,133,2,0,25,125,6,0,124,0,
     0,100,6,0,100,7,0,133,2,0,25,125,7,0,124,5,
-    0,116,0,0,107,3,0,114,168,0,100,8,0,106,1,0,
+    0,116,0,0,107,3,0,114,171,0,100,8,0,106,1,0,
     124,2,0,124,5,0,131,2,0,125,8,0,116,2,0,106,
-    3,0,124,8,0,131,1,0,1,116,4,0,124,8,0,124,
-    4,0,141,1,0,130,1,0,110,119,0,116,5,0,124,6,
-    0,131,1,0,100,5,0,107,3,0,114,229,0,100,9,0,
-    106,1,0,124,2,0,131,1,0,125,8,0,116,2,0,106,
-    3,0,124,8,0,131,1,0,1,116,6,0,124,8,0,131,
-    1,0,130,1,0,110,58,0,116,5,0,124,7,0,131,1,
-    0,100,5,0,107,3,0,114,31,1,100,10,0,106,1,0,
-    124,2,0,131,1,0,125,8,0,116,2,0,106,3,0,124,
-    8,0,131,1,0,1,116,6,0,124,8,0,131,1,0,130,
-    1,0,124,1,0,100,1,0,107,9,0,114,226,1,121,20,
-    0,116,7,0,124,1,0,100,11,0,25,131,1,0,125,9,
-    0,87,110,18,0,4,116,8,0,107,10,0,114,83,1,1,
-    1,1,89,110,62,0,88,116,9,0,124,6,0,131,1,0,
-    124,9,0,107,3,0,114,145,1,100,12,0,106,1,0,124,
-    2,0,131,1,0,125,8,0,116,2,0,106,3,0,124,8,
-    0,131,1,0,1,116,4,0,124,8,0,124,4,0,141,1,
-    0,130,1,0,121,18,0,124,1,0,100,13,0,25,100,14,
-    0,64,125,10,0,87,110,18,0,4,116,8,0,107,10,0,
-    114,183,1,1,1,1,89,110,43,0,88,116,9,0,124,7,
-    0,131,1,0,124,10,0,107,3,0,114,226,1,116,4,0,
-    100,12,0,106,1,0,124,2,0,131,1,0,124,4,0,141,
-    1,0,130,1,0,124,0,0,100,7,0,100,1,0,133,2,
-    0,25,83,41,15,97,122,1,0,0,86,97,108,105,100,97,
-    116,101,32,116,104,101,32,104,101,97,100,101,114,32,111,102,
-    32,116,104,101,32,112,97,115,115,101,100,45,105,110,32,98,
-    121,116,101,99,111,100,101,32,97,103,97,105,110,115,116,32,
-    115,111,117,114,99,101,95,115,116,97,116,115,32,40,105,102,
-    10,32,32,32,32,103,105,118,101,110,41,32,97,110,100,32,
-    114,101,116,117,114,110,105,110,103,32,116,104,101,32,98,121,
-    116,101,99,111,100,101,32,116,104,97,116,32,99,97,110,32,
-    98,101,32,99,111,109,112,105,108,101,100,32,98,121,32,99,
-    111,109,112,105,108,101,40,41,46,10,10,32,32,32,32,65,
-    108,108,32,111,116,104,101,114,32,97,114,103,117,109,101,110,
-    116,115,32,97,114,101,32,117,115,101,100,32,116,111,32,101,
-    110,104,97,110,99,101,32,101,114,114,111,114,32,114,101,112,
-    111,114,116,105,110,103,46,10,10,32,32,32,32,73,109,112,
-    111,114,116,69,114,114,111,114,32,105,115,32,114,97,105,115,
-    101,100,32,119,104,101,110,32,116,104,101,32,109,97,103,105,
-    99,32,110,117,109,98,101,114,32,105,115,32,105,110,99,111,
-    114,114,101,99,116,32,111,114,32,116,104,101,32,98,121,116,
-    101,99,111,100,101,32,105,115,10,32,32,32,32,102,111,117,
-    110,100,32,116,111,32,98,101,32,115,116,97,108,101,46,32,
-    69,79,70,69,114,114,111,114,32,105,115,32,114,97,105,115,
-    101,100,32,119,104,101,110,32,116,104,101,32,100,97,116,97,
-    32,105,115,32,102,111,117,110,100,32,116,111,32,98,101,10,
-    32,32,32,32,116,114,117,110,99,97,116,101,100,46,10,10,
-    32,32,32,32,78,114,98,0,0,0,122,10,60,98,121,116,
-    101,99,111,100,101,62,114,35,0,0,0,114,12,0,0,0,
-    233,8,0,0,0,233,12,0,0,0,122,30,98,97,100,32,
-    109,97,103,105,99,32,110,117,109,98,101,114,32,105,110,32,
-    123,33,114,125,58,32,123,33,114,125,122,43,114,101,97,99,
+    3,0,100,9,0,124,8,0,131,2,0,1,116,4,0,124,
+    8,0,124,4,0,141,1,0,130,1,0,110,125,0,116,5,
+    0,124,6,0,131,1,0,100,5,0,107,3,0,114,235,0,
+    100,10,0,106,1,0,124,2,0,131,1,0,125,8,0,116,
+    2,0,106,3,0,100,9,0,124,8,0,131,2,0,1,116,
+    6,0,124,8,0,131,1,0,130,1,0,110,61,0,116,5,
+    0,124,7,0,131,1,0,100,5,0,107,3,0,114,40,1,
+    100,11,0,106,1,0,124,2,0,131,1,0,125,8,0,116,
+    2,0,106,3,0,100,9,0,124,8,0,131,2,0,1,116,
+    6,0,124,8,0,131,1,0,130,1,0,124,1,0,100,1,
+    0,107,9,0,114,238,1,121,20,0,116,7,0,124,1,0,
+    100,12,0,25,131,1,0,125,9,0,87,110,18,0,4,116,
+    8,0,107,10,0,114,92,1,1,1,1,89,110,65,0,88,
+    116,9,0,124,6,0,131,1,0,124,9,0,107,3,0,114,
+    157,1,100,13,0,106,1,0,124,2,0,131,1,0,125,8,
+    0,116,2,0,106,3,0,100,9,0,124,8,0,131,2,0,
+    1,116,4,0,124,8,0,124,4,0,141,1,0,130,1,0,
+    121,18,0,124,1,0,100,14,0,25,100,15,0,64,125,10,
+    0,87,110,18,0,4,116,8,0,107,10,0,114,195,1,1,
+    1,1,89,110,43,0,88,116,9,0,124,7,0,131,1,0,
+    124,10,0,107,3,0,114,238,1,116,4,0,100,13,0,106,
+    1,0,124,2,0,131,1,0,124,4,0,141,1,0,130,1,
+    0,124,0,0,100,7,0,100,1,0,133,2,0,25,83,41,
+    16,97,122,1,0,0,86,97,108,105,100,97,116,101,32,116,
+    104,101,32,104,101,97,100,101,114,32,111,102,32,116,104,101,
+    32,112,97,115,115,101,100,45,105,110,32,98,121,116,101,99,
+    111,100,101,32,97,103,97,105,110,115,116,32,115,111,117,114,
+    99,101,95,115,116,97,116,115,32,40,105,102,10,32,32,32,
+    32,103,105,118,101,110,41,32,97,110,100,32,114,101,116,117,
+    114,110,105,110,103,32,116,104,101,32,98,121,116,101,99,111,
+    100,101,32,116,104,97,116,32,99,97,110,32,98,101,32,99,
+    111,109,112,105,108,101,100,32,98,121,32,99,111,109,112,105,
+    108,101,40,41,46,10,10,32,32,32,32,65,108,108,32,111,
+    116,104,101,114,32,97,114,103,117,109,101,110,116,115,32,97,
+    114,101,32,117,115,101,100,32,116,111,32,101,110,104,97,110,
+    99,101,32,101,114,114,111,114,32,114,101,112,111,114,116,105,
+    110,103,46,10,10,32,32,32,32,73,109,112,111,114,116,69,
+    114,114,111,114,32,105,115,32,114,97,105,115,101,100,32,119,
+    104,101,110,32,116,104,101,32,109,97,103,105,99,32,110,117,
+    109,98,101,114,32,105,115,32,105,110,99,111,114,114,101,99,
+    116,32,111,114,32,116,104,101,32,98,121,116,101,99,111,100,
+    101,32,105,115,10,32,32,32,32,102,111,117,110,100,32,116,
+    111,32,98,101,32,115,116,97,108,101,46,32,69,79,70,69,
+    114,114,111,114,32,105,115,32,114,97,105,115,101,100,32,119,
+    104,101,110,32,116,104,101,32,100,97,116,97,32,105,115,32,
+    102,111,117,110,100,32,116,111,32,98,101,10,32,32,32,32,
+    116,114,117,110,99,97,116,101,100,46,10,10,32,32,32,32,
+    78,114,98,0,0,0,122,10,60,98,121,116,101,99,111,100,
+    101,62,114,35,0,0,0,114,12,0,0,0,233,8,0,0,
+    0,233,12,0,0,0,122,30,98,97,100,32,109,97,103,105,
+    99,32,110,117,109,98,101,114,32,105,110,32,123,33,114,125,
+    58,32,123,33,114,125,122,2,123,125,122,43,114,101,97,99,
     104,101,100,32,69,79,70,32,119,104,105,108,101,32,114,101,
     97,100,105,110,103,32,116,105,109,101,115,116,97,109,112,32,
     105,110,32,123,33,114,125,122,48,114,101,97,99,104,101,100,
@@ -699,9 +700,9 @@
     97,108,105,100,97,116,101,95,98,121,116,101,99,111,100,101,
     95,104,101,97,100,101,114,152,1,0,0,115,76,0,0,0,
     0,11,6,1,12,1,13,3,6,1,12,1,10,1,16,1,
-    16,1,16,1,12,1,18,1,13,1,18,1,18,1,15,1,
-    13,1,15,1,18,1,15,1,13,1,12,1,12,1,3,1,
-    20,1,13,1,5,2,18,1,15,1,13,1,15,1,3,1,
+    16,1,16,1,12,1,18,1,16,1,18,1,18,1,15,1,
+    16,1,15,1,18,1,15,1,16,1,12,1,12,1,3,1,
+    20,1,13,1,5,2,18,1,15,1,16,1,15,1,3,1,
     18,1,13,1,5,2,18,1,15,1,9,1,114,135,0,0,
     0,99,4,0,0,0,0,0,0,0,5,0,0,0,6,0,
     0,0,67,0,0,0,115,115,0,0,0,116,0,0,106,1,

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  1 23:20:08 2015
From: python-checkins at python.org (victor.stinner)
Date: Thu, 01 Oct 2015 21:20:08 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2325267=3A_The_UTF-?=
 =?utf-8?q?8_encoder_is_now_up_to_75_times_as_fast_for_error?=
Message-ID: <20151001212007.9949.17787@psf.io>

https://hg.python.org/cpython/rev/2b5357b38366
changeset:   98469:2b5357b38366
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Thu Oct 01 21:54:51 2015 +0200
summary:
  Issue #25267: The UTF-8 encoder is now up to 75 times as fast for error
handlers: ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``.
Patch co-written with Serhiy Storchaka.

files:
  Doc/whatsnew/3.6.rst       |    3 +
  Lib/test/test_codecs.py    |   37 ++++-
  Misc/NEWS                  |    4 +
  Objects/stringlib/codecs.h |  149 ++++++++++++++++--------
  Objects/unicodeobject.c    |    7 +-
  5 files changed, 136 insertions(+), 64 deletions(-)


diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst
--- a/Doc/whatsnew/3.6.rst
+++ b/Doc/whatsnew/3.6.rst
@@ -120,6 +120,9 @@
 * The ASCII and the Latin1 encoders are now up to 3 times as fast for the error
   error ``surrogateescape``.
 
+* The UTF-8 encoder is now up to 75 times as fast for error handlers:
+  ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``.
+
 
 Build and C API Changes
 =======================
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -361,6 +361,12 @@
         self.assertEqual("[\uDC80]".encode(self.encoding, "replace"),
                          "[?]".encode(self.encoding))
 
+        # sequential surrogate characters
+        self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "ignore"),
+                         "[]".encode(self.encoding))
+        self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "replace"),
+                         "[??]".encode(self.encoding))
+
         bom = "".encode(self.encoding)
         for before, after in [("\U00010fff", "A"), ("[", "]"),
                               ("A", "\U00010fff")]:
@@ -753,6 +759,7 @@
     encoding = "utf-8"
     ill_formed_sequence = b"\xed\xb2\x80"
     ill_formed_sequence_replace = "\ufffd" * 3
+    BOM = b''
 
     def test_partial(self):
         self.check_partial(
@@ -785,23 +792,32 @@
         super().test_lone_surrogates()
         # not sure if this is making sense for
         # UTF-16 and UTF-32
-        self.assertEqual("[\uDC80]".encode('utf-8', "surrogateescape"),
-                         b'[\x80]')
+        self.assertEqual("[\uDC80]".encode(self.encoding, "surrogateescape"),
+                         self.BOM + b'[\x80]')
+
+        with self.assertRaises(UnicodeEncodeError) as cm:
+            "[\uDC80\uD800\uDFFF]".encode(self.encoding, "surrogateescape")
+        exc = cm.exception
+        self.assertEqual(exc.object[exc.start:exc.end], '\uD800\uDFFF')
 
     def test_surrogatepass_handler(self):
-        self.assertEqual("abc\ud800def".encode("utf-8", "surrogatepass"),
-                         b"abc\xed\xa0\x80def")
-        self.assertEqual(b"abc\xed\xa0\x80def".decode("utf-8", "surrogatepass"),
+        self.assertEqual("abc\ud800def".encode(self.encoding, "surrogatepass"),
+                         self.BOM + b"abc\xed\xa0\x80def")
+        self.assertEqual("\U00010fff\uD800".encode(self.encoding, "surrogatepass"),
+                         self.BOM + b"\xf0\x90\xbf\xbf\xed\xa0\x80")
+        self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "surrogatepass"),
+                         self.BOM + b'[\xed\xa0\x80\xed\xb2\x80]')
+
+        self.assertEqual(b"abc\xed\xa0\x80def".decode(self.encoding, "surrogatepass"),
                          "abc\ud800def")
-        self.assertEqual("\U00010fff\uD800".encode("utf-8", "surrogatepass"),
-                         b"\xf0\x90\xbf\xbf\xed\xa0\x80")
-        self.assertEqual(b"\xf0\x90\xbf\xbf\xed\xa0\x80".decode("utf-8", "surrogatepass"),
+        self.assertEqual(b"\xf0\x90\xbf\xbf\xed\xa0\x80".decode(self.encoding, "surrogatepass"),
                          "\U00010fff\uD800")
+
         self.assertTrue(codecs.lookup_error("surrogatepass"))
         with self.assertRaises(UnicodeDecodeError):
-            b"abc\xed\xa0".decode("utf-8", "surrogatepass")
+            b"abc\xed\xa0".decode(self.encoding, "surrogatepass")
         with self.assertRaises(UnicodeDecodeError):
-            b"abc\xed\xa0z".decode("utf-8", "surrogatepass")
+            b"abc\xed\xa0z".decode(self.encoding, "surrogatepass")
 
 
 @unittest.skipUnless(sys.platform == 'win32',
@@ -1008,6 +1024,7 @@
 
 class UTF8SigTest(UTF8Test, unittest.TestCase):
     encoding = "utf-8-sig"
+    BOM = codecs.BOM_UTF8
 
     def test_partial(self):
         self.check_partial(
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,10 @@
 Core and Builtins
 -----------------
 
+- Issue #25267: The UTF-8 encoder is now up to 75 times as fast for error
+  handlers: ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``.
+  Patch co-written with Serhiy Storchaka.
+
 - Issue #25280: Import trace messages emitted in verbose (-v) mode are no
   longer formatted twice.
 
diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h
--- a/Objects/stringlib/codecs.h
+++ b/Objects/stringlib/codecs.h
@@ -268,9 +268,10 @@
     Py_ssize_t nallocated;      /* number of result bytes allocated */
     Py_ssize_t nneeded;            /* number of result bytes needed */
 #if STRINGLIB_SIZEOF_CHAR > 1
-    PyObject *errorHandler = NULL;
+    PyObject *error_handler_obj = NULL;
     PyObject *exc = NULL;
     PyObject *rep = NULL;
+    _Py_error_handler error_handler = _Py_ERROR_UNKNOWN;
 #endif
 #if STRINGLIB_SIZEOF_CHAR == 1
     const Py_ssize_t max_char_size = 2;
@@ -326,72 +327,116 @@
         }
 #if STRINGLIB_SIZEOF_CHAR > 1
         else if (Py_UNICODE_IS_SURROGATE(ch)) {
-            Py_ssize_t newpos;
-            Py_ssize_t repsize, k, startpos;
+            Py_ssize_t startpos, endpos, newpos;
+            Py_ssize_t repsize, k;
+            if (error_handler == _Py_ERROR_UNKNOWN)
+                error_handler = get_error_handler(errors);
+
             startpos = i-1;
-            rep = unicode_encode_call_errorhandler(
-                  errors, &errorHandler, "utf-8", "surrogates not allowed",
-                  unicode, &exc, startpos, startpos+1, &newpos);
-            if (!rep)
-                goto error;
+            endpos = startpos+1;
 
-            if (PyBytes_Check(rep))
-                repsize = PyBytes_GET_SIZE(rep);
-            else
-                repsize = PyUnicode_GET_LENGTH(rep);
+            while ((endpos < size) && Py_UNICODE_IS_SURROGATE(data[endpos]))
+                endpos++;
 
-            if (repsize > max_char_size) {
-                Py_ssize_t offset;
+            switch (error_handler)
+            {
+            case _Py_ERROR_REPLACE:
+                memset(p, '?', endpos - startpos);
+                p += (endpos - startpos);
+                /* fall through the ignore handler */
+            case _Py_ERROR_IGNORE:
+                i += (endpos - startpos - 1);
+                break;
 
-                if (result == NULL)
-                    offset = p - stackbuf;
+
+            case _Py_ERROR_SURROGATEPASS:
+                for (k=startpos; k<endpos; k++) {
+                    ch = data[k];
+                    *p++ = (char)(0xe0 | (ch >> 12));
+                    *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
+                    *p++ = (char)(0x80 | (ch & 0x3f));
+                }
+                i += (endpos - startpos - 1);
+                break;
+
+            case _Py_ERROR_SURROGATEESCAPE:
+                for (k=startpos; k<endpos; k++) {
+                    ch = data[k];
+                    if (!(0xDC80 <= ch && ch <= 0xDCFF))
+                        break;
+                    *p++ = (char)(ch & 0xff);
+                }
+                if (k >= endpos) {
+                    i += (endpos - startpos - 1);
+                    break;
+                }
+                startpos = k;
+                assert(startpos < endpos);
+                /* fall through the default handler */
+
+            default:
+                rep = unicode_encode_call_errorhandler(
+                      errors, &error_handler_obj, "utf-8", "surrogates not allowed",
+                      unicode, &exc, startpos, endpos, &newpos);
+                if (!rep)
+                    goto error;
+
+                if (PyBytes_Check(rep))
+                    repsize = PyBytes_GET_SIZE(rep);
                 else
-                    offset = p - PyBytes_AS_STRING(result);
+                    repsize = PyUnicode_GET_LENGTH(rep);
 
-                if (nallocated > PY_SSIZE_T_MAX - repsize + max_char_size) {
-                    /* integer overflow */
-                    PyErr_NoMemory();
-                    goto error;
+                if (repsize > max_char_size) {
+                    Py_ssize_t offset;
+
+                    if (result == NULL)
+                        offset = p - stackbuf;
+                    else
+                        offset = p - PyBytes_AS_STRING(result);
+
+                    if (nallocated > PY_SSIZE_T_MAX - repsize + max_char_size) {
+                        /* integer overflow */
+                        PyErr_NoMemory();
+                        goto error;
+                    }
+                    nallocated += repsize - max_char_size;
+                    if (result != NULL) {
+                        if (_PyBytes_Resize(&result, nallocated) < 0)
+                            goto error;
+                    } else {
+                        result = PyBytes_FromStringAndSize(NULL, nallocated);
+                        if (result == NULL)
+                            goto error;
+                        Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset);
+                    }
+                    p = PyBytes_AS_STRING(result) + offset;
                 }
-                nallocated += repsize - max_char_size;
-                if (result != NULL) {
-                    if (_PyBytes_Resize(&result, nallocated) < 0)
+
+                if (PyBytes_Check(rep)) {
+                    memcpy(p, PyBytes_AS_STRING(rep), repsize);
+                    p += repsize;
+                }
+                else {
+                    /* rep is unicode */
+                    if (PyUnicode_READY(rep) < 0)
                         goto error;
-                } else {
-                    result = PyBytes_FromStringAndSize(NULL, nallocated);
-                    if (result == NULL)
-                        goto error;
-                    Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset);
-                }
-                p = PyBytes_AS_STRING(result) + offset;
-            }
 
-            if (PyBytes_Check(rep)) {
-                char *prep = PyBytes_AS_STRING(rep);
-                for(k = repsize; k > 0; k--)
-                    *p++ = *prep++;
-            } else /* rep is unicode */ {
-                enum PyUnicode_Kind repkind;
-                void *repdata;
-
-                if (PyUnicode_READY(rep) < 0)
-                    goto error;
-                repkind = PyUnicode_KIND(rep);
-                repdata = PyUnicode_DATA(rep);
-
-                for(k=0; k<repsize; k++) {
-                    Py_UCS4 c = PyUnicode_READ(repkind, repdata, k);
-                    if (0x80 <= c) {
+                    if (!PyUnicode_IS_ASCII(rep)) {
                         raise_encode_exception(&exc, "utf-8",
                                                unicode,
                                                i-1, i,
                                                "surrogates not allowed");
                         goto error;
                     }
-                    *p++ = (char)c;
+
+                    assert(PyUnicode_KIND(rep) == PyUnicode_1BYTE_KIND);
+                    memcpy(p, PyUnicode_DATA(rep), repsize);
+                    p += repsize;
                 }
+                Py_CLEAR(rep);
+
+                i = newpos;
             }
-            Py_CLEAR(rep);
         }
         else
 #if STRINGLIB_SIZEOF_CHAR > 2
@@ -430,7 +475,7 @@
     }
 
 #if STRINGLIB_SIZEOF_CHAR > 1
-    Py_XDECREF(errorHandler);
+    Py_XDECREF(error_handler_obj);
     Py_XDECREF(exc);
 #endif
     return result;
@@ -438,7 +483,7 @@
 #if STRINGLIB_SIZEOF_CHAR > 1
  error:
     Py_XDECREF(rep);
-    Py_XDECREF(errorHandler);
+    Py_XDECREF(error_handler_obj);
     Py_XDECREF(exc);
     Py_XDECREF(result);
     return NULL;
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -297,6 +297,7 @@
     _Py_ERROR_UNKNOWN=0,
     _Py_ERROR_STRICT,
     _Py_ERROR_SURROGATEESCAPE,
+    _Py_ERROR_SURROGATEPASS,
     _Py_ERROR_REPLACE,
     _Py_ERROR_IGNORE,
     _Py_ERROR_XMLCHARREFREPLACE,
@@ -312,6 +313,8 @@
         return _Py_ERROR_STRICT;
     if (strcmp(errors, "surrogateescape") == 0)
         return _Py_ERROR_SURROGATEESCAPE;
+    if (strcmp(errors, "surrogatepass") == 0)
+        return _Py_ERROR_SURROGATEPASS;
     if (strcmp(errors, "ignore") == 0)
         return _Py_ERROR_IGNORE;
     if (strcmp(errors, "replace") == 0)
@@ -6479,8 +6482,8 @@
                 goto onError;
 
             case _Py_ERROR_REPLACE:
-                while (collstart++ < collend)
-                    *str++ = '?';
+                memset(str, '?', collend - collstart);
+                str += (collend - collstart);
                 /* fall through ignore error handler */
             case _Py_ERROR_IGNORE:
                 pos = collend;

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  2 00:20:41 2015
From: python-checkins at python.org (steve.dower)
Date: Thu, 01 Oct 2015 22:20:41 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy41KTogSXNzdWUgIzI1MTY1?=
 =?utf-8?q?=3A_Windows_uninstallation_should_not_remove_launcher_if_other?=
Message-ID: <20151001222040.82642.8890@psf.io>

https://hg.python.org/cpython/rev/a2d30dfa46a7
changeset:   98473:a2d30dfa46a7
branch:      3.5
user:        Steve Dower <steve.dower at microsoft.com>
date:        Thu Oct 01 15:19:39 2015 -0700
summary:
  Issue #25165: Windows uninstallation should not remove launcher if other versions remain

files:
  Misc/NEWS                           |  5 ++++-
  Tools/msi/common.wxs                |  6 ++++--
  Tools/msi/launcher/launcher.wixproj |  1 +
  Tools/msi/launcher/launcher.wxs     |  7 +++++++
  4 files changed, 16 insertions(+), 3 deletions(-)


diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -244,7 +244,10 @@
 Windows
 -------
 
-- Issues #25112: py.exe launcher is missing icons
+- Issue #25165: Windows uninstallation should not remove launcher if other
+  versions remain
+
+- Issue #25112: py.exe launcher is missing icons
 
 - Issue #25102: Windows installer does not precompile for -O or -OO.
 
diff --git a/Tools/msi/common.wxs b/Tools/msi/common.wxs
--- a/Tools/msi/common.wxs
+++ b/Tools/msi/common.wxs
@@ -22,17 +22,19 @@
         </Upgrade>
         
         <?if $(var.UpgradeCode)!=$(var.CoreUpgradeCode) ?>
+        <?ifndef SkipMissingCore ?>
         <Upgrade Id="$(var.CoreUpgradeCode)">
             <UpgradeVersion Property="MISSING_CORE" Minimum="$(var.Version)" IncludeMinimum="yes" Maximum="$(var.Version)" IncludeMaximum="yes" OnlyDetect="yes" />
         </Upgrade>
+        <Condition Message="!(loc.IncorrectCore)">Installed OR NOT MISSING_CORE</Condition>
+        <?endif ?>
         <?endif ?>
         
         <Condition Message="!(loc.NoDowngrade)">Installed OR NOT DOWNGRADE</Condition>
-        <Condition Message="!(loc.IncorrectCore)">Installed OR NOT MISSING_CORE</Condition>
         <Condition Message="!(loc.NoTargetDir)">Installed OR TARGETDIR OR Suppress_TARGETDIR_Check</Condition>
 
         <InstallExecuteSequence>
-            <RemoveExistingProducts After="InstallInitialize">UPGRADE</RemoveExistingProducts>
+            <RemoveExistingProducts After="InstallInitialize" Overridable="yes">UPGRADE</RemoveExistingProducts>
         </InstallExecuteSequence>
     </Fragment>
     
diff --git a/Tools/msi/launcher/launcher.wixproj b/Tools/msi/launcher/launcher.wixproj
--- a/Tools/msi/launcher/launcher.wixproj
+++ b/Tools/msi/launcher/launcher.wixproj
@@ -5,6 +5,7 @@
         <SchemaVersion>2.0</SchemaVersion>
         <OutputName>launcher</OutputName>
         <OutputType>Package</OutputType>
+        <DefineConstants>SkipMissingCore=1;$(DefineConstants)</DefineConstants>
     </PropertyGroup>
     <Import Project="..\msi.props" />
     <ItemGroup>
diff --git a/Tools/msi/launcher/launcher.wxs b/Tools/msi/launcher/launcher.wxs
--- a/Tools/msi/launcher/launcher.wxs
+++ b/Tools/msi/launcher/launcher.wxs
@@ -26,6 +26,13 @@
         <InstallExecuteSequence>
             <Custom Before="SetLauncherInstallDirectoryLM" Action="SetLauncherInstallDirectoryCU">NOT Installed AND NOT ALLUSERS=1</Custom>
             <Custom Before="CostFinalize" Action="SetLauncherInstallDirectoryLM">NOT Installed AND ALLUSERS=1</Custom>
+
+            <RemoveExistingProducts After="InstallInitialize">UPGRADE or REMOVE_OLD_LAUNCHER</RemoveExistingProducts>
         </InstallExecuteSequence>
+        
+        <!-- Python 3.5.0 shipped with an incorrect UpgradeCode -->
+        <Upgrade Id="A71530B9-E89D-53DB-9C2D-C6D7551876D8">
+            <UpgradeVersion Maximum="$(var.Version)" Property="REMOVE_OLD_LAUNCHER" />
+        </Upgrade>
     </Product>
 </Wix>

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  2 00:20:41 2015
From: python-checkins at python.org (steve.dower)
Date: Thu, 01 Oct 2015 22:20:41 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E5=29=3A_Improves_suppo?=
 =?utf-8?q?rt_for_building_unofficial_versions_of_the_Windows_installer=2E?=
Message-ID: <20151001222040.82662.89218@psf.io>

https://hg.python.org/cpython/rev/26439091dc71
changeset:   98472:26439091dc71
branch:      3.5
parent:      98466:b54528d8d8c3
user:        Steve Dower <steve.dower at microsoft.com>
date:        Thu Oct 01 15:18:53 2015 -0700
summary:
  Improves support for building unofficial versions of the Windows installer.

files:
  Tools/msi/buildrelease.bat                        |  41 ++++++---
  Tools/msi/bundle/bundle.targets                   |   7 +-
  Tools/msi/bundle/bundle.wxs                       |   2 +
  Tools/msi/bundle/packagegroups/launcher.wxs       |   4 +
  Tools/msi/bundle/packagegroups/packageinstall.wxs |  26 ++++++
  Tools/msi/bundle/packagegroups/pip.wxs            |  25 ++++++
  Tools/msi/bundle/packagegroups/postinstall.wxs    |  19 ----
  Tools/msi/msi.props                               |  14 ++-
  8 files changed, 98 insertions(+), 40 deletions(-)


diff --git a/Tools/msi/buildrelease.bat b/Tools/msi/buildrelease.bat
--- a/Tools/msi/buildrelease.bat
+++ b/Tools/msi/buildrelease.bat
@@ -4,17 +4,28 @@
 rem This script is intended for building official releases of Python.
 rem To use it to build alternative releases, you should clone this file
 rem and modify the following three URIs.
+
+rem These two will ensure that your release can be installed
+rem alongside an official Python release, by modifying the GUIDs used
+rem for all components.
 rem
-rem The first two will ensure that your release can be installed
-rem alongside an official Python release, while the second specifies
-rem the URL that will be used to download installation files. The
-rem files available from this URL *will* conflict with your installer.
-rem Trust me, you don't want them, even if it seems like a good idea.
+rem The following substitutions will be applied to the release URI:
+rem     Variable        Description         Example
+rem     {arch}          architecture        amd64, win32
+set RELEASE_URI=http://www.python.org/{arch}
 
-set RELEASE_URI_X86=http://www.python.org/win32
-set RELEASE_URI_X64=http://www.python.org/amd64
-set DOWNLOAD_URL_BASE=https://www.python.org/ftp/python
-set DOWNLOAD_URL=
+rem This is the URL that will be used to download installation files.
+rem The files available from the default URL *will* conflict with your
+rem installer. Trust me, you don't want them, even if it seems like a
+rem good idea.
+rem
+rem The following substitutions will be applied to the download URL:
+rem     Variable        Description         Example
+rem     {version}       version number      3.5.0
+rem     {arch}          architecture        amd64, win32
+rem     {releasename}   release name        a1, b2, rc3 (or blank for final)
+rem     {msi}           MSI filename        core.msi
+set DOWNLOAD_URL=https://www.python.org/ftp/python/{version}/{arch}{releasename}/{msi}
 
 set D=%~dp0
 set PCBUILD=%D%..\..\PCBuild\
@@ -90,14 +101,12 @@
     set BUILD_PLAT=Win32
     set OUTDIR_PLAT=win32
     set OBJDIR_PLAT=x86
-    set RELEASE_URI=%RELEASE_URI_X86%
 ) ELSE (
     call "%PCBUILD%env.bat" x86_amd64
     set BUILD=%PCBUILD%amd64\
     set BUILD_PLAT=x64
     set OUTDIR_PLAT=amd64
     set OBJDIR_PLAT=x64
-    set RELEASE_URI=%RELEASE_URI_X64%
 )
 
 if exist "%BUILD%en-us" (
@@ -157,10 +166,16 @@
 echo    --build (-b)        Incrementally build Python rather than rebuilding
 echo    --skip-build (-B)   Do not build Python (just do the installers)
 echo    --skip-doc (-D)     Do not build documentation
-echo    --download          Specify the full download URL for MSIs (should include {2})
+echo    --download          Specify the full download URL for MSIs
 echo    --test              Specify the test directory to run the installer tests
 echo    -h                  Display this help information
 echo.
 echo If no architecture is specified, all architectures will be built.
 echo If --test is not specified, the installer tests are not run.
-echo.
\ No newline at end of file
+echo.
+echo The following substitutions will be applied to the download URL:
+echo     Variable        Description         Example
+echo     {version}       version number      3.5.0
+echo     {arch}          architecture        amd64, win32
+echo     {releasename}   release name        a1, b2, rc3 (or blank for final)
+echo     {msi}           MSI filename        core.msi
diff --git a/Tools/msi/bundle/bundle.targets b/Tools/msi/bundle/bundle.targets
--- a/Tools/msi/bundle/bundle.targets
+++ b/Tools/msi/bundle/bundle.targets
@@ -16,8 +16,9 @@
         <OutputPath>$(OutputPath)en-us\</OutputPath>
         <OutDir>$(OutputPath)</OutDir>
         
-        <DownloadUrl Condition="'$(DownloadUrl)' == '' and '$(DownloadUrlBase)' != ''">$(DownloadUrlBase.TrimEnd(`/`))/$(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber)/$(ArchName)$(ReleaseLevelName)/</DownloadUrl>
-        <DefineConstants Condition="'$(DownloadUrl)' != ''">$(DefineConstants);DownloadUrl=$(DownloadUrl){2}</DefineConstants>
+        <!-- See Tools/msi/buildrelease.bat for help on configuring the download URL -->
+        <DownloadUrl Condition="'$(DownloadUrl)' == '' and '$(DownloadUrlBase)' != ''">$(DownloadUrlBase.TrimEnd(`/`))/{version}/{arch}{releasename}/{msi}</DownloadUrl>
+        <DefineConstants Condition="'$(DownloadUrl)' != ''">$(DefineConstants);DownloadUrl=$(DownloadUrl.Replace(`{version}`, `$(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber)`).Replace(`{arch}`, `$(ArchName)`).Replace(`{releasename}`, `$(ReleaseName)`).Replace(`{msi}`, `{2}`))</DefineConstants>
         <DefineConstants Condition="'$(DownloadUrl)' == ''">$(DefineConstants);DownloadUrl={2}</DefineConstants>
     </PropertyGroup>
     
@@ -88,7 +89,7 @@
     
     <Target Name="BuildLauncher" BeforeTargets="BeforeBuild" Condition="'$(RebuildAll)' != 'false'">
         <!-- Build the launcher MSI using Exec rather than MSBuild -->
-        <Exec Command='msbuild ..\launcher\launcher.wixproj /p:Platform=x86 /p:OutputPath="$(BuildPath.TrimEnd(`\`))" /p:OutputSuffix=$(Platform) /p:BuildForRelease=$(BuildForRelease) /p:UseTestMarker=$(UseTestMarker)'
+        <Exec Command='msbuild ..\launcher\launcher.wixproj /p:Platform=x86 /p:ReleaseUri="$(ReleaseUri)" /p:OutputPath="$(BuildPath.TrimEnd(`\`))" /p:OutputSuffix=$(Platform) /p:BuildForRelease=$(BuildForRelease) /p:UseTestMarker=$(UseTestMarker)'
               ContinueOnError="false" />
     </Target>
     
diff --git a/Tools/msi/bundle/bundle.wxs b/Tools/msi/bundle/bundle.wxs
--- a/Tools/msi/bundle/bundle.wxs
+++ b/Tools/msi/bundle/bundle.wxs
@@ -85,6 +85,8 @@
       <PackageGroupRef Id="tools" />
       <PackageGroupRef Id="tcltk" />
       <PackageGroupRef Id="launcher" />
+      <PackageGroupRef Id="pip" />
+      <PackageGroupRef Id="packageinstall" />
       <PackageGroupRef Id="postinstall" />
     </Chain>
   </Bundle>
diff --git a/Tools/msi/bundle/packagegroups/launcher.wxs b/Tools/msi/bundle/packagegroups/launcher.wxs
--- a/Tools/msi/bundle/packagegroups/launcher.wxs
+++ b/Tools/msi/bundle/packagegroups/launcher.wxs
@@ -9,6 +9,8 @@
                         DownloadUrl="$(var.DownloadUrl)"
                         ForcePerMachine="yes"
                         EnableFeatureSelection="yes"
+                        Permanent="yes"
+                        Visible="yes"
                         InstallCondition="(InstallAllUsers or InstallLauncherAllUsers) and Include_launcher" />
 
             <MsiPackage Id="launcher_JustForMe"
@@ -17,6 +19,8 @@
                         DownloadUrl="$(var.DownloadUrl)"
                         ForcePerMachine="no"
                         EnableFeatureSelection="yes"
+                        Permanent="yes"
+                        Visible="yes"
                         InstallCondition="not (InstallAllUsers or InstallLauncherAllUsers) and Include_launcher" />
         </PackageGroup>
     </Fragment>
diff --git a/Tools/msi/bundle/packagegroups/packageinstall.wxs b/Tools/msi/bundle/packagegroups/packageinstall.wxs
new file mode 100644
--- /dev/null
+++ b/Tools/msi/bundle/packagegroups/packageinstall.wxs
@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
+    <Fragment>
+        <PackageGroup Id="packageinstall">
+            <!--
+            This is an example of installing a package using pip as part of main install.
+            
+            For a network-only install, remove the Payload element and change the install
+            command to specify the package and (optionally) version specifier.
+            
+            <ExePackage Id="requests"
+                        SourceFile="py.exe"
+                        Compressed="yes"
+                        DisplayName="!(loc.CompileAllDescription)"
+                        InstallCommand='-$(var.ShortVersion)$(var.Suffix32) -m pip install requests-2.7.0-py2.py3-none-any.whl'
+                        UninstallCommand='-$(var.ShortVersion)$(var.Suffix32) -m pip uninstall -y requests'
+                        Vital="no"
+                        InstallCondition="Include_pip and not LauncherOnly">
+                <Payload SourceFile="requests-2.7.0-py2.py3-none-any.whl"
+                         Compressed="$(var.CompressMSI)"
+                         DownloadUrl="$(var.DownloadUrl)" />
+            </ExePackage>
+            -->
+        </PackageGroup>
+    </Fragment>
+</Wix>
\ No newline at end of file
diff --git a/Tools/msi/bundle/packagegroups/pip.wxs b/Tools/msi/bundle/packagegroups/pip.wxs
new file mode 100644
--- /dev/null
+++ b/Tools/msi/bundle/packagegroups/pip.wxs
@@ -0,0 +1,25 @@
+<?xml version="1.0"?>
+<Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
+    <Fragment>
+        <PackageGroup Id="pip">
+            <MsiPackage Id="pip_AllUsers"
+                        SourceFile="pip.msi"
+                        Compressed="$(var.CompressMSI)"
+                        DownloadUrl="$(var.DownloadUrl)"
+                        ForcePerMachine="yes"
+                        InstallCondition="InstallAllUsers and Include_pip and not LauncherOnly">
+                <MsiProperty Name="TARGETDIR" Value="[TargetDir]" />
+                <MsiProperty Name="OPTIONALFEATURESREGISTRYKEY" Value="[OptionalFeaturesRegistryKey]" />
+            </MsiPackage>
+            <MsiPackage Id="pip_JustForMe"
+                        SourceFile="pip.msi"
+                        Compressed="$(var.CompressMSI)"
+                        DownloadUrl="$(var.DownloadUrl)"
+                        ForcePerMachine="no"
+                        InstallCondition="not InstallAllUsers and Include_pip and not LauncherOnly">
+                <MsiProperty Name="TARGETDIR" Value="[TargetDir]" />
+                <MsiProperty Name="OPTIONALFEATURESREGISTRYKEY" Value="[OptionalFeaturesRegistryKey]" />
+            </MsiPackage>
+        </PackageGroup>
+    </Fragment>
+</Wix>
\ No newline at end of file
diff --git a/Tools/msi/bundle/packagegroups/postinstall.wxs b/Tools/msi/bundle/packagegroups/postinstall.wxs
--- a/Tools/msi/bundle/packagegroups/postinstall.wxs
+++ b/Tools/msi/bundle/packagegroups/postinstall.wxs
@@ -2,25 +2,6 @@
 <Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
     <Fragment>
         <PackageGroup Id="postinstall">
-            <MsiPackage Id="pip_AllUsers"
-                        SourceFile="pip.msi"
-                        Compressed="$(var.CompressMSI)"
-                        DownloadUrl="$(var.DownloadUrl)"
-                        ForcePerMachine="yes"
-                        InstallCondition="InstallAllUsers and Include_pip and not LauncherOnly">
-                <MsiProperty Name="TARGETDIR" Value="[TargetDir]" />
-                <MsiProperty Name="OPTIONALFEATURESREGISTRYKEY" Value="[OptionalFeaturesRegistryKey]" />
-            </MsiPackage>
-            <MsiPackage Id="pip_JustForMe"
-                        SourceFile="pip.msi"
-                        Compressed="$(var.CompressMSI)"
-                        DownloadUrl="$(var.DownloadUrl)"
-                        ForcePerMachine="no"
-                        InstallCondition="not InstallAllUsers and Include_pip and not LauncherOnly">
-                <MsiProperty Name="TARGETDIR" Value="[TargetDir]" />
-                <MsiProperty Name="OPTIONALFEATURESREGISTRYKEY" Value="[OptionalFeaturesRegistryKey]" />
-            </MsiPackage>
-            
             <MsiPackage Id="path_AllUsers"
                         SourceFile="path.msi"
                         Compressed="$(var.CompressMSI)"
diff --git a/Tools/msi/msi.props b/Tools/msi/msi.props
--- a/Tools/msi/msi.props
+++ b/Tools/msi/msi.props
@@ -11,6 +11,12 @@
         <Configuration Condition="'$(Configuration)' == ''">Release</Configuration>
         <Platform Condition="'$(Platform)' == ''">x86</Platform>
         <InstallScope Condition="'$(InstallScope)' != 'perMachine'">perUser</InstallScope>
+    </PropertyGroup>
+
+    <Import Project="wix.props" />
+    <Import Project="..\..\PCBuild\tcltk.props" />
+
+    <PropertyGroup>
         <!--
         This URI is used to generate the various GUIDs used by the installer.
         Installers built with the same URI will upgrade each other or block
@@ -21,17 +27,15 @@
         that intend to bundle Python should rebuild these modules with their
         own URI to avoid conflicting with the official releases.
         
-        The official releases are built with http://www.python.org/.
+        The official releases use "http://www.python.org/$(ArchName)"
         
         This is not the same as the DownloadUrl property used in the bundle
         projects.
         -->
-        <ReleaseUri Condition="'$(ReleaseUri)' == ''">$(ComputerName)</ReleaseUri>
+        <ReleaseUri Condition="'$(ReleaseUri)' == ''">$(ComputerName)/$(ArchName)/</ReleaseUri>
         <ReleaseUri Condition="!$(ReleaseUri.EndsWith(`/`))">$(ReleaseUri)/</ReleaseUri>
     </PropertyGroup>
 
-    <Import Project="wix.props" />
-    <Import Project="..\..\PCBuild\tcltk.props" />
     
     <ItemGroup>
         <Compile Include="$(MSBuildThisFileDirectory)common.wxs" />
@@ -150,7 +154,7 @@
     <Target Name="_GenerateGuids" AfterTargets="PrepareForBuild">
         <PropertyGroup>
             <_Uuids>@(_Uuid->'("%(Identity)", "$(MajorVersionNumber).$(MinorVersionNumber)/%(Uri)")',',')</_Uuids>
-            <_GenerateCommand>import uuid; print('\n'.join('{}={}'.format(i, uuid.uuid5(uuid.UUID('c8d9733e-a70c-43ff-ab0c-e26456f11083'), '$(ReleaseUri)' + j)) for i,j in [$(_Uuids.Replace(`"`,`'`))]))</_GenerateCommand>
+            <_GenerateCommand>import uuid; print('\n'.join('{}={}'.format(i, uuid.uuid5(uuid.UUID('c8d9733e-a70c-43ff-ab0c-e26456f11083'), '$(ReleaseUri.Replace(`{arch}`, `$(ArchName)`))' + j)) for i,j in [$(_Uuids.Replace(`"`,`'`))]))</_GenerateCommand>
         </PropertyGroup>
         
         <Exec Command='"$(PythonExe)" -c "$(_GenerateCommand)" &gt; "$(IntermediateOutputPath)$(OutputName)guids.txt"'

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  2 00:20:41 2015
From: python-checkins at python.org (steve.dower)
Date: Thu, 01 Oct 2015 22:20:41 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
	=?utf-8?q?=29=3A_Merge_from_3=2E5?=
Message-ID: <20151001222040.82658.88208@psf.io>

https://hg.python.org/cpython/rev/c98cc9f7e2c5
changeset:   98474:c98cc9f7e2c5
parent:      98471:3bcf60b12094
parent:      98473:a2d30dfa46a7
user:        Steve Dower <steve.dower at microsoft.com>
date:        Thu Oct 01 15:20:11 2015 -0700
summary:
  Merge from 3.5

files:
  Misc/NEWS                                         |   5 +-
  Tools/msi/buildrelease.bat                        |  41 ++++++---
  Tools/msi/bundle/bundle.targets                   |   7 +-
  Tools/msi/bundle/bundle.wxs                       |   2 +
  Tools/msi/bundle/packagegroups/launcher.wxs       |   4 +
  Tools/msi/bundle/packagegroups/packageinstall.wxs |  26 ++++++
  Tools/msi/bundle/packagegroups/pip.wxs            |  25 ++++++
  Tools/msi/bundle/packagegroups/postinstall.wxs    |  19 ----
  Tools/msi/common.wxs                              |   6 +-
  Tools/msi/launcher/launcher.wixproj               |   1 +
  Tools/msi/launcher/launcher.wxs                   |   7 +
  Tools/msi/msi.props                               |  14 ++-
  12 files changed, 114 insertions(+), 43 deletions(-)


diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -358,7 +358,10 @@
 Windows
 -------
 
-- Issues #25112: py.exe launcher is missing icons
+- Issue #25165: Windows uninstallation should not remove launcher if other
+  versions remain
+
+- Issue #25112: py.exe launcher is missing icons
 
 - Issue #25102: Windows installer does not precompile for -O or -OO.
 
diff --git a/Tools/msi/buildrelease.bat b/Tools/msi/buildrelease.bat
--- a/Tools/msi/buildrelease.bat
+++ b/Tools/msi/buildrelease.bat
@@ -4,17 +4,28 @@
 rem This script is intended for building official releases of Python.
 rem To use it to build alternative releases, you should clone this file
 rem and modify the following three URIs.
+
+rem These two will ensure that your release can be installed
+rem alongside an official Python release, by modifying the GUIDs used
+rem for all components.
 rem
-rem The first two will ensure that your release can be installed
-rem alongside an official Python release, while the second specifies
-rem the URL that will be used to download installation files. The
-rem files available from this URL *will* conflict with your installer.
-rem Trust me, you don't want them, even if it seems like a good idea.
+rem The following substitutions will be applied to the release URI:
+rem     Variable        Description         Example
+rem     {arch}          architecture        amd64, win32
+set RELEASE_URI=http://www.python.org/{arch}
 
-set RELEASE_URI_X86=http://www.python.org/win32
-set RELEASE_URI_X64=http://www.python.org/amd64
-set DOWNLOAD_URL_BASE=https://www.python.org/ftp/python
-set DOWNLOAD_URL=
+rem This is the URL that will be used to download installation files.
+rem The files available from the default URL *will* conflict with your
+rem installer. Trust me, you don't want them, even if it seems like a
+rem good idea.
+rem
+rem The following substitutions will be applied to the download URL:
+rem     Variable        Description         Example
+rem     {version}       version number      3.5.0
+rem     {arch}          architecture        amd64, win32
+rem     {releasename}   release name        a1, b2, rc3 (or blank for final)
+rem     {msi}           MSI filename        core.msi
+set DOWNLOAD_URL=https://www.python.org/ftp/python/{version}/{arch}{releasename}/{msi}
 
 set D=%~dp0
 set PCBUILD=%D%..\..\PCBuild\
@@ -90,14 +101,12 @@
     set BUILD_PLAT=Win32
     set OUTDIR_PLAT=win32
     set OBJDIR_PLAT=x86
-    set RELEASE_URI=%RELEASE_URI_X86%
 ) ELSE (
     call "%PCBUILD%env.bat" x86_amd64
     set BUILD=%PCBUILD%amd64\
     set BUILD_PLAT=x64
     set OUTDIR_PLAT=amd64
     set OBJDIR_PLAT=x64
-    set RELEASE_URI=%RELEASE_URI_X64%
 )
 
 if exist "%BUILD%en-us" (
@@ -157,10 +166,16 @@
 echo    --build (-b)        Incrementally build Python rather than rebuilding
 echo    --skip-build (-B)   Do not build Python (just do the installers)
 echo    --skip-doc (-D)     Do not build documentation
-echo    --download          Specify the full download URL for MSIs (should include {2})
+echo    --download          Specify the full download URL for MSIs
 echo    --test              Specify the test directory to run the installer tests
 echo    -h                  Display this help information
 echo.
 echo If no architecture is specified, all architectures will be built.
 echo If --test is not specified, the installer tests are not run.
-echo.
\ No newline at end of file
+echo.
+echo The following substitutions will be applied to the download URL:
+echo     Variable        Description         Example
+echo     {version}       version number      3.5.0
+echo     {arch}          architecture        amd64, win32
+echo     {releasename}   release name        a1, b2, rc3 (or blank for final)
+echo     {msi}           MSI filename        core.msi
diff --git a/Tools/msi/bundle/bundle.targets b/Tools/msi/bundle/bundle.targets
--- a/Tools/msi/bundle/bundle.targets
+++ b/Tools/msi/bundle/bundle.targets
@@ -16,8 +16,9 @@
         <OutputPath>$(OutputPath)en-us\</OutputPath>
         <OutDir>$(OutputPath)</OutDir>
         
-        <DownloadUrl Condition="'$(DownloadUrl)' == '' and '$(DownloadUrlBase)' != ''">$(DownloadUrlBase.TrimEnd(`/`))/$(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber)/$(ArchName)$(ReleaseLevelName)/</DownloadUrl>
-        <DefineConstants Condition="'$(DownloadUrl)' != ''">$(DefineConstants);DownloadUrl=$(DownloadUrl){2}</DefineConstants>
+        <!-- See Tools/msi/buildrelease.bat for help on configuring the download URL -->
+        <DownloadUrl Condition="'$(DownloadUrl)' == '' and '$(DownloadUrlBase)' != ''">$(DownloadUrlBase.TrimEnd(`/`))/{version}/{arch}{releasename}/{msi}</DownloadUrl>
+        <DefineConstants Condition="'$(DownloadUrl)' != ''">$(DefineConstants);DownloadUrl=$(DownloadUrl.Replace(`{version}`, `$(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber)`).Replace(`{arch}`, `$(ArchName)`).Replace(`{releasename}`, `$(ReleaseName)`).Replace(`{msi}`, `{2}`))</DefineConstants>
         <DefineConstants Condition="'$(DownloadUrl)' == ''">$(DefineConstants);DownloadUrl={2}</DefineConstants>
     </PropertyGroup>
     
@@ -88,7 +89,7 @@
     
     <Target Name="BuildLauncher" BeforeTargets="BeforeBuild" Condition="'$(RebuildAll)' != 'false'">
         <!-- Build the launcher MSI using Exec rather than MSBuild -->
-        <Exec Command='msbuild ..\launcher\launcher.wixproj /p:Platform=x86 /p:OutputPath="$(BuildPath.TrimEnd(`\`))" /p:OutputSuffix=$(Platform) /p:BuildForRelease=$(BuildForRelease) /p:UseTestMarker=$(UseTestMarker)'
+        <Exec Command='msbuild ..\launcher\launcher.wixproj /p:Platform=x86 /p:ReleaseUri="$(ReleaseUri)" /p:OutputPath="$(BuildPath.TrimEnd(`\`))" /p:OutputSuffix=$(Platform) /p:BuildForRelease=$(BuildForRelease) /p:UseTestMarker=$(UseTestMarker)'
               ContinueOnError="false" />
     </Target>
     
diff --git a/Tools/msi/bundle/bundle.wxs b/Tools/msi/bundle/bundle.wxs
--- a/Tools/msi/bundle/bundle.wxs
+++ b/Tools/msi/bundle/bundle.wxs
@@ -85,6 +85,8 @@
       <PackageGroupRef Id="tools" />
       <PackageGroupRef Id="tcltk" />
       <PackageGroupRef Id="launcher" />
+      <PackageGroupRef Id="pip" />
+      <PackageGroupRef Id="packageinstall" />
       <PackageGroupRef Id="postinstall" />
     </Chain>
   </Bundle>
diff --git a/Tools/msi/bundle/packagegroups/launcher.wxs b/Tools/msi/bundle/packagegroups/launcher.wxs
--- a/Tools/msi/bundle/packagegroups/launcher.wxs
+++ b/Tools/msi/bundle/packagegroups/launcher.wxs
@@ -9,6 +9,8 @@
                         DownloadUrl="$(var.DownloadUrl)"
                         ForcePerMachine="yes"
                         EnableFeatureSelection="yes"
+                        Permanent="yes"
+                        Visible="yes"
                         InstallCondition="(InstallAllUsers or InstallLauncherAllUsers) and Include_launcher" />
 
             <MsiPackage Id="launcher_JustForMe"
@@ -17,6 +19,8 @@
                         DownloadUrl="$(var.DownloadUrl)"
                         ForcePerMachine="no"
                         EnableFeatureSelection="yes"
+                        Permanent="yes"
+                        Visible="yes"
                         InstallCondition="not (InstallAllUsers or InstallLauncherAllUsers) and Include_launcher" />
         </PackageGroup>
     </Fragment>
diff --git a/Tools/msi/bundle/packagegroups/packageinstall.wxs b/Tools/msi/bundle/packagegroups/packageinstall.wxs
new file mode 100644
--- /dev/null
+++ b/Tools/msi/bundle/packagegroups/packageinstall.wxs
@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
+    <Fragment>
+        <PackageGroup Id="packageinstall">
+            <!--
+            This is an example of installing a package using pip as part of main install.
+            
+            For a network-only install, remove the Payload element and change the install
+            command to specify the package and (optionally) version specifier.
+            
+            <ExePackage Id="requests"
+                        SourceFile="py.exe"
+                        Compressed="yes"
+                        DisplayName="!(loc.CompileAllDescription)"
+                        InstallCommand='-$(var.ShortVersion)$(var.Suffix32) -m pip install requests-2.7.0-py2.py3-none-any.whl'
+                        UninstallCommand='-$(var.ShortVersion)$(var.Suffix32) -m pip uninstall -y requests'
+                        Vital="no"
+                        InstallCondition="Include_pip and not LauncherOnly">
+                <Payload SourceFile="requests-2.7.0-py2.py3-none-any.whl"
+                         Compressed="$(var.CompressMSI)"
+                         DownloadUrl="$(var.DownloadUrl)" />
+            </ExePackage>
+            -->
+        </PackageGroup>
+    </Fragment>
+</Wix>
\ No newline at end of file
diff --git a/Tools/msi/bundle/packagegroups/pip.wxs b/Tools/msi/bundle/packagegroups/pip.wxs
new file mode 100644
--- /dev/null
+++ b/Tools/msi/bundle/packagegroups/pip.wxs
@@ -0,0 +1,25 @@
+<?xml version="1.0"?>
+<Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
+    <Fragment>
+        <PackageGroup Id="pip">
+            <MsiPackage Id="pip_AllUsers"
+                        SourceFile="pip.msi"
+                        Compressed="$(var.CompressMSI)"
+                        DownloadUrl="$(var.DownloadUrl)"
+                        ForcePerMachine="yes"
+                        InstallCondition="InstallAllUsers and Include_pip and not LauncherOnly">
+                <MsiProperty Name="TARGETDIR" Value="[TargetDir]" />
+                <MsiProperty Name="OPTIONALFEATURESREGISTRYKEY" Value="[OptionalFeaturesRegistryKey]" />
+            </MsiPackage>
+            <MsiPackage Id="pip_JustForMe"
+                        SourceFile="pip.msi"
+                        Compressed="$(var.CompressMSI)"
+                        DownloadUrl="$(var.DownloadUrl)"
+                        ForcePerMachine="no"
+                        InstallCondition="not InstallAllUsers and Include_pip and not LauncherOnly">
+                <MsiProperty Name="TARGETDIR" Value="[TargetDir]" />
+                <MsiProperty Name="OPTIONALFEATURESREGISTRYKEY" Value="[OptionalFeaturesRegistryKey]" />
+            </MsiPackage>
+        </PackageGroup>
+    </Fragment>
+</Wix>
\ No newline at end of file
diff --git a/Tools/msi/bundle/packagegroups/postinstall.wxs b/Tools/msi/bundle/packagegroups/postinstall.wxs
--- a/Tools/msi/bundle/packagegroups/postinstall.wxs
+++ b/Tools/msi/bundle/packagegroups/postinstall.wxs
@@ -2,25 +2,6 @@
 <Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
     <Fragment>
         <PackageGroup Id="postinstall">
-            <MsiPackage Id="pip_AllUsers"
-                        SourceFile="pip.msi"
-                        Compressed="$(var.CompressMSI)"
-                        DownloadUrl="$(var.DownloadUrl)"
-                        ForcePerMachine="yes"
-                        InstallCondition="InstallAllUsers and Include_pip and not LauncherOnly">
-                <MsiProperty Name="TARGETDIR" Value="[TargetDir]" />
-                <MsiProperty Name="OPTIONALFEATURESREGISTRYKEY" Value="[OptionalFeaturesRegistryKey]" />
-            </MsiPackage>
-            <MsiPackage Id="pip_JustForMe"
-                        SourceFile="pip.msi"
-                        Compressed="$(var.CompressMSI)"
-                        DownloadUrl="$(var.DownloadUrl)"
-                        ForcePerMachine="no"
-                        InstallCondition="not InstallAllUsers and Include_pip and not LauncherOnly">
-                <MsiProperty Name="TARGETDIR" Value="[TargetDir]" />
-                <MsiProperty Name="OPTIONALFEATURESREGISTRYKEY" Value="[OptionalFeaturesRegistryKey]" />
-            </MsiPackage>
-            
             <MsiPackage Id="path_AllUsers"
                         SourceFile="path.msi"
                         Compressed="$(var.CompressMSI)"
diff --git a/Tools/msi/common.wxs b/Tools/msi/common.wxs
--- a/Tools/msi/common.wxs
+++ b/Tools/msi/common.wxs
@@ -22,17 +22,19 @@
         </Upgrade>
         
         <?if $(var.UpgradeCode)!=$(var.CoreUpgradeCode) ?>
+        <?ifndef SkipMissingCore ?>
         <Upgrade Id="$(var.CoreUpgradeCode)">
             <UpgradeVersion Property="MISSING_CORE" Minimum="$(var.Version)" IncludeMinimum="yes" Maximum="$(var.Version)" IncludeMaximum="yes" OnlyDetect="yes" />
         </Upgrade>
+        <Condition Message="!(loc.IncorrectCore)">Installed OR NOT MISSING_CORE</Condition>
+        <?endif ?>
         <?endif ?>
         
         <Condition Message="!(loc.NoDowngrade)">Installed OR NOT DOWNGRADE</Condition>
-        <Condition Message="!(loc.IncorrectCore)">Installed OR NOT MISSING_CORE</Condition>
         <Condition Message="!(loc.NoTargetDir)">Installed OR TARGETDIR OR Suppress_TARGETDIR_Check</Condition>
 
         <InstallExecuteSequence>
-            <RemoveExistingProducts After="InstallInitialize">UPGRADE</RemoveExistingProducts>
+            <RemoveExistingProducts After="InstallInitialize" Overridable="yes">UPGRADE</RemoveExistingProducts>
         </InstallExecuteSequence>
     </Fragment>
     
diff --git a/Tools/msi/launcher/launcher.wixproj b/Tools/msi/launcher/launcher.wixproj
--- a/Tools/msi/launcher/launcher.wixproj
+++ b/Tools/msi/launcher/launcher.wixproj
@@ -5,6 +5,7 @@
         <SchemaVersion>2.0</SchemaVersion>
         <OutputName>launcher</OutputName>
         <OutputType>Package</OutputType>
+        <DefineConstants>SkipMissingCore=1;$(DefineConstants)</DefineConstants>
     </PropertyGroup>
     <Import Project="..\msi.props" />
     <ItemGroup>
diff --git a/Tools/msi/launcher/launcher.wxs b/Tools/msi/launcher/launcher.wxs
--- a/Tools/msi/launcher/launcher.wxs
+++ b/Tools/msi/launcher/launcher.wxs
@@ -26,6 +26,13 @@
         <InstallExecuteSequence>
             <Custom Before="SetLauncherInstallDirectoryLM" Action="SetLauncherInstallDirectoryCU">NOT Installed AND NOT ALLUSERS=1</Custom>
             <Custom Before="CostFinalize" Action="SetLauncherInstallDirectoryLM">NOT Installed AND ALLUSERS=1</Custom>
+
+            <RemoveExistingProducts After="InstallInitialize">UPGRADE or REMOVE_OLD_LAUNCHER</RemoveExistingProducts>
         </InstallExecuteSequence>
+        
+        <!-- Python 3.5.0 shipped with an incorrect UpgradeCode -->
+        <Upgrade Id="A71530B9-E89D-53DB-9C2D-C6D7551876D8">
+            <UpgradeVersion Maximum="$(var.Version)" Property="REMOVE_OLD_LAUNCHER" />
+        </Upgrade>
     </Product>
 </Wix>
diff --git a/Tools/msi/msi.props b/Tools/msi/msi.props
--- a/Tools/msi/msi.props
+++ b/Tools/msi/msi.props
@@ -11,6 +11,12 @@
         <Configuration Condition="'$(Configuration)' == ''">Release</Configuration>
         <Platform Condition="'$(Platform)' == ''">x86</Platform>
         <InstallScope Condition="'$(InstallScope)' != 'perMachine'">perUser</InstallScope>
+    </PropertyGroup>
+
+    <Import Project="wix.props" />
+    <Import Project="..\..\PCBuild\tcltk.props" />
+
+    <PropertyGroup>
         <!--
         This URI is used to generate the various GUIDs used by the installer.
         Installers built with the same URI will upgrade each other or block
@@ -21,17 +27,15 @@
         that intend to bundle Python should rebuild these modules with their
         own URI to avoid conflicting with the official releases.
         
-        The official releases are built with http://www.python.org/.
+        The official releases use "http://www.python.org/$(ArchName)"
         
         This is not the same as the DownloadUrl property used in the bundle
         projects.
         -->
-        <ReleaseUri Condition="'$(ReleaseUri)' == ''">$(ComputerName)</ReleaseUri>
+        <ReleaseUri Condition="'$(ReleaseUri)' == ''">$(ComputerName)/$(ArchName)/</ReleaseUri>
         <ReleaseUri Condition="!$(ReleaseUri.EndsWith(`/`))">$(ReleaseUri)/</ReleaseUri>
     </PropertyGroup>
 
-    <Import Project="wix.props" />
-    <Import Project="..\..\PCBuild\tcltk.props" />
     
     <ItemGroup>
         <Compile Include="$(MSBuildThisFileDirectory)common.wxs" />
@@ -150,7 +154,7 @@
     <Target Name="_GenerateGuids" AfterTargets="PrepareForBuild">
         <PropertyGroup>
             <_Uuids>@(_Uuid->'("%(Identity)", "$(MajorVersionNumber).$(MinorVersionNumber)/%(Uri)")',',')</_Uuids>
-            <_GenerateCommand>import uuid; print('\n'.join('{}={}'.format(i, uuid.uuid5(uuid.UUID('c8d9733e-a70c-43ff-ab0c-e26456f11083'), '$(ReleaseUri)' + j)) for i,j in [$(_Uuids.Replace(`"`,`'`))]))</_GenerateCommand>
+            <_GenerateCommand>import uuid; print('\n'.join('{}={}'.format(i, uuid.uuid5(uuid.UUID('c8d9733e-a70c-43ff-ab0c-e26456f11083'), '$(ReleaseUri.Replace(`{arch}`, `$(ArchName)`))' + j)) for i,j in [$(_Uuids.Replace(`"`,`'`))]))</_GenerateCommand>
         </PropertyGroup>
         
         <Exec Command='"$(PythonExe)" -c "$(_GenerateCommand)" &gt; "$(IntermediateOutputPath)$(OutputName)guids.txt"'

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  2 04:22:02 2015
From: python-checkins at python.org (ned.deily)
Date: Fri, 02 Oct 2015 02:22:02 +0000
Subject: [Python-checkins] =?utf-8?q?peps=3A_Update_PEP_494_with_a_detaile?=
 =?utf-8?q?d_Python_3=2E6_release_schedule_proposal?=
Message-ID: <20151002022200.31203.12264@psf.io>

https://hg.python.org/peps/rev/742700a03e91
changeset:   6105:742700a03e91
user:        Ned Deily <nad at acm.org>
date:        Thu Oct 01 22:21:48 2015 -0400
summary:
  Update PEP 494 with a detailed Python 3.6 release schedule proposal

files:
  pep-0494.txt |  34 +++++++++++++++++++++++++++-------
  1 files changed, 27 insertions(+), 7 deletions(-)


diff --git a/pep-0494.txt b/pep-0494.txt
--- a/pep-0494.txt
+++ b/pep-0494.txt
@@ -19,7 +19,7 @@
 
 .. Small features may be added up to the first beta
    release.  Bugs may be fixed until the final release,
-   which is planned for September 2015.
+   which is planned for December 2016.
 
 
 Release Manager and Crew
@@ -31,17 +31,37 @@
 - Documentation: Georg Brandl
 
 
+3.6 Lifespan
+============
+
+3.6 will receive bugfix updates approximately every 3-6 months for
+approximately 18 months.  After the release of 3.7.0 final, a final
+3.6 bugfix update will be released.  After that, it is expected that
+security updates (source only) will be released until 5 years after
+the release of 3.6 final, so until approximately December 2021.
+
+
 Release Schedule
 ================
 
-The releases:
+3.6.0 schedule
+--------------
 
-- 3.6.0 alpha 1: TBD
-- 3.6.0 beta 1: TBD
-- 3.6.0 candidate 1: TBD
-- 3.6.0 final: TBD (late 2016?)
+- 3.6 development begins: 2015-05-24
+- 3.6.0 alpha 1: 2016-05-15
+- 3.6.0 alpha 2: 2016-06-12
+- 3.6.0 alpha 3: 2016-07-10
+- 3.6.0 alpha 4: 2016-08-07
+- 3.6.0 beta 1: 2016-09-07
 
-(Beta 1 is also "feature freeze"--no new features beyond this point.)
+(No new features beyond this point.)
+
+- 3.6.0 beta 2: 2016-10-02
+- 3.6.0 beta 3: 2016-10-30
+- 3.6.0 beta 4: 2016-11-20
+- 3.6.0 candidate 1: 2016-12-04
+- 3.6.0 candidate 2 (if needed): 2016-12-11
+- 3.6.0 final: 2016-12-16
 
 
 Features for 3.6

-- 
Repository URL: https://hg.python.org/peps

From solipsis at pitrou.net  Fri Oct  2 10:45:07 2015
From: solipsis at pitrou.net (solipsis at pitrou.net)
Date: Fri, 02 Oct 2015 08:45:07 +0000
Subject: [Python-checkins] Daily reference leaks (c98cc9f7e2c5): sum=17880
Message-ID: <20151002084506.31177.3204@psf.io>

results for c98cc9f7e2c5 on branch "default"
--------------------------------------------

test_asyncio leaked [0, 3, 0] memory blocks, sum=3
test_capi leaked [1598, 1598, 1598] references, sum=4794
test_capi leaked [387, 389, 389] memory blocks, sum=1165
test_functools leaked [0, 2, 2] memory blocks, sum=4
test_threading leaked [3196, 3196, 3196] references, sum=9588
test_threading leaked [774, 776, 776] memory blocks, sum=2326


Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogTFm6JI', '--timeout', '7200']

From python-checkins at python.org  Fri Oct  2 11:51:07 2015
From: python-checkins at python.org (serhiy.storchaka)
Date: Fri, 02 Oct 2015 09:51:07 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy41KTogSXNzdWUgIzI0NDgz?=
 =?utf-8?q?=3A_C_implementation_of_functools=2Elru=5Fcache=28=29_now_calcu?=
 =?utf-8?q?lates_key=27s?=
Message-ID: <20151002095107.31179.4139@psf.io>

https://hg.python.org/cpython/rev/3f4c319a822f
changeset:   98475:3f4c319a822f
branch:      3.5
parent:      98473:a2d30dfa46a7
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Fri Oct 02 12:47:11 2015 +0300
summary:
  Issue #24483: C implementation of functools.lru_cache() now calculates key's
hash only once.

files:
  Include/dictobject.h       |   4 ++
  Misc/NEWS                  |   3 ++
  Modules/_functoolsmodule.c |  26 ++++++++++++++----
  Objects/dictobject.c       |  37 ++++++++++++++++++++++++++
  4 files changed, 64 insertions(+), 6 deletions(-)


diff --git a/Include/dictobject.h b/Include/dictobject.h
--- a/Include/dictobject.h
+++ b/Include/dictobject.h
@@ -72,6 +72,10 @@
                                           PyObject *item, Py_hash_t hash);
 #endif
 PyAPI_FUNC(int) PyDict_DelItem(PyObject *mp, PyObject *key);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(int) _PyDict_DelItem_KnownHash(PyObject *mp, PyObject *key,
+                                          Py_hash_t hash);
+#endif
 PyAPI_FUNC(void) PyDict_Clear(PyObject *mp);
 PyAPI_FUNC(int) PyDict_Next(
     PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value);
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -32,6 +32,9 @@
 Library
 -------
 
+- Issue #24483: C implementation of functools.lru_cache() now calculates key's
+  hash only once.
+
 - Issue #22958: Constructor and update method of weakref.WeakValueDictionary
   now accept the self and the dict keyword arguments.
 
diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c
--- a/Modules/_functoolsmodule.c
+++ b/Modules/_functoolsmodule.c
@@ -601,6 +601,7 @@
 typedef struct lru_list_elem {
     PyObject_HEAD
     struct lru_list_elem *prev, *next;  /* borrowed links */
+    Py_hash_t hash;
     PyObject *key, *result;
 } lru_list_elem;
 
@@ -762,10 +763,14 @@
 infinite_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwds)
 {
     PyObject *result;
+    Py_hash_t hash;
     PyObject *key = lru_cache_make_key(args, kwds, self->typed);
     if (!key)
         return NULL;
-    result = PyDict_GetItemWithError(self->cache, key);
+    hash = PyObject_Hash(key);
+    if (hash == -1)
+        return NULL;
+    result = _PyDict_GetItem_KnownHash(self->cache, key, hash);
     if (result) {
         Py_INCREF(result);
         self->hits++;
@@ -781,7 +786,7 @@
         Py_DECREF(key);
         return NULL;
     }
-    if (PyDict_SetItem(self->cache, key, result) < 0) {
+    if (_PyDict_SetItem_KnownHash(self->cache, key, result, hash) < 0) {
         Py_DECREF(result);
         Py_DECREF(key);
         return NULL;
@@ -813,11 +818,15 @@
 {
     lru_list_elem *link;
     PyObject *key, *result;
+    Py_hash_t hash;
 
     key = lru_cache_make_key(args, kwds, self->typed);
     if (!key)
         return NULL;
-    link  = (lru_list_elem *)PyDict_GetItemWithError(self->cache, key);
+    hash = PyObject_Hash(key);
+    if (hash == -1)
+        return NULL;
+    link  = (lru_list_elem *)_PyDict_GetItem_KnownHash(self->cache, key, hash);
     if (link) {
         lru_cache_extricate_link(link);
         lru_cache_append_link(self, link);
@@ -845,7 +854,8 @@
         /* Remove it from the cache.
            The cache dict holds one reference to the link,
            and the linked list holds yet one reference to it. */
-        if (PyDict_DelItem(self->cache, link->key) < 0) {
+        if (_PyDict_DelItem_KnownHash(self->cache, link->key,
+                                      link->hash) < 0) {
             lru_cache_append_link(self, link);
             Py_DECREF(key);
             Py_DECREF(result);
@@ -859,9 +869,11 @@
         oldkey = link->key;
         oldresult = link->result;
 
+        link->hash = hash;
         link->key = key;
         link->result = result;
-        if (PyDict_SetItem(self->cache, key, (PyObject *)link) < 0) {
+        if (_PyDict_SetItem_KnownHash(self->cache, key, (PyObject *)link,
+                                      hash) < 0) {
             Py_DECREF(link);
             Py_DECREF(oldkey);
             Py_DECREF(oldresult);
@@ -881,10 +893,12 @@
             return NULL;
         }
 
+        link->hash = hash;
         link->key = key;
         link->result = result;
         _PyObject_GC_TRACK(link);
-        if (PyDict_SetItem(self->cache, key, (PyObject *)link) < 0) {
+        if (_PyDict_SetItem_KnownHash(self->cache, key, (PyObject *)link,
+                                      hash) < 0) {
             Py_DECREF(link);
             return NULL;
         }
diff --git a/Objects/dictobject.c b/Objects/dictobject.c
--- a/Objects/dictobject.c
+++ b/Objects/dictobject.c
@@ -1242,6 +1242,7 @@
     }
     assert(key);
     assert(value);
+    assert(hash != -1);
     mp = (PyDictObject *)op;
 
     /* insertdict() handles any resizing that might be necessary */
@@ -1290,6 +1291,42 @@
     return 0;
 }
 
+int
+_PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash)
+{
+    PyDictObject *mp;
+    PyDictKeyEntry *ep;
+    PyObject *old_key, *old_value;
+    PyObject **value_addr;
+
+    if (!PyDict_Check(op)) {
+        PyErr_BadInternalCall();
+        return -1;
+    }
+    assert(key);
+    assert(hash != -1);
+    mp = (PyDictObject *)op;
+    ep = (mp->ma_keys->dk_lookup)(mp, key, hash, &value_addr);
+    if (ep == NULL)
+        return -1;
+    if (*value_addr == NULL) {
+        _PyErr_SetKeyError(key);
+        return -1;
+    }
+    old_value = *value_addr;
+    *value_addr = NULL;
+    mp->ma_used--;
+    if (!_PyDict_HasSplitTable(mp)) {
+        ENSURE_ALLOWS_DELETIONS(mp);
+        old_key = ep->me_key;
+        Py_INCREF(dummy);
+        ep->me_key = dummy;
+        Py_DECREF(old_key);
+    }
+    Py_DECREF(old_value);
+    return 0;
+}
+
 void
 PyDict_Clear(PyObject *op)
 {

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  2 11:51:08 2015
From: python-checkins at python.org (serhiy.storchaka)
Date: Fri, 02 Oct 2015 09:51:08 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2324483=3A_C_implementation_of_functools=2Elru=5F?=
 =?utf-8?q?cache=28=29_now_calculates_key=27s?=
Message-ID: <20151002095107.11704.92168@psf.io>

https://hg.python.org/cpython/rev/5758b85627c9
changeset:   98476:5758b85627c9
parent:      98474:c98cc9f7e2c5
parent:      98475:3f4c319a822f
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Fri Oct 02 12:47:59 2015 +0300
summary:
  Issue #24483: C implementation of functools.lru_cache() now calculates key's
hash only once.

files:
  Include/dictobject.h       |   4 ++
  Misc/NEWS                  |   3 ++
  Modules/_functoolsmodule.c |  26 ++++++++++++++----
  Objects/dictobject.c       |  37 ++++++++++++++++++++++++++
  4 files changed, 64 insertions(+), 6 deletions(-)


diff --git a/Include/dictobject.h b/Include/dictobject.h
--- a/Include/dictobject.h
+++ b/Include/dictobject.h
@@ -72,6 +72,10 @@
                                           PyObject *item, Py_hash_t hash);
 #endif
 PyAPI_FUNC(int) PyDict_DelItem(PyObject *mp, PyObject *key);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(int) _PyDict_DelItem_KnownHash(PyObject *mp, PyObject *key,
+                                          Py_hash_t hash);
+#endif
 PyAPI_FUNC(void) PyDict_Clear(PyObject *mp);
 PyAPI_FUNC(int) PyDict_Next(
     PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value);
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -203,6 +203,9 @@
 Library
 -------
 
+- Issue #24483: C implementation of functools.lru_cache() now calculates key's
+  hash only once.
+
 - Issue #22958: Constructor and update method of weakref.WeakValueDictionary
   now accept the self and the dict keyword arguments.
 
diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c
--- a/Modules/_functoolsmodule.c
+++ b/Modules/_functoolsmodule.c
@@ -601,6 +601,7 @@
 typedef struct lru_list_elem {
     PyObject_HEAD
     struct lru_list_elem *prev, *next;  /* borrowed links */
+    Py_hash_t hash;
     PyObject *key, *result;
 } lru_list_elem;
 
@@ -762,10 +763,14 @@
 infinite_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwds)
 {
     PyObject *result;
+    Py_hash_t hash;
     PyObject *key = lru_cache_make_key(args, kwds, self->typed);
     if (!key)
         return NULL;
-    result = PyDict_GetItemWithError(self->cache, key);
+    hash = PyObject_Hash(key);
+    if (hash == -1)
+        return NULL;
+    result = _PyDict_GetItem_KnownHash(self->cache, key, hash);
     if (result) {
         Py_INCREF(result);
         self->hits++;
@@ -781,7 +786,7 @@
         Py_DECREF(key);
         return NULL;
     }
-    if (PyDict_SetItem(self->cache, key, result) < 0) {
+    if (_PyDict_SetItem_KnownHash(self->cache, key, result, hash) < 0) {
         Py_DECREF(result);
         Py_DECREF(key);
         return NULL;
@@ -813,11 +818,15 @@
 {
     lru_list_elem *link;
     PyObject *key, *result;
+    Py_hash_t hash;
 
     key = lru_cache_make_key(args, kwds, self->typed);
     if (!key)
         return NULL;
-    link  = (lru_list_elem *)PyDict_GetItemWithError(self->cache, key);
+    hash = PyObject_Hash(key);
+    if (hash == -1)
+        return NULL;
+    link  = (lru_list_elem *)_PyDict_GetItem_KnownHash(self->cache, key, hash);
     if (link) {
         lru_cache_extricate_link(link);
         lru_cache_append_link(self, link);
@@ -845,7 +854,8 @@
         /* Remove it from the cache.
            The cache dict holds one reference to the link,
            and the linked list holds yet one reference to it. */
-        if (PyDict_DelItem(self->cache, link->key) < 0) {
+        if (_PyDict_DelItem_KnownHash(self->cache, link->key,
+                                      link->hash) < 0) {
             lru_cache_append_link(self, link);
             Py_DECREF(key);
             Py_DECREF(result);
@@ -859,9 +869,11 @@
         oldkey = link->key;
         oldresult = link->result;
 
+        link->hash = hash;
         link->key = key;
         link->result = result;
-        if (PyDict_SetItem(self->cache, key, (PyObject *)link) < 0) {
+        if (_PyDict_SetItem_KnownHash(self->cache, key, (PyObject *)link,
+                                      hash) < 0) {
             Py_DECREF(link);
             Py_DECREF(oldkey);
             Py_DECREF(oldresult);
@@ -881,10 +893,12 @@
             return NULL;
         }
 
+        link->hash = hash;
         link->key = key;
         link->result = result;
         _PyObject_GC_TRACK(link);
-        if (PyDict_SetItem(self->cache, key, (PyObject *)link) < 0) {
+        if (_PyDict_SetItem_KnownHash(self->cache, key, (PyObject *)link,
+                                      hash) < 0) {
             Py_DECREF(link);
             return NULL;
         }
diff --git a/Objects/dictobject.c b/Objects/dictobject.c
--- a/Objects/dictobject.c
+++ b/Objects/dictobject.c
@@ -1242,6 +1242,7 @@
     }
     assert(key);
     assert(value);
+    assert(hash != -1);
     mp = (PyDictObject *)op;
 
     /* insertdict() handles any resizing that might be necessary */
@@ -1290,6 +1291,42 @@
     return 0;
 }
 
+int
+_PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash)
+{
+    PyDictObject *mp;
+    PyDictKeyEntry *ep;
+    PyObject *old_key, *old_value;
+    PyObject **value_addr;
+
+    if (!PyDict_Check(op)) {
+        PyErr_BadInternalCall();
+        return -1;
+    }
+    assert(key);
+    assert(hash != -1);
+    mp = (PyDictObject *)op;
+    ep = (mp->ma_keys->dk_lookup)(mp, key, hash, &value_addr);
+    if (ep == NULL)
+        return -1;
+    if (*value_addr == NULL) {
+        _PyErr_SetKeyError(key);
+        return -1;
+    }
+    old_value = *value_addr;
+    *value_addr = NULL;
+    mp->ma_used--;
+    if (!_PyDict_HasSplitTable(mp)) {
+        ENSURE_ALLOWS_DELETIONS(mp);
+        old_key = ep->me_key;
+        Py_INCREF(dummy);
+        ep->me_key = dummy;
+        Py_DECREF(old_key);
+    }
+    Py_DECREF(old_value);
+    return 0;
+}
+
 void
 PyDict_Clear(PyObject *op)
 {

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  2 12:15:42 2015
From: python-checkins at python.org (serhiy.storchaka)
Date: Fri, 02 Oct 2015 10:15:42 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzI0ODQ4?=
 =?utf-8?q?=3A_Fixed_bugs_in_UTF-7_decoding_of_misformed_data=3A?=
Message-ID: <20151002101541.31197.99675@psf.io>

https://hg.python.org/cpython/rev/3c13567ea642
changeset:   98477:3c13567ea642
branch:      3.4
parent:      98465:72c57c120c19
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Fri Oct 02 13:07:28 2015 +0300
summary:
  Issue #24848: Fixed bugs in UTF-7 decoding of misformed data:

1. Non-ASCII bytes were accepted after shift sequence.
2. A low surrogate could be emitted in case of error in high surrogate.

files:
  Lib/test/test_codecs.py  |  60 +++++++++++++++++++++++++++-
  Lib/test/test_unicode.py |   3 +-
  Misc/NEWS                |   2 +
  Objects/unicodeobject.c  |  21 +++++----
  4 files changed, 75 insertions(+), 11 deletions(-)


diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -898,6 +898,32 @@
 class UTF7Test(ReadTest, unittest.TestCase):
     encoding = "utf-7"
 
+    def test_ascii(self):
+        # Set D (directly encoded characters)
+        set_d = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+                 'abcdefghijklmnopqrstuvwxyz'
+                 '0123456789'
+                 '\'(),-./:?')
+        self.assertEqual(set_d.encode(self.encoding), set_d.encode('ascii'))
+        self.assertEqual(set_d.encode('ascii').decode(self.encoding), set_d)
+        # Set O (optional direct characters)
+        set_o = ' !"#$%&*;<=>@[]^_`{|}'
+        self.assertEqual(set_o.encode(self.encoding), set_o.encode('ascii'))
+        self.assertEqual(set_o.encode('ascii').decode(self.encoding), set_o)
+        # +
+        self.assertEqual('a+b'.encode(self.encoding), b'a+-b')
+        self.assertEqual(b'a+-b'.decode(self.encoding), 'a+b')
+        # White spaces
+        ws = ' \t\n\r'
+        self.assertEqual(ws.encode(self.encoding), ws.encode('ascii'))
+        self.assertEqual(ws.encode('ascii').decode(self.encoding), ws)
+        # Other ASCII characters
+        other_ascii = ''.join(sorted(set(bytes(range(0x80)).decode()) -
+                                     set(set_d + set_o + '+' + ws)))
+        self.assertEqual(other_ascii.encode(self.encoding),
+                         b'+AAAAAQACAAMABAAFAAYABwAIAAsADAAOAA8AEAARABIAEwAU'
+                         b'ABUAFgAXABgAGQAaABsAHAAdAB4AHwBcAH4Afw-')
+
     def test_partial(self):
         self.check_partial(
             'a+-b\x00c\x80d\u0100e\U00010000f',
@@ -939,7 +965,9 @@
 
     def test_errors(self):
         tests = [
+            (b'\xffb', '\ufffdb'),
             (b'a\xffb', 'a\ufffdb'),
+            (b'a\xff\xffb', 'a\ufffd\ufffdb'),
             (b'a+IK', 'a\ufffd'),
             (b'a+IK-b', 'a\ufffdb'),
             (b'a+IK,b', 'a\ufffdb'),
@@ -955,6 +983,8 @@
             (b'a+//,+IKw-b', 'a\ufffd\u20acb'),
             (b'a+///,+IKw-b', 'a\uffff\ufffd\u20acb'),
             (b'a+////,+IKw-b', 'a\uffff\ufffd\u20acb'),
+            (b'a+IKw-b\xff', 'a\u20acb\ufffd'),
+            (b'a+IKw\xffb', 'a\u20ac\ufffdb'),
         ]
         for raw, expected in tests:
             with self.subTest(raw=raw):
@@ -966,8 +996,36 @@
         self.assertEqual('\U000104A0'.encode(self.encoding), b'+2AHcoA-')
         self.assertEqual('\ud801\udca0'.encode(self.encoding), b'+2AHcoA-')
         self.assertEqual(b'+2AHcoA-'.decode(self.encoding), '\U000104A0')
+        self.assertEqual(b'+2AHcoA'.decode(self.encoding), '\U000104A0')
+        self.assertEqual('\u20ac\U000104A0'.encode(self.encoding), b'+IKzYAdyg-')
+        self.assertEqual(b'+IKzYAdyg-'.decode(self.encoding), '\u20ac\U000104A0')
+        self.assertEqual(b'+IKzYAdyg'.decode(self.encoding), '\u20ac\U000104A0')
+        self.assertEqual('\u20ac\u20ac\U000104A0'.encode(self.encoding),
+                         b'+IKwgrNgB3KA-')
+        self.assertEqual(b'+IKwgrNgB3KA-'.decode(self.encoding),
+                         '\u20ac\u20ac\U000104A0')
+        self.assertEqual(b'+IKwgrNgB3KA'.decode(self.encoding),
+                         '\u20ac\u20ac\U000104A0')
 
-    test_lone_surrogates = None
+    def test_lone_surrogates(self):
+        tests = [
+            (b'a+2AE-b', 'a\ud801b'),
+            (b'a+2AE\xffb', 'a\ufffdb'),
+            (b'a+2AE', 'a\ufffd'),
+            (b'a+2AEA-b', 'a\ufffdb'),
+            (b'a+2AH-b', 'a\ufffdb'),
+            (b'a+IKzYAQ-b', 'a\u20ac\ud801b'),
+            (b'a+IKzYAQ\xffb', 'a\u20ac\ufffdb'),
+            (b'a+IKzYAQA-b', 'a\u20ac\ufffdb'),
+            (b'a+IKzYAd-b', 'a\u20ac\ufffdb'),
+            (b'a+IKwgrNgB-b', 'a\u20ac\u20ac\ud801b'),
+            (b'a+IKwgrNgB\xffb', 'a\u20ac\u20ac\ufffdb'),
+            (b'a+IKwgrNgB', 'a\u20ac\u20ac\ufffd'),
+            (b'a+IKwgrNgBA-b', 'a\u20ac\u20ac\ufffdb'),
+        ]
+        for raw, expected in tests:
+            with self.subTest(raw=raw):
+                self.assertEqual(raw.decode('utf-7', 'replace'), expected)
 
 
 class UTF16ExTest(unittest.TestCase):
diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py
--- a/Lib/test/test_unicode.py
+++ b/Lib/test/test_unicode.py
@@ -1524,7 +1524,7 @@
         self.assertEqual(b'+2AHab9ze-'.decode('utf-7'), '\uD801\U000abcde')
 
         # Issue #2242: crash on some Windows/MSVC versions
-        self.assertEqual(b'+\xc1'.decode('utf-7'), '\xc1')
+        self.assertEqual(b'+\xc1'.decode('utf-7', 'ignore'), '')
 
         # Direct encoded characters
         set_d = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'(),-./:?"
@@ -1966,6 +1966,7 @@
         self.assertRaises(UnicodeError, str, b'Andr\202 x', 'ascii', 'strict')
         self.assertEqual(str(b'Andr\202 x', 'ascii', 'ignore'), "Andr x")
         self.assertEqual(str(b'Andr\202 x', 'ascii', 'replace'), 'Andr\uFFFD x')
+        self.assertEqual(str(b'\202 x', 'ascii', 'replace'), '\uFFFD x')
 
         # Error handling (unknown character names)
         self.assertEqual(b"\\N{foo}xx".decode("unicode-escape", "ignore"), "xx")
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,8 @@
 Core and Builtins
 -----------------
 
+- Issue #24848: Fixed a number of bugs in UTF-7 decoding of misformed data.
+
 - Issue #25280: Import trace messages emitted in verbose (-v) mode are no
   longer formatted twice.
 
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -4381,31 +4381,31 @@
             }
             else { /* now leaving a base-64 section */
                 inShift = 0;
-                s++;
-                if (surrogate) {
-                    if (_PyUnicodeWriter_WriteCharInline(&writer, surrogate) < 0)
-                        goto onError;
-                    surrogate = 0;
-                }
                 if (base64bits > 0) { /* left-over bits */
                     if (base64bits >= 6) {
                         /* We've seen at least one base-64 character */
+                        s++;
                         errmsg = "partial character in shift sequence";
                         goto utf7Error;
                     }
                     else {
                         /* Some bits remain; they should be zero */
                         if (base64buffer != 0) {
+                            s++;
                             errmsg = "non-zero padding bits in shift sequence";
                             goto utf7Error;
                         }
                     }
                 }
-                if (ch != '-') {
+                if (surrogate && DECODE_DIRECT(ch)) {
+                    if (_PyUnicodeWriter_WriteCharInline(&writer, surrogate) < 0)
+                        goto onError;
+                }
+                surrogate = 0;
+                if (ch == '-') {
                     /* '-' is absorbed; other terminating
                        characters are preserved */
-                    if (_PyUnicodeWriter_WriteCharInline(&writer, ch) < 0)
-                        goto onError;
+                    s++;
                 }
             }
         }
@@ -4419,6 +4419,7 @@
             }
             else { /* begin base64-encoded section */
                 inShift = 1;
+                surrogate = 0;
                 shiftOutStart = writer.pos;
                 base64bits = 0;
                 base64buffer = 0;
@@ -4450,6 +4451,7 @@
 
     if (inShift && !consumed) { /* in shift sequence, no more to follow */
         /* if we're in an inconsistent state, that's an error */
+        inShift = 0;
         if (surrogate ||
                 (base64bits >= 6) ||
                 (base64bits > 0 && base64buffer != 0)) {
@@ -13337,6 +13339,7 @@
 
         if (maxchar > writer->maxchar || writer->readonly) {
             /* resize + widen */
+            maxchar = Py_MAX(maxchar, writer->maxchar);
             newbuffer = PyUnicode_New(newlen, maxchar);
             if (newbuffer == NULL)
                 return -1;

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  2 12:15:42 2015
From: python-checkins at python.org (serhiy.storchaka)
Date: Fri, 02 Oct 2015 10:15:42 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
 =?utf-8?q?_Issue_=2324848=3A_Fixed_bugs_in_UTF-7_decoding_of_misformed_da?=
 =?utf-8?b?dGE6?=
Message-ID: <20151002101541.28569.90897@psf.io>

https://hg.python.org/cpython/rev/a61fa2b08f87
changeset:   98478:a61fa2b08f87
branch:      3.5
parent:      98475:3f4c319a822f
parent:      98477:3c13567ea642
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Fri Oct 02 13:13:14 2015 +0300
summary:
  Issue #24848: Fixed bugs in UTF-7 decoding of misformed data:

1. Non-ASCII bytes were accepted after shift sequence.
2. A low surrogate could be emitted in case of error in high surrogate.
3. In some circumstances the '\xfd' character was produced instead of the
replacement character '\ufffd' (due to a bug in _PyUnicodeWriter).

files:
  Lib/test/test_codecs.py  |  62 +++++++++++++++++++++++++++-
  Lib/test/test_unicode.py |   3 +-
  Misc/NEWS                |   2 +
  Objects/unicodeobject.c  |  21 +++++----
  4 files changed, 76 insertions(+), 12 deletions(-)


diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -903,6 +903,32 @@
 class UTF7Test(ReadTest, unittest.TestCase):
     encoding = "utf-7"
 
+    def test_ascii(self):
+        # Set D (directly encoded characters)
+        set_d = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+                 'abcdefghijklmnopqrstuvwxyz'
+                 '0123456789'
+                 '\'(),-./:?')
+        self.assertEqual(set_d.encode(self.encoding), set_d.encode('ascii'))
+        self.assertEqual(set_d.encode('ascii').decode(self.encoding), set_d)
+        # Set O (optional direct characters)
+        set_o = ' !"#$%&*;<=>@[]^_`{|}'
+        self.assertEqual(set_o.encode(self.encoding), set_o.encode('ascii'))
+        self.assertEqual(set_o.encode('ascii').decode(self.encoding), set_o)
+        # +
+        self.assertEqual('a+b'.encode(self.encoding), b'a+-b')
+        self.assertEqual(b'a+-b'.decode(self.encoding), 'a+b')
+        # White spaces
+        ws = ' \t\n\r'
+        self.assertEqual(ws.encode(self.encoding), ws.encode('ascii'))
+        self.assertEqual(ws.encode('ascii').decode(self.encoding), ws)
+        # Other ASCII characters
+        other_ascii = ''.join(sorted(set(bytes(range(0x80)).decode()) -
+                                     set(set_d + set_o + '+' + ws)))
+        self.assertEqual(other_ascii.encode(self.encoding),
+                         b'+AAAAAQACAAMABAAFAAYABwAIAAsADAAOAA8AEAARABIAEwAU'
+                         b'ABUAFgAXABgAGQAaABsAHAAdAB4AHwBcAH4Afw-')
+
     def test_partial(self):
         self.check_partial(
             'a+-b\x00c\x80d\u0100e\U00010000f',
@@ -944,7 +970,9 @@
 
     def test_errors(self):
         tests = [
+            (b'\xffb', '\ufffdb'),
             (b'a\xffb', 'a\ufffdb'),
+            (b'a\xff\xffb', 'a\ufffd\ufffdb'),
             (b'a+IK', 'a\ufffd'),
             (b'a+IK-b', 'a\ufffdb'),
             (b'a+IK,b', 'a\ufffdb'),
@@ -960,6 +988,8 @@
             (b'a+//,+IKw-b', 'a\ufffd\u20acb'),
             (b'a+///,+IKw-b', 'a\uffff\ufffd\u20acb'),
             (b'a+////,+IKw-b', 'a\uffff\ufffd\u20acb'),
+            (b'a+IKw-b\xff', 'a\u20acb\ufffd'),
+            (b'a+IKw\xffb', 'a\u20ac\ufffdb'),
         ]
         for raw, expected in tests:
             with self.subTest(raw=raw):
@@ -971,8 +1001,36 @@
         self.assertEqual('\U000104A0'.encode(self.encoding), b'+2AHcoA-')
         self.assertEqual('\ud801\udca0'.encode(self.encoding), b'+2AHcoA-')
         self.assertEqual(b'+2AHcoA-'.decode(self.encoding), '\U000104A0')
-
-    test_lone_surrogates = None
+        self.assertEqual(b'+2AHcoA'.decode(self.encoding), '\U000104A0')
+        self.assertEqual('\u20ac\U000104A0'.encode(self.encoding), b'+IKzYAdyg-')
+        self.assertEqual(b'+IKzYAdyg-'.decode(self.encoding), '\u20ac\U000104A0')
+        self.assertEqual(b'+IKzYAdyg'.decode(self.encoding), '\u20ac\U000104A0')
+        self.assertEqual('\u20ac\u20ac\U000104A0'.encode(self.encoding),
+                         b'+IKwgrNgB3KA-')
+        self.assertEqual(b'+IKwgrNgB3KA-'.decode(self.encoding),
+                         '\u20ac\u20ac\U000104A0')
+        self.assertEqual(b'+IKwgrNgB3KA'.decode(self.encoding),
+                         '\u20ac\u20ac\U000104A0')
+
+    def test_lone_surrogates(self):
+        tests = [
+            (b'a+2AE-b', 'a\ud801b'),
+            (b'a+2AE\xffb', 'a\ufffdb'),
+            (b'a+2AE', 'a\ufffd'),
+            (b'a+2AEA-b', 'a\ufffdb'),
+            (b'a+2AH-b', 'a\ufffdb'),
+            (b'a+IKzYAQ-b', 'a\u20ac\ud801b'),
+            (b'a+IKzYAQ\xffb', 'a\u20ac\ufffdb'),
+            (b'a+IKzYAQA-b', 'a\u20ac\ufffdb'),
+            (b'a+IKzYAd-b', 'a\u20ac\ufffdb'),
+            (b'a+IKwgrNgB-b', 'a\u20ac\u20ac\ud801b'),
+            (b'a+IKwgrNgB\xffb', 'a\u20ac\u20ac\ufffdb'),
+            (b'a+IKwgrNgB', 'a\u20ac\u20ac\ufffd'),
+            (b'a+IKwgrNgBA-b', 'a\u20ac\u20ac\ufffdb'),
+        ]
+        for raw, expected in tests:
+            with self.subTest(raw=raw):
+                self.assertEqual(raw.decode('utf-7', 'replace'), expected)
 
 
 class UTF16ExTest(unittest.TestCase):
diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py
--- a/Lib/test/test_unicode.py
+++ b/Lib/test/test_unicode.py
@@ -1553,7 +1553,7 @@
         self.assertEqual(b'+2AHab9ze-'.decode('utf-7'), '\uD801\U000abcde')
 
         # Issue #2242: crash on some Windows/MSVC versions
-        self.assertEqual(b'+\xc1'.decode('utf-7'), '\xc1')
+        self.assertEqual(b'+\xc1'.decode('utf-7', 'ignore'), '')
 
         # Direct encoded characters
         set_d = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'(),-./:?"
@@ -1995,6 +1995,7 @@
         self.assertRaises(UnicodeError, str, b'Andr\202 x', 'ascii', 'strict')
         self.assertEqual(str(b'Andr\202 x', 'ascii', 'ignore'), "Andr x")
         self.assertEqual(str(b'Andr\202 x', 'ascii', 'replace'), 'Andr\uFFFD x')
+        self.assertEqual(str(b'\202 x', 'ascii', 'replace'), '\uFFFD x')
 
         # Error handling (unknown character names)
         self.assertEqual(b"\\N{foo}xx".decode("unicode-escape", "ignore"), "xx")
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -11,6 +11,8 @@
 Core and Builtins
 -----------------
 
+- Issue #24848: Fixed a number of bugs in UTF-7 decoding of misformed data.
+
 - Issue #25280: Import trace messages emitted in verbose (-v) mode are no
   longer formatted twice.
 
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -4330,31 +4330,31 @@
             }
             else { /* now leaving a base-64 section */
                 inShift = 0;
-                s++;
-                if (surrogate) {
-                    if (_PyUnicodeWriter_WriteCharInline(&writer, surrogate) < 0)
-                        goto onError;
-                    surrogate = 0;
-                }
                 if (base64bits > 0) { /* left-over bits */
                     if (base64bits >= 6) {
                         /* We've seen at least one base-64 character */
+                        s++;
                         errmsg = "partial character in shift sequence";
                         goto utf7Error;
                     }
                     else {
                         /* Some bits remain; they should be zero */
                         if (base64buffer != 0) {
+                            s++;
                             errmsg = "non-zero padding bits in shift sequence";
                             goto utf7Error;
                         }
                     }
                 }
-                if (ch != '-') {
+                if (surrogate && DECODE_DIRECT(ch)) {
+                    if (_PyUnicodeWriter_WriteCharInline(&writer, surrogate) < 0)
+                        goto onError;
+                }
+                surrogate = 0;
+                if (ch == '-') {
                     /* '-' is absorbed; other terminating
                        characters are preserved */
-                    if (_PyUnicodeWriter_WriteCharInline(&writer, ch) < 0)
-                        goto onError;
+                    s++;
                 }
             }
         }
@@ -4368,6 +4368,7 @@
             }
             else { /* begin base64-encoded section */
                 inShift = 1;
+                surrogate = 0;
                 shiftOutStart = writer.pos;
                 base64bits = 0;
                 base64buffer = 0;
@@ -4399,6 +4400,7 @@
 
     if (inShift && !consumed) { /* in shift sequence, no more to follow */
         /* if we're in an inconsistent state, that's an error */
+        inShift = 0;
         if (surrogate ||
                 (base64bits >= 6) ||
                 (base64bits > 0 && base64buffer != 0)) {
@@ -13291,6 +13293,7 @@
 
         if (maxchar > writer->maxchar || writer->readonly) {
             /* resize + widen */
+            maxchar = Py_MAX(maxchar, writer->maxchar);
             newbuffer = PyUnicode_New(newlen, maxchar);
             if (newbuffer == NULL)
                 return -1;

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  2 12:15:41 2015
From: python-checkins at python.org (serhiy.storchaka)
Date: Fri, 02 Oct 2015 10:15:41 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2324848=3A_Fixed_bugs_in_UTF-7_decoding_of_misfor?=
 =?utf-8?q?med_data=3A?=
Message-ID: <20151002101541.18061.93576@psf.io>

https://hg.python.org/cpython/rev/037253b7cd6d
changeset:   98479:037253b7cd6d
parent:      98476:5758b85627c9
parent:      98478:a61fa2b08f87
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Fri Oct 02 13:14:03 2015 +0300
summary:
  Issue #24848: Fixed bugs in UTF-7 decoding of misformed data:

1. Non-ASCII bytes were accepted after shift sequence.
2. A low surrogate could be emitted in case of error in high surrogate.
3. In some circumstances the '\xfd' character was produced instead of the
replacement character '\ufffd' (due to a bug in _PyUnicodeWriter).

files:
  Lib/test/test_codecs.py  |  62 +++++++++++++++++++++++++++-
  Lib/test/test_unicode.py |   3 +-
  Misc/NEWS                |   2 +
  Objects/unicodeobject.c  |  21 +++++----
  4 files changed, 76 insertions(+), 12 deletions(-)


diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -926,6 +926,32 @@
 class UTF7Test(ReadTest, unittest.TestCase):
     encoding = "utf-7"
 
+    def test_ascii(self):
+        # Set D (directly encoded characters)
+        set_d = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+                 'abcdefghijklmnopqrstuvwxyz'
+                 '0123456789'
+                 '\'(),-./:?')
+        self.assertEqual(set_d.encode(self.encoding), set_d.encode('ascii'))
+        self.assertEqual(set_d.encode('ascii').decode(self.encoding), set_d)
+        # Set O (optional direct characters)
+        set_o = ' !"#$%&*;<=>@[]^_`{|}'
+        self.assertEqual(set_o.encode(self.encoding), set_o.encode('ascii'))
+        self.assertEqual(set_o.encode('ascii').decode(self.encoding), set_o)
+        # +
+        self.assertEqual('a+b'.encode(self.encoding), b'a+-b')
+        self.assertEqual(b'a+-b'.decode(self.encoding), 'a+b')
+        # White spaces
+        ws = ' \t\n\r'
+        self.assertEqual(ws.encode(self.encoding), ws.encode('ascii'))
+        self.assertEqual(ws.encode('ascii').decode(self.encoding), ws)
+        # Other ASCII characters
+        other_ascii = ''.join(sorted(set(bytes(range(0x80)).decode()) -
+                                     set(set_d + set_o + '+' + ws)))
+        self.assertEqual(other_ascii.encode(self.encoding),
+                         b'+AAAAAQACAAMABAAFAAYABwAIAAsADAAOAA8AEAARABIAEwAU'
+                         b'ABUAFgAXABgAGQAaABsAHAAdAB4AHwBcAH4Afw-')
+
     def test_partial(self):
         self.check_partial(
             'a+-b\x00c\x80d\u0100e\U00010000f',
@@ -967,7 +993,9 @@
 
     def test_errors(self):
         tests = [
+            (b'\xffb', '\ufffdb'),
             (b'a\xffb', 'a\ufffdb'),
+            (b'a\xff\xffb', 'a\ufffd\ufffdb'),
             (b'a+IK', 'a\ufffd'),
             (b'a+IK-b', 'a\ufffdb'),
             (b'a+IK,b', 'a\ufffdb'),
@@ -983,6 +1011,8 @@
             (b'a+//,+IKw-b', 'a\ufffd\u20acb'),
             (b'a+///,+IKw-b', 'a\uffff\ufffd\u20acb'),
             (b'a+////,+IKw-b', 'a\uffff\ufffd\u20acb'),
+            (b'a+IKw-b\xff', 'a\u20acb\ufffd'),
+            (b'a+IKw\xffb', 'a\u20ac\ufffdb'),
         ]
         for raw, expected in tests:
             with self.subTest(raw=raw):
@@ -994,8 +1024,36 @@
         self.assertEqual('\U000104A0'.encode(self.encoding), b'+2AHcoA-')
         self.assertEqual('\ud801\udca0'.encode(self.encoding), b'+2AHcoA-')
         self.assertEqual(b'+2AHcoA-'.decode(self.encoding), '\U000104A0')
-
-    test_lone_surrogates = None
+        self.assertEqual(b'+2AHcoA'.decode(self.encoding), '\U000104A0')
+        self.assertEqual('\u20ac\U000104A0'.encode(self.encoding), b'+IKzYAdyg-')
+        self.assertEqual(b'+IKzYAdyg-'.decode(self.encoding), '\u20ac\U000104A0')
+        self.assertEqual(b'+IKzYAdyg'.decode(self.encoding), '\u20ac\U000104A0')
+        self.assertEqual('\u20ac\u20ac\U000104A0'.encode(self.encoding),
+                         b'+IKwgrNgB3KA-')
+        self.assertEqual(b'+IKwgrNgB3KA-'.decode(self.encoding),
+                         '\u20ac\u20ac\U000104A0')
+        self.assertEqual(b'+IKwgrNgB3KA'.decode(self.encoding),
+                         '\u20ac\u20ac\U000104A0')
+
+    def test_lone_surrogates(self):
+        tests = [
+            (b'a+2AE-b', 'a\ud801b'),
+            (b'a+2AE\xffb', 'a\ufffdb'),
+            (b'a+2AE', 'a\ufffd'),
+            (b'a+2AEA-b', 'a\ufffdb'),
+            (b'a+2AH-b', 'a\ufffdb'),
+            (b'a+IKzYAQ-b', 'a\u20ac\ud801b'),
+            (b'a+IKzYAQ\xffb', 'a\u20ac\ufffdb'),
+            (b'a+IKzYAQA-b', 'a\u20ac\ufffdb'),
+            (b'a+IKzYAd-b', 'a\u20ac\ufffdb'),
+            (b'a+IKwgrNgB-b', 'a\u20ac\u20ac\ud801b'),
+            (b'a+IKwgrNgB\xffb', 'a\u20ac\u20ac\ufffdb'),
+            (b'a+IKwgrNgB', 'a\u20ac\u20ac\ufffd'),
+            (b'a+IKwgrNgBA-b', 'a\u20ac\u20ac\ufffdb'),
+        ]
+        for raw, expected in tests:
+            with self.subTest(raw=raw):
+                self.assertEqual(raw.decode('utf-7', 'replace'), expected)
 
 
 class UTF16ExTest(unittest.TestCase):
diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py
--- a/Lib/test/test_unicode.py
+++ b/Lib/test/test_unicode.py
@@ -1553,7 +1553,7 @@
         self.assertEqual(b'+2AHab9ze-'.decode('utf-7'), '\uD801\U000abcde')
 
         # Issue #2242: crash on some Windows/MSVC versions
-        self.assertEqual(b'+\xc1'.decode('utf-7'), '\xc1')
+        self.assertEqual(b'+\xc1'.decode('utf-7', 'ignore'), '')
 
         # Direct encoded characters
         set_d = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'(),-./:?"
@@ -1995,6 +1995,7 @@
         self.assertRaises(UnicodeError, str, b'Andr\202 x', 'ascii', 'strict')
         self.assertEqual(str(b'Andr\202 x', 'ascii', 'ignore'), "Andr x")
         self.assertEqual(str(b'Andr\202 x', 'ascii', 'replace'), 'Andr\uFFFD x')
+        self.assertEqual(str(b'\202 x', 'ascii', 'replace'), '\uFFFD x')
 
         # Error handling (unknown character names)
         self.assertEqual(b"\\N{foo}xx".decode("unicode-escape", "ignore"), "xx")
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,8 @@
 Core and Builtins
 -----------------
 
+- Issue #24848: Fixed a number of bugs in UTF-7 decoding of misformed data.
+
 - Issue #25267: The UTF-8 encoder is now up to 75 times as fast for error
   handlers: ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``.
   Patch co-written with Serhiy Storchaka.
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -4360,31 +4360,31 @@
             }
             else { /* now leaving a base-64 section */
                 inShift = 0;
-                s++;
-                if (surrogate) {
-                    if (_PyUnicodeWriter_WriteCharInline(&writer, surrogate) < 0)
-                        goto onError;
-                    surrogate = 0;
-                }
                 if (base64bits > 0) { /* left-over bits */
                     if (base64bits >= 6) {
                         /* We've seen at least one base-64 character */
+                        s++;
                         errmsg = "partial character in shift sequence";
                         goto utf7Error;
                     }
                     else {
                         /* Some bits remain; they should be zero */
                         if (base64buffer != 0) {
+                            s++;
                             errmsg = "non-zero padding bits in shift sequence";
                             goto utf7Error;
                         }
                     }
                 }
-                if (ch != '-') {
+                if (surrogate && DECODE_DIRECT(ch)) {
+                    if (_PyUnicodeWriter_WriteCharInline(&writer, surrogate) < 0)
+                        goto onError;
+                }
+                surrogate = 0;
+                if (ch == '-') {
                     /* '-' is absorbed; other terminating
                        characters are preserved */
-                    if (_PyUnicodeWriter_WriteCharInline(&writer, ch) < 0)
-                        goto onError;
+                    s++;
                 }
             }
         }
@@ -4398,6 +4398,7 @@
             }
             else { /* begin base64-encoded section */
                 inShift = 1;
+                surrogate = 0;
                 shiftOutStart = writer.pos;
                 base64bits = 0;
                 base64buffer = 0;
@@ -4429,6 +4430,7 @@
 
     if (inShift && !consumed) { /* in shift sequence, no more to follow */
         /* if we're in an inconsistent state, that's an error */
+        inShift = 0;
         if (surrogate ||
                 (base64bits >= 6) ||
                 (base64bits > 0 && base64buffer != 0)) {
@@ -13366,6 +13368,7 @@
 
         if (maxchar > writer->maxchar || writer->readonly) {
             /* resize + widen */
+            maxchar = Py_MAX(maxchar, writer->maxchar);
             newbuffer = PyUnicode_New(newlen, maxchar);
             if (newbuffer == NULL)
                 return -1;

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  2 12:15:42 2015
From: python-checkins at python.org (serhiy.storchaka)
Date: Fri, 02 Oct 2015 10:15:42 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzI0ODQ4?=
 =?utf-8?q?=3A_Fixed_bugs_in_UTF-7_decoding_of_misformed_data=3A?=
Message-ID: <20151002101541.11702.73372@psf.io>

https://hg.python.org/cpython/rev/c6eaa722e2c1
changeset:   98480:c6eaa722e2c1
branch:      2.7
parent:      98454:202c827f86df
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Fri Oct 02 13:14:53 2015 +0300
summary:
  Issue #24848: Fixed bugs in UTF-7 decoding of misformed data:

1. Non-ASCII bytes were accepted after shift sequence.
2. A low surrogate could be emitted in case of error in high surrogate.

files:
  Lib/test/test_codecs.py  |  59 ++++++++++++++++++++++++++++
  Lib/test/test_unicode.py |   1 +
  Misc/NEWS                |   2 +
  Objects/unicodeobject.c  |  16 ++++---
  4 files changed, 71 insertions(+), 7 deletions(-)


diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -642,6 +642,32 @@
 class UTF7Test(ReadTest):
     encoding = "utf-7"
 
+    def test_ascii(self):
+        # Set D (directly encoded characters)
+        set_d = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+                 'abcdefghijklmnopqrstuvwxyz'
+                 '0123456789'
+                 '\'(),-./:?')
+        self.assertEqual(set_d.encode(self.encoding), set_d)
+        self.assertEqual(set_d.decode(self.encoding), set_d)
+        # Set O (optional direct characters)
+        set_o = ' !"#$%&*;<=>@[]^_`{|}'
+        self.assertEqual(set_o.encode(self.encoding), set_o)
+        self.assertEqual(set_o.decode(self.encoding), set_o)
+        # +
+        self.assertEqual(u'a+b'.encode(self.encoding), 'a+-b')
+        self.assertEqual('a+-b'.decode(self.encoding), u'a+b')
+        # White spaces
+        ws = ' \t\n\r'
+        self.assertEqual(ws.encode(self.encoding), ws)
+        self.assertEqual(ws.decode(self.encoding), ws)
+        # Other ASCII characters
+        other_ascii = ''.join(sorted(set(chr(i) for i in range(0x80)) -
+                                     set(set_d + set_o + '+' + ws)))
+        self.assertEqual(other_ascii.encode(self.encoding),
+                         '+AAAAAQACAAMABAAFAAYABwAIAAsADAAOAA8AEAARABIAEwAU'
+                         'ABUAFgAXABgAGQAaABsAHAAdAB4AHwBcAH4Afw-')
+
     def test_partial(self):
         self.check_partial(
             u"a+-b",
@@ -656,7 +682,9 @@
 
     def test_errors(self):
         tests = [
+            ('\xffb', u'\ufffdb'),
             ('a\xffb', u'a\ufffdb'),
+            ('a\xff\xffb', u'a\ufffd\ufffdb'),
             ('a+IK', u'a\ufffd'),
             ('a+IK-b', u'a\ufffdb'),
             ('a+IK,b', u'a\ufffdb'),
@@ -672,6 +700,8 @@
             ('a+//,+IKw-b', u'a\ufffd\u20acb'),
             ('a+///,+IKw-b', u'a\uffff\ufffd\u20acb'),
             ('a+////,+IKw-b', u'a\uffff\ufffd\u20acb'),
+            ('a+IKw-b\xff', u'a\u20acb\ufffd'),
+            ('a+IKw\xffb', u'a\u20ac\ufffdb'),
         ]
         for raw, expected in tests:
             self.assertRaises(UnicodeDecodeError, codecs.utf_7_decode,
@@ -682,6 +712,35 @@
         self.assertEqual(u'\U000104A0'.encode(self.encoding), '+2AHcoA-')
         self.assertEqual(u'\ud801\udca0'.encode(self.encoding), '+2AHcoA-')
         self.assertEqual('+2AHcoA-'.decode(self.encoding), u'\U000104A0')
+        self.assertEqual('+2AHcoA'.decode(self.encoding), u'\U000104A0')
+        self.assertEqual(u'\u20ac\U000104A0'.encode(self.encoding), '+IKzYAdyg-')
+        self.assertEqual('+IKzYAdyg-'.decode(self.encoding), u'\u20ac\U000104A0')
+        self.assertEqual('+IKzYAdyg'.decode(self.encoding), u'\u20ac\U000104A0')
+        self.assertEqual(u'\u20ac\u20ac\U000104A0'.encode(self.encoding),
+                         '+IKwgrNgB3KA-')
+        self.assertEqual('+IKwgrNgB3KA-'.decode(self.encoding),
+                         u'\u20ac\u20ac\U000104A0')
+        self.assertEqual('+IKwgrNgB3KA'.decode(self.encoding),
+                         u'\u20ac\u20ac\U000104A0')
+
+    def test_lone_surrogates(self):
+        tests = [
+            ('a+2AE-b', u'a\ud801b'),
+            ('a+2AE\xffb', u'a\ufffdb'),
+            ('a+2AE', u'a\ufffd'),
+            ('a+2AEA-b', u'a\ufffdb'),
+            ('a+2AH-b', u'a\ufffdb'),
+            ('a+IKzYAQ-b', u'a\u20ac\ud801b'),
+            ('a+IKzYAQ\xffb', u'a\u20ac\ufffdb'),
+            ('a+IKzYAQA-b', u'a\u20ac\ufffdb'),
+            ('a+IKzYAd-b', u'a\u20ac\ufffdb'),
+            ('a+IKwgrNgB-b', u'a\u20ac\u20ac\ud801b'),
+            ('a+IKwgrNgB\xffb', u'a\u20ac\u20ac\ufffdb'),
+            ('a+IKwgrNgB', u'a\u20ac\u20ac\ufffd'),
+            ('a+IKwgrNgBA-b', u'a\u20ac\u20ac\ufffdb'),
+        ]
+        for raw, expected in tests:
+            self.assertEqual(raw.decode('utf-7', 'replace'), expected)
 
 class UTF16ExTest(unittest.TestCase):
 
diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py
--- a/Lib/test/test_unicode.py
+++ b/Lib/test/test_unicode.py
@@ -1036,6 +1036,7 @@
         self.assertRaises(UnicodeError, unicode, 'Andr\202 x', 'ascii','strict')
         self.assertEqual(unicode('Andr\202 x','ascii','ignore'), u"Andr x")
         self.assertEqual(unicode('Andr\202 x','ascii','replace'), u'Andr\uFFFD x')
+        self.assertEqual(unicode('\202 x', 'ascii', 'replace'), u'\uFFFD x')
         self.assertEqual(u'abcde'.decode('ascii', 'ignore'),
                          u'abcde'.decode('ascii', errors='ignore'))
         self.assertEqual(u'abcde'.decode('ascii', 'replace'),
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,8 @@
 Core and Builtins
 -----------------
 
+- Issue #24848: Fixed a number of bugs in UTF-7 decoding of misformed data.
+
 - Issue #25003: os.urandom() doesn't use getentropy() on Solaris because
   getentropy() is blocking, whereas os.urandom() should not block. getentropy()
   is supported since Solaris 11.3.
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -1716,29 +1716,29 @@
             }
             else { /* now leaving a base-64 section */
                 inShift = 0;
-                s++;
-                if (surrogate) {
-                    *p++ = surrogate;
-                    surrogate = 0;
-                }
                 if (base64bits > 0) { /* left-over bits */
                     if (base64bits >= 6) {
                         /* We've seen at least one base-64 character */
+                        s++;
                         errmsg = "partial character in shift sequence";
                         goto utf7Error;
                     }
                     else {
                         /* Some bits remain; they should be zero */
                         if (base64buffer != 0) {
+                            s++;
                             errmsg = "non-zero padding bits in shift sequence";
                             goto utf7Error;
                         }
                     }
                 }
-                if (ch != '-') {
+                if (surrogate && DECODE_DIRECT(ch))
+                    *p++ = surrogate;
+                surrogate = 0;
+                if (ch == '-') {
                     /* '-' is absorbed; other terminating
                        characters are preserved */
-                    *p++ = ch;
+                    s++;
                 }
             }
         }
@@ -1751,6 +1751,7 @@
             }
             else { /* begin base64-encoded section */
                 inShift = 1;
+                surrogate = 0;
                 shiftOutStart = p;
                 base64bits = 0;
                 base64buffer = 0;
@@ -1782,6 +1783,7 @@
 
     if (inShift && !consumed) { /* in shift sequence, no more to follow */
         /* if we're in an inconsistent state, that's an error */
+        inShift = 0;
         if (surrogate ||
                 (base64bits >= 6) ||
                 (base64bits > 0 && base64buffer != 0)) {

-- 
Repository URL: https://hg.python.org/cpython

From lp_benchmark_robot at intel.com  Fri Oct  2 17:05:13 2015
From: lp_benchmark_robot at intel.com (lp_benchmark_robot at intel.com)
Date: Fri, 2 Oct 2015 16:05:13 +0100
Subject: [Python-checkins] Benchmark Results for Python Default 2015-10-02
Message-ID: <201915f0-c6e4-44a7-8467-ee204c9e99ee@irsmsx151.ger.corp.intel.com>

Results for project python_default-nightly, build date 2015-10-02 03:02:00 
commit:		c98cc9f7e2c51bb4c5469754b5a8896c7c6520e1
revision date:	2015-10-01 22:20:11 +0000
environment:	Haswell-EP
	cpu:	Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz 2x18 cores, stepping 2, LLC 45 MB
	mem:	128 GB
	os:	CentOS 7.1
	kernel:	Linux 3.10.0-229.4.2.el7.x86_64

Baseline results were generated using release v3.4.3, with hash
b4cbecbc0781e89a309d03b60a1f75f8499250e6 from 2015-02-25 12:15:33+00:00

------------------------------------------------------------------------------------------
               benchmark    relative      change since      change since  current rev with
                            std_dev*          last run            v3.4.3      regrtest PGO
------------------------------------------------------------------------------------------
:-)            django_v2    0.48178%         -1.74654%          7.03348%         17.29170%
:-|              pybench    0.15028%          0.09730%         -1.62936%          8.80656%
:-(             regex_v8    2.74047%         -0.00311%         -3.98793%          6.26954%
:-|                nbody    0.48580%         -0.20683%         -0.07502%         10.24049%
:-|         json_dump_v2    0.27845%          0.17579%         -0.89919%         11.38912%
:-|       normal_startup    0.65205%          0.02698%          0.25297%          4.87967%
------------------------------------------------------------------------------------------
Note: Benchmark results are measured in seconds.
* Relative Standard Deviation (Standard Deviation/Average)

Our lab does a nightly source pull and build of the Python project and measures
performance changes against the previous stable version and the previous nightly
measurement. This is provided as a service to the community so that quality
issues with current hardware can be identified quickly.

Intel technologies' features and benefits depend on system configuration and may
require enabled hardware, software or service activation. Performance varies
depending on system configuration.


From lp_benchmark_robot at intel.com  Fri Oct  2 17:05:47 2015
From: lp_benchmark_robot at intel.com (lp_benchmark_robot at intel.com)
Date: Fri, 2 Oct 2015 16:05:47 +0100
Subject: [Python-checkins] Benchmark Results for Python 2.7 2015-10-02
Message-ID: <9b1adbb6-1000-4cc1-90e2-52c6cc6cc9b6@irsmsx151.ger.corp.intel.com>

Results for project python_2.7-nightly, build date 2015-10-02 07:24:36 
commit:		202c827f86df311af57d47cde6c39b86b8df3155
revision date:	2015-10-01 07:57:26 +0000
environment:	Haswell-EP
	cpu:	Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz 2x18 cores, stepping 2, LLC 45 MB
	mem:	128 GB
	os:	CentOS 7.1
	kernel:	Linux 3.10.0-229.4.2.el7.x86_64

Baseline results were generated using release v2.7.10, with hash
15c95b7d81dcf821daade360741e00714667653f from 2015-05-23 16:02:14+00:00

------------------------------------------------------------------------------------------
               benchmark    relative      change since      change since  current rev with
                            std_dev*          last run           v2.7.10      regrtest PGO
------------------------------------------------------------------------------------------
:-)            django_v2    0.11813%          0.53079%          4.53292%         10.61265%
:-)              pybench    0.16320%         -0.52636%          6.12360%          7.31444%
:-|             regex_v8    0.98661%          0.06033%         -1.83641%          6.58988%
:-)                nbody    0.13567%         -0.09577%          8.53668%          4.48159%
:-)         json_dump_v2    0.28424%          0.18774%          3.62669%         13.46654%
:-|       normal_startup    1.80744%         -0.37050%         -1.65924%          1.98921%
:-|              ssbench    0.60809%         -0.53691%          1.03661%          2.63734%
------------------------------------------------------------------------------------------
Note: Benchmark results for ssbench are measured in requests/second while all
other are measured in seconds.
* Relative Standard Deviation (Standard Deviation/Average)

Our lab does a nightly source pull and build of the Python project and measures
performance changes against the previous stable version and the previous nightly
measurement. This is provided as a service to the community so that quality
issues with current hardware can be identified quickly.

Intel technologies' features and benefits depend on system configuration and may
require enabled hardware, software or service activation. Performance varies
depending on system configuration.


From python-checkins at python.org  Fri Oct  2 18:25:55 2015
From: python-checkins at python.org (berker.peksag)
Date: Fri, 02 Oct 2015 16:25:55 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2325290=3A_Fix_typo_in_csv=2Ereader=28=29_docstri?=
 =?utf-8?q?ng?=
Message-ID: <20151002162555.8654.48715@psf.io>

https://hg.python.org/cpython/rev/7a3073921687
changeset:   98483:7a3073921687
parent:      98479:037253b7cd6d
parent:      98482:3b565295eba0
user:        Berker Peksag <berker.peksag at gmail.com>
date:        Fri Oct 02 19:26:14 2015 +0300
summary:
  Issue #25290: Fix typo in csv.reader() docstring

Patch by Johannes Niediek.

files:
  Modules/_csv.c |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Modules/_csv.c b/Modules/_csv.c
--- a/Modules/_csv.c
+++ b/Modules/_csv.c
@@ -1562,7 +1562,7 @@
 "provided by the dialect.\n"
 "\n"
 "The returned object is an iterator.  Each iteration returns a row\n"
-"of the CSV file (which can span multiple input lines):\n");
+"of the CSV file (which can span multiple input lines).\n");
 
 PyDoc_STRVAR(csv_writer_doc,
 "    csv_writer = csv.writer(fileobj [, dialect='excel']\n"

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  2 18:25:55 2015
From: python-checkins at python.org (berker.peksag)
Date: Fri, 02 Oct 2015 16:25:55 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
 =?utf-8?q?_Issue_=2325290=3A_Fix_typo_in_csv=2Ereader=28=29_docstring?=
Message-ID: <20151002162554.11700.31496@psf.io>

https://hg.python.org/cpython/rev/3b565295eba0
changeset:   98482:3b565295eba0
branch:      3.5
parent:      98478:a61fa2b08f87
parent:      98481:3940f480ea16
user:        Berker Peksag <berker.peksag at gmail.com>
date:        Fri Oct 02 19:25:53 2015 +0300
summary:
  Issue #25290: Fix typo in csv.reader() docstring

Patch by Johannes Niediek.

files:
  Modules/_csv.c |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Modules/_csv.c b/Modules/_csv.c
--- a/Modules/_csv.c
+++ b/Modules/_csv.c
@@ -1562,7 +1562,7 @@
 "provided by the dialect.\n"
 "\n"
 "The returned object is an iterator.  Each iteration returns a row\n"
-"of the CSV file (which can span multiple input lines):\n");
+"of the CSV file (which can span multiple input lines).\n");
 
 PyDoc_STRVAR(csv_writer_doc,
 "    csv_writer = csv.writer(fileobj [, dialect='excel']\n"

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  2 18:25:54 2015
From: python-checkins at python.org (berker.peksag)
Date: Fri, 02 Oct 2015 16:25:54 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzI1Mjkw?=
 =?utf-8?q?=3A_Fix_typo_in_csv=2Ereader=28=29_docstring?=
Message-ID: <20151002162554.93032.12609@psf.io>

https://hg.python.org/cpython/rev/3940f480ea16
changeset:   98481:3940f480ea16
branch:      3.4
parent:      98477:3c13567ea642
user:        Berker Peksag <berker.peksag at gmail.com>
date:        Fri Oct 02 19:25:32 2015 +0300
summary:
  Issue #25290: Fix typo in csv.reader() docstring

Patch by Johannes Niediek.

files:
  Modules/_csv.c |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Modules/_csv.c b/Modules/_csv.c
--- a/Modules/_csv.c
+++ b/Modules/_csv.c
@@ -1563,7 +1563,7 @@
 "provided by the dialect.\n"
 "\n"
 "The returned object is an iterator.  Each iteration returns a row\n"
-"of the CSV file (which can span multiple input lines):\n");
+"of the CSV file (which can span multiple input lines).\n");
 
 PyDoc_STRVAR(csv_writer_doc,
 "    csv_writer = csv.writer(fileobj [, dialect='excel']\n"

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  2 18:30:05 2015
From: python-checkins at python.org (berker.peksag)
Date: Fri, 02 Oct 2015 16:30:05 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzI1Mjkw?=
 =?utf-8?q?=3A_Fix_typo_in_csv=2Ereader=28=29_docstring?=
Message-ID: <20151002163005.93038.44749@psf.io>

https://hg.python.org/cpython/rev/ceff1babf66e
changeset:   98484:ceff1babf66e
branch:      2.7
parent:      98480:c6eaa722e2c1
user:        Berker Peksag <berker.peksag at gmail.com>
date:        Fri Oct 02 19:30:21 2015 +0300
summary:
  Issue #25290: Fix typo in csv.reader() docstring

Patch by Johannes Niediek.

files:
  Modules/_csv.c |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Modules/_csv.c b/Modules/_csv.c
--- a/Modules/_csv.c
+++ b/Modules/_csv.c
@@ -1520,7 +1520,7 @@
 "provided by the dialect.\n"
 "\n"
 "The returned object is an iterator.  Each iteration returns a row\n"
-"of the CSV file (which can span multiple input lines):\n");
+"of the CSV file (which can span multiple input lines).\n");
 
 PyDoc_STRVAR(csv_writer_doc,
 "    csv_writer = csv.writer(fileobj [, dialect='excel']\n"

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  2 19:24:25 2015
From: python-checkins at python.org (serhiy.storchaka)
Date: Fri, 02 Oct 2015 17:24:25 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_Removed_the_?=
 =?utf-8?q?=22b=22_string_prefix_to_make_test=5Fxpickle_compatible_with_Py?=
 =?utf-8?b?dGhvbiAyLjUu?=
Message-ID: <20151002172423.95970.47257@psf.io>

https://hg.python.org/cpython/rev/8bbc51f97078
changeset:   98485:8bbc51f97078
branch:      2.7
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Fri Oct 02 20:23:46 2015 +0300
summary:
  Removed the "b" string prefix to make test_xpickle compatible with Python 2.5.

files:
  Lib/test/pickletester.py |  66 ++++++++++++++--------------
  1 files changed, 33 insertions(+), 33 deletions(-)


diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py
--- a/Lib/test/pickletester.py
+++ b/Lib/test/pickletester.py
@@ -498,10 +498,10 @@
             self.assertRaises(ValueError, self.loads, buf)
 
     def test_correctly_quoted_string(self):
-        goodpickles = [(b"S''\n.", ''),
-                       (b'S""\n.', ''),
-                       (b'S"\\n"\n.', '\n'),
-                       (b"S'\\n'\n.", '\n')]
+        goodpickles = [("S''\n.", ''),
+                       ('S""\n.', ''),
+                       ('S"\\n"\n.', '\n'),
+                       ("S'\\n'\n.", '\n')]
         for p, expected in goodpickles:
             self.assertEqual(self.loads(p), expected)
 
@@ -521,10 +521,10 @@
             21: b    BUILD
             22: .    STOP
             """
-            pickle0 = (b"(i__main__\n"
-                       b"X\n"
-                       b"p0\n"
-                       b"(dp1\nb.").replace(b'X', xname)
+            pickle0 = ("(i__main__\n"
+                       "X\n"
+                       "p0\n"
+                       "(dp1\nb.").replace('X', xname)
             self.assert_is_copy(X(*args), self.loads(pickle0))
 
             # Protocol 1 (binary mode pickle)
@@ -539,12 +539,12 @@
             21: b    BUILD
             22: .    STOP
             """
-            pickle1 = (b'(c__main__\n'
-                       b'X\n'
-                       b'q\x00oq\x01}q\x02b.').replace(b'X', xname)
+            pickle1 = ('(c__main__\n'
+                       'X\n'
+                       'q\x00oq\x01}q\x02b.').replace('X', xname)
             self.assert_is_copy(X(*args), self.loads(pickle1))
 
-            # Protocol 2 (pickle2 = b'\x80\x02' + pickle1)
+            # Protocol 2 (pickle2 = '\x80\x02' + pickle1)
             """
              0: \x80 PROTO      2
              2: (    MARK
@@ -557,63 +557,63 @@
             23: b    BUILD
             24: .    STOP
             """
-            pickle2 = (b'\x80\x02(c__main__\n'
-                       b'X\n'
-                       b'q\x00oq\x01}q\x02b.').replace(b'X', xname)
+            pickle2 = ('\x80\x02(c__main__\n'
+                       'X\n'
+                       'q\x00oq\x01}q\x02b.').replace('X', xname)
             self.assert_is_copy(X(*args), self.loads(pickle2))
 
     def test_pop_empty_stack(self):
         # Test issue7455
-        s = b'0'
+        s = '0'
         self.assertRaises((cPickle.UnpicklingError, IndexError), self.loads, s)
 
     def test_load_str(self):
         # From Python 2: pickle.dumps('a\x00\xa0', protocol=0)
-        self.assertEqual(self.loads(b"S'a\\x00\\xa0'\n."), b'a\x00\xa0')
+        self.assertEqual(self.loads("S'a\\x00\\xa0'\n."), 'a\x00\xa0')
         # From Python 2: pickle.dumps('a\x00\xa0', protocol=1)
-        self.assertEqual(self.loads(b'U\x03a\x00\xa0.'), b'a\x00\xa0')
+        self.assertEqual(self.loads('U\x03a\x00\xa0.'), 'a\x00\xa0')
         # From Python 2: pickle.dumps('a\x00\xa0', protocol=2)
-        self.assertEqual(self.loads(b'\x80\x02U\x03a\x00\xa0.'), b'a\x00\xa0')
+        self.assertEqual(self.loads('\x80\x02U\x03a\x00\xa0.'), 'a\x00\xa0')
 
     def test_load_unicode(self):
         # From Python 2: pickle.dumps(u'?', protocol=0)
-        self.assertEqual(self.loads(b'V\\u03c0\n.'), u'?')
+        self.assertEqual(self.loads('V\\u03c0\n.'), u'?')
         # From Python 2: pickle.dumps(u'?', protocol=1)
-        self.assertEqual(self.loads(b'X\x02\x00\x00\x00\xcf\x80.'), u'?')
+        self.assertEqual(self.loads('X\x02\x00\x00\x00\xcf\x80.'), u'?')
         # From Python 2: pickle.dumps(u'?', protocol=2)
-        self.assertEqual(self.loads(b'\x80\x02X\x02\x00\x00\x00\xcf\x80.'), u'?')
+        self.assertEqual(self.loads('\x80\x02X\x02\x00\x00\x00\xcf\x80.'), u'?')
 
     def test_constants(self):
-        self.assertIsNone(self.loads(b'N.'))
-        self.assertIs(self.loads(b'\x88.'), True)
-        self.assertIs(self.loads(b'\x89.'), False)
-        self.assertIs(self.loads(b'I01\n.'), True)
-        self.assertIs(self.loads(b'I00\n.'), False)
+        self.assertIsNone(self.loads('N.'))
+        self.assertIs(self.loads('\x88.'), True)
+        self.assertIs(self.loads('\x89.'), False)
+        self.assertIs(self.loads('I01\n.'), True)
+        self.assertIs(self.loads('I00\n.'), False)
 
     def test_misc_get(self):
-        self.assertRaises(self.error, self.loads, b'g0\np0\n')
-        self.assertRaises(self.error, self.loads, b'h\x00q\x00')
+        self.assertRaises(self.error, self.loads, 'g0\np0\n')
+        self.assertRaises(self.error, self.loads, 'h\x00q\x00')
 
     def test_get(self):
-        pickled = b'((lp100000\ng100000\nt.'
+        pickled = '((lp100000\ng100000\nt.'
         unpickled = self.loads(pickled)
         self.assertEqual(unpickled, ([],)*2)
         self.assertIs(unpickled[0], unpickled[1])
 
     def test_binget(self):
-        pickled = b'(]q\xffh\xfft.'
+        pickled = '(]q\xffh\xfft.'
         unpickled = self.loads(pickled)
         self.assertEqual(unpickled, ([],)*2)
         self.assertIs(unpickled[0], unpickled[1])
 
     def test_long_binget(self):
-        pickled = b'(]r\x00\x00\x01\x00j\x00\x00\x01\x00t.'
+        pickled = '(]r\x00\x00\x01\x00j\x00\x00\x01\x00t.'
         unpickled = self.loads(pickled)
         self.assertEqual(unpickled, ([],)*2)
         self.assertIs(unpickled[0], unpickled[1])
 
     def test_dup(self):
-        pickled = b'((l2t.'
+        pickled = '((l2t.'
         unpickled = self.loads(pickled)
         self.assertEqual(unpickled, ([],)*2)
         self.assertIs(unpickled[0], unpickled[1])

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  2 20:45:02 2015
From: python-checkins at python.org (brett.cannon)
Date: Fri, 02 Oct 2015 18:45:02 +0000
Subject: [Python-checkins] =?utf-8?q?benchmarks=3A_Issue_=2325266=3A_the_m?=
 =?utf-8?q?ako_benchmark_does_not_work_in_Python_3=2E6=2E?=
Message-ID: <20151002184500.95982.90038@psf.io>

https://hg.python.org/benchmarks/rev/85edb638dce6
changeset:   222:85edb638dce6
user:        Brett Cannon <brett at python.org>
date:        Fri Oct 02 11:44:56 2015 -0700
summary:
  Issue #25266: the mako benchmark does not work in Python 3.6.

Thanks to Florin Popa of Intel for the patch.

files:
  perf.py |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/perf.py b/perf.py
--- a/perf.py
+++ b/perf.py
@@ -1594,7 +1594,7 @@
     bm_env = BuildEnv({"PYTHONPATH": mako_path}, options.inherit_env)
     return MeasureGeneric(python, options, bm_path, bm_env, iteration_scaling=5)
 
- at VersionRange()
+ at VersionRange(None, '3.5')
 def BM_mako(*args, **kwargs):
     return SimpleBenchmark(MeasureMako, *args, **kwargs)
 

-- 
Repository URL: https://hg.python.org/benchmarks

From python-checkins at python.org  Fri Oct  2 21:10:08 2015
From: python-checkins at python.org (yury.selivanov)
Date: Fri, 02 Oct 2015 19:10:08 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
 =?utf-8?q?_asyncio=3A_Make_ensure=5Ffuture=28=29_accept_all_kinds_of_awai?=
 =?utf-8?q?tables=2E?=
Message-ID: <20151002191007.11714.24234@psf.io>

https://hg.python.org/cpython/rev/901537ff7f80
changeset:   98487:901537ff7f80
branch:      3.5
parent:      98482:3b565295eba0
parent:      98486:b40a61e79893
user:        Yury Selivanov <yselivanov at sprymix.com>
date:        Fri Oct 02 15:05:59 2015 -0400
summary:
  asyncio: Make ensure_future() accept all kinds of awaitables.

files:
  Doc/whatsnew/3.5.rst                |   7 ++++++
  Lib/asyncio/tasks.py                |  16 +++++++++++++-
  Lib/test/test_asyncio/test_tasks.py |  18 +++++++++++++++++
  Misc/NEWS                           |   2 +
  4 files changed, 41 insertions(+), 2 deletions(-)


diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst
--- a/Doc/whatsnew/3.5.rst
+++ b/Doc/whatsnew/3.5.rst
@@ -803,6 +803,13 @@
   :class:`asyncio.Queue` class.
   (Contributed by Victor Stinner.)
 
+Updates in 3.5.1:
+
+* The :func:`~asyncio.ensure_future` function and all functions that
+  use it, such as :meth:`loop.run_until_complete() <asyncio.BaseEventLoop.run_until_complete>`,
+  now accept all kinds of :term:`awaitable objects <awaitable>`.
+  (Contributed by Yury Selivanov.)
+
 
 bz2
 ---
diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py
--- a/Lib/asyncio/tasks.py
+++ b/Lib/asyncio/tasks.py
@@ -512,7 +512,7 @@
 
 
 def ensure_future(coro_or_future, *, loop=None):
-    """Wrap a coroutine in a future.
+    """Wrap a coroutine or an awaitable in a future.
 
     If the argument is a Future, it is returned directly.
     """
@@ -527,8 +527,20 @@
         if task._source_traceback:
             del task._source_traceback[-1]
         return task
+    elif compat.PY35 and inspect.isawaitable(coro_or_future):
+        return ensure_future(_wrap_awaitable(coro_or_future), loop=loop)
     else:
-        raise TypeError('A Future or coroutine is required')
+        raise TypeError('A Future, a coroutine or an awaitable is required')
+
+
+ at coroutine
+def _wrap_awaitable(awaitable):
+    """Helper for asyncio.ensure_future().
+
+    Wraps awaitable (an object with __await__) into a coroutine
+    that will later be wrapped in a Task by ensure_future().
+    """
+    return (yield from awaitable.__await__())
 
 
 class _GatheringFuture(futures.Future):
diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py
--- a/Lib/test/test_asyncio/test_tasks.py
+++ b/Lib/test/test_asyncio/test_tasks.py
@@ -153,6 +153,24 @@
         t = asyncio.ensure_future(t_orig, loop=self.loop)
         self.assertIs(t, t_orig)
 
+    @unittest.skipUnless(PY35, 'need python 3.5 or later')
+    def test_ensure_future_awaitable(self):
+        class Aw:
+            def __init__(self, coro):
+                self.coro = coro
+            def __await__(self):
+                return (yield from self.coro)
+
+        @asyncio.coroutine
+        def coro():
+            return 'ok'
+
+        loop = asyncio.new_event_loop()
+        self.set_event_loop(loop)
+        fut = asyncio.ensure_future(Aw(coro()), loop=loop)
+        loop.run_until_complete(fut)
+        assert fut.result() == 'ok'
+
     def test_ensure_future_neither(self):
         with self.assertRaises(TypeError):
             asyncio.ensure_future('ok')
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -152,6 +152,8 @@
 - Issue #23572: Fixed functools.singledispatch on classes with falsy
   metaclasses.  Patch by Ethan Furman.
 
+- asyncio: ensure_future() now accepts awaitable objects.
+
 IDLE
 ----
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  2 21:10:10 2015
From: python-checkins at python.org (yury.selivanov)
Date: Fri, 02 Oct 2015 19:10:10 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
 =?utf-8?q?=29=3A_asyncio=3A_Make_ensure=5Ffuture=28=29_accept_all_kinds_o?=
 =?utf-8?q?f_awaitables=2E_=28Merge_3=2E5=29?=
Message-ID: <20151002191007.81384.14531@psf.io>

https://hg.python.org/cpython/rev/4d699bf00be0
changeset:   98488:4d699bf00be0
parent:      98483:7a3073921687
parent:      98487:901537ff7f80
user:        Yury Selivanov <yselivanov at sprymix.com>
date:        Fri Oct 02 15:09:51 2015 -0400
summary:
  asyncio: Make ensure_future() accept all kinds of awaitables. (Merge 3.5)

files:
  Doc/whatsnew/3.5.rst                |   7 ++++++
  Lib/asyncio/tasks.py                |  16 +++++++++++++-
  Lib/test/test_asyncio/test_tasks.py |  18 +++++++++++++++++
  3 files changed, 39 insertions(+), 2 deletions(-)


diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst
--- a/Doc/whatsnew/3.5.rst
+++ b/Doc/whatsnew/3.5.rst
@@ -803,6 +803,13 @@
   :class:`asyncio.Queue` class.
   (Contributed by Victor Stinner.)
 
+Updates in 3.5.1:
+
+* The :func:`~asyncio.ensure_future` function and all functions that
+  use it, such as :meth:`loop.run_until_complete() <asyncio.BaseEventLoop.run_until_complete>`,
+  now accept all kinds of :term:`awaitable objects <awaitable>`.
+  (Contributed by Yury Selivanov.)
+
 
 bz2
 ---
diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py
--- a/Lib/asyncio/tasks.py
+++ b/Lib/asyncio/tasks.py
@@ -512,7 +512,7 @@
 
 
 def ensure_future(coro_or_future, *, loop=None):
-    """Wrap a coroutine in a future.
+    """Wrap a coroutine or an awaitable in a future.
 
     If the argument is a Future, it is returned directly.
     """
@@ -527,8 +527,20 @@
         if task._source_traceback:
             del task._source_traceback[-1]
         return task
+    elif compat.PY35 and inspect.isawaitable(coro_or_future):
+        return ensure_future(_wrap_awaitable(coro_or_future), loop=loop)
     else:
-        raise TypeError('A Future or coroutine is required')
+        raise TypeError('A Future, a coroutine or an awaitable is required')
+
+
+ at coroutine
+def _wrap_awaitable(awaitable):
+    """Helper for asyncio.ensure_future().
+
+    Wraps awaitable (an object with __await__) into a coroutine
+    that will later be wrapped in a Task by ensure_future().
+    """
+    return (yield from awaitable.__await__())
 
 
 class _GatheringFuture(futures.Future):
diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py
--- a/Lib/test/test_asyncio/test_tasks.py
+++ b/Lib/test/test_asyncio/test_tasks.py
@@ -153,6 +153,24 @@
         t = asyncio.ensure_future(t_orig, loop=self.loop)
         self.assertIs(t, t_orig)
 
+    @unittest.skipUnless(PY35, 'need python 3.5 or later')
+    def test_ensure_future_awaitable(self):
+        class Aw:
+            def __init__(self, coro):
+                self.coro = coro
+            def __await__(self):
+                return (yield from self.coro)
+
+        @asyncio.coroutine
+        def coro():
+            return 'ok'
+
+        loop = asyncio.new_event_loop()
+        self.set_event_loop(loop)
+        fut = asyncio.ensure_future(Aw(coro()), loop=loop)
+        loop.run_until_complete(fut)
+        assert fut.result() == 'ok'
+
     def test_ensure_future_neither(self):
         with self.assertRaises(TypeError):
             asyncio.ensure_future('ok')

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  2 21:10:13 2015
From: python-checkins at python.org (yury.selivanov)
Date: Fri, 02 Oct 2015 19:10:13 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogYXN5bmNpbzogZW5z?=
 =?utf-8?q?ure=5Ffuture=28=29_now_understands_awaitables?=
Message-ID: <20151002191007.81372.29588@psf.io>

https://hg.python.org/cpython/rev/b40a61e79893
changeset:   98486:b40a61e79893
branch:      3.4
parent:      98481:3940f480ea16
user:        Yury Selivanov <yselivanov at sprymix.com>
date:        Fri Oct 02 15:00:19 2015 -0400
summary:
  asyncio: ensure_future() now understands awaitables

files:
  Lib/asyncio/tasks.py                |  16 +++++++++++++-
  Lib/test/test_asyncio/test_tasks.py |  18 +++++++++++++++++
  2 files changed, 32 insertions(+), 2 deletions(-)


diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py
--- a/Lib/asyncio/tasks.py
+++ b/Lib/asyncio/tasks.py
@@ -512,7 +512,7 @@
 
 
 def ensure_future(coro_or_future, *, loop=None):
-    """Wrap a coroutine in a future.
+    """Wrap a coroutine or an awaitable in a future.
 
     If the argument is a Future, it is returned directly.
     """
@@ -527,8 +527,20 @@
         if task._source_traceback:
             del task._source_traceback[-1]
         return task
+    elif compat.PY35 and inspect.isawaitable(coro_or_future):
+        return ensure_future(_wrap_awaitable(coro_or_future), loop=loop)
     else:
-        raise TypeError('A Future or coroutine is required')
+        raise TypeError('A Future, a coroutine or an awaitable is required')
+
+
+ at coroutine
+def _wrap_awaitable(awaitable):
+    """Helper for asyncio.ensure_future().
+
+    Wraps awaitable (an object with __await__) into a coroutine
+    that will later be wrapped in a Task by ensure_future().
+    """
+    return (yield from awaitable.__await__())
 
 
 class _GatheringFuture(futures.Future):
diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py
--- a/Lib/test/test_asyncio/test_tasks.py
+++ b/Lib/test/test_asyncio/test_tasks.py
@@ -153,6 +153,24 @@
         t = asyncio.ensure_future(t_orig, loop=self.loop)
         self.assertIs(t, t_orig)
 
+    @unittest.skipUnless(PY35, 'need python 3.5 or later')
+    def test_ensure_future_awaitable(self):
+        class Aw:
+            def __init__(self, coro):
+                self.coro = coro
+            def __await__(self):
+                return (yield from self.coro)
+
+        @asyncio.coroutine
+        def coro():
+            return 'ok'
+
+        loop = asyncio.new_event_loop()
+        self.set_event_loop(loop)
+        fut = asyncio.ensure_future(Aw(coro()), loop=loop)
+        loop.run_until_complete(fut)
+        assert fut.result() == 'ok'
+
     def test_ensure_future_neither(self):
         with self.assertRaises(TypeError):
             asyncio.ensure_future('ok')

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  2 22:28:53 2015
From: python-checkins at python.org (berker.peksag)
Date: Fri, 02 Oct 2015 20:28:53 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
	=?utf-8?q?=29=3A_Merge_from_3=2E5?=
Message-ID: <20151002202851.93026.12716@psf.io>

https://hg.python.org/cpython/rev/45640def1e3d
changeset:   98490:45640def1e3d
parent:      98488:4d699bf00be0
parent:      98489:4eb809fa1130
user:        Berker Peksag <berker.peksag at gmail.com>
date:        Fri Oct 02 23:29:13 2015 +0300
summary:
  Merge from 3.5

Hg: --

files:
  Doc/tools/susp-ignored.csv |  10 +++++-----
  1 files changed, 5 insertions(+), 5 deletions(-)


diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv
--- a/Doc/tools/susp-ignored.csv
+++ b/Doc/tools/susp-ignored.csv
@@ -290,8 +290,8 @@
 library/stdtypes,,::,>>> m[::2].tolist()
 library/sys,1115,`,# ``wrapper`` creates a ``wrap(coro)`` coroutine:
 tutorial/venv,77,:c7b9645a6f35,"Python 3.4.3+ (3.4:c7b9645a6f35+, May 22 2015, 09:31:25)"
-whatsnew/3.5,965,:root,'WARNING:root:warning\n'
-whatsnew/3.5,965,:warning,'WARNING:root:warning\n'
-whatsnew/3.5,1292,::,>>> addr6 = ipaddress.IPv6Address('::1')
-whatsnew/3.5,1354,:root,ERROR:root:exception
-whatsnew/3.5,1354,:exception,ERROR:root:exception
+whatsnew/3.5,,:root,'WARNING:root:warning\n'
+whatsnew/3.5,,:warning,'WARNING:root:warning\n'
+whatsnew/3.5,,::,>>> addr6 = ipaddress.IPv6Address('::1')
+whatsnew/3.5,,:root,ERROR:root:exception
+whatsnew/3.5,,:exception,ERROR:root:exception

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  2 22:28:53 2015
From: python-checkins at python.org (berker.peksag)
Date: Fri, 02 Oct 2015 20:28:53 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E5=29=3A_Tweak_susp-ign?=
 =?utf-8?q?ored=2Ecsv_to_make_buildbots_happy?=
Message-ID: <20151002202851.78898.34660@psf.io>

https://hg.python.org/cpython/rev/4eb809fa1130
changeset:   98489:4eb809fa1130
branch:      3.5
parent:      98487:901537ff7f80
user:        Berker Peksag <berker.peksag at gmail.com>
date:        Fri Oct 02 23:28:45 2015 +0300
summary:
  Tweak susp-ignored.csv to make buildbots happy

files:
  Doc/tools/susp-ignored.csv |  10 +++++-----
  1 files changed, 5 insertions(+), 5 deletions(-)


diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv
--- a/Doc/tools/susp-ignored.csv
+++ b/Doc/tools/susp-ignored.csv
@@ -290,8 +290,8 @@
 library/stdtypes,,::,>>> m[::2].tolist()
 library/sys,1115,`,# ``wrapper`` creates a ``wrap(coro)`` coroutine:
 tutorial/venv,77,:c7b9645a6f35,"Python 3.4.3+ (3.4:c7b9645a6f35+, May 22 2015, 09:31:25)"
-whatsnew/3.5,965,:root,'WARNING:root:warning\n'
-whatsnew/3.5,965,:warning,'WARNING:root:warning\n'
-whatsnew/3.5,1292,::,>>> addr6 = ipaddress.IPv6Address('::1')
-whatsnew/3.5,1354,:root,ERROR:root:exception
-whatsnew/3.5,1354,:exception,ERROR:root:exception
+whatsnew/3.5,,:root,'WARNING:root:warning\n'
+whatsnew/3.5,,:warning,'WARNING:root:warning\n'
+whatsnew/3.5,,::,>>> addr6 = ipaddress.IPv6Address('::1')
+whatsnew/3.5,,:root,ERROR:root:exception
+whatsnew/3.5,,:exception,ERROR:root:exception

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  2 23:02:30 2015
From: python-checkins at python.org (victor.stinner)
Date: Fri, 02 Oct 2015 21:02:30 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2325287=3A_Don=27t_?=
 =?utf-8?q?add_crypt=2EMETHOD=5FCRYPT_to_crypt=2Emethods_if_it=27s_not?=
Message-ID: <20151002210230.11706.11619@psf.io>

https://hg.python.org/cpython/rev/4da7edbf78d4
changeset:   98491:4da7edbf78d4
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Fri Oct 02 23:00:39 2015 +0200
summary:
  Issue #25287: Don't add crypt.METHOD_CRYPT to crypt.methods if it's not
supported. Check if it is supported, it may not be supported on OpenBSD for
example.

files:
  Doc/library/crypt.rst |  2 +-
  Lib/crypt.py          |  3 +--
  Misc/NEWS             |  4 ++++
  3 files changed, 6 insertions(+), 3 deletions(-)


diff --git a/Doc/library/crypt.rst b/Doc/library/crypt.rst
--- a/Doc/library/crypt.rst
+++ b/Doc/library/crypt.rst
@@ -64,7 +64,7 @@
 
    A list of available password hashing algorithms, as
    ``crypt.METHOD_*`` objects.  This list is sorted from strongest to
-   weakest, and is guaranteed to have at least ``crypt.METHOD_CRYPT``.
+   weakest.
 
 
 Module Functions
diff --git a/Lib/crypt.py b/Lib/crypt.py
--- a/Lib/crypt.py
+++ b/Lib/crypt.py
@@ -54,9 +54,8 @@
 METHOD_SHA512 = _Method('SHA512', '6', 16, 106)
 
 methods = []
-for _method in (METHOD_SHA512, METHOD_SHA256, METHOD_MD5):
+for _method in (METHOD_SHA512, METHOD_SHA256, METHOD_MD5, METHOD_CRYPT):
     _result = crypt('', _method)
     if _result and len(_result) == _method.total_size:
         methods.append(_method)
-methods.append(METHOD_CRYPT)
 del _result, _method
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -40,6 +40,10 @@
 Library
 -------
 
+- Issue #25287: Don't add crypt.METHOD_CRYPT to crypt.methods if it's not
+  supported. Check if it is supported, it may not be supported on OpenBSD for
+  example.
+
 - Issue #23600: Default implementation of tzinfo.fromutc() was returning
   wrong results in some cases.
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 00:47:42 2015
From: python-checkins at python.org (victor.stinner)
Date: Fri, 02 Oct 2015 22:47:42 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318174=3A_=22pytho?=
 =?utf-8?q?n_-m_test_--huntrleaks_=2E=2E=2E=22_now_also_checks_for_leak_of?=
Message-ID: <20151002224742.93030.91404@psf.io>

https://hg.python.org/cpython/rev/72129c767c92
changeset:   98492:72129c767c92
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Sat Oct 03 00:20:56 2015 +0200
summary:
  Issue #18174: "python -m test --huntrleaks ..." now also checks for leak of
file descriptors. Patch written by Richard Oudkerk.

files:
  Lib/test/libregrtest/refleak.py |  47 ++++++++++++++-
  Lib/test/test_regrtest.py       |  63 ++++++++++++++++----
  Misc/NEWS                       |   3 +
  3 files changed, 94 insertions(+), 19 deletions(-)


diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py
--- a/Lib/test/libregrtest/refleak.py
+++ b/Lib/test/libregrtest/refleak.py
@@ -1,3 +1,4 @@
+import errno
 import os
 import re
 import sys
@@ -6,6 +7,36 @@
 from test import support
 
 
+try:
+    MAXFD = os.sysconf("SC_OPEN_MAX")
+except Exception:
+    MAXFD = 256
+
+
+def fd_count():
+    """Count the number of open file descriptors"""
+    if sys.platform.startswith(('linux', 'freebsd')):
+        try:
+            names = os.listdir("/proc/self/fd")
+            return len(names)
+        except FileNotFoundError:
+            pass
+
+    count = 0
+    for fd in range(MAXFD):
+        try:
+            # Prefer dup() over fstat(). fstat() can require input/output
+            # whereas dup() doesn't.
+            fd2 = os.dup(fd)
+        except OSError as e:
+            if e.errno != errno.EBADF:
+                raise
+        else:
+            os.close(fd2)
+            count += 1
+    return count
+
+
 def dash_R(the_module, test, indirect_test, huntrleaks):
     """Run a test multiple times, looking for reference leaks.
 
@@ -42,20 +73,25 @@
     repcount = nwarmup + ntracked
     rc_deltas = [0] * repcount
     alloc_deltas = [0] * repcount
+    fd_deltas = [0] * repcount
 
     print("beginning", repcount, "repetitions", file=sys.stderr)
     print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr,
           flush=True)
     # initialize variables to make pyflakes quiet
-    rc_before = alloc_before = 0
+    rc_before = alloc_before = fd_before = 0
     for i in range(repcount):
         indirect_test()
-        alloc_after, rc_after = dash_R_cleanup(fs, ps, pic, zdc, abcs)
+        alloc_after, rc_after, fd_after = dash_R_cleanup(fs, ps, pic, zdc,
+                                                         abcs)
         print('.', end='', flush=True)
         if i >= nwarmup:
             rc_deltas[i] = rc_after - rc_before
             alloc_deltas[i] = alloc_after - alloc_before
-        alloc_before, rc_before = alloc_after, rc_after
+            fd_deltas[i] = fd_after - fd_before
+        alloc_before = alloc_after
+        rc_before = rc_after
+        fd_before = fd_after
     print(file=sys.stderr)
     # These checkers return False on success, True on failure
     def check_rc_deltas(deltas):
@@ -71,7 +107,8 @@
     failed = False
     for deltas, item_name, checker in [
         (rc_deltas, 'references', check_rc_deltas),
-        (alloc_deltas, 'memory blocks', check_alloc_deltas)]:
+        (alloc_deltas, 'memory blocks', check_alloc_deltas),
+        (fd_deltas, 'file descriptors', check_rc_deltas)]:
         if checker(deltas):
             msg = '%s leaked %s %s, sum=%s' % (
                 test, deltas[nwarmup:], item_name, sum(deltas))
@@ -151,7 +188,7 @@
     func1 = sys.getallocatedblocks
     func2 = sys.gettotalrefcount
     gc.collect()
-    return func1(), func2()
+    return func1(), func2(), fd_count()
 
 
 def warm_caches():
diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -383,27 +383,32 @@
         self.assertTrue(0 <= randseed <= 10000000, randseed)
         return randseed
 
-    def run_command(self, args, input=None, exitcode=0):
+    def run_command(self, args, input=None, exitcode=0, **kw):
         if not input:
             input = ''
+        if 'stderr' not in kw:
+            kw['stderr'] = subprocess.PIPE
         proc = subprocess.run(args,
                               universal_newlines=True,
                               input=input,
                               stdout=subprocess.PIPE,
-                              stderr=subprocess.PIPE)
+                              **kw)
         if proc.returncode != exitcode:
-            self.fail("Command %s failed with exit code %s\n"
-                      "\n"
-                      "stdout:\n"
-                      "---\n"
-                      "%s\n"
-                      "---\n"
-                      "\n"
-                      "stderr:\n"
-                      "---\n"
-                      "%s"
-                      "---\n"
-                      % (str(args), proc.returncode, proc.stdout, proc.stderr))
+            msg = ("Command %s failed with exit code %s\n"
+                   "\n"
+                   "stdout:\n"
+                   "---\n"
+                   "%s\n"
+                   "---\n"
+                   % (str(args), proc.returncode, proc.stdout))
+            if proc.stderr:
+                msg += ("\n"
+                        "stderr:\n"
+                        "---\n"
+                        "%s"
+                        "---\n"
+                        % proc.stderr)
+            self.fail(msg)
         return proc
 
 
@@ -637,6 +642,36 @@
         output = self.run_tests('--forever', test, exitcode=1)
         self.check_executed_tests(output, [test]*3, failed=test)
 
+    def test_huntrleaks_fd_leak(self):
+        # test --huntrleaks for file descriptor leak
+        code = textwrap.dedent("""
+            import os
+            import unittest
+
+            class FDLeakTest(unittest.TestCase):
+                def test_leak(self):
+                    fd = os.open(__file__, os.O_RDONLY)
+                    # bug: never cloes the file descriptor
+        """)
+        test = self.create_test(code=code)
+
+        filename = 'reflog.txt'
+        self.addCleanup(support.unlink, filename)
+        output = self.run_tests('--huntrleaks', '3:3:', test,
+                                exitcode=1,
+                                stderr=subprocess.STDOUT)
+        self.check_executed_tests(output, [test], failed=test)
+
+        line = 'beginning 6 repetitions\n123456\n......\n'
+        self.check_line(output, re.escape(line))
+
+        line2 = '%s leaked [1, 1, 1] file descriptors, sum=3\n' % test
+        self.check_line(output, re.escape(line2))
+
+        with open(filename) as fp:
+            reflog = fp.read()
+            self.assertEqual(reflog, line2)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -170,6 +170,9 @@
 Tests
 -----
 
+- Issue #18174: ``python -m test --huntrleaks ...`` now also checks for leak of
+  file descriptors. Patch written by Richard Oudkerk.
+
 - Issue #25260: Fix ``python -m test --coverage`` on Windows. Remove the
   list of ignored directories.
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 00:47:42 2015
From: python-checkins at python.org (victor.stinner)
Date: Fri, 02 Oct 2015 22:47:42 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2322806=3A_Add_=60?=
 =?utf-8?q?=60python_-m_test_--list-tests=60=60_command_to_list_tests=2E?=
Message-ID: <20151002224742.93024.92929@psf.io>

https://hg.python.org/cpython/rev/1005573e6a74
changeset:   98493:1005573e6a74
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Sat Oct 03 00:21:12 2015 +0200
summary:
  Issue #22806: Add ``python -m test --list-tests`` command to list tests.

files:
  Lib/test/libregrtest/cmdline.py |  11 +--
  Lib/test/libregrtest/main.py    |  55 +++++++++++++-------
  Lib/test/test_regrtest.py       |   7 ++
  Misc/NEWS                       |   2 +
  4 files changed, 49 insertions(+), 26 deletions(-)


diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py
--- a/Lib/test/libregrtest/cmdline.py
+++ b/Lib/test/libregrtest/cmdline.py
@@ -1,5 +1,4 @@
 import argparse
-import faulthandler
 import os
 from test import support
 
@@ -234,6 +233,9 @@
     group.add_argument('-F', '--forever', action='store_true',
                        help='run the specified tests in a loop, until an '
                             'error happens')
+    group.add_argument('--list-tests', action='store_true',
+                       help="only write the name of tests that will be run, "
+                            "don't execute them")
 
     parser.add_argument('args', nargs=argparse.REMAINDER,
                         help=argparse.SUPPRESS)
@@ -301,12 +303,7 @@
     if ns.quiet:
         ns.verbose = 0
     if ns.timeout is not None:
-        if hasattr(faulthandler, 'dump_traceback_later'):
-            if ns.timeout <= 0:
-                ns.timeout = None
-        else:
-            print("Warning: The timeout option requires "
-                  "faulthandler.dump_traceback_later")
+        if ns.timeout <= 0:
             ns.timeout = None
     if ns.use_mp is not None:
         if ns.use_mp <= 0:
diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -1,3 +1,4 @@
+import faulthandler
 import os
 import platform
 import random
@@ -110,8 +111,13 @@
     def parse_args(self, kwargs):
         ns = _parse_args(sys.argv[1:], **kwargs)
 
+        if ns.timeout and not hasattr(faulthandler, 'dump_traceback_later'):
+            print("Warning: The timeout option requires "
+                  "faulthandler.dump_traceback_later", file=sys.stderr)
+            ns.timeout = None
+
         if ns.threshold is not None and gc is None:
-            print('No GC available, ignore --threshold.')
+            print('No GC available, ignore --threshold.', file=sys.stderr)
             ns.threshold = None
 
         if ns.findleaks:
@@ -122,7 +128,8 @@
                 pass
                 #gc.set_debug(gc.DEBUG_SAVEALL)
             else:
-                print('No GC available, disabling --findleaks')
+                print('No GC available, disabling --findleaks',
+                      file=sys.stderr)
                 ns.findleaks = False
 
         # Strip .py extensions.
@@ -163,20 +170,6 @@
                 nottests.add(arg)
             self.ns.args = []
 
-        # For a partial run, we do not need to clutter the output.
-        if (self.ns.verbose
-            or self.ns.header
-            or not (self.ns.quiet or self.ns.single
-                    or self.tests or self.ns.args)):
-            # Print basic platform information
-            print("==", platform.python_implementation(), *sys.version.split())
-            print("==  ", platform.platform(aliased=True),
-                          "%s-endian" % sys.byteorder)
-            print("==  ", "hash algorithm:", sys.hash_info.algorithm,
-                  "64bit" if sys.maxsize > 2**32 else "32bit")
-            print("==  ", os.getcwd())
-            print("Testing with flags:", sys.flags)
-
         # if testdir is set, then we are not running the python tests suite, so
         # don't add default tests to be executed or skipped (pass empty values)
         if self.ns.testdir:
@@ -199,15 +192,18 @@
                 del self.selected[:self.selected.index(self.ns.start)]
             except ValueError:
                 print("Couldn't find starting test (%s), using all tests"
-                      % self.ns.start)
+                      % self.ns.start, file=sys.stderr)
 
         if self.ns.randomize:
             if self.ns.random_seed is None:
                 self.ns.random_seed = random.randrange(10000000)
             random.seed(self.ns.random_seed)
-            print("Using random seed", self.ns.random_seed)
             random.shuffle(self.selected)
 
+    def list_tests(self):
+        for name in self.selected:
+            print(name)
+
     def rerun_failed_tests(self):
         self.ns.verbose = True
         self.ns.failfast = False
@@ -315,6 +311,23 @@
                     return
 
     def run_tests(self):
+        # For a partial run, we do not need to clutter the output.
+        if (self.ns.verbose
+            or self.ns.header
+            or not (self.ns.quiet or self.ns.single
+                    or self.tests or self.ns.args)):
+            # Print basic platform information
+            print("==", platform.python_implementation(), *sys.version.split())
+            print("==  ", platform.platform(aliased=True),
+                          "%s-endian" % sys.byteorder)
+            print("==  ", "hash algorithm:", sys.hash_info.algorithm,
+                  "64bit" if sys.maxsize > 2**32 else "32bit")
+            print("==  ", os.getcwd())
+            print("Testing with flags:", sys.flags)
+
+        if self.ns.randomize:
+            print("Using random seed", self.ns.random_seed)
+
         if self.ns.forever:
             self.tests = self._test_forever(list(self.selected))
             self.test_count = ''
@@ -359,8 +372,12 @@
         setup_tests(self.ns)
 
         self.find_tests(tests)
+
+        if self.ns.list_tests:
+            self.list_tests()
+            sys.exit(0)
+
         self.run_tests()
-
         self.display_result()
 
         if self.ns.verbose2 and self.bad:
diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -672,6 +672,13 @@
             reflog = fp.read()
             self.assertEqual(reflog, line2)
 
+    def test_list_tests(self):
+        # test --list-tests
+        tests = [self.create_test() for i in range(5)]
+        output = self.run_tests('--list-tests', *tests)
+        self.assertEqual(output.rstrip().splitlines(),
+                         tests)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -170,6 +170,8 @@
 Tests
 -----
 
+- Issue #22806: Add ``python -m test --list-tests`` command to list tests.
+
 - Issue #18174: ``python -m test --huntrleaks ...`` now also checks for leak of
   file descriptors. Patch written by Richard Oudkerk.
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 01:20:55 2015
From: python-checkins at python.org (brett.cannon)
Date: Fri, 02 Oct 2015 23:20:55 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy41KTogSXNzdWUgIzI1MTg4?=
 =?utf-8?q?=3A_Add_a_-P/--pgo_flag_to_regrtest_to_silence_error_output=2E?=
Message-ID: <20151002232054.70970.32628@psf.io>

https://hg.python.org/cpython/rev/fb90425017e3
changeset:   98494:fb90425017e3
branch:      3.5
parent:      98489:4eb809fa1130
user:        Brett Cannon <brett at python.org>
date:        Fri Oct 02 16:16:44 2015 -0700
summary:
  Issue #25188: Add a -P/--pgo flag to regrtest to silence error output.

Since PGO building doesn't care about test failures, silence them when
running the test suite for performance reasons. Initial patch by
Alecsandru Patrascu of Intel.

files:
  Lib/test/regrtest.py |  84 +++++++++++++++++++------------
  Makefile.pre.in      |   2 +-
  2 files changed, 51 insertions(+), 35 deletions(-)


diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py
--- a/Lib/test/regrtest.py
+++ b/Lib/test/regrtest.py
@@ -322,6 +322,8 @@
     group.add_argument('-F', '--forever', action='store_true',
                        help='run the specified tests in a loop, until an '
                             'error happens')
+    group.add_argument('-P', '--pgo', dest='pgo', action='store_true',
+                       help='enable Profile Guided Optimization training')
 
     parser.add_argument('args', nargs=argparse.REMAINDER,
                         help=argparse.SUPPRESS)
@@ -361,7 +363,7 @@
          findleaks=False, use_resources=None, trace=False, coverdir='coverage',
          runleaks=False, huntrleaks=False, verbose2=False, print_slow=False,
          random_seed=None, use_mp=None, verbose3=False, forever=False,
-         header=False, failfast=False, match_tests=None)
+         header=False, failfast=False, match_tests=None, pgo=False)
     for k, v in kwargs.items():
         if not hasattr(ns, k):
             raise TypeError('%r is an invalid keyword argument '
@@ -435,14 +437,16 @@
     from subprocess import Popen, PIPE
     base_cmd = ([sys.executable] + support.args_from_interpreter_flags() +
                 ['-X', 'faulthandler', '-m', 'test.regrtest'])
-
+    # required to spawn a new process with PGO flag on/off
+    if ns.pgo:
+        base_cmd = base_cmd + ['--pgo']
     slaveargs = (
             (testname, ns.verbose, ns.quiet),
             dict(huntrleaks=ns.huntrleaks,
                  use_resources=ns.use_resources,
                  output_on_failure=ns.verbose3,
                  timeout=ns.timeout, failfast=ns.failfast,
-                 match_tests=ns.match_tests))
+                 match_tests=ns.match_tests, pgo=ns.pgo))
     # Running the child from the same working directory as regrtest's original
     # invocation ensures that TEMPDIR for the child is the same when
     # sysconfig.is_python_build() is true. See issue 15300.
@@ -596,13 +600,14 @@
         ns.args = []
 
     # For a partial run, we do not need to clutter the output.
-    if ns.verbose or ns.header or not (ns.quiet or ns.single or tests or ns.args):
+    if (ns.verbose or ns.header or
+            not (ns.pgo or ns.quiet or ns.single or tests or ns.args)):
         # Print basic platform information
         print("==", platform.python_implementation(), *sys.version.split())
         print("==  ", platform.platform(aliased=True),
-                      "%s-endian" % sys.byteorder)
+                        "%s-endian" % sys.byteorder)
         print("==  ", "hash algorithm:", sys.hash_info.algorithm,
-              "64bit" if sys.maxsize > 2**32 else "32bit")
+                "64bit" if sys.maxsize > 2**32 else "32bit")
         print("==  ", os.getcwd())
         print("Testing with flags:", sys.flags)
 
@@ -722,13 +727,16 @@
                     continue
                 accumulate_result(test, result)
                 if not ns.quiet:
-                    fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}"
+                    if bad and not ns.pgo:
+                        fmt = "[{1:{0}}{2}/{3}] {4}"
+                    else:
+                        fmt = "[{1:{0}}{2}] {4}"
                     print(fmt.format(
                         test_count_width, test_index, test_count,
                         len(bad), test))
                 if stdout:
                     print(stdout)
-                if stderr:
+                if stderr and not ns.pgo:
                     print(stderr, file=sys.stderr)
                 sys.stdout.flush()
                 sys.stderr.flush()
@@ -745,7 +753,10 @@
     else:
         for test_index, test in enumerate(tests, 1):
             if not ns.quiet:
-                fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}"
+                if bad and not ns.pgo:
+                    fmt = "[{1:{0}}{2}/{3}] {4}"
+                else:
+                    fmt = "[{1:{0}}{2}] {4}"
                 print(fmt.format(
                     test_count_width, test_index, test_count, len(bad), test))
                 sys.stdout.flush()
@@ -760,7 +771,7 @@
                                      ns.huntrleaks,
                                      output_on_failure=ns.verbose3,
                                      timeout=ns.timeout, failfast=ns.failfast,
-                                     match_tests=ns.match_tests)
+                                     match_tests=ns.match_tests, pgo=ns.pgo)
                     accumulate_result(test, result)
                 except KeyboardInterrupt:
                     interrupted = True
@@ -779,14 +790,14 @@
                 if module not in save_modules and module.startswith("test."):
                     support.unload(module)
 
-    if interrupted:
+    if interrupted and not ns.pgo:
         # print a newline after ^C
         print()
         print("Test suite interrupted by signal SIGINT.")
         omitted = set(selected) - set(good) - set(bad) - set(skipped)
         print(count(len(omitted), "test"), "omitted:")
         printlist(omitted)
-    if good and not ns.quiet:
+    if good and not ns.quiet and not ns.pgo:
         if not bad and not skipped and not interrupted and len(good) > 1:
             print("All", end=' ')
         print(count(len(good), "test"), "OK.")
@@ -795,26 +806,27 @@
         print("10 slowest tests:")
         for time, test in test_times[:10]:
             print("%s: %.1fs" % (test, time))
-    if bad:
+    if bad and not ns.pgo:
         print(count(len(bad), "test"), "failed:")
         printlist(bad)
-    if environment_changed:
+    if environment_changed and not ns.pgo:
         print("{} altered the execution environment:".format(
                  count(len(environment_changed), "test")))
         printlist(environment_changed)
-    if skipped and not ns.quiet:
+    if skipped and not ns.quiet and not ns.pgo:
         print(count(len(skipped), "test"), "skipped:")
         printlist(skipped)
 
     if ns.verbose2 and bad:
         print("Re-running failed tests in verbose mode")
         for test in bad[:]:
-            print("Re-running test %r in verbose mode" % test)
+            if not ns.pgo:
+                print("Re-running test %r in verbose mode" % test)
             sys.stdout.flush()
             try:
                 ns.verbose = True
                 ok = runtest(test, True, ns.quiet, ns.huntrleaks,
-                             timeout=ns.timeout)
+                             timeout=ns.timeout, pgo=ns.pgo)
             except KeyboardInterrupt:
                 # print a newline separate from the ^C
                 print()
@@ -913,7 +925,7 @@
 def runtest(test, verbose, quiet,
             huntrleaks=False, use_resources=None,
             output_on_failure=False, failfast=False, match_tests=None,
-            timeout=None):
+            timeout=None, *, pgo=False):
     """Run a single test.
 
     test -- the name of the test
@@ -926,6 +938,8 @@
     timeout -- dump the traceback and exit if a test takes more than
                timeout seconds
     failfast, match_tests -- See regrtest command-line flags for these.
+    pgo -- if true, do not print unnecessary info when running the test
+           for Profile Guided Optimization build
 
     Returns the tuple result, test_time, where result is one of the constants:
         INTERRUPTED      KeyboardInterrupt when run under -j
@@ -935,7 +949,6 @@
         FAILED           test failed
         PASSED           test passed
     """
-
     if use_resources is not None:
         support.use_resources = use_resources
     use_timeout = (timeout is not None)
@@ -965,8 +978,8 @@
                 sys.stdout = stream
                 sys.stderr = stream
                 result = runtest_inner(test, verbose, quiet, huntrleaks,
-                                       display_failure=False)
-                if result[0] == FAILED:
+                                       display_failure=False, pgo=pgo)
+                if result[0] == FAILED and not pgo:
                     output = stream.getvalue()
                     orig_stderr.write(output)
                     orig_stderr.flush()
@@ -976,7 +989,7 @@
         else:
             support.verbose = verbose  # Tell tests to be moderately quiet
             result = runtest_inner(test, verbose, quiet, huntrleaks,
-                                   display_failure=not verbose)
+                                   display_failure=not verbose, pgo=pgo)
         return result
     finally:
         if use_timeout:
@@ -1008,10 +1021,11 @@
 
     changed = False
 
-    def __init__(self, testname, verbose=0, quiet=False):
+    def __init__(self, testname, verbose=0, quiet=False, *, pgo=False):
         self.testname = testname
         self.verbose = verbose
         self.quiet = quiet
+        self.pgo = pgo
 
     # To add things to save and restore, add a name XXX to the resources list
     # and add corresponding get_XXX/restore_XXX functions.  get_XXX should
@@ -1240,11 +1254,11 @@
             if current != original:
                 self.changed = True
                 restore(original)
-                if not self.quiet:
+                if not self.quiet and not self.pgo:
                     print("Warning -- {} was modified by {}".format(
                                                  name, self.testname),
                                                  file=sys.stderr)
-                    if self.verbose > 1:
+                    if self.verbose > 1 and not self.pgo:
                         print("  Before: {}\n  After:  {} ".format(
                                                   original, current),
                                                   file=sys.stderr)
@@ -1252,7 +1266,7 @@
 
 
 def runtest_inner(test, verbose, quiet,
-                  huntrleaks=False, display_failure=True):
+                  huntrleaks=False, display_failure=True, pgo=False):
     support.unload(test)
 
     test_time = 0.0
@@ -1263,7 +1277,7 @@
         else:
             # Always import it from the test package
             abstest = 'test.' + test
-        with saved_test_environment(test, verbose, quiet) as environment:
+        with saved_test_environment(test, verbose, quiet, pgo=pgo) as environment:
             start_time = time.time()
             the_module = importlib.import_module(abstest)
             # If the test has a test_main, that will run the appropriate
@@ -1283,27 +1297,29 @@
                 refleak = dash_R(the_module, test, test_runner, huntrleaks)
             test_time = time.time() - start_time
     except support.ResourceDenied as msg:
-        if not quiet:
+        if not quiet and not pgo:
             print(test, "skipped --", msg)
             sys.stdout.flush()
         return RESOURCE_DENIED, test_time
     except unittest.SkipTest as msg:
-        if not quiet:
+        if not quiet and not pgo:
             print(test, "skipped --", msg)
             sys.stdout.flush()
         return SKIPPED, test_time
     except KeyboardInterrupt:
         raise
     except support.TestFailed as msg:
-        if display_failure:
-            print("test", test, "failed --", msg, file=sys.stderr)
-        else:
-            print("test", test, "failed", file=sys.stderr)
+        if not pgo:
+            if display_failure:
+                print("test", test, "failed --", msg, file=sys.stderr)
+            else:
+                print("test", test, "failed", file=sys.stderr)
         sys.stderr.flush()
         return FAILED, test_time
     except:
         msg = traceback.format_exc()
-        print("test", test, "crashed --", msg, file=sys.stderr)
+        if not pgo:
+            print("test", test, "crashed --", msg, file=sys.stderr)
         sys.stderr.flush()
         return FAILED, test_time
     else:
diff --git a/Makefile.pre.in b/Makefile.pre.in
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -231,7 +231,7 @@
 TCLTK_LIBS=	@TCLTK_LIBS@
 
 # The task to run while instrument when building the profile-opt target
-PROFILE_TASK=-m test.regrtest >/dev/null 2>&1
+PROFILE_TASK=-m test.regrtest --pgo
 
 # report files for gcov / lcov coverage report
 COVERAGE_INFO=	$(abs_builddir)/coverage.info

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 01:20:58 2015
From: python-checkins at python.org (brett.cannon)
Date: Fri, 02 Oct 2015 23:20:58 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
 =?utf-8?q?=29=3A_Merge_from_3=2E5_for_issue_=2325188=2E?=
Message-ID: <20151002232054.31193.88005@psf.io>

https://hg.python.org/cpython/rev/c1ecb258003b
changeset:   98495:c1ecb258003b
parent:      98493:1005573e6a74
parent:      98494:fb90425017e3
user:        Brett Cannon <brett at python.org>
date:        Fri Oct 02 16:20:49 2015 -0700
summary:
  Merge from 3.5 for issue #25188.

files:
  Lib/test/libregrtest/cmdline.py    |   6 +++-
  Lib/test/libregrtest/main.py       |  12 ++++++-
  Lib/test/libregrtest/runtest.py    |  28 ++++++++++-------
  Lib/test/libregrtest/runtest_mp.py |  11 ++++--
  Lib/test/libregrtest/save_env.py   |   5 +-
  Makefile.pre.in                    |   2 +-
  Misc/NEWS                          |   4 ++
  7 files changed, 47 insertions(+), 21 deletions(-)


diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py
--- a/Lib/test/libregrtest/cmdline.py
+++ b/Lib/test/libregrtest/cmdline.py
@@ -236,6 +236,8 @@
     group.add_argument('--list-tests', action='store_true',
                        help="only write the name of tests that will be run, "
                             "don't execute them")
+    group.add_argument('-P', '--pgo', dest='pgo', action='store_true',
+                       help='enable Profile Guided Optimization training')
 
     parser.add_argument('args', nargs=argparse.REMAINDER,
                         help=argparse.SUPPRESS)
@@ -279,7 +281,7 @@
          findleaks=False, use_resources=None, trace=False, coverdir='coverage',
          runleaks=False, huntrleaks=False, verbose2=False, print_slow=False,
          random_seed=None, use_mp=None, verbose3=False, forever=False,
-         header=False, failfast=False, match_tests=None)
+         header=False, failfast=False, match_tests=None, pgo=False)
     for k, v in kwargs.items():
         if not hasattr(ns, k):
             raise TypeError('%r is an invalid keyword argument '
@@ -299,6 +301,8 @@
         parser.error("-l and -j don't go together!")
     if ns.failfast and not (ns.verbose or ns.verbose3):
         parser.error("-G/--failfast needs either -v or -W")
+    if ns.pgo and (ns.verbose or ns.verbose2 or ns.verbose3):
+        parser.error("--pgo/-v don't go together!")
 
     if ns.quiet:
         ns.verbose = 0
diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -103,7 +103,10 @@
     def display_progress(self, test_index, test):
         if self.ns.quiet:
             return
-        fmt = "[{1:{0}}{2}/{3}] {4}" if self.bad else "[{1:{0}}{2}] {4}"
+        if self.bad and not self.ns.pgo:
+            fmt = "[{1:{0}}{2}/{3}] {4}"
+        else:
+            fmt = "[{1:{0}}{2}] {4}"
         print(fmt.format(self.test_count_width, test_index,
                          self.test_count, len(self.bad), test),
               flush=True)
@@ -238,6 +241,11 @@
             print(count(len(omitted), "test"), "omitted:")
             printlist(omitted)
 
+        # If running the test suite for PGO then no one cares about
+        # results.
+        if self.ns.pgo:
+            return
+
         if self.good and not self.ns.quiet:
             if (not self.bad
                 and not self.skipped
@@ -314,7 +322,7 @@
         # For a partial run, we do not need to clutter the output.
         if (self.ns.verbose
             or self.ns.header
-            or not (self.ns.quiet or self.ns.single
+            or not (self.ns.pgo or self.ns.quiet or self.ns.single
                     or self.tests or self.ns.args)):
             # Print basic platform information
             print("==", platform.python_implementation(), *sys.version.split())
diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py
--- a/Lib/test/libregrtest/runtest.py
+++ b/Lib/test/libregrtest/runtest.py
@@ -65,6 +65,7 @@
     timeout -- dump the traceback and exit if a test takes more than
                timeout seconds
     failfast, match_tests -- See regrtest command-line flags for these.
+    pgo -- if true, suppress any info irrelevant to a generating a PGO build
 
     Returns the tuple result, test_time, where result is one of the constants:
         INTERRUPTED      KeyboardInterrupt when run under -j
@@ -82,6 +83,7 @@
     failfast = ns.failfast
     match_tests = ns.match_tests
     timeout = ns.timeout
+    pgo = ns.pgo
 
     use_timeout = (timeout is not None)
     if use_timeout:
@@ -110,7 +112,7 @@
                 sys.stdout = stream
                 sys.stderr = stream
                 result = runtest_inner(test, verbose, quiet, huntrleaks,
-                                       display_failure=False)
+                                       display_failure=False, pgo=pgo)
                 if result[0] == FAILED:
                     output = stream.getvalue()
                     orig_stderr.write(output)
@@ -121,7 +123,7 @@
         else:
             support.verbose = verbose  # Tell tests to be moderately quiet
             result = runtest_inner(test, verbose, quiet, huntrleaks,
-                                   display_failure=not verbose)
+                                   display_failure=not verbose, pgo=pgo)
         return result
     finally:
         if use_timeout:
@@ -131,7 +133,7 @@
 
 
 def runtest_inner(test, verbose, quiet,
-                  huntrleaks=False, display_failure=True):
+                  huntrleaks=False, display_failure=True, *, pgo=False):
     support.unload(test)
 
     test_time = 0.0
@@ -142,7 +144,7 @@
         else:
             # Always import it from the test package
             abstest = 'test.' + test
-        with saved_test_environment(test, verbose, quiet) as environment:
+        with saved_test_environment(test, verbose, quiet, pgo=pgo) as environment:
             start_time = time.time()
             the_module = importlib.import_module(abstest)
             # If the test has a test_main, that will run the appropriate
@@ -162,24 +164,28 @@
                 refleak = dash_R(the_module, test, test_runner, huntrleaks)
             test_time = time.time() - start_time
     except support.ResourceDenied as msg:
-        if not quiet:
+        if not quiet and not pgo:
             print(test, "skipped --", msg, flush=True)
         return RESOURCE_DENIED, test_time
     except unittest.SkipTest as msg:
-        if not quiet:
+        if not quiet and not pgo:
             print(test, "skipped --", msg, flush=True)
         return SKIPPED, test_time
     except KeyboardInterrupt:
         raise
     except support.TestFailed as msg:
-        if display_failure:
-            print("test", test, "failed --", msg, file=sys.stderr, flush=True)
-        else:
-            print("test", test, "failed", file=sys.stderr, flush=True)
+        if not pgo:
+            if display_failure:
+                print("test", test, "failed --", msg, file=sys.stderr,
+                      flush=True)
+            else:
+                print("test", test, "failed", file=sys.stderr, flush=True)
         return FAILED, test_time
     except:
         msg = traceback.format_exc()
-        print("test", test, "crashed --", msg, file=sys.stderr, flush=True)
+        if not pgo:
+            print("test", test, "crashed --", msg, file=sys.stderr,
+                  flush=True)
         return FAILED, test_time
     else:
         if refleak:
diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py
--- a/Lib/test/libregrtest/runtest_mp.py
+++ b/Lib/test/libregrtest/runtest_mp.py
@@ -42,6 +42,8 @@
            '-X', 'faulthandler',
            '-m', 'test.regrtest',
            '--slaveargs', slaveargs]
+    if ns.pgo:
+        cmd += ['--pgo']
 
     # Running the child from the same working directory as regrtest's original
     # invocation ensures that TEMPDIR for the child is the same when
@@ -175,7 +177,7 @@
                 item = output.get(timeout=timeout)
             except queue.Empty:
                 running = get_running(workers)
-                if running:
+                if running and not regrtest.ns.pgo:
                     print('running: %s' % ', '.join(running))
                 continue
 
@@ -189,17 +191,18 @@
             text = test
             ok, test_time = result
             if (ok not in (CHILD_ERROR, INTERRUPTED)
-                and test_time >= PROGRESS_MIN_TIME):
+                and test_time >= PROGRESS_MIN_TIME
+                and not regrtest.ns.pgo):
                 text += ' (%.0f sec)' % test_time
             running = get_running(workers)
-            if running:
+            if running and not regrtest.ns.pgo:
                 text += ' -- running: %s' % ', '.join(running)
             regrtest.display_progress(test_index, text)
 
             # Copy stdout and stderr from the child process
             if stdout:
                 print(stdout, flush=True)
-            if stderr:
+            if stderr and not regrtest.ns.pgo:
                 print(stderr, file=sys.stderr, flush=True)
 
             if result[0] == INTERRUPTED:
diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py
--- a/Lib/test/libregrtest/save_env.py
+++ b/Lib/test/libregrtest/save_env.py
@@ -41,10 +41,11 @@
 
     changed = False
 
-    def __init__(self, testname, verbose=0, quiet=False):
+    def __init__(self, testname, verbose=0, quiet=False, *, pgo=False):
         self.testname = testname
         self.verbose = verbose
         self.quiet = quiet
+        self.pgo = pgo
 
     # To add things to save and restore, add a name XXX to the resources list
     # and add corresponding get_XXX/restore_XXX functions.  get_XXX should
@@ -273,7 +274,7 @@
             if current != original:
                 self.changed = True
                 restore(original)
-                if not self.quiet:
+                if not self.quiet and not self.pgo:
                     print("Warning -- {} was modified by {}".format(
                                                  name, self.testname),
                                                  file=sys.stderr)
diff --git a/Makefile.pre.in b/Makefile.pre.in
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -231,7 +231,7 @@
 TCLTK_LIBS=	@TCLTK_LIBS@
 
 # The task to run while instrument when building the profile-opt target
-PROFILE_TASK=-m test.regrtest >/dev/null 2>&1
+PROFILE_TASK=-m test.regrtest --pgo
 
 # report files for gcov / lcov coverage report
 COVERAGE_INFO=	$(abs_builddir)/coverage.info
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -170,6 +170,10 @@
 Tests
 -----
 
+- Issue #25188: Add -P/--pgo to test.regrtest to suppress error output when
+  running the test suite for the purposes of a PGO build. Initial patch by
+  Alecsandru Patrascu.
+
 - Issue #22806: Add ``python -m test --list-tests`` command to list tests.
 
 - Issue #18174: ``python -m test --huntrleaks ...`` now also checks for leak of

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 01:22:42 2015
From: python-checkins at python.org (brett.cannon)
Date: Fri, 02 Oct 2015 23:22:42 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzI1MTg4?=
 =?utf-8?q?=3A_Add_-P/--pgo_to_test=2Eregrtest_for_PGO_building=2E?=
Message-ID: <20151002232241.31199.2016@psf.io>

https://hg.python.org/cpython/rev/136ad559fa4f
changeset:   98496:136ad559fa4f
branch:      2.7
parent:      98485:8bbc51f97078
user:        Brett Cannon <brett at python.org>
date:        Fri Oct 02 16:21:34 2015 -0700
summary:
  Issue #25188: Add -P/--pgo to test.regrtest for PGO building.

Initial patch by Alecsandru Patrascu of Intel.

files:
  Lib/test/regrtest.py |  83 +++++++++++++++++++------------
  Makefile.pre.in      |   2 +-
  2 files changed, 51 insertions(+), 34 deletions(-)


diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py
--- a/Lib/test/regrtest.py
+++ b/Lib/test/regrtest.py
@@ -57,6 +57,7 @@
 -t/--threshold THRESHOLD
                 -- call gc.set_threshold(THRESHOLD)
 -F/--forever    -- run the specified tests in a loop, until an error happens
+-P/--pgo        -- enable Profile Guided Optimization training
 
 
 Additional Option Details:
@@ -240,7 +241,7 @@
          findleaks=False, use_resources=None, trace=False, coverdir='coverage',
          runleaks=False, huntrleaks=False, verbose2=False, print_slow=False,
          random_seed=None, use_mp=None, verbose3=False, forever=False,
-         header=False):
+         header=False, pgo=False):
     """Execute a test suite.
 
     This also parses command-line options and modifies its behavior
@@ -266,12 +267,12 @@
 
     test_support.record_original_stdout(sys.stdout)
     try:
-        opts, args = getopt.getopt(sys.argv[1:], 'hvqxsSrf:lu:t:TD:NLR:FwWM:j:',
+        opts, args = getopt.getopt(sys.argv[1:], 'hvqxsSrf:lu:t:TD:NLR:FwWM:j:P',
             ['help', 'verbose', 'verbose2', 'verbose3', 'quiet',
              'exclude', 'single', 'slow', 'randomize', 'fromfile=', 'findleaks',
              'use=', 'threshold=', 'trace', 'coverdir=', 'nocoverdir',
              'runleaks', 'huntrleaks=', 'memlimit=', 'randseed=',
-             'multiprocess=', 'slaveargs=', 'forever', 'header'])
+             'multiprocess=', 'slaveargs=', 'forever', 'header', 'pgo'])
     except getopt.error, msg:
         usage(2, msg)
 
@@ -366,6 +367,8 @@
             print   # Force a newline (just in case)
             print json.dumps(result)
             sys.exit(0)
+        elif o in ('-P', '--pgo'):
+            pgo = True            
         else:
             print >>sys.stderr, ("No handler for option {}.  Please "
                 "report this as a bug at http://bugs.python.org.").format(o)
@@ -431,13 +434,14 @@
 
     # For a partial run, we do not need to clutter the output.
     if verbose or header or not (quiet or single or tests or args):
-        # Print basic platform information
-        print "==", platform.python_implementation(), \
-                    " ".join(sys.version.split())
-        print "==  ", platform.platform(aliased=True), \
-                      "%s-endian" % sys.byteorder
-        print "==  ", os.getcwd()
-        print "Testing with flags:", sys.flags
+        if not pgo:
+            # Print basic platform information
+            print "==", platform.python_implementation(), \
+                        " ".join(sys.version.split())
+            print "==  ", platform.platform(aliased=True), \
+                          "%s-endian" % sys.byteorder
+            print "==  ", os.getcwd()
+            print "Testing with flags:", sys.flags
 
     alltests = findtests(testdir, stdtests, nottests)
     selected = tests or args or alltests
@@ -510,6 +514,9 @@
         pending = tests_and_args()
         opt_args = test_support.args_from_interpreter_flags()
         base_cmd = [sys.executable] + opt_args + ['-m', 'test.regrtest']
+        # required to spawn a new process with PGO flag on/off
+        if pgo:
+            base_cmd = base_cmd + ['--pgo']
         def work():
             # A worker thread.
             try:
@@ -519,6 +526,9 @@
                     except StopIteration:
                         output.put((None, None, None, None))
                         return
+                    # required to permit running tests with PGO flag on/off
+                    if pgo:
+                        args_tuple[1]['pgo']=pgo
                     # -E is needed by some tests, e.g. test_import
                     popen = Popen(base_cmd + ['--slaveargs', json.dumps(args_tuple)],
                                    stdout=PIPE, stderr=PIPE,
@@ -550,7 +560,7 @@
                     continue
                 if stdout:
                     print stdout
-                if stderr:
+                if stderr and not pgo:
                     print >>sys.stderr, stderr
                 sys.stdout.flush()
                 sys.stderr.flush()
@@ -570,7 +580,7 @@
         for worker in workers:
             worker.join()
     else:
-        for test_index, test in enumerate(tests, 1):
+        for test_index, test in enumerate(tests, 1):            
             if not quiet:
                 fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}"
                 print(fmt.format(
@@ -583,11 +593,12 @@
                               globals=globals(), locals=vars())
             else:
                 try:
-                    result = runtest(test, verbose, quiet, huntrleaks)
+                    result = runtest(test, verbose, quiet, huntrleaks, None, pgo)
                     accumulate_result(test, result)
                     if verbose3 and result[0] == FAILED:
-                        print "Re-running test %r in verbose mode" % test
-                        runtest(test, True, quiet, huntrleaks)
+                        if not pgo:
+                            print "Re-running test %r in verbose mode" % test
+                        runtest(test, True, quiet, huntrleaks, None, pgo)
                 except KeyboardInterrupt:
                     interrupted = True
                     break
@@ -607,14 +618,14 @@
                 if module not in save_modules and module.startswith("test."):
                     test_support.unload(module)
 
-    if interrupted:
+    if interrupted and not pgo:
         # print a newline after ^C
         print
         print "Test suite interrupted by signal SIGINT."
         omitted = set(selected) - set(good) - set(bad) - set(skipped)
         print count(len(omitted), "test"), "omitted:"
         printlist(omitted)
-    if good and not quiet:
+    if good and not quiet and not pgo:
         if not bad and not skipped and not interrupted and len(good) > 1:
             print "All",
         print count(len(good), "test"), "OK."
@@ -623,14 +634,14 @@
         print "10 slowest tests:"
         for time, test in test_times[:10]:
             print "%s: %.1fs" % (test, time)
-    if bad:
+    if bad and not pgo:
         print count(len(bad), "test"), "failed:"
         printlist(bad)
-    if environment_changed:
+    if environment_changed and not pgo:
         print "{} altered the execution environment:".format(
             count(len(environment_changed), "test"))
         printlist(environment_changed)
-    if skipped and not quiet:
+    if skipped and not quiet and not pgo:
         print count(len(skipped), "test"), "skipped:"
         printlist(skipped)
 
@@ -655,7 +666,7 @@
             sys.stdout.flush()
             try:
                 test_support.verbose = True
-                ok = runtest(test, True, quiet, huntrleaks)
+                ok = runtest(test, True, quiet, huntrleaks, None, pgo)
             except KeyboardInterrupt:
                 # print a newline separate from the ^C
                 print
@@ -716,7 +727,7 @@
     return stdtests + sorted(tests)
 
 def runtest(test, verbose, quiet,
-            huntrleaks=False, use_resources=None):
+            huntrleaks=False, use_resources=None, pgo=False):
     """Run a single test.
 
     test -- the name of the test
@@ -725,6 +736,9 @@
     test_times -- a list of (time, test_name) pairs
     huntrleaks -- run multiple times to test for leaks; requires a debug
                   build; a triple corresponding to -R's three arguments
+    pgo -- if true, do not print unnecessary info when running the test 
+           for Profile Guided Optimization build
+           
     Returns one of the test result constants:
         INTERRUPTED      KeyboardInterrupt when run under -j
         RESOURCE_DENIED  test skipped because resource denied
@@ -738,7 +752,7 @@
     if use_resources is not None:
         test_support.use_resources = use_resources
     try:
-        return runtest_inner(test, verbose, quiet, huntrleaks)
+        return runtest_inner(test, verbose, quiet, huntrleaks, pgo)
     finally:
         cleanup_test_droppings(test, verbose)
 
@@ -767,10 +781,11 @@
 
     changed = False
 
-    def __init__(self, testname, verbose=0, quiet=False):
+    def __init__(self, testname, verbose=0, quiet=False, pgo=False):
         self.testname = testname
         self.verbose = verbose
         self.quiet = quiet
+        self.pgo = pgo
 
     # To add things to save and restore, add a name XXX to the resources list
     # and add corresponding get_XXX/restore_XXX functions.  get_XXX should
@@ -884,11 +899,11 @@
             if current != original:
                 self.changed = True
                 restore(original)
-                if not self.quiet:
+                if not self.quiet and not self.pgo:
                     print >>sys.stderr, (
                           "Warning -- {} was modified by {}".format(
                                                  name, self.testname))
-                    if self.verbose > 1:
+                    if self.verbose > 1 and not self.pgo:
                         print >>sys.stderr, (
                               "  Before: {}\n  After:  {} ".format(
                                                   original, current))
@@ -899,7 +914,7 @@
         return False
 
 
-def runtest_inner(test, verbose, quiet, huntrleaks=False):
+def runtest_inner(test, verbose, quiet, huntrleaks=False, pgo=False):
     test_support.unload(test)
     if verbose:
         capture_stdout = None
@@ -918,7 +933,7 @@
             else:
                 # Always import it from the test package
                 abstest = 'test.' + test
-            with saved_test_environment(test, verbose, quiet) as environment:
+            with saved_test_environment(test, verbose, quiet, pgo) as environment:
                 start_time = time.time()
                 the_package = __import__(abstest, globals(), locals(), [])
                 the_module = getattr(the_package, test)
@@ -935,26 +950,28 @@
         finally:
             sys.stdout = save_stdout
     except test_support.ResourceDenied, msg:
-        if not quiet:
+        if not quiet and not pgo:
             print test, "skipped --", msg
             sys.stdout.flush()
         return RESOURCE_DENIED, test_time
     except unittest.SkipTest, msg:
-        if not quiet:
+        if not quiet and not pgo:
             print test, "skipped --", msg
             sys.stdout.flush()
         return SKIPPED, test_time
     except KeyboardInterrupt:
         raise
     except test_support.TestFailed, msg:
-        print >>sys.stderr, "test", test, "failed --", msg
+        if not pgo:
+            print >>sys.stderr, "test", test, "failed --", msg
         sys.stderr.flush()
         return FAILED, test_time
     except:
         type, value = sys.exc_info()[:2]
-        print >>sys.stderr, "test", test, "crashed --", str(type) + ":", value
+        if not pgo:
+            print >>sys.stderr, "test", test, "crashed --", str(type) + ":", value
         sys.stderr.flush()
-        if verbose:
+        if verbose and not pgo:
             traceback.print_exc(file=sys.stderr)
             sys.stderr.flush()
         return FAILED, test_time
diff --git a/Makefile.pre.in b/Makefile.pre.in
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -209,7 +209,7 @@
 TCLTK_LIBS=	@TCLTK_LIBS@
 
 # The task to run while instrument when building the profile-opt target
-PROFILE_TASK=-m test.regrtest >/dev/null 2>&1
+PROFILE_TASK=-m test.regrtest --pgo
 
 # === Definitions added by makesetup ===
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 01:22:43 2015
From: python-checkins at python.org (brett.cannon)
Date: Fri, 02 Oct 2015 23:22:43 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_Fix_indentatio?=
	=?utf-8?q?n?=
Message-ID: <20151002232241.55474.47324@psf.io>

https://hg.python.org/cpython/rev/30c143a705dd
changeset:   98497:30c143a705dd
branch:      2.7
user:        Brett Cannon <brett at python.org>
date:        Fri Oct 02 16:22:32 2015 -0700
summary:
  Fix indentation

files:
  Lib/test/regrtest.py |  8 ++++----
  1 files changed, 4 insertions(+), 4 deletions(-)


diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py
--- a/Lib/test/regrtest.py
+++ b/Lib/test/regrtest.py
@@ -368,7 +368,7 @@
             print json.dumps(result)
             sys.exit(0)
         elif o in ('-P', '--pgo'):
-            pgo = True            
+            pgo = True
         else:
             print >>sys.stderr, ("No handler for option {}.  Please "
                 "report this as a bug at http://bugs.python.org.").format(o)
@@ -580,7 +580,7 @@
         for worker in workers:
             worker.join()
     else:
-        for test_index, test in enumerate(tests, 1):            
+        for test_index, test in enumerate(tests, 1):
             if not quiet:
                 fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}"
                 print(fmt.format(
@@ -736,9 +736,9 @@
     test_times -- a list of (time, test_name) pairs
     huntrleaks -- run multiple times to test for leaks; requires a debug
                   build; a triple corresponding to -R's three arguments
-    pgo -- if true, do not print unnecessary info when running the test 
+    pgo -- if true, do not print unnecessary info when running the test
            for Profile Guided Optimization build
-           
+
     Returns one of the test result constants:
         INTERRUPTED      KeyboardInterrupt when run under -j
         RESOURCE_DENIED  test skipped because resource denied

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 01:59:53 2015
From: python-checkins at python.org (victor.stinner)
Date: Fri, 02 Oct 2015 23:59:53 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Fix_=5FPyUnicodeWriter=5FP?=
	=?utf-8?q?repareKind=28=29?=
Message-ID: <20151002235952.7246.8479@psf.io>

https://hg.python.org/cpython/rev/3f6566a49c13
changeset:   98498:3f6566a49c13
parent:      98495:c1ecb258003b
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Sat Oct 03 01:55:51 2015 +0200
summary:
  Fix _PyUnicodeWriter_PrepareKind()

Initialize kind to 0 (PyUnicode_WCHAR_KIND) to ensure that
_PyUnicodeWriter_PrepareKind() handles correctly read-only buffer: copy the
buffer.

files:
  Objects/unicodeobject.c |  25 ++++++++++++++++++-------
  1 files changed, 18 insertions(+), 7 deletions(-)


diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -13294,27 +13294,38 @@
 Py_LOCAL_INLINE(void)
 _PyUnicodeWriter_Update(_PyUnicodeWriter *writer)
 {
-    if (!writer->readonly)
+    writer->maxchar = PyUnicode_MAX_CHAR_VALUE(writer->buffer);
+    writer->data = PyUnicode_DATA(writer->buffer);
+
+    if (!writer->readonly) {
+        writer->kind = PyUnicode_KIND(writer->buffer);
         writer->size = PyUnicode_GET_LENGTH(writer->buffer);
+    }
     else {
+        /* use a value smaller than PyUnicode_1BYTE_KIND() so
+           _PyUnicodeWriter_PrepareKind() will copy the buffer. */
+        writer->kind = PyUnicode_WCHAR_KIND;
+        assert(writer->kind <= PyUnicode_1BYTE_KIND);
+
         /* Copy-on-write mode: set buffer size to 0 so
          * _PyUnicodeWriter_Prepare() will copy (and enlarge) the buffer on
          * next write. */
         writer->size = 0;
     }
-    writer->maxchar = PyUnicode_MAX_CHAR_VALUE(writer->buffer);
-    writer->data = PyUnicode_DATA(writer->buffer);
-    writer->kind = PyUnicode_KIND(writer->buffer);
 }
 
 void
 _PyUnicodeWriter_Init(_PyUnicodeWriter *writer)
 {
     memset(writer, 0, sizeof(*writer));
-#ifdef Py_DEBUG
-    writer->kind = 5;    /* invalid kind */
-#endif
+
+    /* ASCII is the bare minimum */
     writer->min_char = 127;
+
+    /* use a value smaller than PyUnicode_1BYTE_KIND() so
+       _PyUnicodeWriter_PrepareKind() will copy the buffer. */
+    writer->kind = PyUnicode_WCHAR_KIND;
+    assert(writer->kind <= PyUnicode_1BYTE_KIND);
 }
 
 int

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 02:22:17 2015
From: python-checkins at python.org (victor.stinner)
Date: Sat, 03 Oct 2015 00:22:17 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2318174=3A_Fix_test?=
 =?utf-8?q?=5Fregrtest_when_Python_is_compiled_in_release_mode?=
Message-ID: <20151003002216.487.34486@psf.io>

https://hg.python.org/cpython/rev/ec2ef7525fa5
changeset:   98499:ec2ef7525fa5
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Sat Oct 03 02:21:35 2015 +0200
summary:
  Issue #18174: Fix test_regrtest when Python is compiled in release mode

files:
  Lib/test/test_regrtest.py |  1 +
  1 files changed, 1 insertions(+), 0 deletions(-)


diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -642,6 +642,7 @@
         output = self.run_tests('--forever', test, exitcode=1)
         self.check_executed_tests(output, [test]*3, failed=test)
 
+    @unittest.skipUnless(Py_DEBUG, 'need a debug build')
     def test_huntrleaks_fd_leak(self):
         # test --huntrleaks for file descriptor leak
         code = textwrap.dedent("""

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 04:13:21 2015
From: python-checkins at python.org (terry.reedy)
Date: Sat, 03 Oct 2015 02:13:21 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
	=?utf-8?q?=29=3A_Merge_with_3=2E5?=
Message-ID: <20151003021321.128838.36169@psf.io>

https://hg.python.org/cpython/rev/fd904e1e4f29
changeset:   98503:fd904e1e4f29
parent:      98499:ec2ef7525fa5
parent:      98502:ff025cf824d0
user:        Terry Jan Reedy <tjreedy at udel.edu>
date:        Fri Oct 02 22:12:57 2015 -0400
summary:
  Merge with 3.5

files:
  Lib/idlelib/configDialog.py |  30 +++++++++++++-----------
  1 files changed, 16 insertions(+), 14 deletions(-)


diff --git a/Lib/idlelib/configDialog.py b/Lib/idlelib/configDialog.py
--- a/Lib/idlelib/configDialog.py
+++ b/Lib/idlelib/configDialog.py
@@ -43,19 +43,20 @@
         #The first value of the tuple is the sample area tag name.
         #The second value is the display name list sort index.
         self.themeElements={
-            'Normal Text':('normal', '00'),
-            'Python Keywords':('keyword', '01'),
-            'Python Definitions':('definition', '02'),
-            'Python Builtins':('builtin', '03'),
-            'Python Comments':('comment', '04'),
-            'Python Strings':('string', '05'),
-            'Selected Text':('hilite', '06'),
-            'Found Text':('hit', '07'),
-            'Cursor':('cursor', '08'),
-            'Error Text':('error', '09'),
-            'Shell Normal Text':('console', '10'),
-            'Shell Stdout Text':('stdout', '11'),
-            'Shell Stderr Text':('stderr', '12'),
+            'Normal Text': ('normal', '00'),
+            'Python Keywords': ('keyword', '01'),
+            'Python Definitions': ('definition', '02'),
+            'Python Builtins': ('builtin', '03'),
+            'Python Comments': ('comment', '04'),
+            'Python Strings': ('string', '05'),
+            'Selected Text': ('hilite', '06'),
+            'Found Text': ('hit', '07'),
+            'Cursor': ('cursor', '08'),
+            'Editor Breakpoint': ('break', '09'),
+            'Shell Normal Text': ('console', '10'),
+            'Shell Error Text': ('error', '11'),
+            'Shell Stdout Text': ('stdout', '12'),
+            'Shell Stderr Text': ('stderr', '13'),
             }
         self.ResetChangedItems() #load initial values in changed items dict
         self.CreateWidgets()
@@ -219,7 +220,8 @@
             ("'selected'", 'hilite'), ('\n  var2 = ', 'normal'),
             ("'found'", 'hit'), ('\n  var3 = ', 'normal'),
             ('list', 'builtin'), ('(', 'normal'),
-            ('None', 'keyword'), (')\n\n', 'normal'),
+            ('None', 'keyword'), (')\n', 'normal'),
+            ('  breakpoint("line")', 'break'), ('\n\n', 'normal'),
             (' error ', 'error'), (' ', 'normal'),
             ('cursor |', 'cursor'), ('\n ', 'normal'),
             ('shell', 'console'), (' ', 'normal'),

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 04:13:21 2015
From: python-checkins at python.org (terry.reedy)
Date: Sat, 03 Oct 2015 02:13:21 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzI0ODIw?=
 =?utf-8?q?=3A_Users_can_now_set_breakpoint_colors_in_Settings_-=3E?=
Message-ID: <20151003021321.479.42994@psf.io>

https://hg.python.org/cpython/rev/e67da755d614
changeset:   98500:e67da755d614
branch:      2.7
parent:      98497:30c143a705dd
user:        Terry Jan Reedy <tjreedy at udel.edu>
date:        Fri Oct 02 22:12:09 2015 -0400
summary:
  Issue #24820: Users can now set breakpoint colors in Settings ->
Custom Highlighting.  Original patch by Mark Roseman.

files:
  Lib/idlelib/configDialog.py |  30 +++++++++++++-----------
  1 files changed, 16 insertions(+), 14 deletions(-)


diff --git a/Lib/idlelib/configDialog.py b/Lib/idlelib/configDialog.py
--- a/Lib/idlelib/configDialog.py
+++ b/Lib/idlelib/configDialog.py
@@ -41,19 +41,20 @@
         #The first value of the tuple is the sample area tag name.
         #The second value is the display name list sort index.
         self.themeElements={
-            'Normal Text':('normal', '00'),
-            'Python Keywords':('keyword', '01'),
-            'Python Definitions':('definition', '02'),
-            'Python Builtins':('builtin', '03'),
-            'Python Comments':('comment', '04'),
-            'Python Strings':('string', '05'),
-            'Selected Text':('hilite', '06'),
-            'Found Text':('hit', '07'),
-            'Cursor':('cursor', '08'),
-            'Error Text':('error', '09'),
-            'Shell Normal Text':('console', '10'),
-            'Shell Stdout Text':('stdout', '11'),
-            'Shell Stderr Text':('stderr', '12'),
+            'Normal Text': ('normal', '00'),
+            'Python Keywords': ('keyword', '01'),
+            'Python Definitions': ('definition', '02'),
+            'Python Builtins': ('builtin', '03'),
+            'Python Comments': ('comment', '04'),
+            'Python Strings': ('string', '05'),
+            'Selected Text': ('hilite', '06'),
+            'Found Text': ('hit', '07'),
+            'Cursor': ('cursor', '08'),
+            'Editor Breakpoint': ('break', '09'),
+            'Shell Normal Text': ('console', '10'),
+            'Shell Error Text': ('error', '11'),
+            'Shell Stdout Text': ('stdout', '12'),
+            'Shell Stderr Text': ('stderr', '13'),
             }
         self.ResetChangedItems() #load initial values in changed items dict
         self.CreateWidgets()
@@ -217,7 +218,8 @@
             ("'selected'", 'hilite'), ('\n  var2 = ', 'normal'),
             ("'found'", 'hit'), ('\n  var3 = ', 'normal'),
             ('list', 'builtin'), ('(', 'normal'),
-            ('None', 'builtin'), (')\n\n', 'normal'),
+            ('None', 'builtin'), (')\n', 'normal'),
+            ('  breakpoint("line")', 'break'), ('\n\n', 'normal'),
             (' error ', 'error'), (' ', 'normal'),
             ('cursor |', 'cursor'), ('\n ', 'normal'),
             ('shell', 'console'), (' ', 'normal'),

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 04:13:21 2015
From: python-checkins at python.org (terry.reedy)
Date: Sat, 03 Oct 2015 02:13:21 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzI0ODIw?=
 =?utf-8?q?=3A_Users_can_now_set_breakpoint_colors_in_Settings_-=3E?=
Message-ID: <20151003021321.477.93039@psf.io>

https://hg.python.org/cpython/rev/d874a6157223
changeset:   98501:d874a6157223
branch:      3.4
parent:      98486:b40a61e79893
user:        Terry Jan Reedy <tjreedy at udel.edu>
date:        Fri Oct 02 22:12:17 2015 -0400
summary:
  Issue #24820: Users can now set breakpoint colors in Settings ->
Custom Highlighting. Original patch by Mark Roseman.

files:
  Lib/idlelib/configDialog.py |  30 +++++++++++++-----------
  1 files changed, 16 insertions(+), 14 deletions(-)


diff --git a/Lib/idlelib/configDialog.py b/Lib/idlelib/configDialog.py
--- a/Lib/idlelib/configDialog.py
+++ b/Lib/idlelib/configDialog.py
@@ -43,19 +43,20 @@
         #The first value of the tuple is the sample area tag name.
         #The second value is the display name list sort index.
         self.themeElements={
-            'Normal Text':('normal', '00'),
-            'Python Keywords':('keyword', '01'),
-            'Python Definitions':('definition', '02'),
-            'Python Builtins':('builtin', '03'),
-            'Python Comments':('comment', '04'),
-            'Python Strings':('string', '05'),
-            'Selected Text':('hilite', '06'),
-            'Found Text':('hit', '07'),
-            'Cursor':('cursor', '08'),
-            'Error Text':('error', '09'),
-            'Shell Normal Text':('console', '10'),
-            'Shell Stdout Text':('stdout', '11'),
-            'Shell Stderr Text':('stderr', '12'),
+            'Normal Text': ('normal', '00'),
+            'Python Keywords': ('keyword', '01'),
+            'Python Definitions': ('definition', '02'),
+            'Python Builtins': ('builtin', '03'),
+            'Python Comments': ('comment', '04'),
+            'Python Strings': ('string', '05'),
+            'Selected Text': ('hilite', '06'),
+            'Found Text': ('hit', '07'),
+            'Cursor': ('cursor', '08'),
+            'Editor Breakpoint': ('break', '09'),
+            'Shell Normal Text': ('console', '10'),
+            'Shell Error Text': ('error', '11'),
+            'Shell Stdout Text': ('stdout', '12'),
+            'Shell Stderr Text': ('stderr', '13'),
             }
         self.ResetChangedItems() #load initial values in changed items dict
         self.CreateWidgets()
@@ -219,7 +220,8 @@
             ("'selected'", 'hilite'), ('\n  var2 = ', 'normal'),
             ("'found'", 'hit'), ('\n  var3 = ', 'normal'),
             ('list', 'builtin'), ('(', 'normal'),
-            ('None', 'keyword'), (')\n\n', 'normal'),
+            ('None', 'keyword'), (')\n', 'normal'),
+            ('  breakpoint("line")', 'break'), ('\n\n', 'normal'),
             (' error ', 'error'), (' ', 'normal'),
             ('cursor |', 'cursor'), ('\n ', 'normal'),
             ('shell', 'console'), (' ', 'normal'),

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 04:13:22 2015
From: python-checkins at python.org (terry.reedy)
Date: Sat, 03 Oct 2015 02:13:22 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
	=?utf-8?q?_Merge_with_3=2E4?=
Message-ID: <20151003021321.7248.21269@psf.io>

https://hg.python.org/cpython/rev/ff025cf824d0
changeset:   98502:ff025cf824d0
branch:      3.5
parent:      98494:fb90425017e3
parent:      98501:d874a6157223
user:        Terry Jan Reedy <tjreedy at udel.edu>
date:        Fri Oct 02 22:12:39 2015 -0400
summary:
  Merge with 3.4

files:
  Lib/idlelib/configDialog.py |  30 +++++++++++++-----------
  1 files changed, 16 insertions(+), 14 deletions(-)


diff --git a/Lib/idlelib/configDialog.py b/Lib/idlelib/configDialog.py
--- a/Lib/idlelib/configDialog.py
+++ b/Lib/idlelib/configDialog.py
@@ -43,19 +43,20 @@
         #The first value of the tuple is the sample area tag name.
         #The second value is the display name list sort index.
         self.themeElements={
-            'Normal Text':('normal', '00'),
-            'Python Keywords':('keyword', '01'),
-            'Python Definitions':('definition', '02'),
-            'Python Builtins':('builtin', '03'),
-            'Python Comments':('comment', '04'),
-            'Python Strings':('string', '05'),
-            'Selected Text':('hilite', '06'),
-            'Found Text':('hit', '07'),
-            'Cursor':('cursor', '08'),
-            'Error Text':('error', '09'),
-            'Shell Normal Text':('console', '10'),
-            'Shell Stdout Text':('stdout', '11'),
-            'Shell Stderr Text':('stderr', '12'),
+            'Normal Text': ('normal', '00'),
+            'Python Keywords': ('keyword', '01'),
+            'Python Definitions': ('definition', '02'),
+            'Python Builtins': ('builtin', '03'),
+            'Python Comments': ('comment', '04'),
+            'Python Strings': ('string', '05'),
+            'Selected Text': ('hilite', '06'),
+            'Found Text': ('hit', '07'),
+            'Cursor': ('cursor', '08'),
+            'Editor Breakpoint': ('break', '09'),
+            'Shell Normal Text': ('console', '10'),
+            'Shell Error Text': ('error', '11'),
+            'Shell Stdout Text': ('stdout', '12'),
+            'Shell Stderr Text': ('stderr', '13'),
             }
         self.ResetChangedItems() #load initial values in changed items dict
         self.CreateWidgets()
@@ -219,7 +220,8 @@
             ("'selected'", 'hilite'), ('\n  var2 = ', 'normal'),
             ("'found'", 'hit'), ('\n  var3 = ', 'normal'),
             ('list', 'builtin'), ('(', 'normal'),
-            ('None', 'keyword'), (')\n\n', 'normal'),
+            ('None', 'keyword'), (')\n', 'normal'),
+            ('  breakpoint("line")', 'break'), ('\n\n', 'normal'),
             (' error ', 'error'), (' ', 'normal'),
             ('cursor |', 'cursor'), ('\n ', 'normal'),
             ('shell', 'console'), (' ', 'normal'),

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 05:26:20 2015
From: python-checkins at python.org (terry.reedy)
Date: Sat, 03 Oct 2015 03:26:20 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzI1MjI0?=
 =?utf-8?q?=3A_README=2Etxt_is_now_an_idlelib_index_for_IDLE_developers_an?=
 =?utf-8?q?d?=
Message-ID: <20151003032620.7262.61790@psf.io>

https://hg.python.org/cpython/rev/bb1a8d3dd4a1
changeset:   98505:bb1a8d3dd4a1
branch:      3.4
parent:      98501:d874a6157223
user:        Terry Jan Reedy <tjreedy at udel.edu>
date:        Fri Oct 02 23:22:59 2015 -0400
summary:
  Issue #25224: README.txt is now an idlelib index for IDLE developers and
curious users.  The previous user content is now in the IDLE doc and is
redundant.  IDLE now means 'Integrated Development and Learning Environment'.

files:
  Doc/library/idle.rst   |    2 +-
  Lib/idlelib/README.txt |  249 ++++++++++++++++++++++++----
  Lib/idlelib/help.html  |    4 +-
  3 files changed, 212 insertions(+), 43 deletions(-)


diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst
--- a/Doc/library/idle.rst
+++ b/Doc/library/idle.rst
@@ -10,7 +10,7 @@
 
 .. moduleauthor:: Guido van Rossum <guido at python.org>
 
-IDLE is the Python IDE built with the :mod:`tkinter` GUI toolkit.
+IDLE is Python's Integrated Development and Learning Environment.
 
 IDLE has the following features:
 
diff --git a/Lib/idlelib/README.txt b/Lib/idlelib/README.txt
--- a/Lib/idlelib/README.txt
+++ b/Lib/idlelib/README.txt
@@ -1,60 +1,229 @@
-IDLE is Python's Tkinter-based Integrated DeveLopment Environment.
+README.txt: an index to idlelib files and the IDLE menu.
 
-IDLE emphasizes a lightweight, clean design with a simple user interface.
-Although it is suitable for beginners, even advanced users will find that
-IDLE has everything they really need to develop pure Python code.
+IDLE is Python?s Integrated Development and Learning
+Environment.  The user documentation is part of the Library Reference and
+is available in IDLE by selecting Help => IDLE Help.  This README documents
+idlelib for IDLE developers and curious users.
 
-IDLE features a multi-window text editor with multiple undo, Python colorizing,
-and many other capabilities, e.g. smart indent, call tips, and autocompletion.
+IDLELIB FILES lists files alphabetically by category,
+with a short description of each.
 
-The editor has comprehensive search functions, including searching through
-multiple files.  Class browsers and path browsers provide fast access to
-code objects from a top level viewpoint without dealing with code folding.
+IDLE MENU show the menu tree, annotated with the module
+or module object that implements the corresponding function.
 
-There is a Python Shell window which features colorizing and command recall.
+This file is descriptive, not prescriptive, and may have errors
+and omissions and lag behind changes in idlelib.
 
-IDLE executes Python code in a separate process, which is restarted for each
-Run (F5) initiated from an editor window.  The environment can also be 
-restarted from the Shell window without restarting IDLE.
 
-This enhancement has often been requested, and is now finally available.  The
-magic "reload/import *" incantations are no longer required when editing and
-testing a module two or three steps down the import chain.
+IDLELIB FILES
+Implemetation files not in IDLE MENU are marked (nim).
+Deprecated files and objects are listed separately as the end.
 
-(Personal firewall software may warn about the connection IDLE makes to its
-subprocess using this computer's internal loopback interface.  This connection
-is not visible on any external interface and no data is sent to or received
-from the Internet.)
+Startup
+-------
+__init__.py  # import, does nothing
+__main__.py  # -m, starts IDLE
+idle.bat
+idle.py
+idle.pyw
 
-It is possible to interrupt tightly looping user code, even on Windows.
+Implementation
+--------------
+AutoComplete.py   # Complete attribute names or filenames.
+AutoCompleteWindow.py  # Display completions.
+AutoExpand.py     # Expand word with previous word in file.
+Bindings.py       # Define most of IDLE menu.
+CallTipWindow.py  # Display calltip.
+CallTips.py       # Create calltip text.
+ClassBrowser.py   # Create module browser window.
+CodeContext.py    # Show compound statement headers otherwise not visible.
+ColorDelegator.py # Colorize text (nim).
+Debugger.py       # Debug code run from editor; show window.
+Delegator.py      # Define base class for delegators (nim).
+EditorWindow.py   # Define most of editor and utility functions.
+FileList.py       # Open files and manage list of open windows (nim).
+FormatParagraph.py# Re-wrap multiline strings and comments.
+GrepDialog.py     # Find all occurrences of pattern in multiple files.
+HyperParser.py    # Parse code around a given index.
+IOBinding.py      # Open, read, and write files
+IdleHistory.py    # Get previous or next user input in shell (nim)
+MultiCall.py      # Wrap tk widget to allow multiple calls per event (nim).
+MultiStatusBar.py # Define status bar for windows (nim).
+ObjectBrowser.py  # Define class used in StackViewer (nim).
+OutputWindow.py   # Create window for grep output.
+ParenMatch.py     # Match fenceposts: (), [], and {}.
+PathBrowser.py    # Create path browser window.
+Percolator.py     # Manage delegator stack (nim).
+PyParse.py        # Give information on code indentation
+PyShell.py        # Start IDLE, manage shell, complete editor window
+RemoteDebugger.py # Debug code run in remote process.
+RemoteObjectBrowser.py # Communicate objects between processes with rpc (nim).
+ReplaceDialog.py  # Search and replace pattern in text.
+RstripExtension.py# Strip trailing whitespace
+ScriptBinding.py  # Check and run user code.
+ScrolledList.py   # Define ScrolledList widget for IDLE (nim).
+SearchDialog.py   # Search for pattern in text.
+SearchDialogBase.py  # Define base for search, replace, and grep dialogs.
+SearchEngine.py   # Define engine for all 3 search dialogs.
+StackViewer.py    # View stack after exception.
+TreeWidget.py     # Define tree widger, used in browsers (nim).
+UndoDelegator.py  # Manage undo stack.
+WidgetRedirector.py # Intercept widget subcommands (for percolator) (nim).
+WindowList.py     # Manage window list and define listed top level.
+ZoomHeight.py     # Zoom window to full height of screen.
+aboutDialog.py    # Display About IDLE dialog.
+configDialog.py   # Display user configuration dialogs.
+configHandler.py  # Load, fetch, and save configuration (nim).
+configHelpSourceEdit.py  # Specify help source.
+configSectionNameDialog.py  # Spefify user config section name
+dynOptionMenuWidget.py  # define mutable OptionMenu widget (nim).
+help.py           # Display IDLE's html doc.
+keybindingDialog.py  # Change keybindings.
+macosxSupport.py  # Help IDLE run on Macs (nim).
+rpc.py            # Commuicate between idle and user processes (nim).
+run.py            # Manage user code execution subprocess.
+tabbedpages.py    # Define tabbed pages widget (nim).
+textView.py       # Define read-only text widget (nim).
 
-Applications which cannot support subprocesses and/or sockets can still run
-IDLE in a single process.
+Configuration
+-------------
+config-extensions.def # Defaults for extensions
+config-highlight.def  # Defaults for colorizing
+config-keys.def       # Defaults for key bindings
+config-main.def       # Defai;ts fpr font and geneal
 
-IDLE has an integrated debugger with stepping, persistent breakpoints, and call
-stack visibility.
+Text
+----
+CREDITS.txt  # not maintained, displayed by About IDLE
+HISTORY.txt  # NEWS up to July 2001
+NEWS.txt     # commits, displayed by About IDLE
+README.txt   # this file, displeyed by About IDLE
+TODO.txt     # needs review
+extend.txt   # about writing extensions
+help.html    # copy of idle.html in docs, displayed by IDLE Help
 
-There is a GUI configuration manager which makes it easy to select fonts,
-colors, keybindings, and startup options.  This facility includes a feature
-which allows the user to specify additional help sources, either locally or on
-the web.
+Subdirectories
+--------------
+Icons  # small image files
+idle_test  # files for human test and automated unit tests
 
-IDLE is coded in 100% pure Python, using the Tkinter GUI toolkit (Tk/Tcl)
-and is cross-platform, working on Unix, Mac, and Windows.
+Unused and Deprecated files and objects (nim)
+---------------------------------------------
+EditorWindow.py: Helpdialog and helpDialog
+ToolTip.py: unused.
+help.txt
+idlever.py
 
-IDLE accepts command line arguments.  Try idle -h to see the options.
 
+IDLE MENUS
+Top level items and most submenu items are defined in Bindings.
+Extenstions add submenu items when active.  The names given are
+found, quoted, in one of these modules, paired with a '<<pseudoevent>>'.
+Each pseudoevent is bound to an event handler.  Some event handlers
+call another function that does the actual work.  The annotations below
+are intended to at least give the module where the actual work is done.
 
-If you find bugs or have suggestions or patches, let us know about
-them by using the Python issue tracker:
+File  # IOBindig except as noted
+  New File
+  Open...  # IOBinding.open
+  Open Module
+  Recent Files
+  Class Browser  # Class Browser
+  Path Browser  # Path Browser
+  ---
+  Save  # IDBinding.save
+  Save As...  # IOBinding.save_as
+  Save Copy As...  # IOBindling.save_a_copy
+  ---
+  Print Window  # IOBinding.print_window
+  ---
+  Close
+  Exit
 
-http://bugs.python.org
+Edit
+  Undo  # undoDelegator
+  Redo  # undoDelegator
+  ---
+  Cut
+  Copy
+  Paste
+  Select All
+  ---  # Next 5 items use SearchEngine; dialogs use SearchDialogBase
+  Find  # Search Dialog
+  Find Again
+  Find Selection
+  Find in Files...  # GrepDialog
+  Replace...  # ReplaceDialog
+  Go to Line
+  Show Completions  # AutoComplete extension and AutoCompleteWidow (&HP)
+  Expand Word  # AutoExpand extension
+  Show call tip  # Calltips extension and CalltipWindow (& Hyperparser)
+  Show surrounding parens  # ParenMatch (& Hyperparser)
 
-For further details and links, read the Help files and check the IDLE home
-page at
+Shell  # PyShell
+  View Last Restart  # PyShell.?
+  Restart Shell  # PyShell.?
 
-http://www.python.org/idle/
+Debug (Shell only)
+  Go to File/Line
+  Debugger  # Debugger, RemoteDebugger
+  Stack Viewer  # StackViewer
+  Auto-open Stack Viewer  # StackViewer
 
-There is a mail list for IDLE: idle-dev at python.org.  You can join at
+Format (Editor only)
+  Indent Region
+  Dedent Region
+  Comment Out Region
+  Uncomment Region
+  Tabify Region
+  Untabify Region
+  Toggle Tabs
+  New Indent Width
+  Format Paragraph  # FormatParagraph extension
+  ---
+  Strip tailing whitespace  # RstripExtension extension
 
-http://mail.python.org/mailman/listinfo/idle-dev
+Run (Editor only)
+  Python Shell  # PyShell
+  ---
+  Check Module  # ScriptBinding
+  Run Module  # ScriptBinding
+
+Options
+  Configure IDLE  # configDialog
+    (tabs in the dialog)
+    Font tab  # onfig-main.def
+    Highlight tab  # configSectionNameDialog, config-highlight.def
+    Keys tab  # keybindingDialog, configSectionNameDialog, onfig-keus.def
+    General tab  # configHelpSourceEdit, config-main.def
+  Configure Extensions  # configDialog
+    Xyz tab  # xyz.py, config-extensions.def
+  ---
+  Code Context (editor only)  # CodeContext extension
+
+Window
+  Zoomheight  # ZoomHeight extension
+  ---
+  <open windows>  # WindowList
+
+Help
+  About IDLE  # aboutDialog
+  ---
+  IDLE Help  # help
+  Python Doc
+  Turtle Demo
+  ---
+  <other help sources>
+
+<Context Menu> (right click)
+Defined in EditorWindow, PyShell, Output
+   Cut
+   Copy
+   Paste
+   ---
+   Go to file/line (shell and output only)
+   Set Breakpoint (editor only)
+   Clear Breakpoint (editor only)
+ Defined in Debugger
+   Go to source line
+   Show stack frame
diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html
--- a/Lib/idlelib/help.html
+++ b/Lib/idlelib/help.html
@@ -75,7 +75,7 @@
 
   <div class="section" id="idle">
 <span id="id1"></span><h1>25.5. IDLE<a class="headerlink" href="#idle" title="Permalink to this headline">??</a></h1>
-<p id="index-0">IDLE is the Python IDE built with the <a class="reference internal" href="tkinter.html#module-tkinter" title="tkinter: Interface to Tcl/Tk for graphical user interfaces"><tt class="xref py py-mod docutils literal"><span class="pre">tkinter</span></tt></a> GUI toolkit.</p>
+<p id="index-0">IDLE is Python&#8217;s Integrated Development and Learning Environment.</p>
 <p>IDLE has the following features:</p>
 <ul class="simple">
 <li>coded in 100% pure Python, using the <a class="reference internal" href="tkinter.html#module-tkinter" title="tkinter: Interface to Tcl/Tk for graphical user interfaces"><tt class="xref py py-mod docutils literal"><span class="pre">tkinter</span></tt></a> GUI toolkit</li>
@@ -699,7 +699,7 @@
     The Python Software Foundation is a non-profit corporation.
     <a href="https://www.python.org/psf/donations/">Please donate.</a>
     <br />
-    Last updated on Sep 29, 2015.
+    Last updated on Oct 02, 2015.
     <a href="../bugs.html">Found a bug</a>?
     <br />
     Created using <a href="http://sphinx.pocoo.org/">Sphinx</a> 1.2.3.

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 05:26:20 2015
From: python-checkins at python.org (terry.reedy)
Date: Sat, 03 Oct 2015 03:26:20 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzI1MjI0?=
 =?utf-8?q?=3A_README=2Etxt_is_now_an_idlelib_index_for_IDLE_developers_an?=
 =?utf-8?q?d?=
Message-ID: <20151003032620.128840.59379@psf.io>

https://hg.python.org/cpython/rev/4e62989e3688
changeset:   98504:4e62989e3688
branch:      2.7
parent:      98500:e67da755d614
user:        Terry Jan Reedy <tjreedy at udel.edu>
date:        Fri Oct 02 23:22:54 2015 -0400
summary:
  Issue #25224: README.txt is now an idlelib index for IDLE developers and
curious users.  The previous user content is now in the IDLE doc and is
redundant.  IDLE now means 'Integrated Development and Learning Environment'.

files:
  Doc/library/idle.rst   |    2 +-
  Lib/idlelib/README.txt |  250 ++++++++++++++++++++++++----
  Lib/idlelib/help.html  |    4 +-
  3 files changed, 211 insertions(+), 45 deletions(-)


diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst
--- a/Doc/library/idle.rst
+++ b/Doc/library/idle.rst
@@ -10,7 +10,7 @@
 
 .. moduleauthor:: Guido van Rossum <guido at python.org>
 
-IDLE is the Python IDE built with the :mod:`tkinter` GUI toolkit.
+IDLE is Python's Integrated Development and Learning Environment.
 
 IDLE has the following features:
 
diff --git a/Lib/idlelib/README.txt b/Lib/idlelib/README.txt
--- a/Lib/idlelib/README.txt
+++ b/Lib/idlelib/README.txt
@@ -1,63 +1,229 @@
-IDLE is Python's Tkinter-based Integrated DeveLopment Environment.
+README.txt: an index to idlelib files and the IDLE menu.
 
-IDLE emphasizes a lightweight, clean design with a simple user interface.
-Although it is suitable for beginners, even advanced users will find that
-IDLE has everything they really need to develop pure Python code.
+IDLE is Python?s Integrated Development and Learning
+Environment.  The user documentation is part of the Library Reference and
+is available in IDLE by selecting Help => IDLE Help.  This README documents
+idlelib for IDLE developers and curious users.
 
-IDLE features a multi-window text editor with multiple undo, Python colorizing,
-and many other capabilities, e.g. smart indent, call tips, and autocompletion.
+IDLELIB FILES lists files alphabetically by category,
+with a short description of each.
 
-The editor has comprehensive search functions, including searching through
-multiple files.  Class browsers and path browsers provide fast access to
-code objects from a top level viewpoint without dealing with code folding.
+IDLE MENU show the menu tree, annotated with the module
+or module object that implements the corresponding function.
 
-There is a Python Shell window which features colorizing and command recall.
+This file is descriptive, not prescriptive, and may have errors
+and omissions and lag behind changes in idlelib.
 
-IDLE executes Python code in a separate process, which is restarted for each
-Run (F5) initiated from an editor window.  The environment can also be 
-restarted from the Shell window without restarting IDLE.
 
-This enhancement has often been requested, and is now finally available.  The
-magic "reload/import *" incantations are no longer required when editing and
-testing a module two or three steps down the import chain.
+IDLELIB FILES
+Implemetation files not in IDLE MENU are marked (nim).
+Deprecated files and objects are listed separately as the end.
 
-(Personal firewall software may warn about the connection IDLE makes to its
-subprocess using this computer's internal loopback interface.  This connection
-is not visible on any external interface and no data is sent to or received
-from the Internet.)
+Startup
+-------
+__init__.py  # import, does nothing
+__main__.py  # -m, starts IDLE
+idle.bat
+idle.py
+idle.pyw
 
-It is possible to interrupt tightly looping user code, even on Windows.
+Implementation
+--------------
+AutoComplete.py   # Complete attribute names or filenames.
+AutoCompleteWindow.py  # Display completions.
+AutoExpand.py     # Expand word with previous word in file.
+Bindings.py       # Define most of IDLE menu.
+CallTipWindow.py  # Display calltip.
+CallTips.py       # Create calltip text.
+ClassBrowser.py   # Create module browser window.
+CodeContext.py    # Show compound statement headers otherwise not visible.
+ColorDelegator.py # Colorize text (nim).
+Debugger.py       # Debug code run from editor; show window.
+Delegator.py      # Define base class for delegators (nim).
+EditorWindow.py   # Define most of editor and utility functions.
+FileList.py       # Open files and manage list of open windows (nim).
+FormatParagraph.py# Re-wrap multiline strings and comments.
+GrepDialog.py     # Find all occurrences of pattern in multiple files.
+HyperParser.py    # Parse code around a given index.
+IOBinding.py      # Open, read, and write files
+IdleHistory.py    # Get previous or next user input in shell (nim)
+MultiCall.py      # Wrap tk widget to allow multiple calls per event (nim).
+MultiStatusBar.py # Define status bar for windows (nim).
+ObjectBrowser.py  # Define class used in StackViewer (nim).
+OutputWindow.py   # Create window for grep output.
+ParenMatch.py     # Match fenceposts: (), [], and {}.
+PathBrowser.py    # Create path browser window.
+Percolator.py     # Manage delegator stack (nim).
+PyParse.py        # Give information on code indentation
+PyShell.py        # Start IDLE, manage shell, complete editor window
+RemoteDebugger.py # Debug code run in remote process.
+RemoteObjectBrowser.py # Communicate objects between processes with rpc (nim).
+ReplaceDialog.py  # Search and replace pattern in text.
+RstripExtension.py# Strip trailing whitespace
+ScriptBinding.py  # Check and run user code.
+ScrolledList.py   # Define ScrolledList widget for IDLE (nim).
+SearchDialog.py   # Search for pattern in text.
+SearchDialogBase.py  # Define base for search, replace, and grep dialogs.
+SearchEngine.py   # Define engine for all 3 search dialogs.
+StackViewer.py    # View stack after exception.
+TreeWidget.py     # Define tree widger, used in browsers (nim).
+UndoDelegator.py  # Manage undo stack.
+WidgetRedirector.py # Intercept widget subcommands (for percolator) (nim).
+WindowList.py     # Manage window list and define listed top level.
+ZoomHeight.py     # Zoom window to full height of screen.
+aboutDialog.py    # Display About IDLE dialog.
+configDialog.py   # Display user configuration dialogs.
+configHandler.py  # Load, fetch, and save configuration (nim).
+configHelpSourceEdit.py  # Specify help source.
+configSectionNameDialog.py  # Spefify user config section name
+dynOptionMenuWidget.py  # define mutable OptionMenu widget (nim).
+help.py           # Display IDLE's html doc.
+keybindingDialog.py  # Change keybindings.
+macosxSupport.py  # Help IDLE run on Macs (nim).
+rpc.py            # Commuicate between idle and user processes (nim).
+run.py            # Manage user code execution subprocess.
+tabbedpages.py    # Define tabbed pages widget (nim).
+textView.py       # Define read-only text widget (nim).
 
-Applications which cannot support subprocesses and/or sockets can still run
-IDLE in a single process.
+Configuration
+-------------
+config-extensions.def # Defaults for extensions
+config-highlight.def  # Defaults for colorizing
+config-keys.def       # Defaults for key bindings
+config-main.def       # Defai;ts fpr font and geneal
 
-IDLE has an integrated debugger with stepping, persistent breakpoints, and call
-stack visibility.
+Text
+----
+CREDITS.txt  # not maintained, displayed by About IDLE
+HISTORY.txt  # NEWS up to July 2001
+NEWS.txt     # commits, displayed by About IDLE
+README.txt   # this file, displeyed by About IDLE
+TODO.txt     # needs review
+extend.txt   # about writing extensions
+help.html    # copy of idle.html in docs, displayed by IDLE Help
 
-There is a GUI configuration manager which makes it easy to select fonts,
-colors, keybindings, and startup options.  This facility includes a feature
-which allows the user to specify additional help sources, either locally or on
-the web.
+Subdirectories
+--------------
+Icons  # small image files
+idle_test  # files for human test and automated unit tests
 
-IDLE is coded in 100% pure Python, using the Tkinter GUI toolkit (Tk/Tcl)
-and is cross-platform, working on Unix, Mac, and Windows.
+Unused and Deprecated files and objects (nim)
+---------------------------------------------
+EditorWindow.py: Helpdialog and helpDialog
+ToolTip.py: unused.
+help.txt
+idlever.py
 
-IDLE accepts command line arguments.  Try idle -h to see the options.
 
+IDLE MENUS
+Top level items and most submenu items are defined in Bindings.
+Extenstions add submenu items when active.  The names given are
+found, quoted, in one of these modules, paired with a '<<pseudoevent>>'.
+Each pseudoevent is bound to an event handler.  Some event handlers
+call another function that does the actual work.  The annotations below
+are intended to at least give the module where the actual work is done.
 
-If you find bugs or have suggestions, let us know about them by using the
-Python Bug Tracker:
+File  # IOBindig except as noted
+  New File
+  Open...  # IOBinding.open
+  Open Module
+  Recent Files
+  Class Browser  # Class Browser
+  Path Browser  # Path Browser
+  ---
+  Save  # IDBinding.save
+  Save As...  # IOBinding.save_as
+  Save Copy As...  # IOBindling.save_a_copy
+  ---
+  Print Window  # IOBinding.print_window
+  ---
+  Close
+  Exit
 
-http://sourceforge.net/projects/python
+Edit
+  Undo  # undoDelegator
+  Redo  # undoDelegator
+  ---
+  Cut
+  Copy
+  Paste
+  Select All
+  ---  # Next 5 items use SearchEngine; dialogs use SearchDialogBase
+  Find  # Search Dialog
+  Find Again
+  Find Selection
+  Find in Files...  # GrepDialog
+  Replace...  # ReplaceDialog
+  Go to Line
+  Show Completions  # AutoComplete extension and AutoCompleteWidow (&HP)
+  Expand Word  # AutoExpand extension
+  Show call tip  # Calltips extension and CalltipWindow (& Hyperparser)
+  Show surrounding parens  # ParenMatch (& Hyperparser)
 
-Patches are always appreciated at the Python Patch Tracker, and change
-requests should be posted to the RFE Tracker.
+Shell  # PyShell
+  View Last Restart  # PyShell.?
+  Restart Shell  # PyShell.?
 
-For further details and links, read the Help files and check the IDLE home
-page at
+Debug (Shell only)
+  Go to File/Line
+  Debugger  # Debugger, RemoteDebugger
+  Stack Viewer  # StackViewer
+  Auto-open Stack Viewer  # StackViewer
 
-http://www.python.org/idle/
+Format (Editor only)
+  Indent Region
+  Dedent Region
+  Comment Out Region
+  Uncomment Region
+  Tabify Region
+  Untabify Region
+  Toggle Tabs
+  New Indent Width
+  Format Paragraph  # FormatParagraph extension
+  ---
+  Strip tailing whitespace  # RstripExtension extension
 
-There is a mail list for IDLE: idle-dev at python.org.  You can join at
+Run (Editor only)
+  Python Shell  # PyShell
+  ---
+  Check Module  # ScriptBinding
+  Run Module  # ScriptBinding
 
-http://mail.python.org/mailman/listinfo/idle-dev
+Options
+  Configure IDLE  # configDialog
+    (tabs in the dialog)
+    Font tab  # onfig-main.def
+    Highlight tab  # configSectionNameDialog, config-highlight.def
+    Keys tab  # keybindingDialog, configSectionNameDialog, onfig-keus.def
+    General tab  # configHelpSourceEdit, config-main.def
+  Configure Extensions  # configDialog
+    Xyz tab  # xyz.py, config-extensions.def
+  ---
+  Code Context (editor only)  # CodeContext extension
+
+Window
+  Zoomheight  # ZoomHeight extension
+  ---
+  <open windows>  # WindowList
+
+Help
+  About IDLE  # aboutDialog
+  ---
+  IDLE Help  # help
+  Python Doc
+  Turtle Demo
+  ---
+  <other help sources>
+
+<Context Menu> (right click)
+Defined in EditorWindow, PyShell, Output
+   Cut
+   Copy
+   Paste
+   ---
+   Go to file/line (shell and output only)
+   Set Breakpoint (editor only)
+   Clear Breakpoint (editor only)
+ Defined in Debugger
+   Go to source line
+   Show stack frame
diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html
--- a/Lib/idlelib/help.html
+++ b/Lib/idlelib/help.html
@@ -75,7 +75,7 @@
 
   <div class="section" id="idle">
 <span id="id1"></span><h1>24.6. IDLE<a class="headerlink" href="#idle" title="Permalink to this headline">??</a></h1>
-<p id="index-0">IDLE is the Python IDE built with the <tt class="xref py py-mod docutils literal"><span class="pre">tkinter</span></tt> GUI toolkit.</p>
+<p id="index-0">IDLE is Python&#8217;s Integrated Development and Learning Environment.</p>
 <p>IDLE has the following features:</p>
 <ul class="simple">
 <li>coded in 100% pure Python, using the <tt class="xref py py-mod docutils literal"><span class="pre">tkinter</span></tt> GUI toolkit</li>
@@ -699,7 +699,7 @@
     The Python Software Foundation is a non-profit corporation.
     <a href="https://www.python.org/psf/donations/">Please donate.</a>
     <br />
-    Last updated on Sep 29, 2015.
+    Last updated on Oct 02, 2015.
     <a href="../bugs.html">Found a bug</a>?
     <br />
     Created using <a href="http://sphinx.pocoo.org/">Sphinx</a> 1.2.3.

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 05:26:21 2015
From: python-checkins at python.org (terry.reedy)
Date: Sat, 03 Oct 2015 03:26:21 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
	=?utf-8?q?_Merge_with_3=2E4?=
Message-ID: <20151003032621.2685.39202@psf.io>

https://hg.python.org/cpython/rev/227f959f43e7
changeset:   98506:227f959f43e7
branch:      3.5
parent:      98502:ff025cf824d0
parent:      98505:bb1a8d3dd4a1
user:        Terry Jan Reedy <tjreedy at udel.edu>
date:        Fri Oct 02 23:25:14 2015 -0400
summary:
  Merge with 3.4

files:
  Doc/library/idle.rst   |    2 +-
  Lib/idlelib/README.txt |  249 ++++++++++++++++++++++++----
  Lib/idlelib/help.html  |    4 +-
  3 files changed, 212 insertions(+), 43 deletions(-)


diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst
--- a/Doc/library/idle.rst
+++ b/Doc/library/idle.rst
@@ -10,7 +10,7 @@
 
 .. moduleauthor:: Guido van Rossum <guido at python.org>
 
-IDLE is the Python IDE built with the :mod:`tkinter` GUI toolkit.
+IDLE is Python's Integrated Development and Learning Environment.
 
 IDLE has the following features:
 
diff --git a/Lib/idlelib/README.txt b/Lib/idlelib/README.txt
--- a/Lib/idlelib/README.txt
+++ b/Lib/idlelib/README.txt
@@ -1,60 +1,229 @@
-IDLE is Python's Tkinter-based Integrated DeveLopment Environment.
+README.txt: an index to idlelib files and the IDLE menu.
 
-IDLE emphasizes a lightweight, clean design with a simple user interface.
-Although it is suitable for beginners, even advanced users will find that
-IDLE has everything they really need to develop pure Python code.
+IDLE is Python?s Integrated Development and Learning
+Environment.  The user documentation is part of the Library Reference and
+is available in IDLE by selecting Help => IDLE Help.  This README documents
+idlelib for IDLE developers and curious users.
 
-IDLE features a multi-window text editor with multiple undo, Python colorizing,
-and many other capabilities, e.g. smart indent, call tips, and autocompletion.
+IDLELIB FILES lists files alphabetically by category,
+with a short description of each.
 
-The editor has comprehensive search functions, including searching through
-multiple files.  Class browsers and path browsers provide fast access to
-code objects from a top level viewpoint without dealing with code folding.
+IDLE MENU show the menu tree, annotated with the module
+or module object that implements the corresponding function.
 
-There is a Python Shell window which features colorizing and command recall.
+This file is descriptive, not prescriptive, and may have errors
+and omissions and lag behind changes in idlelib.
 
-IDLE executes Python code in a separate process, which is restarted for each
-Run (F5) initiated from an editor window.  The environment can also be
-restarted from the Shell window without restarting IDLE.
 
-This enhancement has often been requested, and is now finally available.  The
-magic "reload/import *" incantations are no longer required when editing and
-testing a module two or three steps down the import chain.
+IDLELIB FILES
+Implemetation files not in IDLE MENU are marked (nim).
+Deprecated files and objects are listed separately as the end.
 
-(Personal firewall software may warn about the connection IDLE makes to its
-subprocess using this computer's internal loopback interface.  This connection
-is not visible on any external interface and no data is sent to or received
-from the Internet.)
+Startup
+-------
+__init__.py  # import, does nothing
+__main__.py  # -m, starts IDLE
+idle.bat
+idle.py
+idle.pyw
 
-It is possible to interrupt tightly looping user code, even on Windows.
+Implementation
+--------------
+AutoComplete.py   # Complete attribute names or filenames.
+AutoCompleteWindow.py  # Display completions.
+AutoExpand.py     # Expand word with previous word in file.
+Bindings.py       # Define most of IDLE menu.
+CallTipWindow.py  # Display calltip.
+CallTips.py       # Create calltip text.
+ClassBrowser.py   # Create module browser window.
+CodeContext.py    # Show compound statement headers otherwise not visible.
+ColorDelegator.py # Colorize text (nim).
+Debugger.py       # Debug code run from editor; show window.
+Delegator.py      # Define base class for delegators (nim).
+EditorWindow.py   # Define most of editor and utility functions.
+FileList.py       # Open files and manage list of open windows (nim).
+FormatParagraph.py# Re-wrap multiline strings and comments.
+GrepDialog.py     # Find all occurrences of pattern in multiple files.
+HyperParser.py    # Parse code around a given index.
+IOBinding.py      # Open, read, and write files
+IdleHistory.py    # Get previous or next user input in shell (nim)
+MultiCall.py      # Wrap tk widget to allow multiple calls per event (nim).
+MultiStatusBar.py # Define status bar for windows (nim).
+ObjectBrowser.py  # Define class used in StackViewer (nim).
+OutputWindow.py   # Create window for grep output.
+ParenMatch.py     # Match fenceposts: (), [], and {}.
+PathBrowser.py    # Create path browser window.
+Percolator.py     # Manage delegator stack (nim).
+PyParse.py        # Give information on code indentation
+PyShell.py        # Start IDLE, manage shell, complete editor window
+RemoteDebugger.py # Debug code run in remote process.
+RemoteObjectBrowser.py # Communicate objects between processes with rpc (nim).
+ReplaceDialog.py  # Search and replace pattern in text.
+RstripExtension.py# Strip trailing whitespace
+ScriptBinding.py  # Check and run user code.
+ScrolledList.py   # Define ScrolledList widget for IDLE (nim).
+SearchDialog.py   # Search for pattern in text.
+SearchDialogBase.py  # Define base for search, replace, and grep dialogs.
+SearchEngine.py   # Define engine for all 3 search dialogs.
+StackViewer.py    # View stack after exception.
+TreeWidget.py     # Define tree widger, used in browsers (nim).
+UndoDelegator.py  # Manage undo stack.
+WidgetRedirector.py # Intercept widget subcommands (for percolator) (nim).
+WindowList.py     # Manage window list and define listed top level.
+ZoomHeight.py     # Zoom window to full height of screen.
+aboutDialog.py    # Display About IDLE dialog.
+configDialog.py   # Display user configuration dialogs.
+configHandler.py  # Load, fetch, and save configuration (nim).
+configHelpSourceEdit.py  # Specify help source.
+configSectionNameDialog.py  # Spefify user config section name
+dynOptionMenuWidget.py  # define mutable OptionMenu widget (nim).
+help.py           # Display IDLE's html doc.
+keybindingDialog.py  # Change keybindings.
+macosxSupport.py  # Help IDLE run on Macs (nim).
+rpc.py            # Commuicate between idle and user processes (nim).
+run.py            # Manage user code execution subprocess.
+tabbedpages.py    # Define tabbed pages widget (nim).
+textView.py       # Define read-only text widget (nim).
 
-Applications which cannot support subprocesses and/or sockets can still run
-IDLE in a single process.
+Configuration
+-------------
+config-extensions.def # Defaults for extensions
+config-highlight.def  # Defaults for colorizing
+config-keys.def       # Defaults for key bindings
+config-main.def       # Defai;ts fpr font and geneal
 
-IDLE has an integrated debugger with stepping, persistent breakpoints, and call
-stack visibility.
+Text
+----
+CREDITS.txt  # not maintained, displayed by About IDLE
+HISTORY.txt  # NEWS up to July 2001
+NEWS.txt     # commits, displayed by About IDLE
+README.txt   # this file, displeyed by About IDLE
+TODO.txt     # needs review
+extend.txt   # about writing extensions
+help.html    # copy of idle.html in docs, displayed by IDLE Help
 
-There is a GUI configuration manager which makes it easy to select fonts,
-colors, keybindings, and startup options.  This facility includes a feature
-which allows the user to specify additional help sources, either locally or on
-the web.
+Subdirectories
+--------------
+Icons  # small image files
+idle_test  # files for human test and automated unit tests
 
-IDLE is coded in 100% pure Python, using the Tkinter GUI toolkit (Tk/Tcl)
-and is cross-platform, working on Unix, Mac, and Windows.
+Unused and Deprecated files and objects (nim)
+---------------------------------------------
+EditorWindow.py: Helpdialog and helpDialog
+ToolTip.py: unused.
+help.txt
+idlever.py
 
-IDLE accepts command line arguments.  Try idle -h to see the options.
 
+IDLE MENUS
+Top level items and most submenu items are defined in Bindings.
+Extenstions add submenu items when active.  The names given are
+found, quoted, in one of these modules, paired with a '<<pseudoevent>>'.
+Each pseudoevent is bound to an event handler.  Some event handlers
+call another function that does the actual work.  The annotations below
+are intended to at least give the module where the actual work is done.
 
-If you find bugs or have suggestions or patches, let us know about
-them by using the Python issue tracker:
+File  # IOBindig except as noted
+  New File
+  Open...  # IOBinding.open
+  Open Module
+  Recent Files
+  Class Browser  # Class Browser
+  Path Browser  # Path Browser
+  ---
+  Save  # IDBinding.save
+  Save As...  # IOBinding.save_as
+  Save Copy As...  # IOBindling.save_a_copy
+  ---
+  Print Window  # IOBinding.print_window
+  ---
+  Close
+  Exit
 
-http://bugs.python.org
+Edit
+  Undo  # undoDelegator
+  Redo  # undoDelegator
+  ---
+  Cut
+  Copy
+  Paste
+  Select All
+  ---  # Next 5 items use SearchEngine; dialogs use SearchDialogBase
+  Find  # Search Dialog
+  Find Again
+  Find Selection
+  Find in Files...  # GrepDialog
+  Replace...  # ReplaceDialog
+  Go to Line
+  Show Completions  # AutoComplete extension and AutoCompleteWidow (&HP)
+  Expand Word  # AutoExpand extension
+  Show call tip  # Calltips extension and CalltipWindow (& Hyperparser)
+  Show surrounding parens  # ParenMatch (& Hyperparser)
 
-For further details and links, read the Help files and check the IDLE home
-page at
+Shell  # PyShell
+  View Last Restart  # PyShell.?
+  Restart Shell  # PyShell.?
 
-http://www.python.org/idle/
+Debug (Shell only)
+  Go to File/Line
+  Debugger  # Debugger, RemoteDebugger
+  Stack Viewer  # StackViewer
+  Auto-open Stack Viewer  # StackViewer
 
-There is a mail list for IDLE: idle-dev at python.org.  You can join at
+Format (Editor only)
+  Indent Region
+  Dedent Region
+  Comment Out Region
+  Uncomment Region
+  Tabify Region
+  Untabify Region
+  Toggle Tabs
+  New Indent Width
+  Format Paragraph  # FormatParagraph extension
+  ---
+  Strip tailing whitespace  # RstripExtension extension
 
-http://mail.python.org/mailman/listinfo/idle-dev
+Run (Editor only)
+  Python Shell  # PyShell
+  ---
+  Check Module  # ScriptBinding
+  Run Module  # ScriptBinding
+
+Options
+  Configure IDLE  # configDialog
+    (tabs in the dialog)
+    Font tab  # onfig-main.def
+    Highlight tab  # configSectionNameDialog, config-highlight.def
+    Keys tab  # keybindingDialog, configSectionNameDialog, onfig-keus.def
+    General tab  # configHelpSourceEdit, config-main.def
+  Configure Extensions  # configDialog
+    Xyz tab  # xyz.py, config-extensions.def
+  ---
+  Code Context (editor only)  # CodeContext extension
+
+Window
+  Zoomheight  # ZoomHeight extension
+  ---
+  <open windows>  # WindowList
+
+Help
+  About IDLE  # aboutDialog
+  ---
+  IDLE Help  # help
+  Python Doc
+  Turtle Demo
+  ---
+  <other help sources>
+
+<Context Menu> (right click)
+Defined in EditorWindow, PyShell, Output
+   Cut
+   Copy
+   Paste
+   ---
+   Go to file/line (shell and output only)
+   Set Breakpoint (editor only)
+   Clear Breakpoint (editor only)
+ Defined in Debugger
+   Go to source line
+   Show stack frame
diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html
--- a/Lib/idlelib/help.html
+++ b/Lib/idlelib/help.html
@@ -75,7 +75,7 @@
 
   <div class="section" id="idle">
 <span id="id1"></span><h1>25.5. IDLE<a class="headerlink" href="#idle" title="Permalink to this headline">??</a></h1>
-<p id="index-0">IDLE is the Python IDE built with the <a class="reference internal" href="tkinter.html#module-tkinter" title="tkinter: Interface to Tcl/Tk for graphical user interfaces"><tt class="xref py py-mod docutils literal"><span class="pre">tkinter</span></tt></a> GUI toolkit.</p>
+<p id="index-0">IDLE is Python&#8217;s Integrated Development and Learning Environment.</p>
 <p>IDLE has the following features:</p>
 <ul class="simple">
 <li>coded in 100% pure Python, using the <a class="reference internal" href="tkinter.html#module-tkinter" title="tkinter: Interface to Tcl/Tk for graphical user interfaces"><tt class="xref py py-mod docutils literal"><span class="pre">tkinter</span></tt></a> GUI toolkit</li>
@@ -699,7 +699,7 @@
     The Python Software Foundation is a non-profit corporation.
     <a href="https://www.python.org/psf/donations/">Please donate.</a>
     <br />
-    Last updated on Sep 29, 2015.
+    Last updated on Oct 02, 2015.
     <a href="../bugs.html">Found a bug</a>?
     <br />
     Created using <a href="http://sphinx.pocoo.org/">Sphinx</a> 1.2.3.

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 05:26:21 2015
From: python-checkins at python.org (terry.reedy)
Date: Sat, 03 Oct 2015 03:26:21 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
	=?utf-8?q?=29=3A_Merge_with_3=2E5?=
Message-ID: <20151003032621.3287.38436@psf.io>

https://hg.python.org/cpython/rev/4202abcf5702
changeset:   98507:4202abcf5702
parent:      98503:fd904e1e4f29
parent:      98506:227f959f43e7
user:        Terry Jan Reedy <tjreedy at udel.edu>
date:        Fri Oct 02 23:25:54 2015 -0400
summary:
  Merge with 3.5

files:
  Doc/library/idle.rst   |    2 +-
  Lib/idlelib/README.txt |  249 ++++++++++++++++++++++++----
  Lib/idlelib/help.html  |    4 +-
  3 files changed, 212 insertions(+), 43 deletions(-)


diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst
--- a/Doc/library/idle.rst
+++ b/Doc/library/idle.rst
@@ -10,7 +10,7 @@
 
 .. moduleauthor:: Guido van Rossum <guido at python.org>
 
-IDLE is the Python IDE built with the :mod:`tkinter` GUI toolkit.
+IDLE is Python's Integrated Development and Learning Environment.
 
 IDLE has the following features:
 
diff --git a/Lib/idlelib/README.txt b/Lib/idlelib/README.txt
--- a/Lib/idlelib/README.txt
+++ b/Lib/idlelib/README.txt
@@ -1,60 +1,229 @@
-IDLE is Python's Tkinter-based Integrated DeveLopment Environment.
+README.txt: an index to idlelib files and the IDLE menu.
 
-IDLE emphasizes a lightweight, clean design with a simple user interface.
-Although it is suitable for beginners, even advanced users will find that
-IDLE has everything they really need to develop pure Python code.
+IDLE is Python?s Integrated Development and Learning
+Environment.  The user documentation is part of the Library Reference and
+is available in IDLE by selecting Help => IDLE Help.  This README documents
+idlelib for IDLE developers and curious users.
 
-IDLE features a multi-window text editor with multiple undo, Python colorizing,
-and many other capabilities, e.g. smart indent, call tips, and autocompletion.
+IDLELIB FILES lists files alphabetically by category,
+with a short description of each.
 
-The editor has comprehensive search functions, including searching through
-multiple files.  Class browsers and path browsers provide fast access to
-code objects from a top level viewpoint without dealing with code folding.
+IDLE MENU show the menu tree, annotated with the module
+or module object that implements the corresponding function.
 
-There is a Python Shell window which features colorizing and command recall.
+This file is descriptive, not prescriptive, and may have errors
+and omissions and lag behind changes in idlelib.
 
-IDLE executes Python code in a separate process, which is restarted for each
-Run (F5) initiated from an editor window.  The environment can also be
-restarted from the Shell window without restarting IDLE.
 
-This enhancement has often been requested, and is now finally available.  The
-magic "reload/import *" incantations are no longer required when editing and
-testing a module two or three steps down the import chain.
+IDLELIB FILES
+Implemetation files not in IDLE MENU are marked (nim).
+Deprecated files and objects are listed separately as the end.
 
-(Personal firewall software may warn about the connection IDLE makes to its
-subprocess using this computer's internal loopback interface.  This connection
-is not visible on any external interface and no data is sent to or received
-from the Internet.)
+Startup
+-------
+__init__.py  # import, does nothing
+__main__.py  # -m, starts IDLE
+idle.bat
+idle.py
+idle.pyw
 
-It is possible to interrupt tightly looping user code, even on Windows.
+Implementation
+--------------
+AutoComplete.py   # Complete attribute names or filenames.
+AutoCompleteWindow.py  # Display completions.
+AutoExpand.py     # Expand word with previous word in file.
+Bindings.py       # Define most of IDLE menu.
+CallTipWindow.py  # Display calltip.
+CallTips.py       # Create calltip text.
+ClassBrowser.py   # Create module browser window.
+CodeContext.py    # Show compound statement headers otherwise not visible.
+ColorDelegator.py # Colorize text (nim).
+Debugger.py       # Debug code run from editor; show window.
+Delegator.py      # Define base class for delegators (nim).
+EditorWindow.py   # Define most of editor and utility functions.
+FileList.py       # Open files and manage list of open windows (nim).
+FormatParagraph.py# Re-wrap multiline strings and comments.
+GrepDialog.py     # Find all occurrences of pattern in multiple files.
+HyperParser.py    # Parse code around a given index.
+IOBinding.py      # Open, read, and write files
+IdleHistory.py    # Get previous or next user input in shell (nim)
+MultiCall.py      # Wrap tk widget to allow multiple calls per event (nim).
+MultiStatusBar.py # Define status bar for windows (nim).
+ObjectBrowser.py  # Define class used in StackViewer (nim).
+OutputWindow.py   # Create window for grep output.
+ParenMatch.py     # Match fenceposts: (), [], and {}.
+PathBrowser.py    # Create path browser window.
+Percolator.py     # Manage delegator stack (nim).
+PyParse.py        # Give information on code indentation
+PyShell.py        # Start IDLE, manage shell, complete editor window
+RemoteDebugger.py # Debug code run in remote process.
+RemoteObjectBrowser.py # Communicate objects between processes with rpc (nim).
+ReplaceDialog.py  # Search and replace pattern in text.
+RstripExtension.py# Strip trailing whitespace
+ScriptBinding.py  # Check and run user code.
+ScrolledList.py   # Define ScrolledList widget for IDLE (nim).
+SearchDialog.py   # Search for pattern in text.
+SearchDialogBase.py  # Define base for search, replace, and grep dialogs.
+SearchEngine.py   # Define engine for all 3 search dialogs.
+StackViewer.py    # View stack after exception.
+TreeWidget.py     # Define tree widger, used in browsers (nim).
+UndoDelegator.py  # Manage undo stack.
+WidgetRedirector.py # Intercept widget subcommands (for percolator) (nim).
+WindowList.py     # Manage window list and define listed top level.
+ZoomHeight.py     # Zoom window to full height of screen.
+aboutDialog.py    # Display About IDLE dialog.
+configDialog.py   # Display user configuration dialogs.
+configHandler.py  # Load, fetch, and save configuration (nim).
+configHelpSourceEdit.py  # Specify help source.
+configSectionNameDialog.py  # Spefify user config section name
+dynOptionMenuWidget.py  # define mutable OptionMenu widget (nim).
+help.py           # Display IDLE's html doc.
+keybindingDialog.py  # Change keybindings.
+macosxSupport.py  # Help IDLE run on Macs (nim).
+rpc.py            # Commuicate between idle and user processes (nim).
+run.py            # Manage user code execution subprocess.
+tabbedpages.py    # Define tabbed pages widget (nim).
+textView.py       # Define read-only text widget (nim).
 
-Applications which cannot support subprocesses and/or sockets can still run
-IDLE in a single process.
+Configuration
+-------------
+config-extensions.def # Defaults for extensions
+config-highlight.def  # Defaults for colorizing
+config-keys.def       # Defaults for key bindings
+config-main.def       # Defai;ts fpr font and geneal
 
-IDLE has an integrated debugger with stepping, persistent breakpoints, and call
-stack visibility.
+Text
+----
+CREDITS.txt  # not maintained, displayed by About IDLE
+HISTORY.txt  # NEWS up to July 2001
+NEWS.txt     # commits, displayed by About IDLE
+README.txt   # this file, displeyed by About IDLE
+TODO.txt     # needs review
+extend.txt   # about writing extensions
+help.html    # copy of idle.html in docs, displayed by IDLE Help
 
-There is a GUI configuration manager which makes it easy to select fonts,
-colors, keybindings, and startup options.  This facility includes a feature
-which allows the user to specify additional help sources, either locally or on
-the web.
+Subdirectories
+--------------
+Icons  # small image files
+idle_test  # files for human test and automated unit tests
 
-IDLE is coded in 100% pure Python, using the Tkinter GUI toolkit (Tk/Tcl)
-and is cross-platform, working on Unix, Mac, and Windows.
+Unused and Deprecated files and objects (nim)
+---------------------------------------------
+EditorWindow.py: Helpdialog and helpDialog
+ToolTip.py: unused.
+help.txt
+idlever.py
 
-IDLE accepts command line arguments.  Try idle -h to see the options.
 
+IDLE MENUS
+Top level items and most submenu items are defined in Bindings.
+Extenstions add submenu items when active.  The names given are
+found, quoted, in one of these modules, paired with a '<<pseudoevent>>'.
+Each pseudoevent is bound to an event handler.  Some event handlers
+call another function that does the actual work.  The annotations below
+are intended to at least give the module where the actual work is done.
 
-If you find bugs or have suggestions or patches, let us know about
-them by using the Python issue tracker:
+File  # IOBindig except as noted
+  New File
+  Open...  # IOBinding.open
+  Open Module
+  Recent Files
+  Class Browser  # Class Browser
+  Path Browser  # Path Browser
+  ---
+  Save  # IDBinding.save
+  Save As...  # IOBinding.save_as
+  Save Copy As...  # IOBindling.save_a_copy
+  ---
+  Print Window  # IOBinding.print_window
+  ---
+  Close
+  Exit
 
-http://bugs.python.org
+Edit
+  Undo  # undoDelegator
+  Redo  # undoDelegator
+  ---
+  Cut
+  Copy
+  Paste
+  Select All
+  ---  # Next 5 items use SearchEngine; dialogs use SearchDialogBase
+  Find  # Search Dialog
+  Find Again
+  Find Selection
+  Find in Files...  # GrepDialog
+  Replace...  # ReplaceDialog
+  Go to Line
+  Show Completions  # AutoComplete extension and AutoCompleteWidow (&HP)
+  Expand Word  # AutoExpand extension
+  Show call tip  # Calltips extension and CalltipWindow (& Hyperparser)
+  Show surrounding parens  # ParenMatch (& Hyperparser)
 
-For further details and links, read the Help files and check the IDLE home
-page at
+Shell  # PyShell
+  View Last Restart  # PyShell.?
+  Restart Shell  # PyShell.?
 
-http://www.python.org/idle/
+Debug (Shell only)
+  Go to File/Line
+  Debugger  # Debugger, RemoteDebugger
+  Stack Viewer  # StackViewer
+  Auto-open Stack Viewer  # StackViewer
 
-There is a mail list for IDLE: idle-dev at python.org.  You can join at
+Format (Editor only)
+  Indent Region
+  Dedent Region
+  Comment Out Region
+  Uncomment Region
+  Tabify Region
+  Untabify Region
+  Toggle Tabs
+  New Indent Width
+  Format Paragraph  # FormatParagraph extension
+  ---
+  Strip tailing whitespace  # RstripExtension extension
 
-http://mail.python.org/mailman/listinfo/idle-dev
+Run (Editor only)
+  Python Shell  # PyShell
+  ---
+  Check Module  # ScriptBinding
+  Run Module  # ScriptBinding
+
+Options
+  Configure IDLE  # configDialog
+    (tabs in the dialog)
+    Font tab  # onfig-main.def
+    Highlight tab  # configSectionNameDialog, config-highlight.def
+    Keys tab  # keybindingDialog, configSectionNameDialog, onfig-keus.def
+    General tab  # configHelpSourceEdit, config-main.def
+  Configure Extensions  # configDialog
+    Xyz tab  # xyz.py, config-extensions.def
+  ---
+  Code Context (editor only)  # CodeContext extension
+
+Window
+  Zoomheight  # ZoomHeight extension
+  ---
+  <open windows>  # WindowList
+
+Help
+  About IDLE  # aboutDialog
+  ---
+  IDLE Help  # help
+  Python Doc
+  Turtle Demo
+  ---
+  <other help sources>
+
+<Context Menu> (right click)
+Defined in EditorWindow, PyShell, Output
+   Cut
+   Copy
+   Paste
+   ---
+   Go to file/line (shell and output only)
+   Set Breakpoint (editor only)
+   Clear Breakpoint (editor only)
+ Defined in Debugger
+   Go to source line
+   Show stack frame
diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html
--- a/Lib/idlelib/help.html
+++ b/Lib/idlelib/help.html
@@ -75,7 +75,7 @@
 
   <div class="section" id="idle">
 <span id="id1"></span><h1>25.5. IDLE<a class="headerlink" href="#idle" title="Permalink to this headline">??</a></h1>
-<p id="index-0">IDLE is the Python IDE built with the <a class="reference internal" href="tkinter.html#module-tkinter" title="tkinter: Interface to Tcl/Tk for graphical user interfaces"><tt class="xref py py-mod docutils literal"><span class="pre">tkinter</span></tt></a> GUI toolkit.</p>
+<p id="index-0">IDLE is Python&#8217;s Integrated Development and Learning Environment.</p>
 <p>IDLE has the following features:</p>
 <ul class="simple">
 <li>coded in 100% pure Python, using the <a class="reference internal" href="tkinter.html#module-tkinter" title="tkinter: Interface to Tcl/Tk for graphical user interfaces"><tt class="xref py py-mod docutils literal"><span class="pre">tkinter</span></tt></a> GUI toolkit</li>
@@ -699,7 +699,7 @@
     The Python Software Foundation is a non-profit corporation.
     <a href="https://www.python.org/psf/donations/">Please donate.</a>
     <br />
-    Last updated on Sep 29, 2015.
+    Last updated on Oct 02, 2015.
     <a href="../bugs.html">Found a bug</a>?
     <br />
     Created using <a href="http://sphinx.pocoo.org/">Sphinx</a> 1.2.3.

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 08:17:40 2015
From: python-checkins at python.org (raymond.hettinger)
Date: Sat, 03 Oct 2015 06:17:40 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Factor_out_common_iterator?=
	=?utf-8?q?_finalization_code?=
Message-ID: <20151003061740.128842.49976@psf.io>

https://hg.python.org/cpython/rev/8c21f32c5882
changeset:   98508:8c21f32c5882
user:        Raymond Hettinger <python at rcn.com>
date:        Fri Oct 02 23:17:33 2015 -0700
summary:
  Factor out common iterator finalization code

files:
  Modules/_collectionsmodule.c |  45 ++++++++++-------------
  1 files changed, 20 insertions(+), 25 deletions(-)


diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c
--- a/Modules/_collectionsmodule.c
+++ b/Modules/_collectionsmodule.c
@@ -350,21 +350,34 @@
 
 PyDoc_STRVAR(appendleft_doc, "Add an element to the left side of the deque.");
 
+static PyObject*
+finalize_iterator(PyObject *it)
+{
+    if (PyErr_Occurred()) {
+        if (PyErr_ExceptionMatches(PyExc_StopIteration))
+            PyErr_Clear();
+        else {
+            Py_DECREF(it);
+            return NULL;
+        }
+    }
+    Py_DECREF(it);
+    Py_RETURN_NONE;
+}
 
 /* Run an iterator to exhaustion.  Shortcut for
    the extend/extendleft methods when maxlen == 0. */
 static PyObject*
 consume_iterator(PyObject *it)
 {
+    PyObject *(*iternext)(PyObject *);
     PyObject *item;
 
-    while ((item = PyIter_Next(it)) != NULL) {
+    iternext = *Py_TYPE(it)->tp_iternext;
+    while ((item = iternext(it)) != NULL) {
         Py_DECREF(item);
     }
-    Py_DECREF(it);
-    if (PyErr_Occurred())
-        return NULL;
-    Py_RETURN_NONE;
+    return finalize_iterator(it);
 }
 
 static PyObject *
@@ -423,16 +436,7 @@
         if (trim)
             deque_trim_left(deque);
     }
-    if (PyErr_Occurred()) {
-        if (PyErr_ExceptionMatches(PyExc_StopIteration))
-            PyErr_Clear();
-        else {
-            Py_DECREF(it);
-            return NULL;
-        }
-    }
-    Py_DECREF(it);
-    Py_RETURN_NONE;
+    return finalize_iterator(it);
 }
 
 PyDoc_STRVAR(extend_doc,
@@ -494,16 +498,7 @@
         if (trim)
             deque_trim_right(deque);
     }
-    if (PyErr_Occurred()) {
-        if (PyErr_ExceptionMatches(PyExc_StopIteration))
-            PyErr_Clear();
-        else {
-            Py_DECREF(it);
-            return NULL;
-        }
-    }
-    Py_DECREF(it);
-    Py_RETURN_NONE;
+    return finalize_iterator(it);
 }
 
 PyDoc_STRVAR(extendleft_doc,

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 08:44:33 2015
From: python-checkins at python.org (martin.panter)
Date: Sat, 03 Oct 2015 06:44:33 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzI1MjMy?=
 =?utf-8?q?=3A_Fix_CGIRequestHandler=27s_splitting_of_URL_query?=
Message-ID: <20151003064433.464.15192@psf.io>

https://hg.python.org/cpython/rev/969afbf501af
changeset:   98509:969afbf501af
branch:      3.4
parent:      98505:bb1a8d3dd4a1
user:        Martin Panter <vadmium+py at gmail.com>
date:        Sat Oct 03 05:38:07 2015 +0000
summary:
  Issue #25232: Fix CGIRequestHandler's splitting of URL query

Patch from Xiang Zhang.

files:
  Lib/http/server.py           |   6 +-----
  Lib/test/test_httpservers.py |  24 ++++++++++++++++++++++++
  Misc/ACKS                    |   1 +
  Misc/NEWS                    |   3 +++
  4 files changed, 29 insertions(+), 5 deletions(-)


diff --git a/Lib/http/server.py b/Lib/http/server.py
--- a/Lib/http/server.py
+++ b/Lib/http/server.py
@@ -1032,11 +1032,7 @@
                 break
 
         # find an explicit query string, if present.
-        i = rest.rfind('?')
-        if i >= 0:
-            rest, query = rest[:i], rest[i+1:]
-        else:
-            query = ''
+        rest, _, query = rest.partition('?')
 
         # dissect the part after the directory name into a script name &
         # a possible additional path, to be stored in PATH_INFO.
diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py
--- a/Lib/test/test_httpservers.py
+++ b/Lib/test/test_httpservers.py
@@ -366,6 +366,16 @@
                           form.getfirst("bacon")))
 """
 
+cgi_file4 = """\
+#!%s
+import os
+
+print("Content-type: text/html")
+print()
+
+print(os.environ["%s"])
+"""
+
 
 @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0,
         "This test can't be run reliably as root (issue #13308).")
@@ -387,6 +397,7 @@
         self.file1_path = None
         self.file2_path = None
         self.file3_path = None
+        self.file4_path = None
 
         # The shebang line should be pure ASCII: use symlink if possible.
         # See issue #7668.
@@ -425,6 +436,11 @@
             file3.write(cgi_file1 % self.pythonexe)
         os.chmod(self.file3_path, 0o777)
 
+        self.file4_path = os.path.join(self.cgi_dir, 'file4.py')
+        with open(self.file4_path, 'w', encoding='utf-8') as file4:
+            file4.write(cgi_file4 % (self.pythonexe, 'QUERY_STRING'))
+        os.chmod(self.file4_path, 0o777)
+
         os.chdir(self.parent_dir)
 
     def tearDown(self):
@@ -440,6 +456,8 @@
                 os.remove(self.file2_path)
             if self.file3_path:
                 os.remove(self.file3_path)
+            if self.file4_path:
+                os.remove(self.file4_path)
             os.rmdir(self.cgi_child_dir)
             os.rmdir(self.cgi_dir)
             os.rmdir(self.parent_dir)
@@ -541,6 +559,12 @@
         self.assertEqual((b'Hello World' + self.linesep, 'text/html', 200),
                 (res.read(), res.getheader('Content-type'), res.status))
 
+    def test_query_with_multiple_question_mark(self):
+        res = self.request('/cgi-bin/file4.py?a=b?c=d')
+        self.assertEqual(
+            (b'a=b?c=d' + self.linesep, 'text/html', 200),
+            (res.read(), res.getheader('Content-type'), res.status))
+
 
 class SocketlessRequestHandler(SimpleHTTPRequestHandler):
     def __init__(self):
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -1531,6 +1531,7 @@
 Daniel Wozniak
 Heiko Wundram
 Doug Wyatt
+Xiang Zhang
 Robert Xiao
 Florent Xicluna
 Hirokazu Yamamoto
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -90,6 +90,9 @@
 Library
 -------
 
+- Issue #25232: Fix CGIRequestHandler to split the query from the URL at the
+  first question mark (?) rather than the last. Patch from Xiang Zhang.
+
 - Issue #22958: Constructor and update method of weakref.WeakValueDictionary
   now accept the self and the dict keyword arguments.
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 08:44:33 2015
From: python-checkins at python.org (martin.panter)
Date: Sat, 03 Oct 2015 06:44:33 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzI0NjU3?=
 =?utf-8?q?=3A_Prevent_CGIRequestHandler_from_collapsing_the_URL_query?=
Message-ID: <20151003064433.128856.78507@psf.io>

https://hg.python.org/cpython/rev/634fe6a90e0c
changeset:   98510:634fe6a90e0c
branch:      3.4
user:        Martin Panter <vadmium+py at gmail.com>
date:        Sat Oct 03 05:55:46 2015 +0000
summary:
  Issue #24657: Prevent CGIRequestHandler from collapsing the URL query

Initial patch from Xiang Zhang. Also fix out-of-date _url_collapse_path() doc
string.

files:
  Lib/http/server.py           |  13 +++++++++----
  Lib/test/test_httpservers.py |   7 +++++++
  Misc/NEWS                    |   3 +++
  3 files changed, 19 insertions(+), 4 deletions(-)


diff --git a/Lib/http/server.py b/Lib/http/server.py
--- a/Lib/http/server.py
+++ b/Lib/http/server.py
@@ -887,13 +887,15 @@
     The utility of this function is limited to is_cgi method and helps
     preventing some security attacks.
 
-    Returns: A tuple of (head, tail) where tail is everything after the final /
-    and head is everything before it.  Head will always start with a '/' and,
-    if it contains anything else, never have a trailing '/'.
+    Returns: The reconstituted URL, which will always start with a '/'.
 
     Raises: IndexError if too many '..' occur within the path.
 
     """
+    # Query component should not be involved.
+    path, _, query = path.partition('?')
+    path = urllib.parse.unquote(path)
+
     # Similar to os.path.split(os.path.normpath(path)) but specific to URL
     # path semantics rather than local operating system semantics.
     path_parts = path.split('/')
@@ -914,6 +916,9 @@
     else:
         tail_part = ''
 
+    if query:
+        tail_part = '?'.join((tail_part, query))
+
     splitpath = ('/' + '/'.join(head_parts), tail_part)
     collapsed_path = "/".join(splitpath)
 
@@ -995,7 +1000,7 @@
         (and the next character is a '/' or the end of the string).
 
         """
-        collapsed_path = _url_collapse_path(urllib.parse.unquote(self.path))
+        collapsed_path = _url_collapse_path(self.path)
         dir_sep = collapsed_path.find('/', 1)
         head, tail = collapsed_path[:dir_sep], collapsed_path[dir_sep+1:]
         if head in self.cgi_directories:
diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py
--- a/Lib/test/test_httpservers.py
+++ b/Lib/test/test_httpservers.py
@@ -565,6 +565,13 @@
             (b'a=b?c=d' + self.linesep, 'text/html', 200),
             (res.read(), res.getheader('Content-type'), res.status))
 
+    def test_query_with_continuous_slashes(self):
+        res = self.request('/cgi-bin/file4.py?k=aa%2F%2Fbb&//q//p//=//a//b//')
+        self.assertEqual(
+            (b'k=aa%2F%2Fbb&//q//p//=//a//b//' + self.linesep,
+             'text/html', 200),
+            (res.read(), res.getheader('Content-type'), res.status))
+
 
 class SocketlessRequestHandler(SimpleHTTPRequestHandler):
     def __init__(self):
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -93,6 +93,9 @@
 - Issue #25232: Fix CGIRequestHandler to split the query from the URL at the
   first question mark (?) rather than the last. Patch from Xiang Zhang.
 
+- Issue #24657: Prevent CGIRequestHandler from collapsing slashes in the
+  query part of the URL as if it were a path. Patch from Xiang Zhang.
+
 - Issue #22958: Constructor and update method of weakref.WeakValueDictionary
   now accept the self and the dict keyword arguments.
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 08:44:33 2015
From: python-checkins at python.org (martin.panter)
Date: Sat, 03 Oct 2015 06:44:33 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy41KTogSXNzdWVzICMyNTIz?=
 =?utf-8?q?2=2C_=2324657=3A_Use_new_enum_status_to_match_rest_of_tests?=
Message-ID: <20151003064433.18388.7153@psf.io>

https://hg.python.org/cpython/rev/88918f2a54df
changeset:   98512:88918f2a54df
branch:      3.5
user:        Martin Panter <vadmium+py at gmail.com>
date:        Sat Oct 03 06:07:22 2015 +0000
summary:
  Issues #25232, #24657: Use new enum status to match rest of tests

files:
  Lib/test/test_httpservers.py |  4 ++--
  1 files changed, 2 insertions(+), 2 deletions(-)


diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py
--- a/Lib/test/test_httpservers.py
+++ b/Lib/test/test_httpservers.py
@@ -627,14 +627,14 @@
     def test_query_with_multiple_question_mark(self):
         res = self.request('/cgi-bin/file4.py?a=b?c=d')
         self.assertEqual(
-            (b'a=b?c=d' + self.linesep, 'text/html', 200),
+            (b'a=b?c=d' + self.linesep, 'text/html', HTTPStatus.OK),
             (res.read(), res.getheader('Content-type'), res.status))
 
     def test_query_with_continuous_slashes(self):
         res = self.request('/cgi-bin/file4.py?k=aa%2F%2Fbb&//q//p//=//a//b//')
         self.assertEqual(
             (b'k=aa%2F%2Fbb&//q//p//=//a//b//' + self.linesep,
-             'text/html', 200),
+             'text/html', HTTPStatus.OK),
             (res.read(), res.getheader('Content-type'), res.status))
 
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 08:44:33 2015
From: python-checkins at python.org (martin.panter)
Date: Sat, 03 Oct 2015 06:44:33 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
 =?utf-8?q?_Issues_=2325232=2C_=2324657=3A_Merge_two_CGI_server_fixes_from?=
 =?utf-8?q?_3=2E4_into_3=2E5?=
Message-ID: <20151003064433.70981.68303@psf.io>

https://hg.python.org/cpython/rev/ba1e3c112e42
changeset:   98511:ba1e3c112e42
branch:      3.5
parent:      98506:227f959f43e7
parent:      98510:634fe6a90e0c
user:        Martin Panter <vadmium+py at gmail.com>
date:        Sat Oct 03 06:03:25 2015 +0000
summary:
  Issues #25232, #24657: Merge two CGI server fixes from 3.4 into 3.5

files:
  Lib/http/server.py           |  19 +++++++------
  Lib/test/test_httpservers.py |  31 ++++++++++++++++++++++++
  Misc/ACKS                    |   1 +
  Misc/NEWS                    |   6 ++++
  4 files changed, 48 insertions(+), 9 deletions(-)


diff --git a/Lib/http/server.py b/Lib/http/server.py
--- a/Lib/http/server.py
+++ b/Lib/http/server.py
@@ -837,13 +837,15 @@
     The utility of this function is limited to is_cgi method and helps
     preventing some security attacks.
 
-    Returns: A tuple of (head, tail) where tail is everything after the final /
-    and head is everything before it.  Head will always start with a '/' and,
-    if it contains anything else, never have a trailing '/'.
+    Returns: The reconstituted URL, which will always start with a '/'.
 
     Raises: IndexError if too many '..' occur within the path.
 
     """
+    # Query component should not be involved.
+    path, _, query = path.partition('?')
+    path = urllib.parse.unquote(path)
+
     # Similar to os.path.split(os.path.normpath(path)) but specific to URL
     # path semantics rather than local operating system semantics.
     path_parts = path.split('/')
@@ -864,6 +866,9 @@
     else:
         tail_part = ''
 
+    if query:
+        tail_part = '?'.join((tail_part, query))
+
     splitpath = ('/' + '/'.join(head_parts), tail_part)
     collapsed_path = "/".join(splitpath)
 
@@ -947,7 +952,7 @@
         (and the next character is a '/' or the end of the string).
 
         """
-        collapsed_path = _url_collapse_path(urllib.parse.unquote(self.path))
+        collapsed_path = _url_collapse_path(self.path)
         dir_sep = collapsed_path.find('/', 1)
         head, tail = collapsed_path[:dir_sep], collapsed_path[dir_sep+1:]
         if head in self.cgi_directories:
@@ -984,11 +989,7 @@
                 break
 
         # find an explicit query string, if present.
-        i = rest.rfind('?')
-        if i >= 0:
-            rest, query = rest[:i], rest[i+1:]
-        else:
-            query = ''
+        rest, _, query = rest.partition('?')
 
         # dissect the part after the directory name into a script name &
         # a possible additional path, to be stored in PATH_INFO.
diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py
--- a/Lib/test/test_httpservers.py
+++ b/Lib/test/test_httpservers.py
@@ -425,6 +425,16 @@
                           form.getfirst("bacon")))
 """
 
+cgi_file4 = """\
+#!%s
+import os
+
+print("Content-type: text/html")
+print()
+
+print(os.environ["%s"])
+"""
+
 
 @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0,
         "This test can't be run reliably as root (issue #13308).")
@@ -446,6 +456,7 @@
         self.file1_path = None
         self.file2_path = None
         self.file3_path = None
+        self.file4_path = None
 
         # The shebang line should be pure ASCII: use symlink if possible.
         # See issue #7668.
@@ -484,6 +495,11 @@
             file3.write(cgi_file1 % self.pythonexe)
         os.chmod(self.file3_path, 0o777)
 
+        self.file4_path = os.path.join(self.cgi_dir, 'file4.py')
+        with open(self.file4_path, 'w', encoding='utf-8') as file4:
+            file4.write(cgi_file4 % (self.pythonexe, 'QUERY_STRING'))
+        os.chmod(self.file4_path, 0o777)
+
         os.chdir(self.parent_dir)
 
     def tearDown(self):
@@ -499,6 +515,8 @@
                 os.remove(self.file2_path)
             if self.file3_path:
                 os.remove(self.file3_path)
+            if self.file4_path:
+                os.remove(self.file4_path)
             os.rmdir(self.cgi_child_dir)
             os.rmdir(self.cgi_dir)
             os.rmdir(self.parent_dir)
@@ -606,6 +624,19 @@
             (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK),
             (res.read(), res.getheader('Content-type'), res.status))
 
+    def test_query_with_multiple_question_mark(self):
+        res = self.request('/cgi-bin/file4.py?a=b?c=d')
+        self.assertEqual(
+            (b'a=b?c=d' + self.linesep, 'text/html', 200),
+            (res.read(), res.getheader('Content-type'), res.status))
+
+    def test_query_with_continuous_slashes(self):
+        res = self.request('/cgi-bin/file4.py?k=aa%2F%2Fbb&//q//p//=//a//b//')
+        self.assertEqual(
+            (b'k=aa%2F%2Fbb&//q//p//=//a//b//' + self.linesep,
+             'text/html', 200),
+            (res.read(), res.getheader('Content-type'), res.status))
+
 
 class SocketlessRequestHandler(SimpleHTTPRequestHandler):
     def __init__(self):
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -1580,6 +1580,7 @@
 Wei Wu
 Heiko Wundram
 Doug Wyatt
+Xiang Zhang
 Robert Xiao
 Florent Xicluna
 Hirokazu Yamamoto
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -34,6 +34,12 @@
 Library
 -------
 
+- Issue #25232: Fix CGIRequestHandler to split the query from the URL at the
+  first question mark (?) rather than the last. Patch from Xiang Zhang.
+
+- Issue #24657: Prevent CGIRequestHandler from collapsing slashes in the
+  query part of the URL as if it were a path. Patch from Xiang Zhang.
+
 - Issue #24483: C implementation of functools.lru_cache() now calculates key's
   hash only once.
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 08:44:34 2015
From: python-checkins at python.org (martin.panter)
Date: Sat, 03 Oct 2015 06:44:34 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
 =?utf-8?q?=29=3A_Issues_=2325232=2C_=2324657=3A_Merge_two_CGI_server_fixe?=
 =?utf-8?q?s_from_3=2E5?=
Message-ID: <20151003064433.7246.58921@psf.io>

https://hg.python.org/cpython/rev/0f03023d4318
changeset:   98513:0f03023d4318
parent:      98508:8c21f32c5882
parent:      98512:88918f2a54df
user:        Martin Panter <vadmium+py at gmail.com>
date:        Sat Oct 03 06:43:19 2015 +0000
summary:
  Issues #25232, #24657: Merge two CGI server fixes from 3.5

files:
  Lib/http/server.py           |  19 +++++++------
  Lib/test/test_httpservers.py |  31 ++++++++++++++++++++++++
  Misc/ACKS                    |   1 +
  Misc/NEWS                    |   6 ++++
  4 files changed, 48 insertions(+), 9 deletions(-)


diff --git a/Lib/http/server.py b/Lib/http/server.py
--- a/Lib/http/server.py
+++ b/Lib/http/server.py
@@ -837,13 +837,15 @@
     The utility of this function is limited to is_cgi method and helps
     preventing some security attacks.
 
-    Returns: A tuple of (head, tail) where tail is everything after the final /
-    and head is everything before it.  Head will always start with a '/' and,
-    if it contains anything else, never have a trailing '/'.
+    Returns: The reconstituted URL, which will always start with a '/'.
 
     Raises: IndexError if too many '..' occur within the path.
 
     """
+    # Query component should not be involved.
+    path, _, query = path.partition('?')
+    path = urllib.parse.unquote(path)
+
     # Similar to os.path.split(os.path.normpath(path)) but specific to URL
     # path semantics rather than local operating system semantics.
     path_parts = path.split('/')
@@ -864,6 +866,9 @@
     else:
         tail_part = ''
 
+    if query:
+        tail_part = '?'.join((tail_part, query))
+
     splitpath = ('/' + '/'.join(head_parts), tail_part)
     collapsed_path = "/".join(splitpath)
 
@@ -947,7 +952,7 @@
         (and the next character is a '/' or the end of the string).
 
         """
-        collapsed_path = _url_collapse_path(urllib.parse.unquote(self.path))
+        collapsed_path = _url_collapse_path(self.path)
         dir_sep = collapsed_path.find('/', 1)
         head, tail = collapsed_path[:dir_sep], collapsed_path[dir_sep+1:]
         if head in self.cgi_directories:
@@ -984,11 +989,7 @@
                 break
 
         # find an explicit query string, if present.
-        i = rest.rfind('?')
-        if i >= 0:
-            rest, query = rest[:i], rest[i+1:]
-        else:
-            query = ''
+        rest, _, query = rest.partition('?')
 
         # dissect the part after the directory name into a script name &
         # a possible additional path, to be stored in PATH_INFO.
diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py
--- a/Lib/test/test_httpservers.py
+++ b/Lib/test/test_httpservers.py
@@ -425,6 +425,16 @@
                           form.getfirst("bacon")))
 """
 
+cgi_file4 = """\
+#!%s
+import os
+
+print("Content-type: text/html")
+print()
+
+print(os.environ["%s"])
+"""
+
 
 @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0,
         "This test can't be run reliably as root (issue #13308).")
@@ -446,6 +456,7 @@
         self.file1_path = None
         self.file2_path = None
         self.file3_path = None
+        self.file4_path = None
 
         # The shebang line should be pure ASCII: use symlink if possible.
         # See issue #7668.
@@ -484,6 +495,11 @@
             file3.write(cgi_file1 % self.pythonexe)
         os.chmod(self.file3_path, 0o777)
 
+        self.file4_path = os.path.join(self.cgi_dir, 'file4.py')
+        with open(self.file4_path, 'w', encoding='utf-8') as file4:
+            file4.write(cgi_file4 % (self.pythonexe, 'QUERY_STRING'))
+        os.chmod(self.file4_path, 0o777)
+
         os.chdir(self.parent_dir)
 
     def tearDown(self):
@@ -499,6 +515,8 @@
                 os.remove(self.file2_path)
             if self.file3_path:
                 os.remove(self.file3_path)
+            if self.file4_path:
+                os.remove(self.file4_path)
             os.rmdir(self.cgi_child_dir)
             os.rmdir(self.cgi_dir)
             os.rmdir(self.parent_dir)
@@ -606,6 +624,19 @@
             (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK),
             (res.read(), res.getheader('Content-type'), res.status))
 
+    def test_query_with_multiple_question_mark(self):
+        res = self.request('/cgi-bin/file4.py?a=b?c=d')
+        self.assertEqual(
+            (b'a=b?c=d' + self.linesep, 'text/html', HTTPStatus.OK),
+            (res.read(), res.getheader('Content-type'), res.status))
+
+    def test_query_with_continuous_slashes(self):
+        res = self.request('/cgi-bin/file4.py?k=aa%2F%2Fbb&//q//p//=//a//b//')
+        self.assertEqual(
+            (b'k=aa%2F%2Fbb&//q//p//=//a//b//' + self.linesep,
+             'text/html', HTTPStatus.OK),
+            (res.read(), res.getheader('Content-type'), res.status))
+
 
 class SocketlessRequestHandler(SimpleHTTPRequestHandler):
     def __init__(self):
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -1582,6 +1582,7 @@
 Wei Wu
 Heiko Wundram
 Doug Wyatt
+Xiang Zhang
 Robert Xiao
 Florent Xicluna
 Hirokazu Yamamoto
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -218,6 +218,12 @@
 Library
 -------
 
+- Issue #25232: Fix CGIRequestHandler to split the query from the URL at the
+  first question mark (?) rather than the last. Patch from Xiang Zhang.
+
+- Issue #24657: Prevent CGIRequestHandler from collapsing slashes in the
+  query part of the URL as if it were a path. Patch from Xiang Zhang.
+
 - Issue #24483: C implementation of functools.lru_cache() now calculates key's
   hash only once.
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 08:44:34 2015
From: python-checkins at python.org (martin.panter)
Date: Sat, 03 Oct 2015 06:44:34 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issues_=2325232=2C_=232465?=
 =?utf-8?q?7=3A_Add_NEWS_to_3=2E6=2E0a1_section?=
Message-ID: <20151003064434.55474.95508@psf.io>

https://hg.python.org/cpython/rev/3c006ee38287
changeset:   98514:3c006ee38287
user:        Martin Panter <vadmium+py at gmail.com>
date:        Sat Oct 03 06:25:43 2015 +0000
summary:
  Issues #25232, #24657: Add NEWS to 3.6.0a1 section

files:
  Misc/NEWS |  6 ++++++
  1 files changed, 6 insertions(+), 0 deletions(-)


diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -40,6 +40,12 @@
 Library
 -------
 
+- Issue #25232: Fix CGIRequestHandler to split the query from the URL at the
+  first question mark (?) rather than the last. Patch from Xiang Zhang.
+
+- Issue #24657: Prevent CGIRequestHandler from collapsing slashes in the
+  query part of the URL as if it were a path. Patch from Xiang Zhang.
+
 - Issue #25287: Don't add crypt.METHOD_CRYPT to crypt.methods if it's not
   supported. Check if it is supported, it may not be supported on OpenBSD for
   example.

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 09:27:37 2015
From: python-checkins at python.org (martin.panter)
Date: Sat, 03 Oct 2015 07:27:37 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzI1MjMy?=
 =?utf-8?q?=3A_Fix_CGIRequestHandler=27s_splitting_of_URL_query?=
Message-ID: <20151003072737.128828.54306@psf.io>

https://hg.python.org/cpython/rev/b12b30dc8617
changeset:   98515:b12b30dc8617
branch:      2.7
parent:      98504:4e62989e3688
user:        Martin Panter <vadmium+py at gmail.com>
date:        Sat Oct 03 05:38:07 2015 +0000
summary:
  Issue #25232: Fix CGIRequestHandler's splitting of URL query

Patch from Xiang Zhang.

files:
  Lib/CGIHTTPServer.py         |   6 +-----
  Lib/test/test_httpservers.py |  22 ++++++++++++++++++++++
  Misc/ACKS                    |   1 +
  Misc/NEWS                    |   3 +++
  4 files changed, 27 insertions(+), 5 deletions(-)


diff --git a/Lib/CGIHTTPServer.py b/Lib/CGIHTTPServer.py
--- a/Lib/CGIHTTPServer.py
+++ b/Lib/CGIHTTPServer.py
@@ -120,11 +120,7 @@
                 break
 
         # find an explicit query string, if present.
-        i = rest.rfind('?')
-        if i >= 0:
-            rest, query = rest[:i], rest[i+1:]
-        else:
-            query = ''
+        rest, _, query = rest.partition('?')
 
         # dissect the part after the directory name into a script name &
         # a possible additional path, to be stored in PATH_INFO.
diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py
--- a/Lib/test/test_httpservers.py
+++ b/Lib/test/test_httpservers.py
@@ -381,6 +381,16 @@
                           form.getfirst("bacon"))
 """
 
+cgi_file4 = """\
+#!%s
+import os
+
+print("Content-type: text/html")
+print()
+
+print(os.environ["%s"])
+"""
+
 
 @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0,
         "This test can't be run reliably as root (issue #13308).")
@@ -424,6 +434,11 @@
             file3.write(cgi_file1 % self.pythonexe)
         os.chmod(self.file3_path, 0777)
 
+        self.file4_path = os.path.join(self.cgi_dir, 'file4.py')
+        with open(self.file4_path, 'w') as file4:
+            file4.write(cgi_file4 % (self.pythonexe, 'QUERY_STRING'))
+        os.chmod(self.file4_path, 0o777)
+
         self.cwd = os.getcwd()
         os.chdir(self.parent_dir)
 
@@ -436,6 +451,7 @@
             os.remove(self.file1_path)
             os.remove(self.file2_path)
             os.remove(self.file3_path)
+            os.remove(self.file4_path)
             os.rmdir(self.cgi_child_dir)
             os.rmdir(self.cgi_dir)
             os.rmdir(self.parent_dir)
@@ -536,6 +552,12 @@
         self.assertEqual((b'Hello World\n', 'text/html', 200),
                 (res.read(), res.getheader('Content-type'), res.status))
 
+    def test_query_with_multiple_question_mark(self):
+        res = self.request('/cgi-bin/file4.py?a=b?c=d')
+        self.assertEqual(
+            (b'a=b?c=d\n', 'text/html', 200),
+            (res.read(), res.getheader('Content-type'), res.status))
+
 
 class SimpleHTTPRequestHandlerTestCase(unittest.TestCase):
     """ Test url parsing """
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -1505,6 +1505,7 @@
 Daniel Wozniak
 Heiko Wundram
 Doug Wyatt
+Xiang Zhang
 Robert Xiao
 Florent Xicluna
 Hirokazu Yamamoto
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -43,6 +43,9 @@
 Library
 -------
 
+- Issue #25232: Fix CGIRequestHandler to split the query from the URL at the
+  first question mark (?) rather than the last. Patch from Xiang Zhang.
+
 - Issue #22958: Constructor and update method of weakref.WeakValueDictionary
   now accept the self keyword argument.
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 09:27:37 2015
From: python-checkins at python.org (martin.panter)
Date: Sat, 03 Oct 2015 07:27:37 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzI0NjU3?=
 =?utf-8?q?=3A_Prevent_CGIRequestHandler_from_collapsing_the_URL_query?=
Message-ID: <20151003072737.453.9704@psf.io>

https://hg.python.org/cpython/rev/a4302005f9a2
changeset:   98516:a4302005f9a2
branch:      2.7
user:        Martin Panter <vadmium+py at gmail.com>
date:        Sat Oct 03 05:55:46 2015 +0000
summary:
  Issue #24657: Prevent CGIRequestHandler from collapsing the URL query

Initial patch from Xiang Zhang. Also fix out-of-date _url_collapse_path() doc
string.

files:
  Lib/CGIHTTPServer.py         |  13 +++++++++----
  Lib/test/test_httpservers.py |   7 +++++++
  Misc/NEWS                    |   3 +++
  3 files changed, 19 insertions(+), 4 deletions(-)


diff --git a/Lib/CGIHTTPServer.py b/Lib/CGIHTTPServer.py
--- a/Lib/CGIHTTPServer.py
+++ b/Lib/CGIHTTPServer.py
@@ -84,7 +84,7 @@
         path begins with one of the strings in self.cgi_directories
         (and the next character is a '/' or the end of the string).
         """
-        collapsed_path = _url_collapse_path(urllib.unquote(self.path))
+        collapsed_path = _url_collapse_path(self.path)
         dir_sep = collapsed_path.find('/', 1)
         head, tail = collapsed_path[:dir_sep], collapsed_path[dir_sep+1:]
         if head in self.cgi_directories:
@@ -304,13 +304,15 @@
     The utility of this function is limited to is_cgi method and helps
     preventing some security attacks.
 
-    Returns: A tuple of (head, tail) where tail is everything after the final /
-    and head is everything before it.  Head will always start with a '/' and,
-    if it contains anything else, never have a trailing '/'.
+    Returns: The reconstituted URL, which will always start with a '/'.
 
     Raises: IndexError if too many '..' occur within the path.
 
     """
+    # Query component should not be involved.
+    path, _, query = path.partition('?')
+    path = urllib.unquote(path)
+
     # Similar to os.path.split(os.path.normpath(path)) but specific to URL
     # path semantics rather than local operating system semantics.
     path_parts = path.split('/')
@@ -331,6 +333,9 @@
     else:
         tail_part = ''
 
+    if query:
+        tail_part = '?'.join((tail_part, query))
+
     splitpath = ('/' + '/'.join(head_parts), tail_part)
     collapsed_path = "/".join(splitpath)
 
diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py
--- a/Lib/test/test_httpservers.py
+++ b/Lib/test/test_httpservers.py
@@ -558,6 +558,13 @@
             (b'a=b?c=d\n', 'text/html', 200),
             (res.read(), res.getheader('Content-type'), res.status))
 
+    def test_query_with_continuous_slashes(self):
+        res = self.request('/cgi-bin/file4.py?k=aa%2F%2Fbb&//q//p//=//a//b//')
+        self.assertEqual(
+            (b'k=aa%2F%2Fbb&//q//p//=//a//b//\n',
+             'text/html', 200),
+            (res.read(), res.getheader('Content-type'), res.status))
+
 
 class SimpleHTTPRequestHandlerTestCase(unittest.TestCase):
     """ Test url parsing """
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -46,6 +46,9 @@
 - Issue #25232: Fix CGIRequestHandler to split the query from the URL at the
   first question mark (?) rather than the last. Patch from Xiang Zhang.
 
+- Issue #24657: Prevent CGIRequestHandler from collapsing slashes in the
+  query part of the URL as if it were a path. Patch from Xiang Zhang.
+
 - Issue #22958: Constructor and update method of weakref.WeakValueDictionary
   now accept the self keyword argument.
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 10:02:22 2015
From: python-checkins at python.org (martin.panter)
Date: Sat, 03 Oct 2015 08:02:22 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzE2NzAx?=
 =?utf-8?q?=3A_Document_+=3D_and_*=3D_for_mutable_sequences?=
Message-ID: <20151003080222.128844.82741@psf.io>

https://hg.python.org/cpython/rev/f83db23bec7f
changeset:   98518:f83db23bec7f
branch:      3.4
parent:      98510:634fe6a90e0c
user:        Martin Panter <vadmium+py at gmail.com>
date:        Sat Oct 03 07:46:04 2015 +0000
summary:
  Issue #16701: Document += and *= for mutable sequences

files:
  Doc/library/stdtypes.rst |  14 ++++++++++++--
  1 files changed, 12 insertions(+), 2 deletions(-)


diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst
--- a/Doc/library/stdtypes.rst
+++ b/Doc/library/stdtypes.rst
@@ -1063,10 +1063,14 @@
 | ``s.copy()``                 | creates a shallow copy of ``s``| \(5)                |
 |                              | (same as ``s[:]``)             |                     |
 +------------------------------+--------------------------------+---------------------+
-| ``s.extend(t)``              | extends *s* with the           |                     |
-|                              | contents of *t* (same as       |                     |
+| ``s.extend(t)`` or           | extends *s* with the           |                     |
+| ``s += t``                   | contents of *t* (for the       |                     |
+|                              | most part the same as          |                     |
 |                              | ``s[len(s):len(s)] = t``)      |                     |
 +------------------------------+--------------------------------+---------------------+
+| ``s *= n``                   | updates *s* with its contents  | \(6)                |
+|                              | repeated *n* times             |                     |
++------------------------------+--------------------------------+---------------------+
 | ``s.insert(i, x)``           | inserts *x* into *s* at the    |                     |
 |                              | index given by *i*             |                     |
 |                              | (same as ``s[i:i] = [x]``)     |                     |
@@ -1107,6 +1111,12 @@
    .. versionadded:: 3.3
       :meth:`clear` and :meth:`!copy` methods.
 
+(6)
+   The value *n* is an integer, or an object implementing
+   :meth:`~object.__index__`.  Zero and negative values of *n* clear
+   the sequence.  Items in the sequence are not copied; they are referenced
+   multiple times, as explained for ``s * n`` under :ref:`typesseq-common`.
+
 
 .. _typesseq-list:
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 10:02:22 2015
From: python-checkins at python.org (martin.panter)
Date: Sat, 03 Oct 2015 08:02:22 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzE2NzAx?=
 =?utf-8?q?=3A_Document_+=3D_and_*=3D_for_mutable_sequences?=
Message-ID: <20151003080222.2677.42413@psf.io>

https://hg.python.org/cpython/rev/ec373d762213
changeset:   98517:ec373d762213
branch:      2.7
user:        Martin Panter <vadmium+py at gmail.com>
date:        Sat Oct 03 07:37:22 2015 +0000
summary:
  Issue #16701: Document += and *= for mutable sequences

files:
  Doc/library/stdtypes.rst |  13 +++++++++++--
  1 files changed, 11 insertions(+), 2 deletions(-)


diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst
--- a/Doc/library/stdtypes.rst
+++ b/Doc/library/stdtypes.rst
@@ -1613,8 +1613,11 @@
 | ``s.append(x)``              | same as ``s[len(s):len(s)] =   | \(2)                |
 |                              | [x]``                          |                     |
 +------------------------------+--------------------------------+---------------------+
-| ``s.extend(x)``              | same as ``s[len(s):len(s)] =   | \(3)                |
-|                              | x``                            |                     |
+| ``s.extend(x)`` or           | for the most part the same as  | \(3)                |
+| ``s += t``                   | ``s[len(s):len(s)] = x``       |                     |
++------------------------------+--------------------------------+---------------------+
+| ``s *= n``                   | updates *s* with its contents  | \(11)               |
+|                              | repeated *n* times             |                     |
 +------------------------------+--------------------------------+---------------------+
 | ``s.count(x)``               | return number of *i*'s for     |                     |
 |                              | which ``s[i] == x``            |                     |
@@ -1720,6 +1723,12 @@
       :exc:`ValueError` if it can detect that the list has been mutated during a
       sort.
 
+(11)
+   The value *n* is an integer, or an object implementing
+   :meth:`~object.__index__`.  Zero and negative values of *n* clear
+   the sequence.  Items in the sequence are not copied; they are referenced
+   multiple times, as explained for ``s * n`` under :ref:`typesseq`.
+
 
 .. _types-set:
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 10:02:23 2015
From: python-checkins at python.org (martin.panter)
Date: Sat, 03 Oct 2015 08:02:23 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
 =?utf-8?q?_Issue_=2316701=3A_Merge_sequence_docs_from_3=2E4_into_3=2E5?=
Message-ID: <20151003080222.70990.91299@psf.io>

https://hg.python.org/cpython/rev/6e43a3833293
changeset:   98519:6e43a3833293
branch:      3.5
parent:      98512:88918f2a54df
parent:      98518:f83db23bec7f
user:        Martin Panter <vadmium+py at gmail.com>
date:        Sat Oct 03 07:53:49 2015 +0000
summary:
  Issue #16701: Merge sequence docs from 3.4 into 3.5

files:
  Doc/library/stdtypes.rst |  14 ++++++++++++--
  1 files changed, 12 insertions(+), 2 deletions(-)


diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst
--- a/Doc/library/stdtypes.rst
+++ b/Doc/library/stdtypes.rst
@@ -1063,10 +1063,14 @@
 | ``s.copy()``                 | creates a shallow copy of ``s``| \(5)                |
 |                              | (same as ``s[:]``)             |                     |
 +------------------------------+--------------------------------+---------------------+
-| ``s.extend(t)``              | extends *s* with the           |                     |
-|                              | contents of *t* (same as       |                     |
+| ``s.extend(t)`` or           | extends *s* with the           |                     |
+| ``s += t``                   | contents of *t* (for the       |                     |
+|                              | most part the same as          |                     |
 |                              | ``s[len(s):len(s)] = t``)      |                     |
 +------------------------------+--------------------------------+---------------------+
+| ``s *= n``                   | updates *s* with its contents  | \(6)                |
+|                              | repeated *n* times             |                     |
++------------------------------+--------------------------------+---------------------+
 | ``s.insert(i, x)``           | inserts *x* into *s* at the    |                     |
 |                              | index given by *i*             |                     |
 |                              | (same as ``s[i:i] = [x]``)     |                     |
@@ -1107,6 +1111,12 @@
    .. versionadded:: 3.3
       :meth:`clear` and :meth:`!copy` methods.
 
+(6)
+   The value *n* is an integer, or an object implementing
+   :meth:`~object.__index__`.  Zero and negative values of *n* clear
+   the sequence.  Items in the sequence are not copied; they are referenced
+   multiple times, as explained for ``s * n`` under :ref:`typesseq-common`.
+
 
 .. _typesseq-list:
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 10:02:23 2015
From: python-checkins at python.org (martin.panter)
Date: Sat, 03 Oct 2015 08:02:23 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2316701=3A_Merge_sequence_docs_from_3=2E5?=
Message-ID: <20151003080222.55460.3299@psf.io>

https://hg.python.org/cpython/rev/a92466bf16cc
changeset:   98520:a92466bf16cc
parent:      98514:3c006ee38287
parent:      98519:6e43a3833293
user:        Martin Panter <vadmium+py at gmail.com>
date:        Sat Oct 03 07:54:08 2015 +0000
summary:
  Issue #16701: Merge sequence docs from 3.5

files:
  Doc/library/stdtypes.rst |  14 ++++++++++++--
  1 files changed, 12 insertions(+), 2 deletions(-)


diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst
--- a/Doc/library/stdtypes.rst
+++ b/Doc/library/stdtypes.rst
@@ -1063,10 +1063,14 @@
 | ``s.copy()``                 | creates a shallow copy of ``s``| \(5)                |
 |                              | (same as ``s[:]``)             |                     |
 +------------------------------+--------------------------------+---------------------+
-| ``s.extend(t)``              | extends *s* with the           |                     |
-|                              | contents of *t* (same as       |                     |
+| ``s.extend(t)`` or           | extends *s* with the           |                     |
+| ``s += t``                   | contents of *t* (for the       |                     |
+|                              | most part the same as          |                     |
 |                              | ``s[len(s):len(s)] = t``)      |                     |
 +------------------------------+--------------------------------+---------------------+
+| ``s *= n``                   | updates *s* with its contents  | \(6)                |
+|                              | repeated *n* times             |                     |
++------------------------------+--------------------------------+---------------------+
 | ``s.insert(i, x)``           | inserts *x* into *s* at the    |                     |
 |                              | index given by *i*             |                     |
 |                              | (same as ``s[i:i] = [x]``)     |                     |
@@ -1107,6 +1111,12 @@
    .. versionadded:: 3.3
       :meth:`clear` and :meth:`!copy` methods.
 
+(6)
+   The value *n* is an integer, or an object implementing
+   :meth:`~object.__index__`.  Zero and negative values of *n* clear
+   the sequence.  Items in the sequence are not copied; they are referenced
+   multiple times, as explained for ``s * n`` under :ref:`typesseq-common`.
+
 
 .. _typesseq-list:
 

-- 
Repository URL: https://hg.python.org/cpython

From solipsis at pitrou.net  Sat Oct  3 10:44:16 2015
From: solipsis at pitrou.net (solipsis at pitrou.net)
Date: Sat, 03 Oct 2015 08:44:16 +0000
Subject: [Python-checkins] Daily reference leaks (4202abcf5702): sum=61491
Message-ID: <20151003084416.3279.19397@psf.io>

results for 4202abcf5702 on branch "default"
--------------------------------------------

test_capi leaked [5410, 5410, 5410] references, sum=16230
test_capi leaked [1421, 1423, 1423] memory blocks, sum=4267
test_functools leaked [0, 2, 2] memory blocks, sum=4
test_threading leaked [10820, 10820, 10820] references, sum=32460
test_threading leaked [2842, 2844, 2844] memory blocks, sum=8530


Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogP1RQ4e', '--timeout', '7200']

From python-checkins at python.org  Sat Oct  3 17:35:55 2015
From: python-checkins at python.org (guido.van.rossum)
Date: Sat, 03 Oct 2015 15:35:55 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
 =?utf-8?q?_Issue_=2325304=3A_Add_asyncio=2Erun=5Fcoroutine=5Fthreadsafe?=
 =?utf-8?q?=28=29=2E_By_Vincent_Michel=2E_=28Merge?=
Message-ID: <20151003153555.20759.28452@psf.io>

https://hg.python.org/cpython/rev/e0db10d8c95e
changeset:   98522:e0db10d8c95e
branch:      3.5
parent:      98519:6e43a3833293
parent:      98521:25e05b3e1869
user:        Guido van Rossum <guido at python.org>
date:        Sat Oct 03 08:34:34 2015 -0700
summary:
  Issue #25304: Add asyncio.run_coroutine_threadsafe(). By Vincent Michel. (Merge 3.4->3.5.)

files:
  Lib/asyncio/futures.py                |  74 +++++++++++---
  Lib/asyncio/tasks.py                  |  18 +++-
  Lib/test/test_asyncio/test_futures.py |   2 -
  Lib/test/test_asyncio/test_tasks.py   |  67 +++++++++++++
  Misc/ACKS                             |   1 +
  Misc/NEWS                             |   4 +
  6 files changed, 147 insertions(+), 19 deletions(-)


diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py
--- a/Lib/asyncio/futures.py
+++ b/Lib/asyncio/futures.py
@@ -390,22 +390,64 @@
         __await__ = __iter__ # make compatible with 'await' expression
 
 
-def wrap_future(fut, *, loop=None):
+def _set_concurrent_future_state(concurrent, source):
+    """Copy state from a future to a concurrent.futures.Future."""
+    assert source.done()
+    if source.cancelled():
+        concurrent.cancel()
+    if not concurrent.set_running_or_notify_cancel():
+        return
+    exception = source.exception()
+    if exception is not None:
+        concurrent.set_exception(exception)
+    else:
+        result = source.result()
+        concurrent.set_result(result)
+
+
+def _chain_future(source, destination):
+    """Chain two futures so that when one completes, so does the other.
+
+    The result (or exception) of source will be copied to destination.
+    If destination is cancelled, source gets cancelled too.
+    Compatible with both asyncio.Future and concurrent.futures.Future.
+    """
+    if not isinstance(source, (Future, concurrent.futures.Future)):
+        raise TypeError('A future is required for source argument')
+    if not isinstance(destination, (Future, concurrent.futures.Future)):
+        raise TypeError('A future is required for destination argument')
+    source_loop = source._loop if isinstance(source, Future) else None
+    dest_loop = destination._loop if isinstance(destination, Future) else None
+
+    def _set_state(future, other):
+        if isinstance(future, Future):
+            future._copy_state(other)
+        else:
+            _set_concurrent_future_state(future, other)
+
+    def _call_check_cancel(destination):
+        if destination.cancelled():
+            if source_loop is None or source_loop is dest_loop:
+                source.cancel()
+            else:
+                source_loop.call_soon_threadsafe(source.cancel)
+
+    def _call_set_state(source):
+        if dest_loop is None or dest_loop is source_loop:
+            _set_state(destination, source)
+        else:
+            dest_loop.call_soon_threadsafe(_set_state, destination, source)
+
+    destination.add_done_callback(_call_check_cancel)
+    source.add_done_callback(_call_set_state)
+
+
+def wrap_future(future, *, loop=None):
     """Wrap concurrent.futures.Future object."""
-    if isinstance(fut, Future):
-        return fut
-    assert isinstance(fut, concurrent.futures.Future), \
-        'concurrent.futures.Future is expected, got {!r}'.format(fut)
-    if loop is None:
-        loop = events.get_event_loop()
+    if isinstance(future, Future):
+        return future
+    assert isinstance(future, concurrent.futures.Future), \
+        'concurrent.futures.Future is expected, got {!r}'.format(future)
     new_future = Future(loop=loop)
-
-    def _check_cancel_other(f):
-        if f.cancelled():
-            fut.cancel()
-
-    new_future.add_done_callback(_check_cancel_other)
-    fut.add_done_callback(
-        lambda future: loop.call_soon_threadsafe(
-            new_future._copy_state, future))
+    _chain_future(future, new_future)
     return new_future
diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py
--- a/Lib/asyncio/tasks.py
+++ b/Lib/asyncio/tasks.py
@@ -3,7 +3,7 @@
 __all__ = ['Task',
            'FIRST_COMPLETED', 'FIRST_EXCEPTION', 'ALL_COMPLETED',
            'wait', 'wait_for', 'as_completed', 'sleep', 'async',
-           'gather', 'shield', 'ensure_future',
+           'gather', 'shield', 'ensure_future', 'run_coroutine_threadsafe',
            ]
 
 import concurrent.futures
@@ -692,3 +692,19 @@
 
     inner.add_done_callback(_done_callback)
     return outer
+
+
+def run_coroutine_threadsafe(coro, loop):
+    """Submit a coroutine object to a given event loop.
+
+    Return a concurrent.futures.Future to access the result.
+    """
+    if not coroutines.iscoroutine(coro):
+        raise TypeError('A coroutine object is required')
+    future = concurrent.futures.Future()
+
+    def callback():
+        futures._chain_future(ensure_future(coro, loop=loop), future)
+
+    loop.call_soon_threadsafe(callback)
+    return future
diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py
--- a/Lib/test/test_asyncio/test_futures.py
+++ b/Lib/test/test_asyncio/test_futures.py
@@ -174,8 +174,6 @@
                          '<Future cancelled>')
 
     def test_copy_state(self):
-        # Test the internal _copy_state method since it's being directly
-        # invoked in other modules.
         f = asyncio.Future(loop=self.loop)
         f.set_result(10)
 
diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py
--- a/Lib/test/test_asyncio/test_tasks.py
+++ b/Lib/test/test_asyncio/test_tasks.py
@@ -2100,5 +2100,72 @@
         self.assertIsInstance(f.exception(), RuntimeError)
 
 
+class RunCoroutineThreadsafeTests(test_utils.TestCase):
+    """Test case for futures.submit_to_loop."""
+
+    def setUp(self):
+        self.loop = self.new_test_loop(self.time_gen)
+
+    def time_gen(self):
+        """Handle the timer."""
+        yield 0  # second
+        yield 1  # second
+
+    @asyncio.coroutine
+    def add(self, a, b, fail=False, cancel=False):
+        """Wait 1 second and return a + b."""
+        yield from asyncio.sleep(1, loop=self.loop)
+        if fail:
+            raise RuntimeError("Fail!")
+        if cancel:
+            asyncio.tasks.Task.current_task(self.loop).cancel()
+            yield
+        return a + b
+
+    def target(self, fail=False, cancel=False, timeout=None):
+        """Run add coroutine in the event loop."""
+        coro = self.add(1, 2, fail=fail, cancel=cancel)
+        future = asyncio.run_coroutine_threadsafe(coro, self.loop)
+        try:
+            return future.result(timeout)
+        finally:
+            future.done() or future.cancel()
+
+    def test_run_coroutine_threadsafe(self):
+        """Test coroutine submission from a thread to an event loop."""
+        future = self.loop.run_in_executor(None, self.target)
+        result = self.loop.run_until_complete(future)
+        self.assertEqual(result, 3)
+
+    def test_run_coroutine_threadsafe_with_exception(self):
+        """Test coroutine submission from a thread to an event loop
+        when an exception is raised."""
+        future = self.loop.run_in_executor(None, self.target, True)
+        with self.assertRaises(RuntimeError) as exc_context:
+            self.loop.run_until_complete(future)
+        self.assertIn("Fail!", exc_context.exception.args)
+
+    def test_run_coroutine_threadsafe_with_timeout(self):
+        """Test coroutine submission from a thread to an event loop
+        when a timeout is raised."""
+        callback = lambda: self.target(timeout=0)
+        future = self.loop.run_in_executor(None, callback)
+        with self.assertRaises(asyncio.TimeoutError):
+            self.loop.run_until_complete(future)
+        # Clear the time generator and tasks
+        test_utils.run_briefly(self.loop)
+        # Check that there's no pending task (add has been cancelled)
+        for task in asyncio.Task.all_tasks(self.loop):
+            self.assertTrue(task.done())
+
+    def test_run_coroutine_threadsafe_task_cancelled(self):
+        """Test coroutine submission from a tread to an event loop
+        when the task is cancelled."""
+        callback = lambda: self.target(cancel=True)
+        future = self.loop.run_in_executor(None, callback)
+        with self.assertRaises(asyncio.CancelledError):
+            self.loop.run_until_complete(future)
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -957,6 +957,7 @@
 Trent Mick
 Jason Michalski
 Franck Michea
+Vincent Michel
 Tom Middleton
 Thomas Miedema
 Stan Mihai
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -34,6 +34,10 @@
 Library
 -------
 
+- Issue #25304: Add asyncio.run_coroutine_threadsafe().  This lets you
+  submit a coroutine to a loop from another thread, returning a
+  concurrent.futures.Future.  By Vincent Michel.
+
 - Issue #25232: Fix CGIRequestHandler to split the query from the URL at the
   first question mark (?) rather than the last. Patch from Xiang Zhang.
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 17:35:55 2015
From: python-checkins at python.org (guido.van.rossum)
Date: Sat, 03 Oct 2015 15:35:55 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2325304=3A_Add_asyncio=2Erun=5Fcoroutine=5Fthread?=
 =?utf-8?q?safe=28=29=2E_By_Vincent_Michel=2E_=28Merge?=
Message-ID: <20151003153555.20769.35597@psf.io>

https://hg.python.org/cpython/rev/69829a7fccde
changeset:   98523:69829a7fccde
parent:      98520:a92466bf16cc
parent:      98522:e0db10d8c95e
user:        Guido van Rossum <guido at python.org>
date:        Sat Oct 03 08:35:28 2015 -0700
summary:
  Issue #25304: Add asyncio.run_coroutine_threadsafe(). By Vincent Michel. (Merge 3.5->3.6.)

files:
  Lib/asyncio/futures.py                |  74 +++++++++++---
  Lib/asyncio/tasks.py                  |  18 +++-
  Lib/test/test_asyncio/test_futures.py |   2 -
  Lib/test/test_asyncio/test_tasks.py   |  67 +++++++++++++
  Misc/ACKS                             |   1 +
  Misc/NEWS                             |   4 +
  6 files changed, 147 insertions(+), 19 deletions(-)


diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py
--- a/Lib/asyncio/futures.py
+++ b/Lib/asyncio/futures.py
@@ -390,22 +390,64 @@
         __await__ = __iter__ # make compatible with 'await' expression
 
 
-def wrap_future(fut, *, loop=None):
+def _set_concurrent_future_state(concurrent, source):
+    """Copy state from a future to a concurrent.futures.Future."""
+    assert source.done()
+    if source.cancelled():
+        concurrent.cancel()
+    if not concurrent.set_running_or_notify_cancel():
+        return
+    exception = source.exception()
+    if exception is not None:
+        concurrent.set_exception(exception)
+    else:
+        result = source.result()
+        concurrent.set_result(result)
+
+
+def _chain_future(source, destination):
+    """Chain two futures so that when one completes, so does the other.
+
+    The result (or exception) of source will be copied to destination.
+    If destination is cancelled, source gets cancelled too.
+    Compatible with both asyncio.Future and concurrent.futures.Future.
+    """
+    if not isinstance(source, (Future, concurrent.futures.Future)):
+        raise TypeError('A future is required for source argument')
+    if not isinstance(destination, (Future, concurrent.futures.Future)):
+        raise TypeError('A future is required for destination argument')
+    source_loop = source._loop if isinstance(source, Future) else None
+    dest_loop = destination._loop if isinstance(destination, Future) else None
+
+    def _set_state(future, other):
+        if isinstance(future, Future):
+            future._copy_state(other)
+        else:
+            _set_concurrent_future_state(future, other)
+
+    def _call_check_cancel(destination):
+        if destination.cancelled():
+            if source_loop is None or source_loop is dest_loop:
+                source.cancel()
+            else:
+                source_loop.call_soon_threadsafe(source.cancel)
+
+    def _call_set_state(source):
+        if dest_loop is None or dest_loop is source_loop:
+            _set_state(destination, source)
+        else:
+            dest_loop.call_soon_threadsafe(_set_state, destination, source)
+
+    destination.add_done_callback(_call_check_cancel)
+    source.add_done_callback(_call_set_state)
+
+
+def wrap_future(future, *, loop=None):
     """Wrap concurrent.futures.Future object."""
-    if isinstance(fut, Future):
-        return fut
-    assert isinstance(fut, concurrent.futures.Future), \
-        'concurrent.futures.Future is expected, got {!r}'.format(fut)
-    if loop is None:
-        loop = events.get_event_loop()
+    if isinstance(future, Future):
+        return future
+    assert isinstance(future, concurrent.futures.Future), \
+        'concurrent.futures.Future is expected, got {!r}'.format(future)
     new_future = Future(loop=loop)
-
-    def _check_cancel_other(f):
-        if f.cancelled():
-            fut.cancel()
-
-    new_future.add_done_callback(_check_cancel_other)
-    fut.add_done_callback(
-        lambda future: loop.call_soon_threadsafe(
-            new_future._copy_state, future))
+    _chain_future(future, new_future)
     return new_future
diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py
--- a/Lib/asyncio/tasks.py
+++ b/Lib/asyncio/tasks.py
@@ -3,7 +3,7 @@
 __all__ = ['Task',
            'FIRST_COMPLETED', 'FIRST_EXCEPTION', 'ALL_COMPLETED',
            'wait', 'wait_for', 'as_completed', 'sleep', 'async',
-           'gather', 'shield', 'ensure_future',
+           'gather', 'shield', 'ensure_future', 'run_coroutine_threadsafe',
            ]
 
 import concurrent.futures
@@ -692,3 +692,19 @@
 
     inner.add_done_callback(_done_callback)
     return outer
+
+
+def run_coroutine_threadsafe(coro, loop):
+    """Submit a coroutine object to a given event loop.
+
+    Return a concurrent.futures.Future to access the result.
+    """
+    if not coroutines.iscoroutine(coro):
+        raise TypeError('A coroutine object is required')
+    future = concurrent.futures.Future()
+
+    def callback():
+        futures._chain_future(ensure_future(coro, loop=loop), future)
+
+    loop.call_soon_threadsafe(callback)
+    return future
diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py
--- a/Lib/test/test_asyncio/test_futures.py
+++ b/Lib/test/test_asyncio/test_futures.py
@@ -174,8 +174,6 @@
                          '<Future cancelled>')
 
     def test_copy_state(self):
-        # Test the internal _copy_state method since it's being directly
-        # invoked in other modules.
         f = asyncio.Future(loop=self.loop)
         f.set_result(10)
 
diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py
--- a/Lib/test/test_asyncio/test_tasks.py
+++ b/Lib/test/test_asyncio/test_tasks.py
@@ -2100,5 +2100,72 @@
         self.assertIsInstance(f.exception(), RuntimeError)
 
 
+class RunCoroutineThreadsafeTests(test_utils.TestCase):
+    """Test case for futures.submit_to_loop."""
+
+    def setUp(self):
+        self.loop = self.new_test_loop(self.time_gen)
+
+    def time_gen(self):
+        """Handle the timer."""
+        yield 0  # second
+        yield 1  # second
+
+    @asyncio.coroutine
+    def add(self, a, b, fail=False, cancel=False):
+        """Wait 1 second and return a + b."""
+        yield from asyncio.sleep(1, loop=self.loop)
+        if fail:
+            raise RuntimeError("Fail!")
+        if cancel:
+            asyncio.tasks.Task.current_task(self.loop).cancel()
+            yield
+        return a + b
+
+    def target(self, fail=False, cancel=False, timeout=None):
+        """Run add coroutine in the event loop."""
+        coro = self.add(1, 2, fail=fail, cancel=cancel)
+        future = asyncio.run_coroutine_threadsafe(coro, self.loop)
+        try:
+            return future.result(timeout)
+        finally:
+            future.done() or future.cancel()
+
+    def test_run_coroutine_threadsafe(self):
+        """Test coroutine submission from a thread to an event loop."""
+        future = self.loop.run_in_executor(None, self.target)
+        result = self.loop.run_until_complete(future)
+        self.assertEqual(result, 3)
+
+    def test_run_coroutine_threadsafe_with_exception(self):
+        """Test coroutine submission from a thread to an event loop
+        when an exception is raised."""
+        future = self.loop.run_in_executor(None, self.target, True)
+        with self.assertRaises(RuntimeError) as exc_context:
+            self.loop.run_until_complete(future)
+        self.assertIn("Fail!", exc_context.exception.args)
+
+    def test_run_coroutine_threadsafe_with_timeout(self):
+        """Test coroutine submission from a thread to an event loop
+        when a timeout is raised."""
+        callback = lambda: self.target(timeout=0)
+        future = self.loop.run_in_executor(None, callback)
+        with self.assertRaises(asyncio.TimeoutError):
+            self.loop.run_until_complete(future)
+        # Clear the time generator and tasks
+        test_utils.run_briefly(self.loop)
+        # Check that there's no pending task (add has been cancelled)
+        for task in asyncio.Task.all_tasks(self.loop):
+            self.assertTrue(task.done())
+
+    def test_run_coroutine_threadsafe_task_cancelled(self):
+        """Test coroutine submission from a tread to an event loop
+        when the task is cancelled."""
+        callback = lambda: self.target(cancel=True)
+        future = self.loop.run_in_executor(None, callback)
+        with self.assertRaises(asyncio.CancelledError):
+            self.loop.run_until_complete(future)
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -958,6 +958,7 @@
 Trent Mick
 Jason Michalski
 Franck Michea
+Vincent Michel
 Tom Middleton
 Thomas Miedema
 Stan Mihai
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -40,6 +40,10 @@
 Library
 -------
 
+- Issue #25304: Add asyncio.run_coroutine_threadsafe().  This lets you
+  submit a coroutine to a loop from another thread, returning a
+  concurrent.futures.Future.  By Vincent Michel.
+
 - Issue #25232: Fix CGIRequestHandler to split the query from the URL at the
   first question mark (?) rather than the last. Patch from Xiang Zhang.
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 17:35:56 2015
From: python-checkins at python.org (guido.van.rossum)
Date: Sat, 03 Oct 2015 15:35:56 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzI1MzA0?=
 =?utf-8?q?=3A_Add_asyncio=2Erun=5Fcoroutine=5Fthreadsafe=28=29=2E_By_Vinc?=
 =?utf-8?q?ent_Michel=2E?=
Message-ID: <20151003153555.20757.60480@psf.io>

https://hg.python.org/cpython/rev/25e05b3e1869
changeset:   98521:25e05b3e1869
branch:      3.4
parent:      98518:f83db23bec7f
user:        Guido van Rossum <guido at python.org>
date:        Sat Oct 03 08:31:42 2015 -0700
summary:
  Issue #25304: Add asyncio.run_coroutine_threadsafe(). By Vincent Michel.

files:
  Lib/asyncio/futures.py                |  74 +++++++++++---
  Lib/asyncio/tasks.py                  |  18 +++-
  Lib/test/test_asyncio/test_futures.py |   2 -
  Lib/test/test_asyncio/test_tasks.py   |  67 +++++++++++++
  Misc/ACKS                             |   1 +
  Misc/NEWS                             |   4 +
  6 files changed, 147 insertions(+), 19 deletions(-)


diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py
--- a/Lib/asyncio/futures.py
+++ b/Lib/asyncio/futures.py
@@ -390,22 +390,64 @@
         __await__ = __iter__ # make compatible with 'await' expression
 
 
-def wrap_future(fut, *, loop=None):
+def _set_concurrent_future_state(concurrent, source):
+    """Copy state from a future to a concurrent.futures.Future."""
+    assert source.done()
+    if source.cancelled():
+        concurrent.cancel()
+    if not concurrent.set_running_or_notify_cancel():
+        return
+    exception = source.exception()
+    if exception is not None:
+        concurrent.set_exception(exception)
+    else:
+        result = source.result()
+        concurrent.set_result(result)
+
+
+def _chain_future(source, destination):
+    """Chain two futures so that when one completes, so does the other.
+
+    The result (or exception) of source will be copied to destination.
+    If destination is cancelled, source gets cancelled too.
+    Compatible with both asyncio.Future and concurrent.futures.Future.
+    """
+    if not isinstance(source, (Future, concurrent.futures.Future)):
+        raise TypeError('A future is required for source argument')
+    if not isinstance(destination, (Future, concurrent.futures.Future)):
+        raise TypeError('A future is required for destination argument')
+    source_loop = source._loop if isinstance(source, Future) else None
+    dest_loop = destination._loop if isinstance(destination, Future) else None
+
+    def _set_state(future, other):
+        if isinstance(future, Future):
+            future._copy_state(other)
+        else:
+            _set_concurrent_future_state(future, other)
+
+    def _call_check_cancel(destination):
+        if destination.cancelled():
+            if source_loop is None or source_loop is dest_loop:
+                source.cancel()
+            else:
+                source_loop.call_soon_threadsafe(source.cancel)
+
+    def _call_set_state(source):
+        if dest_loop is None or dest_loop is source_loop:
+            _set_state(destination, source)
+        else:
+            dest_loop.call_soon_threadsafe(_set_state, destination, source)
+
+    destination.add_done_callback(_call_check_cancel)
+    source.add_done_callback(_call_set_state)
+
+
+def wrap_future(future, *, loop=None):
     """Wrap concurrent.futures.Future object."""
-    if isinstance(fut, Future):
-        return fut
-    assert isinstance(fut, concurrent.futures.Future), \
-        'concurrent.futures.Future is expected, got {!r}'.format(fut)
-    if loop is None:
-        loop = events.get_event_loop()
+    if isinstance(future, Future):
+        return future
+    assert isinstance(future, concurrent.futures.Future), \
+        'concurrent.futures.Future is expected, got {!r}'.format(future)
     new_future = Future(loop=loop)
-
-    def _check_cancel_other(f):
-        if f.cancelled():
-            fut.cancel()
-
-    new_future.add_done_callback(_check_cancel_other)
-    fut.add_done_callback(
-        lambda future: loop.call_soon_threadsafe(
-            new_future._copy_state, future))
+    _chain_future(future, new_future)
     return new_future
diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py
--- a/Lib/asyncio/tasks.py
+++ b/Lib/asyncio/tasks.py
@@ -3,7 +3,7 @@
 __all__ = ['Task',
            'FIRST_COMPLETED', 'FIRST_EXCEPTION', 'ALL_COMPLETED',
            'wait', 'wait_for', 'as_completed', 'sleep', 'async',
-           'gather', 'shield', 'ensure_future',
+           'gather', 'shield', 'ensure_future', 'run_coroutine_threadsafe',
            ]
 
 import concurrent.futures
@@ -692,3 +692,19 @@
 
     inner.add_done_callback(_done_callback)
     return outer
+
+
+def run_coroutine_threadsafe(coro, loop):
+    """Submit a coroutine object to a given event loop.
+
+    Return a concurrent.futures.Future to access the result.
+    """
+    if not coroutines.iscoroutine(coro):
+        raise TypeError('A coroutine object is required')
+    future = concurrent.futures.Future()
+
+    def callback():
+        futures._chain_future(ensure_future(coro, loop=loop), future)
+
+    loop.call_soon_threadsafe(callback)
+    return future
diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py
--- a/Lib/test/test_asyncio/test_futures.py
+++ b/Lib/test/test_asyncio/test_futures.py
@@ -174,8 +174,6 @@
                          '<Future cancelled>')
 
     def test_copy_state(self):
-        # Test the internal _copy_state method since it's being directly
-        # invoked in other modules.
         f = asyncio.Future(loop=self.loop)
         f.set_result(10)
 
diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py
--- a/Lib/test/test_asyncio/test_tasks.py
+++ b/Lib/test/test_asyncio/test_tasks.py
@@ -2100,5 +2100,72 @@
         self.assertIsInstance(f.exception(), RuntimeError)
 
 
+class RunCoroutineThreadsafeTests(test_utils.TestCase):
+    """Test case for futures.submit_to_loop."""
+
+    def setUp(self):
+        self.loop = self.new_test_loop(self.time_gen)
+
+    def time_gen(self):
+        """Handle the timer."""
+        yield 0  # second
+        yield 1  # second
+
+    @asyncio.coroutine
+    def add(self, a, b, fail=False, cancel=False):
+        """Wait 1 second and return a + b."""
+        yield from asyncio.sleep(1, loop=self.loop)
+        if fail:
+            raise RuntimeError("Fail!")
+        if cancel:
+            asyncio.tasks.Task.current_task(self.loop).cancel()
+            yield
+        return a + b
+
+    def target(self, fail=False, cancel=False, timeout=None):
+        """Run add coroutine in the event loop."""
+        coro = self.add(1, 2, fail=fail, cancel=cancel)
+        future = asyncio.run_coroutine_threadsafe(coro, self.loop)
+        try:
+            return future.result(timeout)
+        finally:
+            future.done() or future.cancel()
+
+    def test_run_coroutine_threadsafe(self):
+        """Test coroutine submission from a thread to an event loop."""
+        future = self.loop.run_in_executor(None, self.target)
+        result = self.loop.run_until_complete(future)
+        self.assertEqual(result, 3)
+
+    def test_run_coroutine_threadsafe_with_exception(self):
+        """Test coroutine submission from a thread to an event loop
+        when an exception is raised."""
+        future = self.loop.run_in_executor(None, self.target, True)
+        with self.assertRaises(RuntimeError) as exc_context:
+            self.loop.run_until_complete(future)
+        self.assertIn("Fail!", exc_context.exception.args)
+
+    def test_run_coroutine_threadsafe_with_timeout(self):
+        """Test coroutine submission from a thread to an event loop
+        when a timeout is raised."""
+        callback = lambda: self.target(timeout=0)
+        future = self.loop.run_in_executor(None, callback)
+        with self.assertRaises(asyncio.TimeoutError):
+            self.loop.run_until_complete(future)
+        # Clear the time generator and tasks
+        test_utils.run_briefly(self.loop)
+        # Check that there's no pending task (add has been cancelled)
+        for task in asyncio.Task.all_tasks(self.loop):
+            self.assertTrue(task.done())
+
+    def test_run_coroutine_threadsafe_task_cancelled(self):
+        """Test coroutine submission from a tread to an event loop
+        when the task is cancelled."""
+        callback = lambda: self.target(cancel=True)
+        future = self.loop.run_in_executor(None, callback)
+        with self.assertRaises(asyncio.CancelledError):
+            self.loop.run_until_complete(future)
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -929,6 +929,7 @@
 Trent Mick
 Jason Michalski
 Franck Michea
+Vincent Michel
 Tom Middleton
 Thomas Miedema
 Stan Mihai
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -90,6 +90,10 @@
 Library
 -------
 
+- Issue #25304: Add asyncio.run_coroutine_threadsafe().  This lets you
+  submit a coroutine to a loop from another thread, returning a
+  concurrent.futures.Future.  By Vincent Michel.
+
 - Issue #25232: Fix CGIRequestHandler to split the query from the URL at the
   first question mark (?) rather than the last. Patch from Xiang Zhang.
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 21:22:48 2015
From: python-checkins at python.org (victor.stinner)
Date: Sat, 03 Oct 2015 19:22:48 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2325306=3A_Skip_tes?=
 =?utf-8?q?t=5Fhuntrleaks=5Ffd=5Fleak=28=29_of_test=5Fregrtest_until_the_b?=
 =?utf-8?q?ug_is?=
Message-ID: <20151003192247.97700.36068@psf.io>

https://hg.python.org/cpython/rev/fd915645627a
changeset:   98524:fd915645627a
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Sat Oct 03 21:20:41 2015 +0200
summary:
  Issue #25306: Skip test_huntrleaks_fd_leak() of test_regrtest until the bug is
fixed.

files:
  Lib/test/test_regrtest.py |  2 ++
  1 files changed, 2 insertions(+), 0 deletions(-)


diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -643,6 +643,8 @@
         self.check_executed_tests(output, [test]*3, failed=test)
 
     @unittest.skipUnless(Py_DEBUG, 'need a debug build')
+    # Issue #25306: the test hangs sometimes on Windows
+    @unittest.skipIf(sys.platform == 'win32', 'test broken on Windows')
     def test_huntrleaks_fd_leak(self):
         # test --huntrleaks for file descriptor leak
         code = textwrap.dedent("""

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sat Oct  3 21:41:01 2015
From: python-checkins at python.org (victor.stinner)
Date: Sat, 03 Oct 2015 19:41:01 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2325306=3A_Try_to_f?=
 =?utf-8?q?ix_test=5Fhuntrleaks=5Ffd=5Fleak=28=29_on_Windows?=
Message-ID: <20151003194101.18376.99223@psf.io>

https://hg.python.org/cpython/rev/850efcc9155c
changeset:   98525:850efcc9155c
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Sat Oct 03 21:40:21 2015 +0200
summary:
  Issue #25306: Try to fix test_huntrleaks_fd_leak() on Windows

Issue #25306: Disable popup and logs to stderr on assertion failures in MSCRT.

files:
  Lib/test/test_regrtest.py |  14 ++++++++++++--
  1 files changed, 12 insertions(+), 2 deletions(-)


diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -643,14 +643,24 @@
         self.check_executed_tests(output, [test]*3, failed=test)
 
     @unittest.skipUnless(Py_DEBUG, 'need a debug build')
-    # Issue #25306: the test hangs sometimes on Windows
-    @unittest.skipIf(sys.platform == 'win32', 'test broken on Windows')
     def test_huntrleaks_fd_leak(self):
         # test --huntrleaks for file descriptor leak
         code = textwrap.dedent("""
             import os
             import unittest
 
+            # Issue #25306: Disable popups and logs to stderr on assertion
+            # failures in MSCRT
+            try:
+                import msvcrt
+                msvcrt.CrtSetReportMode
+            except (ImportError, AttributeError):
+                # no Windows, o release build
+                pass
+            else:
+                for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]:
+                    msvcrt.CrtSetReportMode(m, 0)
+
             class FDLeakTest(unittest.TestCase):
                 def test_leak(self):
                     fd = os.open(__file__, os.O_RDONLY)

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sun Oct  4 05:03:38 2015
From: python-checkins at python.org (terry.reedy)
Date: Sun, 04 Oct 2015 03:03:38 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=239232=3A_Escape_rst_markup_char_in_NEWS_entry_to?=
 =?utf-8?q?_avoid_Sphinx_warning=2E?=
Message-ID: <20151004030338.97722.10650@psf.io>

https://hg.python.org/cpython/rev/6db349fac3ec
changeset:   98527:6db349fac3ec
parent:      98525:850efcc9155c
parent:      98526:20e0906a808e
user:        Terry Jan Reedy <tjreedy at udel.edu>
date:        Sat Oct 03 23:03:15 2015 -0400
summary:
  Issue #9232: Escape rst markup char in NEWS entry to avoid Sphinx warning.

files:
  Misc/NEWS |  6 +++---
  1 files changed, 3 insertions(+), 3 deletions(-)


diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -29,7 +29,7 @@
   doesn't need such high-quality entropy.
 
 - Issue #9232: Modify Python's grammar to allow trailing commas in the
-  argument list of a function declaration.  For example, "def f(*, a =
+  argument list of a function declaration.  For example, "def f(\*, a =
   3,): pass" is now legal. Patch from Mark Dickinson.
 
 - Issue #24965: Implement PEP 498 "Literal String Interpolation". This
@@ -555,8 +555,8 @@
 
 - Issue #17527: Add PATCH to wsgiref.validator. Patch from Luca Sbardella.
 
-- Issue #24791: Fix grammar regression for call syntax: 'g(*a or b)'.
-
+- Issue #24791: Fix grammar regression for call syntax: 'g(\*a or b)'.
+p
 IDLE
 ----
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sun Oct  4 05:03:38 2015
From: python-checkins at python.org (terry.reedy)
Date: Sun, 04 Oct 2015 03:03:38 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy41KTogSXNzdWUgIzI0Nzkx?=
 =?utf-8?q?=3A_Escape_rst_markup_char_in_NEWS_entry_to_avoid_Sphinx_warnin?=
 =?utf-8?q?g=2E?=
Message-ID: <20151004030338.2673.92117@psf.io>

https://hg.python.org/cpython/rev/20e0906a808e
changeset:   98526:20e0906a808e
branch:      3.5
parent:      98522:e0db10d8c95e
user:        Terry Jan Reedy <tjreedy at udel.edu>
date:        Sat Oct 03 23:01:46 2015 -0400
summary:
  Issue #24791: Escape rst markup char in NEWS entry to avoid Sphinx warning.

files:
  Misc/NEWS |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -423,7 +423,7 @@
 
 - Issue #17527: Add PATCH to wsgiref.validator. Patch from Luca Sbardella.
 
-- Issue #24791: Fix grammar regression for call syntax: 'g(*a or b)'.
+- Issue #24791: Fix grammar regression for call syntax: 'g(\*a or b)'.
 
 IDLE
 ----

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sun Oct  4 06:32:03 2015
From: python-checkins at python.org (terry.reedy)
Date: Sun, 04 Oct 2015 04:32:03 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzI0ODIw?=
 =?utf-8?q?=3A_Add_=27IDLE_Dark=27_text_color_theme=2C_warning=2C_and_solu?=
 =?utf-8?q?tion=2E?=
Message-ID: <20151004043203.7248.13110@psf.io>

https://hg.python.org/cpython/rev/1de01a63f360
changeset:   98529:1de01a63f360
branch:      3.4
parent:      98521:25e05b3e1869
user:        Terry Jan Reedy <tjreedy at udel.edu>
date:        Sun Oct 04 00:31:05 2015 -0400
summary:
  Issue #24820: Add 'IDLE Dark' text color theme, warning, and solution.

files:
  Lib/idlelib/config-highlight.def |  29 ++++++++++++++++++++
  Lib/idlelib/configDialog.py      |  14 +++++++++
  2 files changed, 43 insertions(+), 0 deletions(-)


diff --git a/Lib/idlelib/config-highlight.def b/Lib/idlelib/config-highlight.def
--- a/Lib/idlelib/config-highlight.def
+++ b/Lib/idlelib/config-highlight.def
@@ -62,3 +62,32 @@
 stderr-background= #ffffff
 console-foreground= #770000
 console-background= #ffffff
+
+[IDLE Dark]
+comment-foreground = #dd0000
+console-foreground = #ff4d4d
+error-foreground = #FFFFFF
+hilite-background = #7e7e7e
+string-foreground = #02ff02
+stderr-background = #002240
+stderr-foreground = #ffb3b3
+console-background = #002240
+hit-background = #fbfbfb
+string-background = #002240
+normal-background = #002240
+hilite-foreground = #FFFFFF
+keyword-foreground = #ff8000
+error-background = #c86464
+keyword-background = #002240
+builtin-background = #002240
+break-background = #808000
+builtin-foreground = #ff00ff
+definition-foreground = #5e5eff
+stdout-foreground = #c2d1fa
+definition-background = #002240
+normal-foreground = #FFFFFF
+cursor-foreground = #ffffff
+stdout-background = #002240
+hit-foreground = #002240
+comment-background = #002240
+break-foreground = #FFFFFF
diff --git a/Lib/idlelib/configDialog.py b/Lib/idlelib/configDialog.py
--- a/Lib/idlelib/configDialog.py
+++ b/Lib/idlelib/configDialog.py
@@ -507,6 +507,20 @@
 
     def VarChanged_builtinTheme(self, *params):
         value = self.builtinTheme.get()
+        if value == 'IDLE Dark':
+            tkMessageBox.showwarning(
+                title="The 'IDLE Dark' Text Color Theme",
+                message="IDLE Dark is new in October, 2015.  Trying to "
+                        "run earlier versions of IDLE with it selected "
+                        "will disable colorizing, or worse.\n\n"
+                        "If you might ever run an earlier release of IDLE, "
+                        "then before exiting this version, "
+                        "either switch to another theme or "
+                        "hit the 'Save as New Custom Theme' button.  "
+                        "The latter requires a new name, such as "
+                        "'Custom Dark', but the custom theme will work "
+                        "with any IDLE release, and can be modified.",
+                parent=self)
         self.AddChangedItem('main', 'Theme', 'name', value)
         self.PaintThemeSample()
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sun Oct  4 06:32:03 2015
From: python-checkins at python.org (terry.reedy)
Date: Sun, 04 Oct 2015 04:32:03 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
	=?utf-8?q?=29=3A_Merge_with_3=2E5?=
Message-ID: <20151004043203.3275.40945@psf.io>

https://hg.python.org/cpython/rev/254cd94b653b
changeset:   98531:254cd94b653b
parent:      98527:6db349fac3ec
parent:      98530:16e3fe295243
user:        Terry Jan Reedy <tjreedy at udel.edu>
date:        Sun Oct 04 00:31:36 2015 -0400
summary:
  Merge with 3.5

files:
  Lib/idlelib/config-highlight.def |  29 ++++++++++++++++++++
  Lib/idlelib/configDialog.py      |  14 +++++++++
  2 files changed, 43 insertions(+), 0 deletions(-)


diff --git a/Lib/idlelib/config-highlight.def b/Lib/idlelib/config-highlight.def
--- a/Lib/idlelib/config-highlight.def
+++ b/Lib/idlelib/config-highlight.def
@@ -62,3 +62,32 @@
 stderr-background= #ffffff
 console-foreground= #770000
 console-background= #ffffff
+
+[IDLE Dark]
+comment-foreground = #dd0000
+console-foreground = #ff4d4d
+error-foreground = #FFFFFF
+hilite-background = #7e7e7e
+string-foreground = #02ff02
+stderr-background = #002240
+stderr-foreground = #ffb3b3
+console-background = #002240
+hit-background = #fbfbfb
+string-background = #002240
+normal-background = #002240
+hilite-foreground = #FFFFFF
+keyword-foreground = #ff8000
+error-background = #c86464
+keyword-background = #002240
+builtin-background = #002240
+break-background = #808000
+builtin-foreground = #ff00ff
+definition-foreground = #5e5eff
+stdout-foreground = #c2d1fa
+definition-background = #002240
+normal-foreground = #FFFFFF
+cursor-foreground = #ffffff
+stdout-background = #002240
+hit-foreground = #002240
+comment-background = #002240
+break-foreground = #FFFFFF
diff --git a/Lib/idlelib/configDialog.py b/Lib/idlelib/configDialog.py
--- a/Lib/idlelib/configDialog.py
+++ b/Lib/idlelib/configDialog.py
@@ -507,6 +507,20 @@
 
     def VarChanged_builtinTheme(self, *params):
         value = self.builtinTheme.get()
+        if value == 'IDLE Dark':
+            tkMessageBox.showwarning(
+                title="The 'IDLE Dark' Text Color Theme",
+                message="IDLE Dark is new in October, 2015.  Trying to "
+                        "run earlier versions of IDLE with it selected "
+                        "will disable colorizing, or worse.\n\n"
+                        "If you might ever run an earlier release of IDLE, "
+                        "then before exiting this version, "
+                        "either switch to another theme or "
+                        "hit the 'Save as New Custom Theme' button.  "
+                        "The latter requires a new name, such as "
+                        "'Custom Dark', but the custom theme will work "
+                        "with any IDLE release, and can be modified.",
+                parent=self)
         self.AddChangedItem('main', 'Theme', 'name', value)
         self.PaintThemeSample()
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sun Oct  4 06:32:03 2015
From: python-checkins at python.org (terry.reedy)
Date: Sun, 04 Oct 2015 04:32:03 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
	=?utf-8?q?_Merge_with_3=2E4?=
Message-ID: <20151004043203.7266.77280@psf.io>

https://hg.python.org/cpython/rev/16e3fe295243
changeset:   98530:16e3fe295243
branch:      3.5
parent:      98526:20e0906a808e
parent:      98529:1de01a63f360
user:        Terry Jan Reedy <tjreedy at udel.edu>
date:        Sun Oct 04 00:31:23 2015 -0400
summary:
  Merge with 3.4

files:
  Lib/idlelib/config-highlight.def |  29 ++++++++++++++++++++
  Lib/idlelib/configDialog.py      |  14 +++++++++
  2 files changed, 43 insertions(+), 0 deletions(-)


diff --git a/Lib/idlelib/config-highlight.def b/Lib/idlelib/config-highlight.def
--- a/Lib/idlelib/config-highlight.def
+++ b/Lib/idlelib/config-highlight.def
@@ -62,3 +62,32 @@
 stderr-background= #ffffff
 console-foreground= #770000
 console-background= #ffffff
+
+[IDLE Dark]
+comment-foreground = #dd0000
+console-foreground = #ff4d4d
+error-foreground = #FFFFFF
+hilite-background = #7e7e7e
+string-foreground = #02ff02
+stderr-background = #002240
+stderr-foreground = #ffb3b3
+console-background = #002240
+hit-background = #fbfbfb
+string-background = #002240
+normal-background = #002240
+hilite-foreground = #FFFFFF
+keyword-foreground = #ff8000
+error-background = #c86464
+keyword-background = #002240
+builtin-background = #002240
+break-background = #808000
+builtin-foreground = #ff00ff
+definition-foreground = #5e5eff
+stdout-foreground = #c2d1fa
+definition-background = #002240
+normal-foreground = #FFFFFF
+cursor-foreground = #ffffff
+stdout-background = #002240
+hit-foreground = #002240
+comment-background = #002240
+break-foreground = #FFFFFF
diff --git a/Lib/idlelib/configDialog.py b/Lib/idlelib/configDialog.py
--- a/Lib/idlelib/configDialog.py
+++ b/Lib/idlelib/configDialog.py
@@ -507,6 +507,20 @@
 
     def VarChanged_builtinTheme(self, *params):
         value = self.builtinTheme.get()
+        if value == 'IDLE Dark':
+            tkMessageBox.showwarning(
+                title="The 'IDLE Dark' Text Color Theme",
+                message="IDLE Dark is new in October, 2015.  Trying to "
+                        "run earlier versions of IDLE with it selected "
+                        "will disable colorizing, or worse.\n\n"
+                        "If you might ever run an earlier release of IDLE, "
+                        "then before exiting this version, "
+                        "either switch to another theme or "
+                        "hit the 'Save as New Custom Theme' button.  "
+                        "The latter requires a new name, such as "
+                        "'Custom Dark', but the custom theme will work "
+                        "with any IDLE release, and can be modified.",
+                parent=self)
         self.AddChangedItem('main', 'Theme', 'name', value)
         self.PaintThemeSample()
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sun Oct  4 06:32:04 2015
From: python-checkins at python.org (terry.reedy)
Date: Sun, 04 Oct 2015 04:32:04 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzI0ODIw?=
 =?utf-8?q?=3A_Add_=27IDLE_Dark=27_text_color_theme=2C_warning=2C_and_solu?=
 =?utf-8?q?tion=2E?=
Message-ID: <20151004043203.55472.96354@psf.io>

https://hg.python.org/cpython/rev/afa95f032de1
changeset:   98528:afa95f032de1
branch:      2.7
parent:      98517:ec373d762213
user:        Terry Jan Reedy <tjreedy at udel.edu>
date:        Sun Oct 04 00:30:59 2015 -0400
summary:
  Issue #24820: Add 'IDLE Dark' text color theme, warning, and solution.

files:
  Lib/idlelib/config-highlight.def |  29 ++++++++++++++++++++
  Lib/idlelib/configDialog.py      |  14 +++++++++
  2 files changed, 43 insertions(+), 0 deletions(-)


diff --git a/Lib/idlelib/config-highlight.def b/Lib/idlelib/config-highlight.def
--- a/Lib/idlelib/config-highlight.def
+++ b/Lib/idlelib/config-highlight.def
@@ -62,3 +62,32 @@
 stderr-background= #ffffff
 console-foreground= #770000
 console-background= #ffffff
+
+[IDLE Dark]
+comment-foreground = #dd0000
+console-foreground = #ff4d4d
+error-foreground = #FFFFFF
+hilite-background = #7e7e7e
+string-foreground = #02ff02
+stderr-background = #002240
+stderr-foreground = #ffb3b3
+console-background = #002240
+hit-background = #fbfbfb
+string-background = #002240
+normal-background = #002240
+hilite-foreground = #FFFFFF
+keyword-foreground = #ff8000
+error-background = #c86464
+keyword-background = #002240
+builtin-background = #002240
+break-background = #808000
+builtin-foreground = #ff00ff
+definition-foreground = #5e5eff
+stdout-foreground = #c2d1fa
+definition-background = #002240
+normal-foreground = #FFFFFF
+cursor-foreground = #ffffff
+stdout-background = #002240
+hit-foreground = #002240
+comment-background = #002240
+break-foreground = #FFFFFF
diff --git a/Lib/idlelib/configDialog.py b/Lib/idlelib/configDialog.py
--- a/Lib/idlelib/configDialog.py
+++ b/Lib/idlelib/configDialog.py
@@ -524,6 +524,20 @@
 
     def VarChanged_builtinTheme(self, *params):
         value = self.builtinTheme.get()
+        if value == 'IDLE Dark':
+            tkMessageBox.showwarning(
+                title="The 'IDLE Dark' Text Color Theme",
+                message="IDLE Dark is new in October, 2015.  Trying to "
+                        "run earlier versions of IDLE with it selected "
+                        "will disable colorizing, or worse.\n\n"
+                        "If you might ever run an earlier release of IDLE, "
+                        "then before exiting this version, "
+                        "either switch to another theme or "
+                        "hit the 'Save as New Custom Theme' button.  "
+                        "The latter requires a new name, such as "
+                        "'Custom Dark', but the custom theme will work "
+                        "with any IDLE release, and can be modified.",
+                parent=self)
         self.AddChangedItem('main', 'Theme', 'name', value)
         self.PaintThemeSample()
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sun Oct  4 07:20:01 2015
From: python-checkins at python.org (terry.reedy)
Date: Sun, 04 Oct 2015 05:20:01 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzI0ODIw?=
 =?utf-8?q?=3A_Update_IDLE_NEWS_items=2E?=
Message-ID: <20151004052001.55465.21539@psf.io>

https://hg.python.org/cpython/rev/739cc9ca55cd
changeset:   98532:739cc9ca55cd
branch:      2.7
parent:      98528:afa95f032de1
user:        Terry Jan Reedy <tjreedy at udel.edu>
date:        Sun Oct 04 01:14:45 2015 -0400
summary:
  Issue #24820: Update IDLE NEWS items.

files:
  Lib/idlelib/NEWS.txt |  19 +++++++++++++++++++
  Misc/NEWS            |  19 +++++++++++++++++++
  2 files changed, 38 insertions(+), 0 deletions(-)


diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt
--- a/Lib/idlelib/NEWS.txt
+++ b/Lib/idlelib/NEWS.txt
@@ -2,6 +2,25 @@
 =========================
 *Release date:
 
+- Issue #24820: IDLE now has an 'IDLE Dark' built-in text color theme.
+  It is more or less IDLE Classic inverted, with a cobalt blue background.
+  Strings, comments, keywords, ... are still green, red, orange, ... .
+  Selecting it displays the following warning and solution.
+  "IDLE Dark is new in October, 2015.  Trying to run earlier versions
+  of IDLE with it selected will disable colorizing, or worse.
+  If you might ever run an earlier release of IDLE, then before
+  exiting this version, either switch to another theme or hit the
+  'Save as New Custom Theme' button.  The latter requires a new name,
+  such as 'Custom Dark', but the custom theme will work with any IDLE
+  release, and can be modified."
+
+- Issue #25224: README.txt is now an idlelib index for IDLE developers and
+  curious users.  The previous user content is now in the IDLE doc and is
+  redundant.  IDLE now means 'Integrated Development and Learning Environment'.
+
+- Issue #24820: Users can now set breakpoint colors in
+  Settings -> Custom Highlighting.  Original patch by Mark Roseman.
+
 - Issue #24972: Inactive selection background now matches active selection
   background, as configured by user, on all systems.  Found items are now
   always highlighted on Windows.  Initial patch by Mark Roseman.
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -177,6 +177,25 @@
 IDLE
 ----
 
+- Issue #24820: IDLE now has an 'IDLE Dark' built-in text color theme.
+  It is more or less IDLE Classic inverted, with a cobalt blue background.
+  Strings, comments, keywords, ... are still green, red, orange, ... .
+  Selecting it displays the following warning and solution.
+  "IDLE Dark is new in October, 2015.  Trying to run earlier versions
+  of IDLE with it selected will disable colorizing, or worse.
+  If you might ever run an earlier release of IDLE, then before
+  exiting this version, either switch to another theme or hit the
+  'Save as New Custom Theme' button.  The latter requires a new name,
+  such as 'Custom Dark', but the custom theme will work with any IDLE
+  release, and can be modified."
+
+- Issue #25224: README.txt is now an idlelib index for IDLE developers and
+  curious users.  The previous user content is now in the IDLE doc and is
+  redundant.  IDLE now means 'Integrated Development and Learning Environment'.
+
+- Issue #24820: Users can now set breakpoint colors in
+  Settings -> Custom Highlighting.  Original patch by Mark Roseman.
+
 - Issue #24972: Inactive selection background now matches active selection
   background, as configured by user, on all systems.  Found items are now
   always highlighted on Windows.  Initial patch by Mark Roseman.

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sun Oct  4 07:20:01 2015
From: python-checkins at python.org (terry.reedy)
Date: Sun, 04 Oct 2015 05:20:01 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
 =?utf-8?q?_Issue_=2324820=3A_Update_IDLE_NEWS_items=2E?=
Message-ID: <20151004052001.2689.53755@psf.io>

https://hg.python.org/cpython/rev/89a1e03b4639
changeset:   98534:89a1e03b4639
branch:      3.5
parent:      98530:16e3fe295243
parent:      98533:233974dfda03
user:        Terry Jan Reedy <tjreedy at udel.edu>
date:        Sun Oct 04 01:17:13 2015 -0400
summary:
  Issue #24820: Update IDLE NEWS items.

files:
  Lib/idlelib/NEWS.txt |  19 +++++++++++++++++++
  Misc/NEWS            |  19 +++++++++++++++++++
  2 files changed, 38 insertions(+), 0 deletions(-)


diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt
--- a/Lib/idlelib/NEWS.txt
+++ b/Lib/idlelib/NEWS.txt
@@ -2,6 +2,25 @@
 =========================
 *Release date: 2015-09-13*
 
+- Issue #24820: IDLE now has an 'IDLE Dark' built-in text color theme.
+  It is more or less IDLE Classic inverted, with a cobalt blue background.
+  Strings, comments, keywords, ... are still green, red, orange, ... .
+  Selecting it displays the following warning and solution.
+  "IDLE Dark is new in October, 2015.  Trying to run earlier versions
+  of IDLE with it selected will disable colorizing, or worse.
+  If you might ever run an earlier release of IDLE, then before
+  exiting this version, either switch to another theme or hit the
+  'Save as New Custom Theme' button.  The latter requires a new name,
+  such as 'Custom Dark', but the custom theme will work with any IDLE
+  release, and can be modified."
+
+- Issue #25224: README.txt is now an idlelib index for IDLE developers and
+  curious users.  The previous user content is now in the IDLE doc and is
+  redundant.  IDLE now means 'Integrated Development and Learning Environment'.
+
+- Issue #24820: Users can now set breakpoint colors in
+  Settings -> Custom Highlighting.  Original patch by Mark Roseman.
+
 - Issue #24972: Inactive selection background now matches active selection
   background, as configured by user, on all systems.  Found items are now
   always highlighted on Windows.  Initial patch by Mark Roseman.
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -167,6 +167,25 @@
 IDLE
 ----
 
+- Issue #24820: IDLE now has an 'IDLE Dark' built-in text color theme.
+  It is more or less IDLE Classic inverted, with a cobalt blue background.
+  Strings, comments, keywords, ... are still green, red, orange, ... .
+  Selecting it displays the following warning and solution.
+  "IDLE Dark is new in October, 2015.  Trying to run earlier versions
+  of IDLE with it selected will disable colorizing, or worse.
+  If you might ever run an earlier release of IDLE, then before
+  exiting this version, either switch to another theme or hit the
+  'Save as New Custom Theme' button.  The latter requires a new name,
+  such as 'Custom Dark', but the custom theme will work with any IDLE
+  release, and can be modified."
+
+- Issue #25224: README.txt is now an idlelib index for IDLE developers and
+  curious users.  The previous user content is now in the IDLE doc and is
+  redundant.  IDLE now means 'Integrated Development and Learning Environment'.
+
+- Issue #24820: Users can now set breakpoint colors in
+  Settings -> Custom Highlighting.  Original patch by Mark Roseman.
+
 - Issue #24972: Inactive selection background now matches active selection
   background, as configured by user, on all systems.  Found items are now
   always highlighted on Windows.  Initial patch by Mark Roseman.

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sun Oct  4 07:20:01 2015
From: python-checkins at python.org (terry.reedy)
Date: Sun, 04 Oct 2015 05:20:01 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
	=?utf-8?b?KTogbWVyZ2UgMy41?=
Message-ID: <20151004052001.20777.20260@psf.io>

https://hg.python.org/cpython/rev/f51921883f50
changeset:   98535:f51921883f50
parent:      98531:254cd94b653b
parent:      98534:89a1e03b4639
user:        Terry Jan Reedy <tjreedy at udel.edu>
date:        Sun Oct 04 01:19:36 2015 -0400
summary:
  merge 3.5

files:
  Lib/idlelib/NEWS.txt |  19 +++++++++++++++++++
  Misc/NEWS            |  21 ++++++++++++++++++++-
  2 files changed, 39 insertions(+), 1 deletions(-)


diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt
--- a/Lib/idlelib/NEWS.txt
+++ b/Lib/idlelib/NEWS.txt
@@ -2,6 +2,25 @@
 ===========================
 *Release date: 2017?*
 
+- Issue #24820: IDLE now has an 'IDLE Dark' built-in text color theme.
+  It is more or less IDLE Classic inverted, with a cobalt blue background.
+  Strings, comments, keywords, ... are still green, red, orange, ... .
+  Selecting it displays the following warning and solution.
+  "IDLE Dark is new in October, 2015.  Trying to run earlier versions
+  of IDLE with it selected will disable colorizing, or worse.
+  If you might ever run an earlier release of IDLE, then before
+  exiting this version, either switch to another theme or hit the
+  'Save as New Custom Theme' button.  The latter requires a new name,
+  such as 'Custom Dark', but the custom theme will work with any IDLE
+  release, and can be modified."
+
+- Issue #25224: README.txt is now an idlelib index for IDLE developers and
+  curious users.  The previous user content is now in the IDLE doc and is
+  redundant.  IDLE now means 'Integrated Development and Learning Environment'.
+
+- Issue #24820: Users can now set breakpoint colors in
+  Settings -> Custom Highlighting.  Original patch by Mark Roseman.
+
 - Issue #24972: Inactive selection background now matches active selection
   background, as configured by user, on all systems.  Found items are now
   always highlighted on Windows.  Initial patch by Mark Roseman.
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -119,6 +119,25 @@
 IDLE
 ----
 
+- Issue #24820: IDLE now has an 'IDLE Dark' built-in text color theme.
+  It is more or less IDLE Classic inverted, with a cobalt blue background.
+  Strings, comments, keywords, ... are still green, red, orange, ... .
+  Selecting it displays the following warning and solution.
+  "IDLE Dark is new in October, 2015.  Trying to run earlier versions
+  of IDLE with it selected will disable colorizing, or worse.
+  If you might ever run an earlier release of IDLE, then before
+  exiting this version, either switch to another theme or hit the
+  'Save as New Custom Theme' button.  The latter requires a new name,
+  such as 'Custom Dark', but the custom theme will work with any IDLE
+  release, and can be modified."
+
+- Issue #25224: README.txt is now an idlelib index for IDLE developers and
+  curious users.  The previous user content is now in the IDLE doc and is
+  redundant.  IDLE now means 'Integrated Development and Learning Environment'.
+
+- Issue #24820: Users can now set breakpoint colors in
+  Settings -> Custom Highlighting.  Original patch by Mark Roseman.
+
 - Issue #24972: Inactive selection background now matches active selection
   background, as configured by user, on all systems.  Found items are now
   always highlighted on Windows.  Initial patch by Mark Roseman.
@@ -556,7 +575,7 @@
 - Issue #17527: Add PATCH to wsgiref.validator. Patch from Luca Sbardella.
 
 - Issue #24791: Fix grammar regression for call syntax: 'g(\*a or b)'.
-p
+
 IDLE
 ----
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Sun Oct  4 07:20:02 2015
From: python-checkins at python.org (terry.reedy)
Date: Sun, 04 Oct 2015 05:20:02 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzI0ODIw?=
 =?utf-8?q?=3A_Update_IDLE_NEWS_items=2E?=
Message-ID: <20151004052001.20765.35169@psf.io>

https://hg.python.org/cpython/rev/233974dfda03
changeset:   98533:233974dfda03
branch:      3.4
parent:      98529:1de01a63f360
user:        Terry Jan Reedy <tjreedy at udel.edu>
date:        Sun Oct 04 01:14:51 2015 -0400
summary:
  Issue #24820: Update IDLE NEWS items.

files:
  Lib/idlelib/NEWS.txt |  19 +++++++++++++++++++
  Misc/NEWS            |  19 +++++++++++++++++++
  2 files changed, 38 insertions(+), 0 deletions(-)


diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt
--- a/Lib/idlelib/NEWS.txt
+++ b/Lib/idlelib/NEWS.txt
@@ -2,6 +2,25 @@
 =========================
 *Release date: 2015-??-??*
 
+- Issue #24820: IDLE now has an 'IDLE Dark' built-in text color theme.
+  It is more or less IDLE Classic inverted, with a cobalt blue background.
+  Strings, comments, keywords, ... are still green, red, orange, ... .
+  Selecting it displays the following warning and solution.
+  "IDLE Dark is new in October, 2015.  Trying to run earlier versions
+  of IDLE with it selected will disable colorizing, or worse.
+  If you might ever run an earlier release of IDLE, then before
+  exiting this version, either switch to another theme or hit the
+  'Save as New Custom Theme' button.  The latter requires a new name,
+  such as 'Custom Dark', but the custom theme will work with any IDLE
+  release, and can be modified."
+
+- Issue #25224: README.txt is now an idlelib index for IDLE developers and
+  curious users.  The previous user content is now in the IDLE doc and is
+  redundant.  IDLE now means 'Integrated Development and Learning Environment'.
+
+- Issue #24820: Users can now set breakpoint colors in
+  Settings -> Custom Highlighting.  Original patch by Mark Roseman.
+
 - Issue #24972: Inactive selection background now matches active selection
   background, as configured by user, on all systems.  Found items are now
   always highlighted on Windows.  Initial patch by Mark Roseman.
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -492,6 +492,25 @@
 IDLE
 ----
 
+- Issue #24820: IDLE now has an 'IDLE Dark' built-in text color theme.
+  It is more or less IDLE Classic inverted, with a cobalt blue background.
+  Strings, comments, keywords, ... are still green, red, orange, ... .
+  Selecting it displays the following warning and solution.
+  "IDLE Dark is new in October, 2015.  Trying to run earlier versions
+  of IDLE with it selected will disable colorizing, or worse.
+  If you might ever run an earlier release of IDLE, then before
+  exiting this version, either switch to another theme or hit the
+  'Save as New Custom Theme' button.  The latter requires a new name,
+  such as 'Custom Dark', but the custom theme will work with any IDLE
+  release, and can be modified."
+
+- Issue #25224: README.txt is now an idlelib index for IDLE developers and
+  curious users.  The previous user content is now in the IDLE doc and is
+  redundant.  IDLE now means 'Integrated Development and Learning Environment'.
+
+- Issue #24820: Users can now set breakpoint colors in
+  Settings -> Custom Highlighting.  Original patch by Mark Roseman.
+
 - Issue #24972: Inactive selection background now matches active selection
   background, as configured by user, on all systems.  Found items are now
   always highlighted on Windows.  Initial patch by Mark Roseman.

-- 
Repository URL: https://hg.python.org/cpython

From solipsis at pitrou.net  Sun Oct  4 10:46:22 2015
From: solipsis at pitrou.net (solipsis at pitrou.net)
Date: Sun, 04 Oct 2015 08:46:22 +0000
Subject: [Python-checkins] Daily reference leaks (f51921883f50): sum=61494
Message-ID: <20151004084622.20765.66976@psf.io>

results for f51921883f50 on branch "default"
--------------------------------------------

test_asyncio leaked [3, 0, 0] memory blocks, sum=3
test_capi leaked [5410, 5410, 5410] references, sum=16230
test_capi leaked [1421, 1423, 1423] memory blocks, sum=4267
test_functools leaked [0, 2, 2] memory blocks, sum=4
test_threading leaked [10820, 10820, 10820] references, sum=32460
test_threading leaked [2842, 2844, 2844] memory blocks, sum=8530


Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogAfUcYa', '--timeout', '7200']

From python-checkins at python.org  Sun Oct  4 12:53:15 2015
From: python-checkins at python.org (serhiy.storchaka)
Date: Sun, 04 Oct 2015 10:53:15 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_Make_error_rep?=
 =?utf-8?q?ort_in_test=5Fcodecs_more_informative=2E?=
Message-ID: <20151004105315.128830.66026@psf.io>

https://hg.python.org/cpython/rev/45a04eadefd6
changeset:   98536:45a04eadefd6
branch:      2.7
parent:      98532:739cc9ca55cd
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Sun Oct 04 13:52:40 2015 +0300
summary:
  Make error report in test_codecs more informative.

files:
  Lib/test/test_codecs.py |  36 ++++++++++++++++++++--------
  1 files changed, 26 insertions(+), 10 deletions(-)


diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -573,9 +573,13 @@
             (b'\x00\xdcA\x00', u'\ufffdA'),
         ]
         for raw, expected in tests:
-            self.assertRaises(UnicodeDecodeError, codecs.utf_16_le_decode,
-                              raw, 'strict', True)
-            self.assertEqual(raw.decode('utf-16le', 'replace'), expected)
+            try:
+                with self.assertRaises(UnicodeDecodeError):
+                    codecs.utf_16_le_decode(raw, 'strict', True)
+                self.assertEqual(raw.decode('utf-16le', 'replace'), expected)
+            except:
+                print 'raw=%r' % raw
+                raise
 
 class UTF16BETest(ReadTest):
     encoding = "utf-16-be"
@@ -610,9 +614,13 @@
             (b'\xdc\x00\x00A', u'\ufffdA'),
         ]
         for raw, expected in tests:
-            self.assertRaises(UnicodeDecodeError, codecs.utf_16_be_decode,
-                              raw, 'strict', True)
-            self.assertEqual(raw.decode('utf-16be', 'replace'), expected)
+            try:
+                with self.assertRaises(UnicodeDecodeError):
+                    codecs.utf_16_be_decode(raw, 'strict', True)
+                self.assertEqual(raw.decode('utf-16be', 'replace'), expected)
+            except:
+                print 'raw=%r' % raw
+                raise
 
 class UTF8Test(ReadTest):
     encoding = "utf-8"
@@ -704,9 +712,13 @@
             ('a+IKw\xffb', u'a\u20ac\ufffdb'),
         ]
         for raw, expected in tests:
-            self.assertRaises(UnicodeDecodeError, codecs.utf_7_decode,
-                              raw, 'strict', True)
-            self.assertEqual(raw.decode('utf-7', 'replace'), expected)
+            try:
+                with self.assertRaises(UnicodeDecodeError):
+                    codecs.utf_7_decode(raw, 'strict', True)
+                self.assertEqual(raw.decode('utf-7', 'replace'), expected)
+            except:
+                print 'raw=%r' % raw
+                raise
 
     def test_nonbmp(self):
         self.assertEqual(u'\U000104A0'.encode(self.encoding), '+2AHcoA-')
@@ -740,7 +752,11 @@
             ('a+IKwgrNgBA-b', u'a\u20ac\u20ac\ufffdb'),
         ]
         for raw, expected in tests:
-            self.assertEqual(raw.decode('utf-7', 'replace'), expected)
+            try:
+                self.assertEqual(raw.decode('utf-7', 'replace'), expected)
+            except:
+                print 'raw=%r' % raw
+                raise
 
 class UTF16ExTest(unittest.TestCase):
 

-- 
Repository URL: https://hg.python.org/cpython

From solipsis at pitrou.net  Mon Oct  5 10:48:11 2015
From: solipsis at pitrou.net (solipsis at pitrou.net)
Date: Mon, 05 Oct 2015 08:48:11 +0000
Subject: [Python-checkins] Daily reference leaks (f51921883f50): sum=61491
Message-ID: <20151005084810.7258.29002@psf.io>

results for f51921883f50 on branch "default"
--------------------------------------------

test_capi leaked [5410, 5410, 5410] references, sum=16230
test_capi leaked [1421, 1423, 1423] memory blocks, sum=4267
test_functools leaked [0, 2, 2] memory blocks, sum=4
test_threading leaked [10820, 10820, 10820] references, sum=32460
test_threading leaked [2842, 2844, 2844] memory blocks, sum=8530


Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogZ7hcsa', '--timeout', '7200']

From python-checkins at python.org  Mon Oct  5 13:44:01 2015
From: python-checkins at python.org (victor.stinner)
Date: Mon, 05 Oct 2015 11:44:01 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2325301=3A_The_UTF-?=
 =?utf-8?q?8_decoder_is_now_up_to_15_times_as_fast_for_error?=
Message-ID: <20151005114359.97718.6957@psf.io>

https://hg.python.org/cpython/rev/3152e4038d97
changeset:   98537:3152e4038d97
parent:      98535:f51921883f50
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Mon Oct 05 13:43:50 2015 +0200
summary:
  Issue #25301: The UTF-8 decoder is now up to 15 times as fast for error
handlers: ``ignore``, ``replace`` and ``surrogateescape``.

files:
  Doc/whatsnew/3.6.rst    |   3 +
  Lib/test/test_codecs.py |  12 +++++++
  Misc/NEWS               |   3 +
  Objects/unicodeobject.c |  48 +++++++++++++++++++++++-----
  4 files changed, 57 insertions(+), 9 deletions(-)


diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst
--- a/Doc/whatsnew/3.6.rst
+++ b/Doc/whatsnew/3.6.rst
@@ -123,6 +123,9 @@
 * The UTF-8 encoder is now up to 75 times as fast for error handlers:
   ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``.
 
+* The UTF-8 decoder is now up to 15 times as fast for error handlers:
+  ``ignore``, ``replace`` and ``surrogateescape``.
+
 
 Build and C API Changes
 =======================
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -788,6 +788,18 @@
         self.check_state_handling_decode(self.encoding,
                                          u, u.encode(self.encoding))
 
+    def test_decode_error(self):
+        for data, error_handler, expected in (
+            (b'[\x80\xff]', 'ignore', '[]'),
+            (b'[\x80\xff]', 'replace', '[\ufffd\ufffd]'),
+            (b'[\x80\xff]', 'surrogateescape', '[\udc80\udcff]'),
+            (b'[\x80\xff]', 'backslashreplace', '[\\x80\\xff]'),
+        ):
+            with self.subTest(data=data, error_handler=error_handler,
+                              expected=expected):
+                self.assertEqual(data.decode(self.encoding, error_handler),
+                                 expected)
+
     def test_lone_surrogates(self):
         super().test_lone_surrogates()
         # not sure if this is making sense for
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,9 @@
 Core and Builtins
 -----------------
 
+* Issue #25301: The UTF-8 decoder is now up to 15 times as fast for error
+  handlers: ``ignore``, ``replace`` and ``surrogateescape``.
+
 - Issue #24848: Fixed a number of bugs in UTF-7 decoding of misformed data.
 
 - Issue #25267: The UTF-8 encoder is now up to 75 times as fast for error
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -4714,8 +4714,9 @@
     Py_ssize_t startinpos;
     Py_ssize_t endinpos;
     const char *errmsg = "";
-    PyObject *errorHandler = NULL;
+    PyObject *error_handler_obj = NULL;
     PyObject *exc = NULL;
+    _Py_error_handler error_handler = _Py_ERROR_UNKNOWN;
 
     if (size == 0) {
         if (consumed)
@@ -4740,6 +4741,7 @@
     while (s < end) {
         Py_UCS4 ch;
         int kind = writer.kind;
+
         if (kind == PyUnicode_1BYTE_KIND) {
             if (PyUnicode_IS_ASCII(writer.buffer))
                 ch = asciilib_utf8_decode(&s, end, writer.data, &writer.pos);
@@ -4778,24 +4780,52 @@
             continue;
         }
 
-        if (unicode_decode_call_errorhandler_writer(
-                errors, &errorHandler,
-                "utf-8", errmsg,
-                &starts, &end, &startinpos, &endinpos, &exc, &s,
-                &writer))
-            goto onError;
+        if (error_handler == _Py_ERROR_UNKNOWN)
+            error_handler = get_error_handler(errors);
+
+        switch (error_handler) {
+        case _Py_ERROR_IGNORE:
+            s += (endinpos - startinpos);
+            break;
+
+        case _Py_ERROR_REPLACE:
+            if (_PyUnicodeWriter_WriteCharInline(&writer, 0xfffd) < 0)
+                goto onError;
+            s += (endinpos - startinpos);
+            break;
+
+        case _Py_ERROR_SURROGATEESCAPE:
+            if (_PyUnicodeWriter_PrepareKind(&writer, PyUnicode_2BYTE_KIND) < 0)
+                goto onError;
+            for (Py_ssize_t i=startinpos; i<endinpos; i++) {
+                ch = (Py_UCS4)(unsigned char)(starts[i]);
+                PyUnicode_WRITE(writer.kind, writer.data, writer.pos,
+                                ch + 0xdc00);
+                writer.pos++;
+            }
+            s += (endinpos - startinpos);
+            break;
+
+        default:
+            if (unicode_decode_call_errorhandler_writer(
+                    errors, &error_handler_obj,
+                    "utf-8", errmsg,
+                    &starts, &end, &startinpos, &endinpos, &exc, &s,
+                    &writer))
+                goto onError;
+        }
     }
 
 End:
     if (consumed)
         *consumed = s - starts;
 
-    Py_XDECREF(errorHandler);
+    Py_XDECREF(error_handler_obj);
     Py_XDECREF(exc);
     return _PyUnicodeWriter_Finish(&writer);
 
 onError:
-    Py_XDECREF(errorHandler);
+    Py_XDECREF(error_handler_obj);
     Py_XDECREF(exc);
     _PyUnicodeWriter_Dealloc(&writer);
     return NULL;

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Mon Oct  5 13:49:33 2015
From: python-checkins at python.org (victor.stinner)
Date: Mon, 05 Oct 2015 11:49:33 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2325301=3A_Fix_comp?=
	=?utf-8?q?atibility_with_ISO_C90?=
Message-ID: <20151005114933.70990.39106@psf.io>

https://hg.python.org/cpython/rev/5b9ffea7e7c3
changeset:   98538:5b9ffea7e7c3
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Mon Oct 05 13:49:26 2015 +0200
summary:
  Issue #25301: Fix compatibility with ISO C90

files:
  Objects/unicodeobject.c |  6 +++++-
  1 files changed, 5 insertions(+), 1 deletions(-)


diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -4795,9 +4795,12 @@
             break;
 
         case _Py_ERROR_SURROGATEESCAPE:
+        {
+            Py_ssize_t i;
+
             if (_PyUnicodeWriter_PrepareKind(&writer, PyUnicode_2BYTE_KIND) < 0)
                 goto onError;
-            for (Py_ssize_t i=startinpos; i<endinpos; i++) {
+            for (i=startinpos; i<endinpos; i++) {
                 ch = (Py_UCS4)(unsigned char)(starts[i]);
                 PyUnicode_WRITE(writer.kind, writer.data, writer.pos,
                                 ch + 0xdc00);
@@ -4805,6 +4808,7 @@
             }
             s += (endinpos - startinpos);
             break;
+        }
 
         default:
             if (unicode_decode_call_errorhandler_writer(

-- 
Repository URL: https://hg.python.org/cpython

From lp_benchmark_robot at intel.com  Mon Oct  5 14:42:42 2015
From: lp_benchmark_robot at intel.com (lp_benchmark_robot at intel.com)
Date: Mon, 5 Oct 2015 13:42:42 +0100
Subject: [Python-checkins] Benchmark Results for Python Default 2015-10-05
Message-ID: <4a79339d-9940-4437-a373-db904e3c8a12@irsmsx102.ger.corp.intel.com>

Results for project python_default-nightly, build date 2015-10-05 03:02:04 
commit:		f51921883f50be87405c81030bf01e6a29211c5e
revision date:	2015-10-04 05:19:36 +0000
environment:	Haswell-EP
	cpu:	Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz 2x18 cores, stepping 2, LLC 45 MB
	mem:	128 GB
	os:	CentOS 7.1
	kernel:	Linux 3.10.0-229.4.2.el7.x86_64

Baseline results were generated using release v3.4.3, with hash
b4cbecbc0781e89a309d03b60a1f75f8499250e6 from 2015-02-25 12:15:33+00:00

------------------------------------------------------------------------------------------
               benchmark    relative      change since      change since  current rev with
                            std_dev*          last run            v3.4.3      regrtest PGO
------------------------------------------------------------------------------------------
:-)            django_v2    0.46435%         -0.26186%          7.78646%         16.61379%
:-|              pybench    0.13488%          0.02614%         -1.91868%          8.48274%
:-(             regex_v8    2.58012%          0.03680%         -5.19978%          0.37598%
:-|                nbody    0.13849%          0.77933%         -0.26838%          8.96712%
:-|         json_dump_v2    0.35644%          1.12972%         -1.09199%         10.21158%
:-|       normal_startup    0.76398%          0.35423%          0.50319%          5.00412%
------------------------------------------------------------------------------------------
Note: Benchmark results are measured in seconds.
* Relative Standard Deviation (Standard Deviation/Average)

Our lab does a nightly source pull and build of the Python project and measures
performance changes against the previous stable version and the previous nightly
measurement. This is provided as a service to the community so that quality
issues with current hardware can be identified quickly.

Intel technologies' features and benefits depend on system configuration and may
require enabled hardware, software or service activation. Performance varies
depending on system configuration.


From lp_benchmark_robot at intel.com  Mon Oct  5 14:44:12 2015
From: lp_benchmark_robot at intel.com (lp_benchmark_robot at intel.com)
Date: Mon, 5 Oct 2015 13:44:12 +0100
Subject: [Python-checkins] Benchmark Results for Python 2.7 2015-10-05
Message-ID: <030f3e1a-f52f-427e-9f24-1fb43d4a8726@irsmsx102.ger.corp.intel.com>

No new revisions. Here are the previous results:

Results for project python_2.7-nightly, build date 2015-10-05 08:13:47 
commit:		45a04eadefd6ed13c110059375d2932f6b0d7490
revision date:	2015-10-04 10:52:40 +0000
environment:	Haswell-EP
	cpu:	Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz 2x18 cores, stepping 2, LLC 45 MB
	mem:	128 GB
	os:	CentOS 7.1
	kernel:	Linux 3.10.0-229.4.2.el7.x86_64

Baseline results were generated using release v2.7.10, with hash
15c95b7d81dcf821daade360741e00714667653f from 2015-05-23 16:02:14+00:00

------------------------------------------------------------------------------------------
               benchmark    relative      change since      change since  current rev with
                            std_dev*          last run           v2.7.10      regrtest PGO
------------------------------------------------------------------------------------------
:-)            django_v2    0.11335%          0.94969%          5.43955%          8.53025%
:-)              pybench    0.16785%          0.69147%          6.77273%          6.45979%
:-|             regex_v8    1.07392%          0.01169%         -1.82451%          7.81632%
:-)                nbody    0.15352%          0.15454%          8.67803%          3.87488%
:-)         json_dump_v2    0.21320%         -0.26509%          3.37121%         12.71810%
:-(       normal_startup    1.91283%         -0.47298%         -2.14007%          3.41653%
:-|              ssbench    0.10483%          0.27749%          1.31698%          1.08129%
------------------------------------------------------------------------------------------
Note: Benchmark results for ssbench are measured in requests/second while all
other are measured in seconds.
* Relative Standard Deviation (Standard Deviation/Average)

Our lab does a nightly source pull and build of the Python project and measures
performance changes against the previous stable version and the previous nightly
measurement. This is provided as a service to the community so that quality
issues with current hardware can be identified quickly.

Intel technologies' features and benefits depend on system configuration and may
require enabled hardware, software or service activation. Performance varies
depending on system configuration.


From python-checkins at python.org  Mon Oct  5 18:30:47 2015
From: python-checkins at python.org (guido.van.rossum)
Date: Mon, 05 Oct 2015 16:30:47 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzIzOTcy?=
 =?utf-8?q?=3A_updates_to_asyncio_datagram_API=2E_By_Chris_Laws=2E?=
Message-ID: <20151005163044.7250.52332@psf.io>

https://hg.python.org/cpython/rev/5e7e9b131904
changeset:   98539:5e7e9b131904
branch:      3.4
parent:      98533:233974dfda03
user:        Guido van Rossum <guido at python.org>
date:        Mon Oct 05 09:15:28 2015 -0700
summary:
  Issue #23972: updates to asyncio datagram API. By Chris Laws.

files:
  Doc/library/asyncio-eventloop.rst         |   46 ++-
  Lib/asyncio/base_events.py                |  160 ++++++---
  Lib/asyncio/events.py                     |   40 ++-
  Lib/test/test_asyncio/test_base_events.py |  140 ++++++++-
  Lib/test/test_asyncio/test_events.py      |   52 +++
  Misc/ACKS                                 |    1 +
  Misc/NEWS                                 |    6 +
  7 files changed, 378 insertions(+), 67 deletions(-)


diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst
--- a/Doc/library/asyncio-eventloop.rst
+++ b/Doc/library/asyncio-eventloop.rst
@@ -283,17 +283,50 @@
       (:class:`StreamReader`, :class:`StreamWriter`) instead of a protocol.
 
 
-.. coroutinemethod:: BaseEventLoop.create_datagram_endpoint(protocol_factory, local_addr=None, remote_addr=None, \*, family=0, proto=0, flags=0)
+.. coroutinemethod:: BaseEventLoop.create_datagram_endpoint(protocol_factory, local_addr=None, remote_addr=None, \*, family=0, proto=0, flags=0, reuse_address=None, reuse_port=None, allow_broadcast=None, sock=None)
 
    Create datagram connection: socket family :py:data:`~socket.AF_INET` or
    :py:data:`~socket.AF_INET6` depending on *host* (or *family* if specified),
-   socket type :py:data:`~socket.SOCK_DGRAM`.
+   socket type :py:data:`~socket.SOCK_DGRAM`. *protocol_factory* must be a
+   callable returning a :ref:`protocol <asyncio-protocol>` instance.
 
    This method is a :ref:`coroutine <coroutine>` which will try to
    establish the connection in the background.  When successful, the
    coroutine returns a ``(transport, protocol)`` pair.
 
-   See the :meth:`BaseEventLoop.create_connection` method for parameters.
+   Options changing how the connection is created:
+
+   * *local_addr*, if given, is a ``(local_host, local_port)`` tuple used
+     to bind the socket to locally.  The *local_host* and *local_port*
+     are looked up using :meth:`getaddrinfo`.
+
+   * *remote_addr*, if given, is a ``(remote_host, remote_port)`` tuple used
+     to connect the socket to a remote address.  The *remote_host* and
+     *remote_port* are looked up using :meth:`getaddrinfo`.
+
+   * *family*, *proto*, *flags* are the optional address family, protocol
+     and flags to be passed through to :meth:`getaddrinfo` for *host*
+     resolution. If given, these should all be integers from the
+     corresponding :mod:`socket` module constants.
+
+   * *reuse_address* tells the kernel to reuse a local socket in
+     TIME_WAIT state, without waiting for its natural timeout to
+     expire. If not specified will automatically be set to True on
+     UNIX.
+
+   * *reuse_port* tells the kernel to allow this endpoint to be bound to the
+     same port as other existing endpoints are bound to, so long as they all
+     set this flag when being created. This option is not supported on Windows
+     and some UNIX's. If the :py:data:`~socket.SO_REUSEPORT` constant is not
+     defined then this capability is unsupported.
+
+   * *allow_broadcast* tells the kernel to allow this endpoint to send
+     messages to the broadcast address.
+
+   * *sock* can optionally be specified in order to use a preexisting,
+     already connected, :class:`socket.socket` object to be used by the
+     transport. If specified, *local_addr* and *remote_addr* should be omitted
+     (must be :const:`None`).
 
    On Windows with :class:`ProactorEventLoop`, this method is not supported.
 
@@ -320,7 +353,7 @@
 Creating listening connections
 ------------------------------
 
-.. coroutinemethod:: BaseEventLoop.create_server(protocol_factory, host=None, port=None, \*, family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE, sock=None, backlog=100, ssl=None, reuse_address=None)
+.. coroutinemethod:: BaseEventLoop.create_server(protocol_factory, host=None, port=None, \*, family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE, sock=None, backlog=100, ssl=None, reuse_address=None, reuse_port=None)
 
    Create a TCP server (socket type :data:`~socket.SOCK_STREAM`) bound to
    *host* and *port*.
@@ -359,6 +392,11 @@
      expire. If not specified will automatically be set to True on
      UNIX.
 
+   * *reuse_port* tells the kernel to allow this endpoint to be bound to the
+     same port as other existing endpoints are bound to, so long as they all
+     set this flag when being created. This option is not supported on
+     Windows.
+
    This method is a :ref:`coroutine <coroutine>`.
 
    On Windows with :class:`ProactorEventLoop`, SSL/TLS is not supported.
diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py
--- a/Lib/asyncio/base_events.py
+++ b/Lib/asyncio/base_events.py
@@ -700,75 +700,109 @@
     @coroutine
     def create_datagram_endpoint(self, protocol_factory,
                                  local_addr=None, remote_addr=None, *,
-                                 family=0, proto=0, flags=0):
+                                 family=0, proto=0, flags=0,
+                                 reuse_address=None, reuse_port=None,
+                                 allow_broadcast=None, sock=None):
         """Create datagram connection."""
-        if not (local_addr or remote_addr):
-            if family == 0:
-                raise ValueError('unexpected address family')
-            addr_pairs_info = (((family, proto), (None, None)),)
+        if sock is not None:
+            if (local_addr or remote_addr or
+                    family or proto or flags or
+                    reuse_address or reuse_port or allow_broadcast):
+                # show the problematic kwargs in exception msg
+                opts = dict(local_addr=local_addr, remote_addr=remote_addr,
+                            family=family, proto=proto, flags=flags,
+                            reuse_address=reuse_address, reuse_port=reuse_port,
+                            allow_broadcast=allow_broadcast)
+                problems = ', '.join(
+                    '{}={}'.format(k, v) for k, v in opts.items() if v)
+                raise ValueError(
+                    'socket modifier keyword arguments can not be used '
+                    'when sock is specified. ({})'.format(problems))
+            sock.setblocking(False)
+            r_addr = None
         else:
-            # join address by (family, protocol)
-            addr_infos = collections.OrderedDict()
-            for idx, addr in ((0, local_addr), (1, remote_addr)):
-                if addr is not None:
-                    assert isinstance(addr, tuple) and len(addr) == 2, (
-                        '2-tuple is expected')
+            if not (local_addr or remote_addr):
+                if family == 0:
+                    raise ValueError('unexpected address family')
+                addr_pairs_info = (((family, proto), (None, None)),)
+            else:
+                # join address by (family, protocol)
+                addr_infos = collections.OrderedDict()
+                for idx, addr in ((0, local_addr), (1, remote_addr)):
+                    if addr is not None:
+                        assert isinstance(addr, tuple) and len(addr) == 2, (
+                            '2-tuple is expected')
 
-                    infos = yield from self.getaddrinfo(
-                        *addr, family=family, type=socket.SOCK_DGRAM,
-                        proto=proto, flags=flags)
-                    if not infos:
-                        raise OSError('getaddrinfo() returned empty list')
+                        infos = yield from self.getaddrinfo(
+                            *addr, family=family, type=socket.SOCK_DGRAM,
+                            proto=proto, flags=flags)
+                        if not infos:
+                            raise OSError('getaddrinfo() returned empty list')
 
-                    for fam, _, pro, _, address in infos:
-                        key = (fam, pro)
-                        if key not in addr_infos:
-                            addr_infos[key] = [None, None]
-                        addr_infos[key][idx] = address
+                        for fam, _, pro, _, address in infos:
+                            key = (fam, pro)
+                            if key not in addr_infos:
+                                addr_infos[key] = [None, None]
+                            addr_infos[key][idx] = address
 
-            # each addr has to have info for each (family, proto) pair
-            addr_pairs_info = [
-                (key, addr_pair) for key, addr_pair in addr_infos.items()
-                if not ((local_addr and addr_pair[0] is None) or
-                        (remote_addr and addr_pair[1] is None))]
+                # each addr has to have info for each (family, proto) pair
+                addr_pairs_info = [
+                    (key, addr_pair) for key, addr_pair in addr_infos.items()
+                    if not ((local_addr and addr_pair[0] is None) or
+                            (remote_addr and addr_pair[1] is None))]
 
-            if not addr_pairs_info:
-                raise ValueError('can not get address information')
+                if not addr_pairs_info:
+                    raise ValueError('can not get address information')
 
-        exceptions = []
+            exceptions = []
 
-        for ((family, proto),
-             (local_address, remote_address)) in addr_pairs_info:
-            sock = None
-            r_addr = None
-            try:
-                sock = socket.socket(
-                    family=family, type=socket.SOCK_DGRAM, proto=proto)
-                sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-                sock.setblocking(False)
+            if reuse_address is None:
+                reuse_address = os.name == 'posix' and sys.platform != 'cygwin'
 
-                if local_addr:
-                    sock.bind(local_address)
-                if remote_addr:
-                    yield from self.sock_connect(sock, remote_address)
-                    r_addr = remote_address
-            except OSError as exc:
-                if sock is not None:
-                    sock.close()
-                exceptions.append(exc)
-            except:
-                if sock is not None:
-                    sock.close()
-                raise
+            for ((family, proto),
+                 (local_address, remote_address)) in addr_pairs_info:
+                sock = None
+                r_addr = None
+                try:
+                    sock = socket.socket(
+                        family=family, type=socket.SOCK_DGRAM, proto=proto)
+                    if reuse_address:
+                        sock.setsockopt(
+                            socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+                    if reuse_port:
+                        if not hasattr(socket, 'SO_REUSEPORT'):
+                            raise ValueError(
+                                'reuse_port not supported by socket module')
+                        else:
+                            sock.setsockopt(
+                                socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+                    if allow_broadcast:
+                        sock.setsockopt(
+                            socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
+                    sock.setblocking(False)
+
+                    if local_addr:
+                        sock.bind(local_address)
+                    if remote_addr:
+                        yield from self.sock_connect(sock, remote_address)
+                        r_addr = remote_address
+                except OSError as exc:
+                    if sock is not None:
+                        sock.close()
+                    exceptions.append(exc)
+                except:
+                    if sock is not None:
+                        sock.close()
+                    raise
+                else:
+                    break
             else:
-                break
-        else:
-            raise exceptions[0]
+                raise exceptions[0]
 
         protocol = protocol_factory()
         waiter = futures.Future(loop=self)
-        transport = self._make_datagram_transport(sock, protocol, r_addr,
-                                                  waiter)
+        transport = self._make_datagram_transport(
+            sock, protocol, r_addr, waiter)
         if self._debug:
             if local_addr:
                 logger.info("Datagram endpoint local_addr=%r remote_addr=%r "
@@ -804,7 +838,8 @@
                       sock=None,
                       backlog=100,
                       ssl=None,
-                      reuse_address=None):
+                      reuse_address=None,
+                      reuse_port=None):
         """Create a TCP server.
 
         The host parameter can be a string, in that case the TCP server is bound
@@ -857,8 +892,15 @@
                         continue
                     sockets.append(sock)
                     if reuse_address:
-                        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,
-                                        True)
+                        sock.setsockopt(
+                            socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
+                    if reuse_port:
+                        if not hasattr(socket, 'SO_REUSEPORT'):
+                            raise ValueError(
+                                'reuse_port not supported by socket module')
+                        else:
+                            sock.setsockopt(
+                                socket.SOL_SOCKET, socket.SO_REUSEPORT, True)
                     # Disable IPv4/IPv6 dual stack support (enabled by
                     # default on Linux) which makes a single socket
                     # listen on both address families.
diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py
--- a/Lib/asyncio/events.py
+++ b/Lib/asyncio/events.py
@@ -297,7 +297,8 @@
 
     def create_server(self, protocol_factory, host=None, port=None, *,
                       family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE,
-                      sock=None, backlog=100, ssl=None, reuse_address=None):
+                      sock=None, backlog=100, ssl=None, reuse_address=None,
+                      reuse_port=None):
         """A coroutine which creates a TCP server bound to host and port.
 
         The return value is a Server object which can be used to stop
@@ -327,6 +328,11 @@
         TIME_WAIT state, without waiting for its natural timeout to
         expire. If not specified will automatically be set to True on
         UNIX.
+
+        reuse_port tells the kernel to allow this endpoint to be bound to
+        the same port as other existing endpoints are bound to, so long as
+        they all set this flag when being created. This option is not
+        supported on Windows.
         """
         raise NotImplementedError
 
@@ -358,7 +364,37 @@
 
     def create_datagram_endpoint(self, protocol_factory,
                                  local_addr=None, remote_addr=None, *,
-                                 family=0, proto=0, flags=0):
+                                 family=0, proto=0, flags=0,
+                                 reuse_address=None, reuse_port=None,
+                                 allow_broadcast=None, sock=None):
+        """A coroutine which creates a datagram endpoint.
+
+        This method will try to establish the endpoint in the background.
+        When successful, the coroutine returns a (transport, protocol) pair.
+
+        protocol_factory must be a callable returning a protocol instance.
+
+        socket family AF_INET or socket.AF_INET6 depending on host (or
+        family if specified), socket type SOCK_DGRAM.
+
+        reuse_address tells the kernel to reuse a local socket in
+        TIME_WAIT state, without waiting for its natural timeout to
+        expire. If not specified it will automatically be set to True on
+        UNIX.
+
+        reuse_port tells the kernel to allow this endpoint to be bound to
+        the same port as other existing endpoints are bound to, so long as
+        they all set this flag when being created. This option is not
+        supported on Windows and some UNIX's. If the
+        :py:data:`~socket.SO_REUSEPORT` constant is not defined then this
+        capability is unsupported.
+
+        allow_broadcast tells the kernel to allow this endpoint to send
+        messages to the broadcast address.
+
+        sock can optionally be specified in order to use a preexisting
+        socket object.
+        """
         raise NotImplementedError
 
     # Pipes and subprocesses.
diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py
--- a/Lib/test/test_asyncio/test_base_events.py
+++ b/Lib/test/test_asyncio/test_base_events.py
@@ -3,6 +3,7 @@
 import errno
 import logging
 import math
+import os
 import socket
 import sys
 import threading
@@ -790,11 +791,11 @@
 class MyDatagramProto(asyncio.DatagramProtocol):
     done = None
 
-    def __init__(self, create_future=False):
+    def __init__(self, create_future=False, loop=None):
         self.state = 'INITIAL'
         self.nbytes = 0
         if create_future:
-            self.done = asyncio.Future()
+            self.done = asyncio.Future(loop=loop)
 
     def connection_made(self, transport):
         self.transport = transport
@@ -1100,6 +1101,19 @@
         self.assertRaises(OSError, self.loop.run_until_complete, f)
 
     @mock.patch('asyncio.base_events.socket')
+    def test_create_server_nosoreuseport(self, m_socket):
+        m_socket.getaddrinfo = socket.getaddrinfo
+        m_socket.SOCK_STREAM = socket.SOCK_STREAM
+        m_socket.SOL_SOCKET = socket.SOL_SOCKET
+        del m_socket.SO_REUSEPORT
+        m_socket.socket.return_value = mock.Mock()
+
+        f = self.loop.create_server(
+            MyProto, '0.0.0.0', 0, reuse_port=True)
+
+        self.assertRaises(ValueError, self.loop.run_until_complete, f)
+
+    @mock.patch('asyncio.base_events.socket')
     def test_create_server_cant_bind(self, m_socket):
 
         class Err(OSError):
@@ -1199,6 +1213,128 @@
         self.assertRaises(Err, self.loop.run_until_complete, fut)
         self.assertTrue(m_sock.close.called)
 
+    def test_create_datagram_endpoint_sock(self):
+        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        fut = self.loop.create_datagram_endpoint(
+            lambda: MyDatagramProto(create_future=True, loop=self.loop),
+            sock=sock)
+        transport, protocol = self.loop.run_until_complete(fut)
+        transport.close()
+        self.loop.run_until_complete(protocol.done)
+        self.assertEqual('CLOSED', protocol.state)
+
+    def test_create_datagram_endpoint_sock_sockopts(self):
+        fut = self.loop.create_datagram_endpoint(
+            MyDatagramProto, local_addr=('127.0.0.1', 0), sock=object())
+        self.assertRaises(ValueError, self.loop.run_until_complete, fut)
+
+        fut = self.loop.create_datagram_endpoint(
+            MyDatagramProto, remote_addr=('127.0.0.1', 0), sock=object())
+        self.assertRaises(ValueError, self.loop.run_until_complete, fut)
+
+        fut = self.loop.create_datagram_endpoint(
+            MyDatagramProto, family=1, sock=object())
+        self.assertRaises(ValueError, self.loop.run_until_complete, fut)
+
+        fut = self.loop.create_datagram_endpoint(
+            MyDatagramProto, proto=1, sock=object())
+        self.assertRaises(ValueError, self.loop.run_until_complete, fut)
+
+        fut = self.loop.create_datagram_endpoint(
+            MyDatagramProto, flags=1, sock=object())
+        self.assertRaises(ValueError, self.loop.run_until_complete, fut)
+
+        fut = self.loop.create_datagram_endpoint(
+            MyDatagramProto, reuse_address=True, sock=object())
+        self.assertRaises(ValueError, self.loop.run_until_complete, fut)
+
+        fut = self.loop.create_datagram_endpoint(
+            MyDatagramProto, reuse_port=True, sock=object())
+        self.assertRaises(ValueError, self.loop.run_until_complete, fut)
+
+        fut = self.loop.create_datagram_endpoint(
+            MyDatagramProto, allow_broadcast=True, sock=object())
+        self.assertRaises(ValueError, self.loop.run_until_complete, fut)
+
+    def test_create_datagram_endpoint_sockopts(self):
+        # Socket options should not be applied unless asked for.
+        # SO_REUSEADDR defaults to on for UNIX.
+        # SO_REUSEPORT is not available on all platforms.
+
+        coro = self.loop.create_datagram_endpoint(
+            lambda: MyDatagramProto(create_future=True, loop=self.loop),
+            local_addr=('127.0.0.1', 0))
+        transport, protocol = self.loop.run_until_complete(coro)
+        sock = transport.get_extra_info('socket')
+
+        reuse_address_default_on = (
+            os.name == 'posix' and sys.platform != 'cygwin')
+        reuseport_supported = hasattr(socket, 'SO_REUSEPORT')
+
+        if reuse_address_default_on:
+            self.assertTrue(
+                sock.getsockopt(
+                    socket.SOL_SOCKET, socket.SO_REUSEADDR))
+        else:
+            self.assertFalse(
+                sock.getsockopt(
+                    socket.SOL_SOCKET, socket.SO_REUSEADDR))
+        if reuseport_supported:
+            self.assertFalse(
+                sock.getsockopt(
+                    socket.SOL_SOCKET, socket.SO_REUSEPORT))
+        self.assertFalse(
+            sock.getsockopt(
+                socket.SOL_SOCKET, socket.SO_BROADCAST))
+
+        transport.close()
+        self.loop.run_until_complete(protocol.done)
+        self.assertEqual('CLOSED', protocol.state)
+
+        coro = self.loop.create_datagram_endpoint(
+            lambda: MyDatagramProto(create_future=True, loop=self.loop),
+            local_addr=('127.0.0.1', 0),
+            reuse_address=True,
+            reuse_port=reuseport_supported,
+            allow_broadcast=True)
+        transport, protocol = self.loop.run_until_complete(coro)
+        sock = transport.get_extra_info('socket')
+
+        self.assertTrue(
+            sock.getsockopt(
+                socket.SOL_SOCKET, socket.SO_REUSEADDR))
+        if reuseport_supported:
+            self.assertTrue(
+                sock.getsockopt(
+                    socket.SOL_SOCKET, socket.SO_REUSEPORT))
+        else:
+            self.assertFalse(
+                sock.getsockopt(
+                    socket.SOL_SOCKET, socket.SO_REUSEPORT))
+        self.assertTrue(
+            sock.getsockopt(
+                socket.SOL_SOCKET, socket.SO_BROADCAST))
+
+        transport.close()
+        self.loop.run_until_complete(protocol.done)
+        self.assertEqual('CLOSED', protocol.state)
+
+    @mock.patch('asyncio.base_events.socket')
+    def test_create_datagram_endpoint_nosoreuseport(self, m_socket):
+        m_socket.getaddrinfo = socket.getaddrinfo
+        m_socket.SOCK_DGRAM = socket.SOCK_DGRAM
+        m_socket.SOL_SOCKET = socket.SOL_SOCKET
+        del m_socket.SO_REUSEPORT
+        m_socket.socket.return_value = mock.Mock()
+
+        coro = self.loop.create_datagram_endpoint(
+            lambda: MyDatagramProto(loop=self.loop),
+            local_addr=('127.0.0.1', 0),
+            reuse_address=False,
+            reuse_port=True)
+
+        self.assertRaises(ValueError, self.loop.run_until_complete, coro)
+
     def test_accept_connection_retry(self):
         sock = mock.Mock()
         sock.accept.side_effect = BlockingIOError()
diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py
--- a/Lib/test/test_asyncio/test_events.py
+++ b/Lib/test/test_asyncio/test_events.py
@@ -814,6 +814,32 @@
         # close server
         server.close()
 
+    @unittest.skipUnless(hasattr(socket, 'SO_REUSEPORT'), 'No SO_REUSEPORT')
+    def test_create_server_reuse_port(self):
+        proto = MyProto(self.loop)
+        f = self.loop.create_server(
+            lambda: proto, '0.0.0.0', 0)
+        server = self.loop.run_until_complete(f)
+        self.assertEqual(len(server.sockets), 1)
+        sock = server.sockets[0]
+        self.assertFalse(
+            sock.getsockopt(
+                socket.SOL_SOCKET, socket.SO_REUSEPORT))
+        server.close()
+
+        test_utils.run_briefly(self.loop)
+
+        proto = MyProto(self.loop)
+        f = self.loop.create_server(
+            lambda: proto, '0.0.0.0', 0, reuse_port=True)
+        server = self.loop.run_until_complete(f)
+        self.assertEqual(len(server.sockets), 1)
+        sock = server.sockets[0]
+        self.assertTrue(
+            sock.getsockopt(
+                socket.SOL_SOCKET, socket.SO_REUSEPORT))
+        server.close()
+
     def _make_unix_server(self, factory, **kwargs):
         path = test_utils.gen_unix_socket_path()
         self.addCleanup(lambda: os.path.exists(path) and os.unlink(path))
@@ -1264,6 +1290,32 @@
         self.assertEqual('CLOSED', client.state)
         server.transport.close()
 
+    def test_create_datagram_endpoint_sock(self):
+        sock = None
+        local_address = ('127.0.0.1', 0)
+        infos = self.loop.run_until_complete(
+            self.loop.getaddrinfo(
+                *local_address, type=socket.SOCK_DGRAM))
+        for family, type, proto, cname, address in infos:
+            try:
+                sock = socket.socket(family=family, type=type, proto=proto)
+                sock.setblocking(False)
+                sock.bind(address)
+            except:
+                pass
+            else:
+                break
+        else:
+            assert False, 'Can not create socket.'
+
+        f = self.loop.create_connection(
+            lambda: MyDatagramProto(loop=self.loop), sock=sock)
+        tr, pr = self.loop.run_until_complete(f)
+        self.assertIsInstance(tr, asyncio.Transport)
+        self.assertIsInstance(pr, MyDatagramProto)
+        tr.close()
+        self.loop.run_until_complete(pr.done)
+
     def test_internal_fds(self):
         loop = self.create_event_loop()
         if not isinstance(loop, selector_events.BaseSelectorEventLoop):
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -789,6 +789,7 @@
 Simon Law
 Julia Lawall
 Chris Lawrence
+Chris Laws
 Brian Leair
 Mathieu Leduc-Hamel
 Amandine Lee
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -90,6 +90,12 @@
 Library
 -------
 
+- Issue #23972: Updates asyncio datagram create method allowing reuseport
+  and reuseaddr socket options to be set prior to binding the socket.
+  Mirroring the existing asyncio create_server method the reuseaddr option
+  for datagram sockets defaults to True if the O/S is 'posix' (except if the
+  platform is Cygwin). Patch by Chris Laws.
+
 - Issue #25304: Add asyncio.run_coroutine_threadsafe().  This lets you
   submit a coroutine to a loop from another thread, returning a
   concurrent.futures.Future.  By Vincent Michel.

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Mon Oct  5 18:30:47 2015
From: python-checkins at python.org (guido.van.rossum)
Date: Mon, 05 Oct 2015 16:30:47 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
 =?utf-8?q?_Issue_=2323972=3A_updates_to_asyncio_datagram_API=2E_By_Chris_?=
 =?utf-8?b?TGF3cy4gKE1lcmdlIDMuNC0+My41Lik=?=
Message-ID: <20151005163044.3273.59596@psf.io>

https://hg.python.org/cpython/rev/ba956289fe66
changeset:   98540:ba956289fe66
branch:      3.5
parent:      98534:89a1e03b4639
parent:      98539:5e7e9b131904
user:        Guido van Rossum <guido at python.org>
date:        Mon Oct 05 09:19:11 2015 -0700
summary:
  Issue #23972: updates to asyncio datagram API. By Chris Laws. (Merge 3.4->3.5.)

files:
  Doc/library/asyncio-eventloop.rst         |   46 ++-
  Lib/asyncio/base_events.py                |  160 ++++++---
  Lib/asyncio/events.py                     |   40 ++-
  Lib/test/test_asyncio/test_base_events.py |  140 ++++++++-
  Lib/test/test_asyncio/test_events.py      |   52 +++
  Misc/ACKS                                 |    1 +
  Misc/NEWS                                 |    6 +
  7 files changed, 378 insertions(+), 67 deletions(-)


diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst
--- a/Doc/library/asyncio-eventloop.rst
+++ b/Doc/library/asyncio-eventloop.rst
@@ -285,17 +285,50 @@
       (:class:`StreamReader`, :class:`StreamWriter`) instead of a protocol.
 
 
-.. coroutinemethod:: BaseEventLoop.create_datagram_endpoint(protocol_factory, local_addr=None, remote_addr=None, \*, family=0, proto=0, flags=0)
+.. coroutinemethod:: BaseEventLoop.create_datagram_endpoint(protocol_factory, local_addr=None, remote_addr=None, \*, family=0, proto=0, flags=0, reuse_address=None, reuse_port=None, allow_broadcast=None, sock=None)
 
    Create datagram connection: socket family :py:data:`~socket.AF_INET` or
    :py:data:`~socket.AF_INET6` depending on *host* (or *family* if specified),
-   socket type :py:data:`~socket.SOCK_DGRAM`.
+   socket type :py:data:`~socket.SOCK_DGRAM`. *protocol_factory* must be a
+   callable returning a :ref:`protocol <asyncio-protocol>` instance.
 
    This method is a :ref:`coroutine <coroutine>` which will try to
    establish the connection in the background.  When successful, the
    coroutine returns a ``(transport, protocol)`` pair.
 
-   See the :meth:`BaseEventLoop.create_connection` method for parameters.
+   Options changing how the connection is created:
+
+   * *local_addr*, if given, is a ``(local_host, local_port)`` tuple used
+     to bind the socket to locally.  The *local_host* and *local_port*
+     are looked up using :meth:`getaddrinfo`.
+
+   * *remote_addr*, if given, is a ``(remote_host, remote_port)`` tuple used
+     to connect the socket to a remote address.  The *remote_host* and
+     *remote_port* are looked up using :meth:`getaddrinfo`.
+
+   * *family*, *proto*, *flags* are the optional address family, protocol
+     and flags to be passed through to :meth:`getaddrinfo` for *host*
+     resolution. If given, these should all be integers from the
+     corresponding :mod:`socket` module constants.
+
+   * *reuse_address* tells the kernel to reuse a local socket in
+     TIME_WAIT state, without waiting for its natural timeout to
+     expire. If not specified will automatically be set to True on
+     UNIX.
+
+   * *reuse_port* tells the kernel to allow this endpoint to be bound to the
+     same port as other existing endpoints are bound to, so long as they all
+     set this flag when being created. This option is not supported on Windows
+     and some UNIX's. If the :py:data:`~socket.SO_REUSEPORT` constant is not
+     defined then this capability is unsupported.
+
+   * *allow_broadcast* tells the kernel to allow this endpoint to send
+     messages to the broadcast address.
+
+   * *sock* can optionally be specified in order to use a preexisting,
+     already connected, :class:`socket.socket` object to be used by the
+     transport. If specified, *local_addr* and *remote_addr* should be omitted
+     (must be :const:`None`).
 
    On Windows with :class:`ProactorEventLoop`, this method is not supported.
 
@@ -322,7 +355,7 @@
 Creating listening connections
 ------------------------------
 
-.. coroutinemethod:: BaseEventLoop.create_server(protocol_factory, host=None, port=None, \*, family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE, sock=None, backlog=100, ssl=None, reuse_address=None)
+.. coroutinemethod:: BaseEventLoop.create_server(protocol_factory, host=None, port=None, \*, family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE, sock=None, backlog=100, ssl=None, reuse_address=None, reuse_port=None)
 
    Create a TCP server (socket type :data:`~socket.SOCK_STREAM`) bound to
    *host* and *port*.
@@ -361,6 +394,11 @@
      expire. If not specified will automatically be set to True on
      UNIX.
 
+   * *reuse_port* tells the kernel to allow this endpoint to be bound to the
+     same port as other existing endpoints are bound to, so long as they all
+     set this flag when being created. This option is not supported on
+     Windows.
+
    This method is a :ref:`coroutine <coroutine>`.
 
    .. versionchanged:: 3.5
diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py
--- a/Lib/asyncio/base_events.py
+++ b/Lib/asyncio/base_events.py
@@ -700,75 +700,109 @@
     @coroutine
     def create_datagram_endpoint(self, protocol_factory,
                                  local_addr=None, remote_addr=None, *,
-                                 family=0, proto=0, flags=0):
+                                 family=0, proto=0, flags=0,
+                                 reuse_address=None, reuse_port=None,
+                                 allow_broadcast=None, sock=None):
         """Create datagram connection."""
-        if not (local_addr or remote_addr):
-            if family == 0:
-                raise ValueError('unexpected address family')
-            addr_pairs_info = (((family, proto), (None, None)),)
+        if sock is not None:
+            if (local_addr or remote_addr or
+                    family or proto or flags or
+                    reuse_address or reuse_port or allow_broadcast):
+                # show the problematic kwargs in exception msg
+                opts = dict(local_addr=local_addr, remote_addr=remote_addr,
+                            family=family, proto=proto, flags=flags,
+                            reuse_address=reuse_address, reuse_port=reuse_port,
+                            allow_broadcast=allow_broadcast)
+                problems = ', '.join(
+                    '{}={}'.format(k, v) for k, v in opts.items() if v)
+                raise ValueError(
+                    'socket modifier keyword arguments can not be used '
+                    'when sock is specified. ({})'.format(problems))
+            sock.setblocking(False)
+            r_addr = None
         else:
-            # join address by (family, protocol)
-            addr_infos = collections.OrderedDict()
-            for idx, addr in ((0, local_addr), (1, remote_addr)):
-                if addr is not None:
-                    assert isinstance(addr, tuple) and len(addr) == 2, (
-                        '2-tuple is expected')
+            if not (local_addr or remote_addr):
+                if family == 0:
+                    raise ValueError('unexpected address family')
+                addr_pairs_info = (((family, proto), (None, None)),)
+            else:
+                # join address by (family, protocol)
+                addr_infos = collections.OrderedDict()
+                for idx, addr in ((0, local_addr), (1, remote_addr)):
+                    if addr is not None:
+                        assert isinstance(addr, tuple) and len(addr) == 2, (
+                            '2-tuple is expected')
 
-                    infos = yield from self.getaddrinfo(
-                        *addr, family=family, type=socket.SOCK_DGRAM,
-                        proto=proto, flags=flags)
-                    if not infos:
-                        raise OSError('getaddrinfo() returned empty list')
+                        infos = yield from self.getaddrinfo(
+                            *addr, family=family, type=socket.SOCK_DGRAM,
+                            proto=proto, flags=flags)
+                        if not infos:
+                            raise OSError('getaddrinfo() returned empty list')
 
-                    for fam, _, pro, _, address in infos:
-                        key = (fam, pro)
-                        if key not in addr_infos:
-                            addr_infos[key] = [None, None]
-                        addr_infos[key][idx] = address
+                        for fam, _, pro, _, address in infos:
+                            key = (fam, pro)
+                            if key not in addr_infos:
+                                addr_infos[key] = [None, None]
+                            addr_infos[key][idx] = address
 
-            # each addr has to have info for each (family, proto) pair
-            addr_pairs_info = [
-                (key, addr_pair) for key, addr_pair in addr_infos.items()
-                if not ((local_addr and addr_pair[0] is None) or
-                        (remote_addr and addr_pair[1] is None))]
+                # each addr has to have info for each (family, proto) pair
+                addr_pairs_info = [
+                    (key, addr_pair) for key, addr_pair in addr_infos.items()
+                    if not ((local_addr and addr_pair[0] is None) or
+                            (remote_addr and addr_pair[1] is None))]
 
-            if not addr_pairs_info:
-                raise ValueError('can not get address information')
+                if not addr_pairs_info:
+                    raise ValueError('can not get address information')
 
-        exceptions = []
+            exceptions = []
 
-        for ((family, proto),
-             (local_address, remote_address)) in addr_pairs_info:
-            sock = None
-            r_addr = None
-            try:
-                sock = socket.socket(
-                    family=family, type=socket.SOCK_DGRAM, proto=proto)
-                sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-                sock.setblocking(False)
+            if reuse_address is None:
+                reuse_address = os.name == 'posix' and sys.platform != 'cygwin'
 
-                if local_addr:
-                    sock.bind(local_address)
-                if remote_addr:
-                    yield from self.sock_connect(sock, remote_address)
-                    r_addr = remote_address
-            except OSError as exc:
-                if sock is not None:
-                    sock.close()
-                exceptions.append(exc)
-            except:
-                if sock is not None:
-                    sock.close()
-                raise
+            for ((family, proto),
+                 (local_address, remote_address)) in addr_pairs_info:
+                sock = None
+                r_addr = None
+                try:
+                    sock = socket.socket(
+                        family=family, type=socket.SOCK_DGRAM, proto=proto)
+                    if reuse_address:
+                        sock.setsockopt(
+                            socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+                    if reuse_port:
+                        if not hasattr(socket, 'SO_REUSEPORT'):
+                            raise ValueError(
+                                'reuse_port not supported by socket module')
+                        else:
+                            sock.setsockopt(
+                                socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+                    if allow_broadcast:
+                        sock.setsockopt(
+                            socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
+                    sock.setblocking(False)
+
+                    if local_addr:
+                        sock.bind(local_address)
+                    if remote_addr:
+                        yield from self.sock_connect(sock, remote_address)
+                        r_addr = remote_address
+                except OSError as exc:
+                    if sock is not None:
+                        sock.close()
+                    exceptions.append(exc)
+                except:
+                    if sock is not None:
+                        sock.close()
+                    raise
+                else:
+                    break
             else:
-                break
-        else:
-            raise exceptions[0]
+                raise exceptions[0]
 
         protocol = protocol_factory()
         waiter = futures.Future(loop=self)
-        transport = self._make_datagram_transport(sock, protocol, r_addr,
-                                                  waiter)
+        transport = self._make_datagram_transport(
+            sock, protocol, r_addr, waiter)
         if self._debug:
             if local_addr:
                 logger.info("Datagram endpoint local_addr=%r remote_addr=%r "
@@ -804,7 +838,8 @@
                       sock=None,
                       backlog=100,
                       ssl=None,
-                      reuse_address=None):
+                      reuse_address=None,
+                      reuse_port=None):
         """Create a TCP server.
 
         The host parameter can be a string, in that case the TCP server is bound
@@ -857,8 +892,15 @@
                         continue
                     sockets.append(sock)
                     if reuse_address:
-                        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,
-                                        True)
+                        sock.setsockopt(
+                            socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
+                    if reuse_port:
+                        if not hasattr(socket, 'SO_REUSEPORT'):
+                            raise ValueError(
+                                'reuse_port not supported by socket module')
+                        else:
+                            sock.setsockopt(
+                                socket.SOL_SOCKET, socket.SO_REUSEPORT, True)
                     # Disable IPv4/IPv6 dual stack support (enabled by
                     # default on Linux) which makes a single socket
                     # listen on both address families.
diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py
--- a/Lib/asyncio/events.py
+++ b/Lib/asyncio/events.py
@@ -297,7 +297,8 @@
 
     def create_server(self, protocol_factory, host=None, port=None, *,
                       family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE,
-                      sock=None, backlog=100, ssl=None, reuse_address=None):
+                      sock=None, backlog=100, ssl=None, reuse_address=None,
+                      reuse_port=None):
         """A coroutine which creates a TCP server bound to host and port.
 
         The return value is a Server object which can be used to stop
@@ -327,6 +328,11 @@
         TIME_WAIT state, without waiting for its natural timeout to
         expire. If not specified will automatically be set to True on
         UNIX.
+
+        reuse_port tells the kernel to allow this endpoint to be bound to
+        the same port as other existing endpoints are bound to, so long as
+        they all set this flag when being created. This option is not
+        supported on Windows.
         """
         raise NotImplementedError
 
@@ -358,7 +364,37 @@
 
     def create_datagram_endpoint(self, protocol_factory,
                                  local_addr=None, remote_addr=None, *,
-                                 family=0, proto=0, flags=0):
+                                 family=0, proto=0, flags=0,
+                                 reuse_address=None, reuse_port=None,
+                                 allow_broadcast=None, sock=None):
+        """A coroutine which creates a datagram endpoint.
+
+        This method will try to establish the endpoint in the background.
+        When successful, the coroutine returns a (transport, protocol) pair.
+
+        protocol_factory must be a callable returning a protocol instance.
+
+        socket family AF_INET or socket.AF_INET6 depending on host (or
+        family if specified), socket type SOCK_DGRAM.
+
+        reuse_address tells the kernel to reuse a local socket in
+        TIME_WAIT state, without waiting for its natural timeout to
+        expire. If not specified it will automatically be set to True on
+        UNIX.
+
+        reuse_port tells the kernel to allow this endpoint to be bound to
+        the same port as other existing endpoints are bound to, so long as
+        they all set this flag when being created. This option is not
+        supported on Windows and some UNIX's. If the
+        :py:data:`~socket.SO_REUSEPORT` constant is not defined then this
+        capability is unsupported.
+
+        allow_broadcast tells the kernel to allow this endpoint to send
+        messages to the broadcast address.
+
+        sock can optionally be specified in order to use a preexisting
+        socket object.
+        """
         raise NotImplementedError
 
     # Pipes and subprocesses.
diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py
--- a/Lib/test/test_asyncio/test_base_events.py
+++ b/Lib/test/test_asyncio/test_base_events.py
@@ -3,6 +3,7 @@
 import errno
 import logging
 import math
+import os
 import socket
 import sys
 import threading
@@ -790,11 +791,11 @@
 class MyDatagramProto(asyncio.DatagramProtocol):
     done = None
 
-    def __init__(self, create_future=False):
+    def __init__(self, create_future=False, loop=None):
         self.state = 'INITIAL'
         self.nbytes = 0
         if create_future:
-            self.done = asyncio.Future()
+            self.done = asyncio.Future(loop=loop)
 
     def connection_made(self, transport):
         self.transport = transport
@@ -1100,6 +1101,19 @@
         self.assertRaises(OSError, self.loop.run_until_complete, f)
 
     @mock.patch('asyncio.base_events.socket')
+    def test_create_server_nosoreuseport(self, m_socket):
+        m_socket.getaddrinfo = socket.getaddrinfo
+        m_socket.SOCK_STREAM = socket.SOCK_STREAM
+        m_socket.SOL_SOCKET = socket.SOL_SOCKET
+        del m_socket.SO_REUSEPORT
+        m_socket.socket.return_value = mock.Mock()
+
+        f = self.loop.create_server(
+            MyProto, '0.0.0.0', 0, reuse_port=True)
+
+        self.assertRaises(ValueError, self.loop.run_until_complete, f)
+
+    @mock.patch('asyncio.base_events.socket')
     def test_create_server_cant_bind(self, m_socket):
 
         class Err(OSError):
@@ -1199,6 +1213,128 @@
         self.assertRaises(Err, self.loop.run_until_complete, fut)
         self.assertTrue(m_sock.close.called)
 
+    def test_create_datagram_endpoint_sock(self):
+        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        fut = self.loop.create_datagram_endpoint(
+            lambda: MyDatagramProto(create_future=True, loop=self.loop),
+            sock=sock)
+        transport, protocol = self.loop.run_until_complete(fut)
+        transport.close()
+        self.loop.run_until_complete(protocol.done)
+        self.assertEqual('CLOSED', protocol.state)
+
+    def test_create_datagram_endpoint_sock_sockopts(self):
+        fut = self.loop.create_datagram_endpoint(
+            MyDatagramProto, local_addr=('127.0.0.1', 0), sock=object())
+        self.assertRaises(ValueError, self.loop.run_until_complete, fut)
+
+        fut = self.loop.create_datagram_endpoint(
+            MyDatagramProto, remote_addr=('127.0.0.1', 0), sock=object())
+        self.assertRaises(ValueError, self.loop.run_until_complete, fut)
+
+        fut = self.loop.create_datagram_endpoint(
+            MyDatagramProto, family=1, sock=object())
+        self.assertRaises(ValueError, self.loop.run_until_complete, fut)
+
+        fut = self.loop.create_datagram_endpoint(
+            MyDatagramProto, proto=1, sock=object())
+        self.assertRaises(ValueError, self.loop.run_until_complete, fut)
+
+        fut = self.loop.create_datagram_endpoint(
+            MyDatagramProto, flags=1, sock=object())
+        self.assertRaises(ValueError, self.loop.run_until_complete, fut)
+
+        fut = self.loop.create_datagram_endpoint(
+            MyDatagramProto, reuse_address=True, sock=object())
+        self.assertRaises(ValueError, self.loop.run_until_complete, fut)
+
+        fut = self.loop.create_datagram_endpoint(
+            MyDatagramProto, reuse_port=True, sock=object())
+        self.assertRaises(ValueError, self.loop.run_until_complete, fut)
+
+        fut = self.loop.create_datagram_endpoint(
+            MyDatagramProto, allow_broadcast=True, sock=object())
+        self.assertRaises(ValueError, self.loop.run_until_complete, fut)
+
+    def test_create_datagram_endpoint_sockopts(self):
+        # Socket options should not be applied unless asked for.
+        # SO_REUSEADDR defaults to on for UNIX.
+        # SO_REUSEPORT is not available on all platforms.
+
+        coro = self.loop.create_datagram_endpoint(
+            lambda: MyDatagramProto(create_future=True, loop=self.loop),
+            local_addr=('127.0.0.1', 0))
+        transport, protocol = self.loop.run_until_complete(coro)
+        sock = transport.get_extra_info('socket')
+
+        reuse_address_default_on = (
+            os.name == 'posix' and sys.platform != 'cygwin')
+        reuseport_supported = hasattr(socket, 'SO_REUSEPORT')
+
+        if reuse_address_default_on:
+            self.assertTrue(
+                sock.getsockopt(
+                    socket.SOL_SOCKET, socket.SO_REUSEADDR))
+        else:
+            self.assertFalse(
+                sock.getsockopt(
+                    socket.SOL_SOCKET, socket.SO_REUSEADDR))
+        if reuseport_supported:
+            self.assertFalse(
+                sock.getsockopt(
+                    socket.SOL_SOCKET, socket.SO_REUSEPORT))
+        self.assertFalse(
+            sock.getsockopt(
+                socket.SOL_SOCKET, socket.SO_BROADCAST))
+
+        transport.close()
+        self.loop.run_until_complete(protocol.done)
+        self.assertEqual('CLOSED', protocol.state)
+
+        coro = self.loop.create_datagram_endpoint(
+            lambda: MyDatagramProto(create_future=True, loop=self.loop),
+            local_addr=('127.0.0.1', 0),
+            reuse_address=True,
+            reuse_port=reuseport_supported,
+            allow_broadcast=True)
+        transport, protocol = self.loop.run_until_complete(coro)
+        sock = transport.get_extra_info('socket')
+
+        self.assertTrue(
+            sock.getsockopt(
+                socket.SOL_SOCKET, socket.SO_REUSEADDR))
+        if reuseport_supported:
+            self.assertTrue(
+                sock.getsockopt(
+                    socket.SOL_SOCKET, socket.SO_REUSEPORT))
+        else:
+            self.assertFalse(
+                sock.getsockopt(
+                    socket.SOL_SOCKET, socket.SO_REUSEPORT))
+        self.assertTrue(
+            sock.getsockopt(
+                socket.SOL_SOCKET, socket.SO_BROADCAST))
+
+        transport.close()
+        self.loop.run_until_complete(protocol.done)
+        self.assertEqual('CLOSED', protocol.state)
+
+    @mock.patch('asyncio.base_events.socket')
+    def test_create_datagram_endpoint_nosoreuseport(self, m_socket):
+        m_socket.getaddrinfo = socket.getaddrinfo
+        m_socket.SOCK_DGRAM = socket.SOCK_DGRAM
+        m_socket.SOL_SOCKET = socket.SOL_SOCKET
+        del m_socket.SO_REUSEPORT
+        m_socket.socket.return_value = mock.Mock()
+
+        coro = self.loop.create_datagram_endpoint(
+            lambda: MyDatagramProto(loop=self.loop),
+            local_addr=('127.0.0.1', 0),
+            reuse_address=False,
+            reuse_port=True)
+
+        self.assertRaises(ValueError, self.loop.run_until_complete, coro)
+
     def test_accept_connection_retry(self):
         sock = mock.Mock()
         sock.accept.side_effect = BlockingIOError()
diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py
--- a/Lib/test/test_asyncio/test_events.py
+++ b/Lib/test/test_asyncio/test_events.py
@@ -814,6 +814,32 @@
         # close server
         server.close()
 
+    @unittest.skipUnless(hasattr(socket, 'SO_REUSEPORT'), 'No SO_REUSEPORT')
+    def test_create_server_reuse_port(self):
+        proto = MyProto(self.loop)
+        f = self.loop.create_server(
+            lambda: proto, '0.0.0.0', 0)
+        server = self.loop.run_until_complete(f)
+        self.assertEqual(len(server.sockets), 1)
+        sock = server.sockets[0]
+        self.assertFalse(
+            sock.getsockopt(
+                socket.SOL_SOCKET, socket.SO_REUSEPORT))
+        server.close()
+
+        test_utils.run_briefly(self.loop)
+
+        proto = MyProto(self.loop)
+        f = self.loop.create_server(
+            lambda: proto, '0.0.0.0', 0, reuse_port=True)
+        server = self.loop.run_until_complete(f)
+        self.assertEqual(len(server.sockets), 1)
+        sock = server.sockets[0]
+        self.assertTrue(
+            sock.getsockopt(
+                socket.SOL_SOCKET, socket.SO_REUSEPORT))
+        server.close()
+
     def _make_unix_server(self, factory, **kwargs):
         path = test_utils.gen_unix_socket_path()
         self.addCleanup(lambda: os.path.exists(path) and os.unlink(path))
@@ -1264,6 +1290,32 @@
         self.assertEqual('CLOSED', client.state)
         server.transport.close()
 
+    def test_create_datagram_endpoint_sock(self):
+        sock = None
+        local_address = ('127.0.0.1', 0)
+        infos = self.loop.run_until_complete(
+            self.loop.getaddrinfo(
+                *local_address, type=socket.SOCK_DGRAM))
+        for family, type, proto, cname, address in infos:
+            try:
+                sock = socket.socket(family=family, type=type, proto=proto)
+                sock.setblocking(False)
+                sock.bind(address)
+            except:
+                pass
+            else:
+                break
+        else:
+            assert False, 'Can not create socket.'
+
+        f = self.loop.create_connection(
+            lambda: MyDatagramProto(loop=self.loop), sock=sock)
+        tr, pr = self.loop.run_until_complete(f)
+        self.assertIsInstance(tr, asyncio.Transport)
+        self.assertIsInstance(pr, MyDatagramProto)
+        tr.close()
+        self.loop.run_until_complete(pr.done)
+
     def test_internal_fds(self):
         loop = self.create_event_loop()
         if not isinstance(loop, selector_events.BaseSelectorEventLoop):
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -813,6 +813,7 @@
 Julia Lawall
 Chris Lawrence
 Mark Lawrence
+Chris Laws
 Brian Leair
 Mathieu Leduc-Hamel
 Amandine Lee
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -34,6 +34,12 @@
 Library
 -------
 
+- Issue #23972: Updates asyncio datagram create method allowing reuseport
+  and reuseaddr socket options to be set prior to binding the socket.
+  Mirroring the existing asyncio create_server method the reuseaddr option
+  for datagram sockets defaults to True if the O/S is 'posix' (except if the
+  platform is Cygwin). Patch by Chris Laws.
+
 - Issue #25304: Add asyncio.run_coroutine_threadsafe().  This lets you
   submit a coroutine to a loop from another thread, returning a
   concurrent.futures.Future.  By Vincent Michel.

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Mon Oct  5 18:30:49 2015
From: python-checkins at python.org (guido.van.rossum)
Date: Mon, 05 Oct 2015 16:30:49 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2323972=3A_updates_to_asyncio_datagram_API=2E_By_?=
 =?utf-8?b?Q2hyaXMgTGF3cy4gKE1lcmdlIDMuNS0+My42Lik=?=
Message-ID: <20151005163045.128852.41658@psf.io>

https://hg.python.org/cpython/rev/c0f1f882737c
changeset:   98541:c0f1f882737c
parent:      98538:5b9ffea7e7c3
parent:      98540:ba956289fe66
user:        Guido van Rossum <guido at dropbox.com>
date:        Mon Oct 05 09:29:32 2015 -0700
summary:
  Issue #23972: updates to asyncio datagram API. By Chris Laws. (Merge 3.5->3.6.)

files:
  Doc/library/asyncio-eventloop.rst         |   46 ++-
  Lib/asyncio/base_events.py                |  160 ++++++---
  Lib/asyncio/events.py                     |   40 ++-
  Lib/test/test_asyncio/test_base_events.py |  140 ++++++++-
  Lib/test/test_asyncio/test_events.py      |   52 +++
  Misc/ACKS                                 |    1 +
  Misc/NEWS                                 |    6 +
  7 files changed, 378 insertions(+), 67 deletions(-)


diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst
--- a/Doc/library/asyncio-eventloop.rst
+++ b/Doc/library/asyncio-eventloop.rst
@@ -285,17 +285,50 @@
       (:class:`StreamReader`, :class:`StreamWriter`) instead of a protocol.
 
 
-.. coroutinemethod:: BaseEventLoop.create_datagram_endpoint(protocol_factory, local_addr=None, remote_addr=None, \*, family=0, proto=0, flags=0)
+.. coroutinemethod:: BaseEventLoop.create_datagram_endpoint(protocol_factory, local_addr=None, remote_addr=None, \*, family=0, proto=0, flags=0, reuse_address=None, reuse_port=None, allow_broadcast=None, sock=None)
 
    Create datagram connection: socket family :py:data:`~socket.AF_INET` or
    :py:data:`~socket.AF_INET6` depending on *host* (or *family* if specified),
-   socket type :py:data:`~socket.SOCK_DGRAM`.
+   socket type :py:data:`~socket.SOCK_DGRAM`. *protocol_factory* must be a
+   callable returning a :ref:`protocol <asyncio-protocol>` instance.
 
    This method is a :ref:`coroutine <coroutine>` which will try to
    establish the connection in the background.  When successful, the
    coroutine returns a ``(transport, protocol)`` pair.
 
-   See the :meth:`BaseEventLoop.create_connection` method for parameters.
+   Options changing how the connection is created:
+
+   * *local_addr*, if given, is a ``(local_host, local_port)`` tuple used
+     to bind the socket to locally.  The *local_host* and *local_port*
+     are looked up using :meth:`getaddrinfo`.
+
+   * *remote_addr*, if given, is a ``(remote_host, remote_port)`` tuple used
+     to connect the socket to a remote address.  The *remote_host* and
+     *remote_port* are looked up using :meth:`getaddrinfo`.
+
+   * *family*, *proto*, *flags* are the optional address family, protocol
+     and flags to be passed through to :meth:`getaddrinfo` for *host*
+     resolution. If given, these should all be integers from the
+     corresponding :mod:`socket` module constants.
+
+   * *reuse_address* tells the kernel to reuse a local socket in
+     TIME_WAIT state, without waiting for its natural timeout to
+     expire. If not specified will automatically be set to True on
+     UNIX.
+
+   * *reuse_port* tells the kernel to allow this endpoint to be bound to the
+     same port as other existing endpoints are bound to, so long as they all
+     set this flag when being created. This option is not supported on Windows
+     and some UNIX's. If the :py:data:`~socket.SO_REUSEPORT` constant is not
+     defined then this capability is unsupported.
+
+   * *allow_broadcast* tells the kernel to allow this endpoint to send
+     messages to the broadcast address.
+
+   * *sock* can optionally be specified in order to use a preexisting,
+     already connected, :class:`socket.socket` object to be used by the
+     transport. If specified, *local_addr* and *remote_addr* should be omitted
+     (must be :const:`None`).
 
    On Windows with :class:`ProactorEventLoop`, this method is not supported.
 
@@ -322,7 +355,7 @@
 Creating listening connections
 ------------------------------
 
-.. coroutinemethod:: BaseEventLoop.create_server(protocol_factory, host=None, port=None, \*, family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE, sock=None, backlog=100, ssl=None, reuse_address=None)
+.. coroutinemethod:: BaseEventLoop.create_server(protocol_factory, host=None, port=None, \*, family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE, sock=None, backlog=100, ssl=None, reuse_address=None, reuse_port=None)
 
    Create a TCP server (socket type :data:`~socket.SOCK_STREAM`) bound to
    *host* and *port*.
@@ -361,6 +394,11 @@
      expire. If not specified will automatically be set to True on
      UNIX.
 
+   * *reuse_port* tells the kernel to allow this endpoint to be bound to the
+     same port as other existing endpoints are bound to, so long as they all
+     set this flag when being created. This option is not supported on
+     Windows.
+
    This method is a :ref:`coroutine <coroutine>`.
 
    .. versionchanged:: 3.5
diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py
--- a/Lib/asyncio/base_events.py
+++ b/Lib/asyncio/base_events.py
@@ -700,75 +700,109 @@
     @coroutine
     def create_datagram_endpoint(self, protocol_factory,
                                  local_addr=None, remote_addr=None, *,
-                                 family=0, proto=0, flags=0):
+                                 family=0, proto=0, flags=0,
+                                 reuse_address=None, reuse_port=None,
+                                 allow_broadcast=None, sock=None):
         """Create datagram connection."""
-        if not (local_addr or remote_addr):
-            if family == 0:
-                raise ValueError('unexpected address family')
-            addr_pairs_info = (((family, proto), (None, None)),)
+        if sock is not None:
+            if (local_addr or remote_addr or
+                    family or proto or flags or
+                    reuse_address or reuse_port or allow_broadcast):
+                # show the problematic kwargs in exception msg
+                opts = dict(local_addr=local_addr, remote_addr=remote_addr,
+                            family=family, proto=proto, flags=flags,
+                            reuse_address=reuse_address, reuse_port=reuse_port,
+                            allow_broadcast=allow_broadcast)
+                problems = ', '.join(
+                    '{}={}'.format(k, v) for k, v in opts.items() if v)
+                raise ValueError(
+                    'socket modifier keyword arguments can not be used '
+                    'when sock is specified. ({})'.format(problems))
+            sock.setblocking(False)
+            r_addr = None
         else:
-            # join address by (family, protocol)
-            addr_infos = collections.OrderedDict()
-            for idx, addr in ((0, local_addr), (1, remote_addr)):
-                if addr is not None:
-                    assert isinstance(addr, tuple) and len(addr) == 2, (
-                        '2-tuple is expected')
+            if not (local_addr or remote_addr):
+                if family == 0:
+                    raise ValueError('unexpected address family')
+                addr_pairs_info = (((family, proto), (None, None)),)
+            else:
+                # join address by (family, protocol)
+                addr_infos = collections.OrderedDict()
+                for idx, addr in ((0, local_addr), (1, remote_addr)):
+                    if addr is not None:
+                        assert isinstance(addr, tuple) and len(addr) == 2, (
+                            '2-tuple is expected')
 
-                    infos = yield from self.getaddrinfo(
-                        *addr, family=family, type=socket.SOCK_DGRAM,
-                        proto=proto, flags=flags)
-                    if not infos:
-                        raise OSError('getaddrinfo() returned empty list')
+                        infos = yield from self.getaddrinfo(
+                            *addr, family=family, type=socket.SOCK_DGRAM,
+                            proto=proto, flags=flags)
+                        if not infos:
+                            raise OSError('getaddrinfo() returned empty list')
 
-                    for fam, _, pro, _, address in infos:
-                        key = (fam, pro)
-                        if key not in addr_infos:
-                            addr_infos[key] = [None, None]
-                        addr_infos[key][idx] = address
+                        for fam, _, pro, _, address in infos:
+                            key = (fam, pro)
+                            if key not in addr_infos:
+                                addr_infos[key] = [None, None]
+                            addr_infos[key][idx] = address
 
-            # each addr has to have info for each (family, proto) pair
-            addr_pairs_info = [
-                (key, addr_pair) for key, addr_pair in addr_infos.items()
-                if not ((local_addr and addr_pair[0] is None) or
-                        (remote_addr and addr_pair[1] is None))]
+                # each addr has to have info for each (family, proto) pair
+                addr_pairs_info = [
+                    (key, addr_pair) for key, addr_pair in addr_infos.items()
+                    if not ((local_addr and addr_pair[0] is None) or
+                            (remote_addr and addr_pair[1] is None))]
 
-            if not addr_pairs_info:
-                raise ValueError('can not get address information')
+                if not addr_pairs_info:
+                    raise ValueError('can not get address information')
 
-        exceptions = []
+            exceptions = []
 
-        for ((family, proto),
-             (local_address, remote_address)) in addr_pairs_info:
-            sock = None
-            r_addr = None
-            try:
-                sock = socket.socket(
-                    family=family, type=socket.SOCK_DGRAM, proto=proto)
-                sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-                sock.setblocking(False)
+            if reuse_address is None:
+                reuse_address = os.name == 'posix' and sys.platform != 'cygwin'
 
-                if local_addr:
-                    sock.bind(local_address)
-                if remote_addr:
-                    yield from self.sock_connect(sock, remote_address)
-                    r_addr = remote_address
-            except OSError as exc:
-                if sock is not None:
-                    sock.close()
-                exceptions.append(exc)
-            except:
-                if sock is not None:
-                    sock.close()
-                raise
+            for ((family, proto),
+                 (local_address, remote_address)) in addr_pairs_info:
+                sock = None
+                r_addr = None
+                try:
+                    sock = socket.socket(
+                        family=family, type=socket.SOCK_DGRAM, proto=proto)
+                    if reuse_address:
+                        sock.setsockopt(
+                            socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+                    if reuse_port:
+                        if not hasattr(socket, 'SO_REUSEPORT'):
+                            raise ValueError(
+                                'reuse_port not supported by socket module')
+                        else:
+                            sock.setsockopt(
+                                socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+                    if allow_broadcast:
+                        sock.setsockopt(
+                            socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
+                    sock.setblocking(False)
+
+                    if local_addr:
+                        sock.bind(local_address)
+                    if remote_addr:
+                        yield from self.sock_connect(sock, remote_address)
+                        r_addr = remote_address
+                except OSError as exc:
+                    if sock is not None:
+                        sock.close()
+                    exceptions.append(exc)
+                except:
+                    if sock is not None:
+                        sock.close()
+                    raise
+                else:
+                    break
             else:
-                break
-        else:
-            raise exceptions[0]
+                raise exceptions[0]
 
         protocol = protocol_factory()
         waiter = futures.Future(loop=self)
-        transport = self._make_datagram_transport(sock, protocol, r_addr,
-                                                  waiter)
+        transport = self._make_datagram_transport(
+            sock, protocol, r_addr, waiter)
         if self._debug:
             if local_addr:
                 logger.info("Datagram endpoint local_addr=%r remote_addr=%r "
@@ -804,7 +838,8 @@
                       sock=None,
                       backlog=100,
                       ssl=None,
-                      reuse_address=None):
+                      reuse_address=None,
+                      reuse_port=None):
         """Create a TCP server.
 
         The host parameter can be a string, in that case the TCP server is bound
@@ -857,8 +892,15 @@
                         continue
                     sockets.append(sock)
                     if reuse_address:
-                        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,
-                                        True)
+                        sock.setsockopt(
+                            socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
+                    if reuse_port:
+                        if not hasattr(socket, 'SO_REUSEPORT'):
+                            raise ValueError(
+                                'reuse_port not supported by socket module')
+                        else:
+                            sock.setsockopt(
+                                socket.SOL_SOCKET, socket.SO_REUSEPORT, True)
                     # Disable IPv4/IPv6 dual stack support (enabled by
                     # default on Linux) which makes a single socket
                     # listen on both address families.
diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py
--- a/Lib/asyncio/events.py
+++ b/Lib/asyncio/events.py
@@ -297,7 +297,8 @@
 
     def create_server(self, protocol_factory, host=None, port=None, *,
                       family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE,
-                      sock=None, backlog=100, ssl=None, reuse_address=None):
+                      sock=None, backlog=100, ssl=None, reuse_address=None,
+                      reuse_port=None):
         """A coroutine which creates a TCP server bound to host and port.
 
         The return value is a Server object which can be used to stop
@@ -327,6 +328,11 @@
         TIME_WAIT state, without waiting for its natural timeout to
         expire. If not specified will automatically be set to True on
         UNIX.
+
+        reuse_port tells the kernel to allow this endpoint to be bound to
+        the same port as other existing endpoints are bound to, so long as
+        they all set this flag when being created. This option is not
+        supported on Windows.
         """
         raise NotImplementedError
 
@@ -358,7 +364,37 @@
 
     def create_datagram_endpoint(self, protocol_factory,
                                  local_addr=None, remote_addr=None, *,
-                                 family=0, proto=0, flags=0):
+                                 family=0, proto=0, flags=0,
+                                 reuse_address=None, reuse_port=None,
+                                 allow_broadcast=None, sock=None):
+        """A coroutine which creates a datagram endpoint.
+
+        This method will try to establish the endpoint in the background.
+        When successful, the coroutine returns a (transport, protocol) pair.
+
+        protocol_factory must be a callable returning a protocol instance.
+
+        socket family AF_INET or socket.AF_INET6 depending on host (or
+        family if specified), socket type SOCK_DGRAM.
+
+        reuse_address tells the kernel to reuse a local socket in
+        TIME_WAIT state, without waiting for its natural timeout to
+        expire. If not specified it will automatically be set to True on
+        UNIX.
+
+        reuse_port tells the kernel to allow this endpoint to be bound to
+        the same port as other existing endpoints are bound to, so long as
+        they all set this flag when being created. This option is not
+        supported on Windows and some UNIX's. If the
+        :py:data:`~socket.SO_REUSEPORT` constant is not defined then this
+        capability is unsupported.
+
+        allow_broadcast tells the kernel to allow this endpoint to send
+        messages to the broadcast address.
+
+        sock can optionally be specified in order to use a preexisting
+        socket object.
+        """
         raise NotImplementedError
 
     # Pipes and subprocesses.
diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py
--- a/Lib/test/test_asyncio/test_base_events.py
+++ b/Lib/test/test_asyncio/test_base_events.py
@@ -3,6 +3,7 @@
 import errno
 import logging
 import math
+import os
 import socket
 import sys
 import threading
@@ -790,11 +791,11 @@
 class MyDatagramProto(asyncio.DatagramProtocol):
     done = None
 
-    def __init__(self, create_future=False):
+    def __init__(self, create_future=False, loop=None):
         self.state = 'INITIAL'
         self.nbytes = 0
         if create_future:
-            self.done = asyncio.Future()
+            self.done = asyncio.Future(loop=loop)
 
     def connection_made(self, transport):
         self.transport = transport
@@ -1100,6 +1101,19 @@
         self.assertRaises(OSError, self.loop.run_until_complete, f)
 
     @mock.patch('asyncio.base_events.socket')
+    def test_create_server_nosoreuseport(self, m_socket):
+        m_socket.getaddrinfo = socket.getaddrinfo
+        m_socket.SOCK_STREAM = socket.SOCK_STREAM
+        m_socket.SOL_SOCKET = socket.SOL_SOCKET
+        del m_socket.SO_REUSEPORT
+        m_socket.socket.return_value = mock.Mock()
+
+        f = self.loop.create_server(
+            MyProto, '0.0.0.0', 0, reuse_port=True)
+
+        self.assertRaises(ValueError, self.loop.run_until_complete, f)
+
+    @mock.patch('asyncio.base_events.socket')
     def test_create_server_cant_bind(self, m_socket):
 
         class Err(OSError):
@@ -1199,6 +1213,128 @@
         self.assertRaises(Err, self.loop.run_until_complete, fut)
         self.assertTrue(m_sock.close.called)
 
+    def test_create_datagram_endpoint_sock(self):
+        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        fut = self.loop.create_datagram_endpoint(
+            lambda: MyDatagramProto(create_future=True, loop=self.loop),
+            sock=sock)
+        transport, protocol = self.loop.run_until_complete(fut)
+        transport.close()
+        self.loop.run_until_complete(protocol.done)
+        self.assertEqual('CLOSED', protocol.state)
+
+    def test_create_datagram_endpoint_sock_sockopts(self):
+        fut = self.loop.create_datagram_endpoint(
+            MyDatagramProto, local_addr=('127.0.0.1', 0), sock=object())
+        self.assertRaises(ValueError, self.loop.run_until_complete, fut)
+
+        fut = self.loop.create_datagram_endpoint(
+            MyDatagramProto, remote_addr=('127.0.0.1', 0), sock=object())
+        self.assertRaises(ValueError, self.loop.run_until_complete, fut)
+
+        fut = self.loop.create_datagram_endpoint(
+            MyDatagramProto, family=1, sock=object())
+        self.assertRaises(ValueError, self.loop.run_until_complete, fut)
+
+        fut = self.loop.create_datagram_endpoint(
+            MyDatagramProto, proto=1, sock=object())
+        self.assertRaises(ValueError, self.loop.run_until_complete, fut)
+
+        fut = self.loop.create_datagram_endpoint(
+            MyDatagramProto, flags=1, sock=object())
+        self.assertRaises(ValueError, self.loop.run_until_complete, fut)
+
+        fut = self.loop.create_datagram_endpoint(
+            MyDatagramProto, reuse_address=True, sock=object())
+        self.assertRaises(ValueError, self.loop.run_until_complete, fut)
+
+        fut = self.loop.create_datagram_endpoint(
+            MyDatagramProto, reuse_port=True, sock=object())
+        self.assertRaises(ValueError, self.loop.run_until_complete, fut)
+
+        fut = self.loop.create_datagram_endpoint(
+            MyDatagramProto, allow_broadcast=True, sock=object())
+        self.assertRaises(ValueError, self.loop.run_until_complete, fut)
+
+    def test_create_datagram_endpoint_sockopts(self):
+        # Socket options should not be applied unless asked for.
+        # SO_REUSEADDR defaults to on for UNIX.
+        # SO_REUSEPORT is not available on all platforms.
+
+        coro = self.loop.create_datagram_endpoint(
+            lambda: MyDatagramProto(create_future=True, loop=self.loop),
+            local_addr=('127.0.0.1', 0))
+        transport, protocol = self.loop.run_until_complete(coro)
+        sock = transport.get_extra_info('socket')
+
+        reuse_address_default_on = (
+            os.name == 'posix' and sys.platform != 'cygwin')
+        reuseport_supported = hasattr(socket, 'SO_REUSEPORT')
+
+        if reuse_address_default_on:
+            self.assertTrue(
+                sock.getsockopt(
+                    socket.SOL_SOCKET, socket.SO_REUSEADDR))
+        else:
+            self.assertFalse(
+                sock.getsockopt(
+                    socket.SOL_SOCKET, socket.SO_REUSEADDR))
+        if reuseport_supported:
+            self.assertFalse(
+                sock.getsockopt(
+                    socket.SOL_SOCKET, socket.SO_REUSEPORT))
+        self.assertFalse(
+            sock.getsockopt(
+                socket.SOL_SOCKET, socket.SO_BROADCAST))
+
+        transport.close()
+        self.loop.run_until_complete(protocol.done)
+        self.assertEqual('CLOSED', protocol.state)
+
+        coro = self.loop.create_datagram_endpoint(
+            lambda: MyDatagramProto(create_future=True, loop=self.loop),
+            local_addr=('127.0.0.1', 0),
+            reuse_address=True,
+            reuse_port=reuseport_supported,
+            allow_broadcast=True)
+        transport, protocol = self.loop.run_until_complete(coro)
+        sock = transport.get_extra_info('socket')
+
+        self.assertTrue(
+            sock.getsockopt(
+                socket.SOL_SOCKET, socket.SO_REUSEADDR))
+        if reuseport_supported:
+            self.assertTrue(
+                sock.getsockopt(
+                    socket.SOL_SOCKET, socket.SO_REUSEPORT))
+        else:
+            self.assertFalse(
+                sock.getsockopt(
+                    socket.SOL_SOCKET, socket.SO_REUSEPORT))
+        self.assertTrue(
+            sock.getsockopt(
+                socket.SOL_SOCKET, socket.SO_BROADCAST))
+
+        transport.close()
+        self.loop.run_until_complete(protocol.done)
+        self.assertEqual('CLOSED', protocol.state)
+
+    @mock.patch('asyncio.base_events.socket')
+    def test_create_datagram_endpoint_nosoreuseport(self, m_socket):
+        m_socket.getaddrinfo = socket.getaddrinfo
+        m_socket.SOCK_DGRAM = socket.SOCK_DGRAM
+        m_socket.SOL_SOCKET = socket.SOL_SOCKET
+        del m_socket.SO_REUSEPORT
+        m_socket.socket.return_value = mock.Mock()
+
+        coro = self.loop.create_datagram_endpoint(
+            lambda: MyDatagramProto(loop=self.loop),
+            local_addr=('127.0.0.1', 0),
+            reuse_address=False,
+            reuse_port=True)
+
+        self.assertRaises(ValueError, self.loop.run_until_complete, coro)
+
     def test_accept_connection_retry(self):
         sock = mock.Mock()
         sock.accept.side_effect = BlockingIOError()
diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py
--- a/Lib/test/test_asyncio/test_events.py
+++ b/Lib/test/test_asyncio/test_events.py
@@ -814,6 +814,32 @@
         # close server
         server.close()
 
+    @unittest.skipUnless(hasattr(socket, 'SO_REUSEPORT'), 'No SO_REUSEPORT')
+    def test_create_server_reuse_port(self):
+        proto = MyProto(self.loop)
+        f = self.loop.create_server(
+            lambda: proto, '0.0.0.0', 0)
+        server = self.loop.run_until_complete(f)
+        self.assertEqual(len(server.sockets), 1)
+        sock = server.sockets[0]
+        self.assertFalse(
+            sock.getsockopt(
+                socket.SOL_SOCKET, socket.SO_REUSEPORT))
+        server.close()
+
+        test_utils.run_briefly(self.loop)
+
+        proto = MyProto(self.loop)
+        f = self.loop.create_server(
+            lambda: proto, '0.0.0.0', 0, reuse_port=True)
+        server = self.loop.run_until_complete(f)
+        self.assertEqual(len(server.sockets), 1)
+        sock = server.sockets[0]
+        self.assertTrue(
+            sock.getsockopt(
+                socket.SOL_SOCKET, socket.SO_REUSEPORT))
+        server.close()
+
     def _make_unix_server(self, factory, **kwargs):
         path = test_utils.gen_unix_socket_path()
         self.addCleanup(lambda: os.path.exists(path) and os.unlink(path))
@@ -1264,6 +1290,32 @@
         self.assertEqual('CLOSED', client.state)
         server.transport.close()
 
+    def test_create_datagram_endpoint_sock(self):
+        sock = None
+        local_address = ('127.0.0.1', 0)
+        infos = self.loop.run_until_complete(
+            self.loop.getaddrinfo(
+                *local_address, type=socket.SOCK_DGRAM))
+        for family, type, proto, cname, address in infos:
+            try:
+                sock = socket.socket(family=family, type=type, proto=proto)
+                sock.setblocking(False)
+                sock.bind(address)
+            except:
+                pass
+            else:
+                break
+        else:
+            assert False, 'Can not create socket.'
+
+        f = self.loop.create_connection(
+            lambda: MyDatagramProto(loop=self.loop), sock=sock)
+        tr, pr = self.loop.run_until_complete(f)
+        self.assertIsInstance(tr, asyncio.Transport)
+        self.assertIsInstance(pr, MyDatagramProto)
+        tr.close()
+        self.loop.run_until_complete(pr.done)
+
     def test_internal_fds(self):
         loop = self.create_event_loop()
         if not isinstance(loop, selector_events.BaseSelectorEventLoop):
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -813,6 +813,7 @@
 Julia Lawall
 Chris Lawrence
 Mark Lawrence
+Chris Laws
 Brian Leair
 Mathieu Leduc-Hamel
 Amandine Lee
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -43,6 +43,12 @@
 Library
 -------
 
+- Issue #23972: Updates asyncio datagram create method allowing reuseport
+  and reuseaddr socket options to be set prior to binding the socket.
+  Mirroring the existing asyncio create_server method the reuseaddr option
+  for datagram sockets defaults to True if the O/S is 'posix' (except if the
+  platform is Cygwin). Patch by Chris Laws.
+
 - Issue #25304: Add asyncio.run_coroutine_threadsafe().  This lets you
   submit a coroutine to a loop from another thread, returning a
   concurrent.futures.Future.  By Vincent Michel.

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Mon Oct  5 19:35:44 2015
From: python-checkins at python.org (steve.dower)
Date: Mon, 05 Oct 2015 17:35:44 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2325316=3A_distutils_raises_OSError_instead_of_Di?=
 =?utf-8?q?stutilsPlatformError_when?=
Message-ID: <20151005173544.3273.18324@psf.io>

https://hg.python.org/cpython/rev/07161dd8a078
changeset:   98543:07161dd8a078
parent:      98541:c0f1f882737c
parent:      98542:a2016b29762c
user:        Steve Dower <steve.dower at microsoft.com>
date:        Mon Oct 05 10:35:19 2015 -0700
summary:
  Issue #25316: distutils raises OSError instead of DistutilsPlatformError when MSVC is not installed.

files:
  Lib/distutils/_msvccompiler.py |  18 ++++++++++--------
  Misc/NEWS                      |   3 +++
  2 files changed, 13 insertions(+), 8 deletions(-)


diff --git a/Lib/distutils/_msvccompiler.py b/Lib/distutils/_msvccompiler.py
--- a/Lib/distutils/_msvccompiler.py
+++ b/Lib/distutils/_msvccompiler.py
@@ -28,15 +28,17 @@
 from itertools import count
 
 def _find_vcvarsall(plat_spec):
-    with winreg.OpenKeyEx(
-        winreg.HKEY_LOCAL_MACHINE,
-        r"Software\Microsoft\VisualStudio\SxS\VC7",
-        access=winreg.KEY_READ | winreg.KEY_WOW64_32KEY
-    ) as key:
-        if not key:
-            log.debug("Visual C++ is not registered")
-            return None, None
+    try:
+        key = winreg.OpenKeyEx(
+            winreg.HKEY_LOCAL_MACHINE,
+            r"Software\Microsoft\VisualStudio\SxS\VC7",
+            access=winreg.KEY_READ | winreg.KEY_WOW64_32KEY
+        )
+    except OSError:
+        log.debug("Visual C++ is not registered")
+        return None, None
 
+    with key:
         best_version = 0
         best_dir = None
         for i in count():
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -43,6 +43,9 @@
 Library
 -------
 
+- Issue #25316: distutils raises OSError instead of DistutilsPlatformError
+  when MSVC is not installed.
+
 - Issue #23972: Updates asyncio datagram create method allowing reuseport
   and reuseaddr socket options to be set prior to binding the socket.
   Mirroring the existing asyncio create_server method the reuseaddr option

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Mon Oct  5 19:35:44 2015
From: python-checkins at python.org (steve.dower)
Date: Mon, 05 Oct 2015 17:35:44 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy41KTogSXNzdWUgIzI1MzE2?=
 =?utf-8?q?=3A_distutils_raises_OSError_instead_of_DistutilsPlatformError_?=
 =?utf-8?q?when?=
Message-ID: <20151005173544.20783.81966@psf.io>

https://hg.python.org/cpython/rev/a2016b29762c
changeset:   98542:a2016b29762c
branch:      3.5
parent:      98540:ba956289fe66
user:        Steve Dower <steve.dower at microsoft.com>
date:        Mon Oct 05 10:35:00 2015 -0700
summary:
  Issue #25316: distutils raises OSError instead of DistutilsPlatformError when MSVC is not installed.

files:
  Lib/distutils/_msvccompiler.py |  18 ++++++++++--------
  Misc/NEWS                      |   3 +++
  2 files changed, 13 insertions(+), 8 deletions(-)


diff --git a/Lib/distutils/_msvccompiler.py b/Lib/distutils/_msvccompiler.py
--- a/Lib/distutils/_msvccompiler.py
+++ b/Lib/distutils/_msvccompiler.py
@@ -28,15 +28,17 @@
 from itertools import count
 
 def _find_vcvarsall(plat_spec):
-    with winreg.OpenKeyEx(
-        winreg.HKEY_LOCAL_MACHINE,
-        r"Software\Microsoft\VisualStudio\SxS\VC7",
-        access=winreg.KEY_READ | winreg.KEY_WOW64_32KEY
-    ) as key:
-        if not key:
-            log.debug("Visual C++ is not registered")
-            return None, None
+    try:
+        key = winreg.OpenKeyEx(
+            winreg.HKEY_LOCAL_MACHINE,
+            r"Software\Microsoft\VisualStudio\SxS\VC7",
+            access=winreg.KEY_READ | winreg.KEY_WOW64_32KEY
+        )
+    except OSError:
+        log.debug("Visual C++ is not registered")
+        return None, None
 
+    with key:
         best_version = 0
         best_dir = None
         for i in count():
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -34,6 +34,9 @@
 Library
 -------
 
+- Issue #25316: distutils raises OSError instead of DistutilsPlatformError
+  when MSVC is not installed.
+
 - Issue #23972: Updates asyncio datagram create method allowing reuseport
   and reuseaddr socket options to be set prior to binding the socket.
   Mirroring the existing asyncio create_server method the reuseaddr option

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Tue Oct  6 01:27:05 2015
From: python-checkins at python.org (guido.van.rossum)
Date: Mon, 05 Oct 2015 23:27:05 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
 =?utf-8?q?=29=3A_Docs_and_one_small_improvement_for_issue_=2325304=2C_by_?=
 =?utf-8?q?Vincent_Michel=2E_=28Merge?=
Message-ID: <20151005232704.20771.8010@psf.io>

https://hg.python.org/cpython/rev/cba27498a2f7
changeset:   98546:cba27498a2f7
parent:      98543:07161dd8a078
parent:      98545:28fcd7f13613
user:        Guido van Rossum <guido at dropbox.com>
date:        Mon Oct 05 16:26:00 2015 -0700
summary:
  Docs and one small improvement for issue #25304, by Vincent Michel. (Merge 3.5->3.6.)

files:
  Doc/library/asyncio-dev.rst         |  12 +++-
  Doc/library/asyncio-task.rst        |  39 +++++++++++++++++
  Lib/asyncio/tasks.py                |   7 ++-
  Lib/test/test_asyncio/test_tasks.py |  21 +++++++++
  4 files changed, 75 insertions(+), 4 deletions(-)


diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst
--- a/Doc/library/asyncio-dev.rst
+++ b/Doc/library/asyncio-dev.rst
@@ -96,10 +96,9 @@
 and the event loop executes the next task.
 
 To schedule a callback from a different thread, the
-:meth:`BaseEventLoop.call_soon_threadsafe` method should be used. Example to
-schedule a coroutine from a different thread::
+:meth:`BaseEventLoop.call_soon_threadsafe` method should be used. Example::
 
-    loop.call_soon_threadsafe(asyncio.ensure_future, coro_func())
+    loop.call_soon_threadsafe(callback, *args)
 
 Most asyncio objects are not thread safe. You should only worry if you access
 objects outside the event loop. For example, to cancel a future, don't call
@@ -110,6 +109,13 @@
 To handle signals and to execute subprocesses, the event loop must be run in
 the main thread.
 
+To schedule a coroutine object from a different thread, the
+:func:`run_coroutine_threadsafe` function should be used. It returns a
+:class:`concurrent.futures.Future` to access the result::
+
+     future = asyncio.run_coroutine_threadsafe(coro_func(), loop)
+     result = future.result(timeout)  # Wait for the result with a timeout
+
 The :meth:`BaseEventLoop.run_in_executor` method can be used with a thread pool
 executor to execute a callback in different thread to not block the thread of
 the event loop.
diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst
--- a/Doc/library/asyncio-task.rst
+++ b/Doc/library/asyncio-task.rst
@@ -683,3 +683,42 @@
    .. versionchanged:: 3.4.3
       If the wait is cancelled, the future *fut* is now also cancelled.
 
+
+.. function:: run_coroutine_threadsafe(coro, loop)
+
+   Submit a :ref:`coroutine object <coroutine>` to a given event loop.
+
+   Return a :class:`concurrent.futures.Future` to access the result.
+
+   This function is meant to be called from a different thread than the one
+   where the event loop is running. Usage::
+
+     # Create a coroutine
+     coro = asyncio.sleep(1, result=3)
+     # Submit the coroutine to a given loop
+     future = asyncio.run_coroutine_threadsafe(coro, loop)
+     # Wait for the result with an optional timeout argument
+     assert future.result(timeout) == 3
+
+   If an exception is raised in the coroutine, the returned future will be
+   notified. It can also be used to cancel the task in the event loop::
+
+     try:
+         result = future.result(timeout)
+     except asyncio.TimeoutError:
+         print('The coroutine took too long, cancelling the task...')
+         future.cancel()
+     except Exception as exc:
+         print('The coroutine raised an exception: {!r}'.format(exc))
+     else:
+         print('The coroutine returned: {!r}'.format(result))
+
+   See the :ref:`concurrency and multithreading <asyncio-multithreading>`
+   section of the documentation.
+
+   .. note::
+
+      Unlike the functions above, :func:`run_coroutine_threadsafe` requires the
+      *loop* argument to be passed explicitely.
+
+   .. versionadded:: 3.4.4
diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py
--- a/Lib/asyncio/tasks.py
+++ b/Lib/asyncio/tasks.py
@@ -704,7 +704,12 @@
     future = concurrent.futures.Future()
 
     def callback():
-        futures._chain_future(ensure_future(coro, loop=loop), future)
+        try:
+            futures._chain_future(ensure_future(coro, loop=loop), future)
+        except Exception as exc:
+            if future.set_running_or_notify_cancel():
+                future.set_exception(exc)
+            raise
 
     loop.call_soon_threadsafe(callback)
     return future
diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py
--- a/Lib/test/test_asyncio/test_tasks.py
+++ b/Lib/test/test_asyncio/test_tasks.py
@@ -2166,6 +2166,27 @@
         with self.assertRaises(asyncio.CancelledError):
             self.loop.run_until_complete(future)
 
+    def test_run_coroutine_threadsafe_task_factory_exception(self):
+        """Test coroutine submission from a tread to an event loop
+        when the task factory raise an exception."""
+        # Clear the time generator
+        asyncio.ensure_future(self.add(1, 2), loop=self.loop)
+        # Schedule the target
+        future = self.loop.run_in_executor(None, self.target)
+        # Set corrupted task factory
+        self.loop.set_task_factory(lambda loop, coro: wrong_name)
+        # Set exception handler
+        callback = test_utils.MockCallback()
+        self.loop.set_exception_handler(callback)
+        # Run event loop
+        with self.assertRaises(NameError) as exc_context:
+            self.loop.run_until_complete(future)
+        # Check exceptions
+        self.assertIn('wrong_name', exc_context.exception.args[0])
+        self.assertEqual(len(callback.call_args_list), 1)
+        (loop, context), kwargs = callback.call_args
+        self.assertEqual(context['exception'], exc_context.exception)
+
 
 if __name__ == '__main__':
     unittest.main()

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Tue Oct  6 01:27:05 2015
From: python-checkins at python.org (guido.van.rossum)
Date: Mon, 05 Oct 2015 23:27:05 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
 =?utf-8?q?_Docs_and_one_small_improvement_for_issue_=2325304=2C_by_Vincen?=
 =?utf-8?q?t_Michel=2E_=28Merge?=
Message-ID: <20151005232704.457.50658@psf.io>

https://hg.python.org/cpython/rev/28fcd7f13613
changeset:   98545:28fcd7f13613
branch:      3.5
parent:      98542:a2016b29762c
parent:      98544:54c77fdcdb2e
user:        Guido van Rossum <guido at python.org>
date:        Mon Oct 05 16:23:13 2015 -0700
summary:
  Docs and one small improvement for issue #25304, by Vincent Michel. (Merge 3.4->3.5.)

files:
  Doc/library/asyncio-dev.rst         |  12 +++-
  Doc/library/asyncio-task.rst        |  39 +++++++++++++++++
  Lib/asyncio/tasks.py                |   7 ++-
  Lib/test/test_asyncio/test_tasks.py |  21 +++++++++
  4 files changed, 75 insertions(+), 4 deletions(-)


diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst
--- a/Doc/library/asyncio-dev.rst
+++ b/Doc/library/asyncio-dev.rst
@@ -96,10 +96,9 @@
 and the event loop executes the next task.
 
 To schedule a callback from a different thread, the
-:meth:`BaseEventLoop.call_soon_threadsafe` method should be used. Example to
-schedule a coroutine from a different thread::
+:meth:`BaseEventLoop.call_soon_threadsafe` method should be used. Example::
 
-    loop.call_soon_threadsafe(asyncio.ensure_future, coro_func())
+    loop.call_soon_threadsafe(callback, *args)
 
 Most asyncio objects are not thread safe. You should only worry if you access
 objects outside the event loop. For example, to cancel a future, don't call
@@ -110,6 +109,13 @@
 To handle signals and to execute subprocesses, the event loop must be run in
 the main thread.
 
+To schedule a coroutine object from a different thread, the
+:func:`run_coroutine_threadsafe` function should be used. It returns a
+:class:`concurrent.futures.Future` to access the result::
+
+     future = asyncio.run_coroutine_threadsafe(coro_func(), loop)
+     result = future.result(timeout)  # Wait for the result with a timeout
+
 The :meth:`BaseEventLoop.run_in_executor` method can be used with a thread pool
 executor to execute a callback in different thread to not block the thread of
 the event loop.
diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst
--- a/Doc/library/asyncio-task.rst
+++ b/Doc/library/asyncio-task.rst
@@ -683,3 +683,42 @@
    .. versionchanged:: 3.4.3
       If the wait is cancelled, the future *fut* is now also cancelled.
 
+
+.. function:: run_coroutine_threadsafe(coro, loop)
+
+   Submit a :ref:`coroutine object <coroutine>` to a given event loop.
+
+   Return a :class:`concurrent.futures.Future` to access the result.
+
+   This function is meant to be called from a different thread than the one
+   where the event loop is running. Usage::
+
+     # Create a coroutine
+     coro = asyncio.sleep(1, result=3)
+     # Submit the coroutine to a given loop
+     future = asyncio.run_coroutine_threadsafe(coro, loop)
+     # Wait for the result with an optional timeout argument
+     assert future.result(timeout) == 3
+
+   If an exception is raised in the coroutine, the returned future will be
+   notified. It can also be used to cancel the task in the event loop::
+
+     try:
+         result = future.result(timeout)
+     except asyncio.TimeoutError:
+         print('The coroutine took too long, cancelling the task...')
+         future.cancel()
+     except Exception as exc:
+         print('The coroutine raised an exception: {!r}'.format(exc))
+     else:
+         print('The coroutine returned: {!r}'.format(result))
+
+   See the :ref:`concurrency and multithreading <asyncio-multithreading>`
+   section of the documentation.
+
+   .. note::
+
+      Unlike the functions above, :func:`run_coroutine_threadsafe` requires the
+      *loop* argument to be passed explicitely.
+
+   .. versionadded:: 3.4.4
diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py
--- a/Lib/asyncio/tasks.py
+++ b/Lib/asyncio/tasks.py
@@ -704,7 +704,12 @@
     future = concurrent.futures.Future()
 
     def callback():
-        futures._chain_future(ensure_future(coro, loop=loop), future)
+        try:
+            futures._chain_future(ensure_future(coro, loop=loop), future)
+        except Exception as exc:
+            if future.set_running_or_notify_cancel():
+                future.set_exception(exc)
+            raise
 
     loop.call_soon_threadsafe(callback)
     return future
diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py
--- a/Lib/test/test_asyncio/test_tasks.py
+++ b/Lib/test/test_asyncio/test_tasks.py
@@ -2166,6 +2166,27 @@
         with self.assertRaises(asyncio.CancelledError):
             self.loop.run_until_complete(future)
 
+    def test_run_coroutine_threadsafe_task_factory_exception(self):
+        """Test coroutine submission from a tread to an event loop
+        when the task factory raise an exception."""
+        # Clear the time generator
+        asyncio.ensure_future(self.add(1, 2), loop=self.loop)
+        # Schedule the target
+        future = self.loop.run_in_executor(None, self.target)
+        # Set corrupted task factory
+        self.loop.set_task_factory(lambda loop, coro: wrong_name)
+        # Set exception handler
+        callback = test_utils.MockCallback()
+        self.loop.set_exception_handler(callback)
+        # Run event loop
+        with self.assertRaises(NameError) as exc_context:
+            self.loop.run_until_complete(future)
+        # Check exceptions
+        self.assertIn('wrong_name', exc_context.exception.args[0])
+        self.assertEqual(len(callback.call_args_list), 1)
+        (loop, context), kwargs = callback.call_args
+        self.assertEqual(context['exception'], exc_context.exception)
+
 
 if __name__ == '__main__':
     unittest.main()

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Tue Oct  6 01:27:05 2015
From: python-checkins at python.org (guido.van.rossum)
Date: Mon, 05 Oct 2015 23:27:05 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E4=29=3A_Docs_and_one_s?=
 =?utf-8?q?mall_improvement_for_issue_=2325304=2C_by_Vincent_Michel=2E?=
Message-ID: <20151005232704.2677.14135@psf.io>

https://hg.python.org/cpython/rev/54c77fdcdb2e
changeset:   98544:54c77fdcdb2e
branch:      3.4
parent:      98539:5e7e9b131904
user:        Guido van Rossum <guido at python.org>
date:        Mon Oct 05 16:20:00 2015 -0700
summary:
  Docs and one small improvement for issue #25304, by Vincent Michel.

files:
  Doc/library/asyncio-dev.rst         |  12 +++-
  Doc/library/asyncio-task.rst        |  39 +++++++++++++++++
  Lib/asyncio/tasks.py                |   7 ++-
  Lib/test/test_asyncio/test_tasks.py |  21 +++++++++
  4 files changed, 75 insertions(+), 4 deletions(-)


diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst
--- a/Doc/library/asyncio-dev.rst
+++ b/Doc/library/asyncio-dev.rst
@@ -96,10 +96,9 @@
 and the event loop executes the next task.
 
 To schedule a callback from a different thread, the
-:meth:`BaseEventLoop.call_soon_threadsafe` method should be used. Example to
-schedule a coroutine from a different thread::
+:meth:`BaseEventLoop.call_soon_threadsafe` method should be used. Example::
 
-    loop.call_soon_threadsafe(asyncio.async, coro_func())
+    loop.call_soon_threadsafe(callback, *args)
 
 Most asyncio objects are not thread safe. You should only worry if you access
 objects outside the event loop. For example, to cancel a future, don't call
@@ -107,6 +106,13 @@
 
     loop.call_soon_threadsafe(fut.cancel)
 
+To schedule a coroutine object from a different thread, the
+:func:`run_coroutine_threadsafe` function should be used. It returns a
+:class:`concurrent.futures.Future` to access the result::
+
+     future = asyncio.run_coroutine_threadsafe(coro_func(), loop)
+     result = future.result(timeout)  # Wait for the result with a timeout
+
 To handle signals and to execute subprocesses, the event loop must be run in
 the main thread.
 
diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst
--- a/Doc/library/asyncio-task.rst
+++ b/Doc/library/asyncio-task.rst
@@ -661,3 +661,42 @@
    .. versionchanged:: 3.4.3
       If the wait is cancelled, the future *fut* is now also cancelled.
 
+
+.. function:: run_coroutine_threadsafe(coro, loop)
+
+   Submit a :ref:`coroutine object <coroutine>` to a given event loop.
+
+   Return a :class:`concurrent.futures.Future` to access the result.
+
+   This function is meant to be called from a different thread than the one
+   where the event loop is running. Usage::
+
+     # Create a coroutine
+     coro = asyncio.sleep(1, result=3)
+     # Submit the coroutine to a given loop
+     future = asyncio.run_coroutine_threadsafe(coro, loop)
+     # Wait for the result with an optional timeout argument
+     assert future.result(timeout) == 3
+
+   If an exception is raised in the coroutine, the returned future will be
+   notified. It can also be used to cancel the task in the event loop::
+
+     try:
+         result = future.result(timeout)
+     except asyncio.TimeoutError:
+         print('The coroutine took too long, cancelling the task...')
+         future.cancel()
+     except Exception as exc:
+         print('The coroutine raised an exception: {!r}'.format(exc))
+     else:
+         print('The coroutine returned: {!r}'.format(result))
+
+   See the :ref:`concurrency and multithreading <asyncio-multithreading>`
+   section of the documentation.
+
+   .. note::
+
+      Unlike the functions above, :func:`run_coroutine_threadsafe` requires the
+      *loop* argument to be passed explicitely.
+
+   .. versionadded:: 3.4.4
diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py
--- a/Lib/asyncio/tasks.py
+++ b/Lib/asyncio/tasks.py
@@ -704,7 +704,12 @@
     future = concurrent.futures.Future()
 
     def callback():
-        futures._chain_future(ensure_future(coro, loop=loop), future)
+        try:
+            futures._chain_future(ensure_future(coro, loop=loop), future)
+        except Exception as exc:
+            if future.set_running_or_notify_cancel():
+                future.set_exception(exc)
+            raise
 
     loop.call_soon_threadsafe(callback)
     return future
diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py
--- a/Lib/test/test_asyncio/test_tasks.py
+++ b/Lib/test/test_asyncio/test_tasks.py
@@ -2166,6 +2166,27 @@
         with self.assertRaises(asyncio.CancelledError):
             self.loop.run_until_complete(future)
 
+    def test_run_coroutine_threadsafe_task_factory_exception(self):
+        """Test coroutine submission from a tread to an event loop
+        when the task factory raise an exception."""
+        # Clear the time generator
+        asyncio.ensure_future(self.add(1, 2), loop=self.loop)
+        # Schedule the target
+        future = self.loop.run_in_executor(None, self.target)
+        # Set corrupted task factory
+        self.loop.set_task_factory(lambda loop, coro: wrong_name)
+        # Set exception handler
+        callback = test_utils.MockCallback()
+        self.loop.set_exception_handler(callback)
+        # Run event loop
+        with self.assertRaises(NameError) as exc_context:
+            self.loop.run_until_complete(future)
+        # Check exceptions
+        self.assertIn('wrong_name', exc_context.exception.args[0])
+        self.assertEqual(len(callback.call_args_list), 1)
+        (loop, context), kwargs = callback.call_args
+        self.assertEqual(context['exception'], exc_context.exception)
+
 
 if __name__ == '__main__':
     unittest.main()

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Tue Oct  6 04:52:41 2015
From: python-checkins at python.org (raymond.hettinger)
Date: Tue, 06 Oct 2015 02:52:41 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Eliminate_unnecessary_test?=
Message-ID: <20151006025241.464.75509@psf.io>

https://hg.python.org/cpython/rev/d00c0544880a
changeset:   98547:d00c0544880a
user:        Raymond Hettinger <python at rcn.com>
date:        Mon Oct 05 22:52:37 2015 -0400
summary:
  Eliminate unnecessary test

files:
  Modules/_collectionsmodule.c |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c
--- a/Modules/_collectionsmodule.c
+++ b/Modules/_collectionsmodule.c
@@ -530,7 +530,7 @@
             return NULL;
         new_deque->maxlen = old_deque->maxlen;
         /* Fast path for the deque_repeat() common case where len(deque) == 1 */
-        if (Py_SIZE(deque) == 1 && new_deque->maxlen != 0) {
+        if (Py_SIZE(deque) == 1) {
             PyObject *item = old_deque->leftblock->data[old_deque->leftindex];
             rv = deque_append(new_deque, item);
         } else {

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Tue Oct  6 07:01:38 2015
From: python-checkins at python.org (benjamin.peterson)
Date: Tue, 06 Oct 2015 05:01:38 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_reinitialize_a?=
 =?utf-8?q?n_Event=27s_Condition_with_a_regular_lock_=28closes_=2325319=29?=
Message-ID: <20151006050136.451.13572@psf.io>

https://hg.python.org/cpython/rev/69a26f0800b3
changeset:   98548:69a26f0800b3
branch:      2.7
parent:      98536:45a04eadefd6
user:        Benjamin Peterson <benjamin at python.org>
date:        Mon Oct 05 21:56:22 2015 -0700
summary:
  reinitialize an Event's Condition with a regular lock (closes #25319)

files:
  Lib/test/lock_tests.py |  8 ++++++++
  Lib/threading.py       |  2 +-
  Misc/ACKS              |  1 +
  Misc/NEWS              |  3 +++
  4 files changed, 13 insertions(+), 1 deletions(-)


diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py
--- a/Lib/test/lock_tests.py
+++ b/Lib/test/lock_tests.py
@@ -305,6 +305,14 @@
         for r, dt in results2:
             self.assertTrue(r)
 
+    def test_reset_internal_locks(self):
+        evt = self.eventtype()
+        old_lock = evt._Event__cond._Condition__lock
+        evt._reset_internal_locks()
+        new_lock = evt._Event__cond._Condition__lock
+        self.assertIsNot(new_lock, old_lock)
+        self.assertIs(type(new_lock), type(old_lock))
+
 
 class ConditionTests(BaseTestCase):
     """
diff --git a/Lib/threading.py b/Lib/threading.py
--- a/Lib/threading.py
+++ b/Lib/threading.py
@@ -565,7 +565,7 @@
 
     def _reset_internal_locks(self):
         # private!  called by Thread._reset_internal_locks by _after_fork()
-        self.__cond.__init__()
+        self.__cond.__init__(Lock())
 
     def isSet(self):
         'Return true if and only if the internal flag is true.'
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -1290,6 +1290,7 @@
 Rafal Smotrzyk
 Eric Snow
 Dirk Soede
+Nir Soffer
 Paul Sokolovsky
 Evgeny Sologubov
 Cody Somerville
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -43,6 +43,9 @@
 Library
 -------
 
+- Issue #25319: When threading.Event is reinitialized, the underlying condition
+  should use a regular lock rather than a recursive lock.
+
 - Issue #25232: Fix CGIRequestHandler to split the query from the URL at the
   first question mark (?) rather than the last. Patch from Xiang Zhang.
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Tue Oct  6 07:01:37 2015
From: python-checkins at python.org (benjamin.peterson)
Date: Tue, 06 Oct 2015 05:01:37 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E4=29=3A_reinitialize_a?=
 =?utf-8?q?n_Event=27s_Condition_with_a_regular_lock_=28closes_=2325319=29?=
Message-ID: <20151006050136.3291.68053@psf.io>

https://hg.python.org/cpython/rev/6108d30dde21
changeset:   98549:6108d30dde21
branch:      3.4
parent:      98544:54c77fdcdb2e
user:        Benjamin Peterson <benjamin at python.org>
date:        Mon Oct 05 21:56:22 2015 -0700
summary:
  reinitialize an Event's Condition with a regular lock (closes #25319)

files:
  Lib/test/lock_tests.py |  8 ++++++++
  Lib/threading.py       |  2 +-
  Misc/ACKS              |  1 +
  Misc/NEWS              |  3 +++
  4 files changed, 13 insertions(+), 1 deletions(-)


diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py
--- a/Lib/test/lock_tests.py
+++ b/Lib/test/lock_tests.py
@@ -388,6 +388,14 @@
         b.wait_for_finished()
         self.assertEqual(results, [True] * N)
 
+    def test_reset_internal_locks(self):
+        evt = self.eventtype()
+        old_lock = evt._cond._lock
+        evt._reset_internal_locks()
+        new_lock = evt._cond._lock
+        self.assertIsNot(new_lock, old_lock)
+        self.assertIs(type(new_lock), type(old_lock))
+
 
 class ConditionTests(BaseTestCase):
     """
diff --git a/Lib/threading.py b/Lib/threading.py
--- a/Lib/threading.py
+++ b/Lib/threading.py
@@ -496,7 +496,7 @@
 
     def _reset_internal_locks(self):
         # private!  called by Thread._reset_internal_locks by _after_fork()
-        self._cond.__init__()
+        self._cond.__init__(Lock())
 
     def is_set(self):
         """Return true if and only if the internal flag is true."""
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -1315,6 +1315,7 @@
 Rafal Smotrzyk
 Eric Snow
 Dirk Soede
+Nir Soffer
 Paul Sokolovsky
 Evgeny Sologubov
 Cody Somerville
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -100,6 +100,9 @@
   submit a coroutine to a loop from another thread, returning a
   concurrent.futures.Future.  By Vincent Michel.
 
+- Issue #25319: When threading.Event is reinitialized, the underlying condition
+  should use a regular lock rather than a recursive lock.
+
 - Issue #25232: Fix CGIRequestHandler to split the query from the URL at the
   first question mark (?) rather than the last. Patch from Xiang Zhang.
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Tue Oct  6 07:01:52 2015
From: python-checkins at python.org (benjamin.peterson)
Date: Tue, 06 Oct 2015 05:01:52 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
 =?utf-8?b?KTogbWVyZ2UgMy41ICgjMjUzMTkp?=
Message-ID: <20151006050137.55460.52904@psf.io>

https://hg.python.org/cpython/rev/41f29bbf520d
changeset:   98551:41f29bbf520d
parent:      98547:d00c0544880a
parent:      98550:3719e842a7b1
user:        Benjamin Peterson <benjamin at python.org>
date:        Mon Oct 05 22:01:29 2015 -0700
summary:
  merge 3.5 (#25319)

files:
  Lib/test/lock_tests.py |  8 ++++++++
  Lib/threading.py       |  2 +-
  Misc/ACKS              |  1 +
  Misc/NEWS              |  3 +++
  4 files changed, 13 insertions(+), 1 deletions(-)


diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py
--- a/Lib/test/lock_tests.py
+++ b/Lib/test/lock_tests.py
@@ -394,6 +394,14 @@
         b.wait_for_finished()
         self.assertEqual(results, [True] * N)
 
+    def test_reset_internal_locks(self):
+        evt = self.eventtype()
+        old_lock = evt._cond._lock
+        evt._reset_internal_locks()
+        new_lock = evt._cond._lock
+        self.assertIsNot(new_lock, old_lock)
+        self.assertIs(type(new_lock), type(old_lock))
+
 
 class ConditionTests(BaseTestCase):
     """
diff --git a/Lib/threading.py b/Lib/threading.py
--- a/Lib/threading.py
+++ b/Lib/threading.py
@@ -499,7 +499,7 @@
 
     def _reset_internal_locks(self):
         # private!  called by Thread._reset_internal_locks by _after_fork()
-        self._cond.__init__()
+        self._cond.__init__(Lock())
 
     def is_set(self):
         """Return true if and only if the internal flag is true."""
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -1353,6 +1353,7 @@
 Rafal Smotrzyk
 Eric Snow
 Dirk Soede
+Nir Soffer
 Paul Sokolovsky
 Evgeny Sologubov
 Cody Somerville
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -128,6 +128,9 @@
 - Issue #13248: Remove deprecated inspect.getargspec and inspect.getmoduleinfo
   functions.
 
+- Issue #25319: When threading.Event is reinitialized, the underlying condition
+  should use a regular lock rather than a recursive lock.
+
 IDLE
 ----
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Tue Oct  6 07:01:52 2015
From: python-checkins at python.org (benjamin.peterson)
Date: Tue, 06 Oct 2015 05:01:52 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
 =?utf-8?q?_merge_3=2E4_=28=2325319=29?=
Message-ID: <20151006050137.2667.40281@psf.io>

https://hg.python.org/cpython/rev/3719e842a7b1
changeset:   98550:3719e842a7b1
branch:      3.5
parent:      98545:28fcd7f13613
parent:      98549:6108d30dde21
user:        Benjamin Peterson <benjamin at python.org>
date:        Mon Oct 05 22:00:33 2015 -0700
summary:
  merge 3.4 (#25319)

files:
  Lib/test/lock_tests.py |  8 ++++++++
  Lib/threading.py       |  2 +-
  Misc/ACKS              |  1 +
  Misc/NEWS              |  3 +++
  4 files changed, 13 insertions(+), 1 deletions(-)


diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py
--- a/Lib/test/lock_tests.py
+++ b/Lib/test/lock_tests.py
@@ -394,6 +394,14 @@
         b.wait_for_finished()
         self.assertEqual(results, [True] * N)
 
+    def test_reset_internal_locks(self):
+        evt = self.eventtype()
+        old_lock = evt._cond._lock
+        evt._reset_internal_locks()
+        new_lock = evt._cond._lock
+        self.assertIsNot(new_lock, old_lock)
+        self.assertIs(type(new_lock), type(old_lock))
+
 
 class ConditionTests(BaseTestCase):
     """
diff --git a/Lib/threading.py b/Lib/threading.py
--- a/Lib/threading.py
+++ b/Lib/threading.py
@@ -499,7 +499,7 @@
 
     def _reset_internal_locks(self):
         # private!  called by Thread._reset_internal_locks by _after_fork()
-        self._cond.__init__()
+        self._cond.__init__(Lock())
 
     def is_set(self):
         """Return true if and only if the internal flag is true."""
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -1352,6 +1352,7 @@
 Rafal Smotrzyk
 Eric Snow
 Dirk Soede
+Nir Soffer
 Paul Sokolovsky
 Evgeny Sologubov
 Cody Somerville
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -150,6 +150,9 @@
 - Issue #24881: Fixed setting binary mode in Python implementation of FileIO
   on Windows and Cygwin.  Patch from Akira Li.
 
+- Issue #25319: When threading.Event is reinitialized, the underlying condition
+  should use a regular lock rather than a recursive lock.
+
 - Issue #21112: Fix regression in unittest.expectedFailure on subclasses.
   Patch from Berker Peksag.
 

-- 
Repository URL: https://hg.python.org/cpython

From solipsis at pitrou.net  Tue Oct  6 10:45:15 2015
From: solipsis at pitrou.net (solipsis at pitrou.net)
Date: Tue, 06 Oct 2015 08:45:15 +0000
Subject: [Python-checkins] Daily reference leaks (41f29bbf520d): sum=61491
Message-ID: <20151006084514.3293.31633@psf.io>

results for 41f29bbf520d on branch "default"
--------------------------------------------

test_capi leaked [5410, 5410, 5410] references, sum=16230
test_capi leaked [1421, 1423, 1423] memory blocks, sum=4267
test_functools leaked [0, 2, 2] memory blocks, sum=4
test_threading leaked [10820, 10820, 10820] references, sum=32460
test_threading leaked [2842, 2844, 2844] memory blocks, sum=8530


Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogHMljtJ', '--timeout', '7200']

From python-checkins at python.org  Tue Oct  6 14:11:21 2015
From: python-checkins at python.org (steven.daprano)
Date: Tue, 06 Oct 2015 12:11:21 +0000
Subject: [Python-checkins] =?utf-8?q?peps_=28merge_default_-=3E_default=29?=
 =?utf-8?q?=3A_add_discussion_on_defaults=2C_naming_conventions=2C_possibl?=
 =?utf-8?q?e_implementations?=
Message-ID: <20151006121121.2685.95754@psf.io>

https://hg.python.org/peps/rev/ce879f833f82
changeset:   6108:ce879f833f82
parent:      6105:742700a03e91
parent:      6107:850897ef2790
user:        Steven D'Aprano <steve+python at pearwood.info>
date:        Tue Oct 06 23:10:55 2015 +1100
summary:
  add discussion on defaults, naming conventions, possible implementations

files:
  pep-0506.txt |  140 +++++++++++++++++++++++++++++---------
  1 files changed, 107 insertions(+), 33 deletions(-)


diff --git a/pep-0506.txt b/pep-0506.txt
--- a/pep-0506.txt
+++ b/pep-0506.txt
@@ -50,7 +50,7 @@
 and expressed some concern [1]_ about the use of MT for generating sensitive
 information such as passwords, secure tokens, session keys and similar.
 
-Although the documentation for the random module explicitly states that
+Although the documentation for the ``random`` module explicitly states that
 the default is not suitable for security purposes [2]_, it is strongly
 believed that this warning may be missed, ignored or misunderstood by
 many Python developers.  In particular:
@@ -58,7 +58,7 @@
 * developers may not have read the documentation and consequently
   not seen the warning;
 
-* they may not realise that their specific use of it has security
+* they may not realise that their specific use of the module has security
   implications; or
 
 * not realising that there could be a problem, they have copied code
@@ -140,20 +140,71 @@
 the ``random`` module to support these uses, ``SystemRandom`` will be
 sufficient.
 
-Some illustrative implementations have been given by Nick Coghlan [10]_.
-This idea has also been discussed on the issue tracker for the
-"cryptography" module [11]_.
+Some illustrative implementations have been given by Nick Coghlan [10]_
+and a minimalist API by Tim Peters [11]_. This idea has also been discussed
+on the issue tracker for the "cryptography" module [12]_.  The following
+pseudo-code can be taken as a possible starting point for the real
+implementation::
+
+    from random import SystemRandom
+    from hmac import compare_digest as equal
+
+    _sysrand = SystemRandom()
+
+    randrange = _sysrand.randrange
+    randint = _sysrand.randint
+    randbits = _sysrand.getrandbits
+    choice = _sysrand.choice
+
+    def randbelow(exclusive_upper_bound):
+        return _sysrand._randbelow(exclusive_upper_bound)
+
+    DEFAULT_ENTROPY = 32  # bytes
+
+    def token_bytes(nbytes=None):
+        if nbytes is None:
+            nbytes = DEFAULT_ENTROPY
+        return os.urandom(nbytes)
+
+    def token_hex(nbytes=None):
+        return binascii.hexlify(token_bytes(nbytes)).decode('ascii')
+
+    def token_url(nbytes=None):
+        tok = token_bytes(nbytes)
+        return base64.urlsafe_b64encode(tok).rstrip(b'=').decode('ascii')
+
 
 The ``secrets`` module itself will be pure Python, and other Python
 implementations can easily make use of it unchanged, or adapt it as
 necessary.
 
+Default arguments
+~~~~~~~~~~~~~~~~~
+
+One difficult question is "How many bytes should my token be?".  We can
+help with this question by providing a default amount of entropy for the
+"token_*" functions. If the ``nbytes`` argument is None or not given, the
+default entropy will be used. This default value should be large enough
+to be expected to be secure for medium-security uses, but is expected to
+change in the future, possibly even in a maintenance release [13]_.
+
+Naming conventions
+~~~~~~~~~~~~~~~~~~
+
+One question is the naming conventions used in the module [14]_, whether to
+use C-like naming conventions such as "randrange" or more Pythonic names
+such as "random_range".
+
+Functions which are simply bound methods of the private ``SystemRandom``
+instance (e.g. ``randrange``), or a thin wrapper around such, should keep
+the familiar names. Those which are something new (such as the various
+``token_*`` functions) will use more Pythonic names.
 
 Alternatives
 ============
 
 One alternative is to change the default PRNG provided by the ``random``
-module [12]_.  This received considerable scepticism and outright opposition:
+module [15]_.  This received considerable scepticism and outright opposition:
 
 * There is fear that a CSPRNG may be slower than the current PRNG (which
   in the case of MT is already quite slow).
@@ -172,13 +223,13 @@
 
 * Demonstrated attacks against MT are typically against PHP applications.
   It is believed that PHP's version of MT is a significantly softer target
-  than Python's version, due to a poor seeding technique [13]_.  Consequently,
+  than Python's version, due to a poor seeding technique [16]_.  Consequently,
   without a proven attack against Python applications, many people object
   to a backwards-incompatible change.
 
 Nick Coghlan made an earlier suggestion for a globally configurable PRNG
-which uses the system CSPRNG by default [14]_, but has since hinted that he
-may withdraw it in favour of this proposal [15]_.
+which uses the system CSPRNG by default [17]_, but has since withdrawn it
+in favour of this proposal.
 
 
 Comparison To Other Languages
@@ -186,7 +237,7 @@
 
 * PHP
 
-  PHP includes a function ``uniqid`` [16]_ which by default returns a
+  PHP includes a function ``uniqid`` [18]_ which by default returns a
   thirteen character string based on the current time in microseconds.
   Translated into Python syntax, it has the following signature::
 
@@ -197,7 +248,7 @@
   applications use it for that purpose (citation needed).
 
   PHP 5.3 and better also includes a function ``openssl_random_pseudo_bytes``
-  [17]_.  Translated into Python syntax, it has roughly the following
+  [19]_.  Translated into Python syntax, it has roughly the following
   signature::
 
     def openssl_random_pseudo_bytes(length:int)->Tuple[str, bool]
@@ -209,16 +260,16 @@
 
 * Javascript
 
-  Based on a rather cursory search [18]_, there doesn't appear to be any
+  Based on a rather cursory search [20]_, there do not appear to be any
   well-known standard functions for producing strong random values in
   Javascript, although there may be good quality third-party libraries.
   Standard Javascript doesn't seem to include an interface to the
   system CSPRNG either, and people have extensively written about the
-  weaknesses of Javascript's ``Math.random`` [19]_.
+  weaknesses of Javascript's ``Math.random`` [21]_.
 
 * Ruby
 
-  The Ruby standard library includes a module ``SecureRandom`` [20]_
+  The Ruby standard library includes a module ``SecureRandom`` [22]_
   which includes the following methods:
 
   * base64 - returns a Base64 encoded random string.
@@ -240,12 +291,15 @@
 
 There was a proposal to add a "random.safe" submodule, quoting the Zen
 of Python "Namespaces are one honking great idea" koan.  However, the
-author of the Zen, Tim Peters, has come out against this idea [21]_, and
+author of the Zen, Tim Peters, has come out against this idea [23]_, and
 recommends a top-level module.
 
 In discussion on the python-ideas mailing list so far, the name "secrets"
 has received some approval, and no strong opposition.
 
+There is already an existing third-party module with the same name [24]_,
+but it appears to be unused and abandoned.
+
 
 Frequently Asked Questions
 ==========================
@@ -255,9 +309,9 @@
 
   A: The consensus among security professionals is that MT is not safe
   in security contexts.  It is not difficult to reconstruct the internal
-  state of MT [22]_ [23]_ and so predict all past and future values.  There
+  state of MT [25]_ [26]_ and so predict all past and future values.  There
   are a number of known, practical attacks on systems using MT for
-  randomness [24]_.
+  randomness [27]_.
 
   While there are currently no known direct attacks on applications
   written in Python due to the use of MT, there is widespread agreement
@@ -268,7 +322,7 @@
   A: No. This is a "batteries included" solution, not a full-featured
   "nuclear reactor".  It is intended to mitigate against some basic
   security errors, not be a solution to all security-related issues. To
-  quote Nick Coghlan referring to his earlier proposal [25]_::
+  quote Nick Coghlan referring to his earlier proposal [28]_::
 
       "...folks really are better off learning to use things like
       cryptography.io for security sensitive software, so this change
@@ -276,6 +330,14 @@
       non-trivial proportion of the millions of current and future
       Python developers won't do that."
 
+* Q: What about a password generator?
+
+  A: The consensus is that the requirements for password generators are too
+     variable for it to be a good match for the standard library [29]_. No
+     password generator will be included in the initial release of the
+     module, instead it will be given in the documentation as a recipe (? la
+     the recipes in the ``itertools`` module) [30]_.
+
 
 References
 ==========
@@ -305,38 +367,50 @@
 
 .. [10] https://mail.python.org/pipermail/python-ideas/2015-September/036271.html
 
-.. [11] https://github.com/pyca/cryptography/issues/2347
+.. [11] https://mail.python.org/pipermail/python-ideas/2015-September/036350.html
 
-.. [12] Link needed.
+.. [12] https://github.com/pyca/cryptography/issues/2347
 
-.. [13] By default PHP seeds the MT PRNG with the time (citation needed),
+.. [13] https://mail.python.org/pipermail/python-ideas/2015-September/036517.html
+        https://mail.python.org/pipermail/python-ideas/2015-September/036515.html
+
+.. [14] https://mail.python.org/pipermail/python-ideas/2015-September/036474.html
+
+.. [15] Link needed.
+
+.. [16] By default PHP seeds the MT PRNG with the time (citation needed),
         which is exploitable by attackers, while Python seeds the PRNG with
         output from the system CSPRNG, which is believed to be much harder to
         exploit.
 
-.. [14] http://legacy.python.org/dev/peps/pep-0504/
+.. [17] http://legacy.python.org/dev/peps/pep-0504/
 
-.. [15] https://mail.python.org/pipermail/python-ideas/2015-September/036243.html
+.. [18] http://php.net/manual/en/function.uniqid.php
 
-.. [16] http://php.net/manual/en/function.uniqid.php
+.. [19] http://php.net/manual/en/function.openssl-random-pseudo-bytes.php
 
-.. [17] http://php.net/manual/en/function.openssl-random-pseudo-bytes.php
+.. [20] Volunteers and patches are welcome.
 
-.. [18] Volunteers and patches are welcome.
+.. [21] http://ifsec.blogspot.fr/2012/05/cross-domain-mathrandom-prediction.html
 
-.. [19] http://ifsec.blogspot.fr/2012/05/cross-domain-mathrandom-prediction.html
+.. [22] http://ruby-doc.org/stdlib-2.1.2/libdoc/securerandom/rdoc/SecureRandom.html
 
-.. [20] http://ruby-doc.org/stdlib-2.1.2/libdoc/securerandom/rdoc/SecureRandom.html
+.. [23] https://mail.python.org/pipermail/python-ideas/2015-September/036254.html
 
-.. [21] https://mail.python.org/pipermail/python-ideas/2015-September/036254.html
+.. [24] https://pypi.python.org/pypi/secrets
 
-.. [22] https://jazzy.id.au/2010/09/22/cracking_random_number_generators_part_3.html
+.. [25] https://jazzy.id.au/2010/09/22/cracking_random_number_generators_part_3.html
 
-.. [23] https://mail.python.org/pipermail/python-ideas/2015-September/036077.html
+.. [26] https://mail.python.org/pipermail/python-ideas/2015-September/036077.html
 
-.. [24] https://media.blackhat.com/bh-us-12/Briefings/Argyros/BH_US_12_Argyros_PRNG_WP.pdf
+.. [27] https://media.blackhat.com/bh-us-12/Briefings/Argyros/BH_US_12_Argyros_PRNG_WP.pdf
 
-.. [25] https://mail.python.org/pipermail/python-ideas/2015-September/036157.html
+.. [28] https://mail.python.org/pipermail/python-ideas/2015-September/036157.html
+
+.. [29] https://mail.python.org/pipermail/python-ideas/2015-September/036476.html
+        https://mail.python.org/pipermail/python-ideas/2015-September/036478.html
+
+.. [30] https://mail.python.org/pipermail/python-ideas/2015-September/036488.html
 
 
 Copyright

-- 
Repository URL: https://hg.python.org/peps

From python-checkins at python.org  Tue Oct  6 14:11:23 2015
From: python-checkins at python.org (steven.daprano)
Date: Tue, 06 Oct 2015 12:11:23 +0000
Subject: [Python-checkins] =?utf-8?q?peps=3A_Add_default_entropy=2E?=
Message-ID: <20151006121120.18380.93612@psf.io>

https://hg.python.org/peps/rev/850897ef2790
changeset:   6107:850897ef2790
user:        Steven D'Aprano <steve+python at pearwood.info>
date:        Mon Oct 05 03:11:17 2015 +1100
summary:
  Add default entropy.

files:
  pep-0506.txt |  96 ++++++++++++++++++++-------------------
  1 files changed, 50 insertions(+), 46 deletions(-)


diff --git a/pep-0506.txt b/pep-0506.txt
--- a/pep-0506.txt
+++ b/pep-0506.txt
@@ -159,14 +159,19 @@
     def randbelow(exclusive_upper_bound):
         return _sysrand._randbelow(exclusive_upper_bound)
 
-    def token_bytes(nbytes=32):
+    DEFAULT_ENTROPY = 32  # bytes
+
+    def token_bytes(nbytes=None):
+        if nbytes is None:
+            nbytes = DEFAULT_ENTROPY
         return os.urandom(nbytes)
 
-    def token_hex(nbytes=32):
+    def token_hex(nbytes=None):
         return binascii.hexlify(token_bytes(nbytes)).decode('ascii')
 
-    def token_url(nbytes=32):
-        return base64.urlsafe_b64encode(token_bytes(nbytes)).decode('ascii')
+    def token_url(nbytes=None):
+        tok = token_bytes(nbytes)
+        return base64.urlsafe_b64encode(tok).rstrip(b'=').decode('ascii')
 
 
 The ``secrets`` module itself will be pure Python, and other Python
@@ -176,18 +181,17 @@
 Default arguments
 ~~~~~~~~~~~~~~~~~
 
-One difficult question is "How many bytes should my token be?" We can help
-with this question by giving the "token_*" functions a sensible default for
-the ``nbytes`` argument. This default value should be large enough to be
-expected to be secure for medium-security uses [xxx]_.
-
-It is expected that future versions will need to increase those default
-values, possibly even during 
+One difficult question is "How many bytes should my token be?".  We can
+help with this question by providing a default amount of entropy for the
+"token_*" functions. If the ``nbytes`` argument is None or not given, the
+default entropy will be used. This default value should be large enough
+to be expected to be secure for medium-security uses, but is expected to
+change in the future, possibly even in a maintenance release [13]_.
 
 Naming conventions
 ~~~~~~~~~~~~~~~~~~
 
-One question is the naming conventions used in the module [13]_, whether to
+One question is the naming conventions used in the module [14]_, whether to
 use C-like naming conventions such as "randrange" or more Pythonic names
 such as "random_range".
 
@@ -200,7 +204,7 @@
 ============
 
 One alternative is to change the default PRNG provided by the ``random``
-module [14]_.  This received considerable scepticism and outright opposition:
+module [15]_.  This received considerable scepticism and outright opposition:
 
 * There is fear that a CSPRNG may be slower than the current PRNG (which
   in the case of MT is already quite slow).
@@ -219,12 +223,12 @@
 
 * Demonstrated attacks against MT are typically against PHP applications.
   It is believed that PHP's version of MT is a significantly softer target
-  than Python's version, due to a poor seeding technique [15]_.  Consequently,
+  than Python's version, due to a poor seeding technique [16]_.  Consequently,
   without a proven attack against Python applications, many people object
   to a backwards-incompatible change.
 
 Nick Coghlan made an earlier suggestion for a globally configurable PRNG
-which uses the system CSPRNG by default [16]_, but has since withdrawn it
+which uses the system CSPRNG by default [17]_, but has since withdrawn it
 in favour of this proposal.
 
 
@@ -233,7 +237,7 @@
 
 * PHP
 
-  PHP includes a function ``uniqid`` [17]_ which by default returns a
+  PHP includes a function ``uniqid`` [18]_ which by default returns a
   thirteen character string based on the current time in microseconds.
   Translated into Python syntax, it has the following signature::
 
@@ -244,7 +248,7 @@
   applications use it for that purpose (citation needed).
 
   PHP 5.3 and better also includes a function ``openssl_random_pseudo_bytes``
-  [18]_.  Translated into Python syntax, it has roughly the following
+  [19]_.  Translated into Python syntax, it has roughly the following
   signature::
 
     def openssl_random_pseudo_bytes(length:int)->Tuple[str, bool]
@@ -256,16 +260,16 @@
 
 * Javascript
 
-  Based on a rather cursory search [19]_, there do not appear to be any
+  Based on a rather cursory search [20]_, there do not appear to be any
   well-known standard functions for producing strong random values in
   Javascript, although there may be good quality third-party libraries.
   Standard Javascript doesn't seem to include an interface to the
   system CSPRNG either, and people have extensively written about the
-  weaknesses of Javascript's ``Math.random`` [20]_.
+  weaknesses of Javascript's ``Math.random`` [21]_.
 
 * Ruby
 
-  The Ruby standard library includes a module ``SecureRandom`` [21]_
+  The Ruby standard library includes a module ``SecureRandom`` [22]_
   which includes the following methods:
 
   * base64 - returns a Base64 encoded random string.
@@ -287,13 +291,13 @@
 
 There was a proposal to add a "random.safe" submodule, quoting the Zen
 of Python "Namespaces are one honking great idea" koan.  However, the
-author of the Zen, Tim Peters, has come out against this idea [22]_, and
+author of the Zen, Tim Peters, has come out against this idea [23]_, and
 recommends a top-level module.
 
 In discussion on the python-ideas mailing list so far, the name "secrets"
 has received some approval, and no strong opposition.
 
-There is already an existing third-party module with the same name [23]_,
+There is already an existing third-party module with the same name [24]_,
 but it appears to be unused and abandoned.
 
 
@@ -305,9 +309,9 @@
 
   A: The consensus among security professionals is that MT is not safe
   in security contexts.  It is not difficult to reconstruct the internal
-  state of MT [24]_ [25]_ and so predict all past and future values.  There
+  state of MT [25]_ [26]_ and so predict all past and future values.  There
   are a number of known, practical attacks on systems using MT for
-  randomness [26]_.
+  randomness [27]_.
 
   While there are currently no known direct attacks on applications
   written in Python due to the use of MT, there is widespread agreement
@@ -318,7 +322,7 @@
   A: No. This is a "batteries included" solution, not a full-featured
   "nuclear reactor".  It is intended to mitigate against some basic
   security errors, not be a solution to all security-related issues. To
-  quote Nick Coghlan referring to his earlier proposal [27]_::
+  quote Nick Coghlan referring to his earlier proposal [28]_::
 
       "...folks really are better off learning to use things like
       cryptography.io for security sensitive software, so this change
@@ -329,10 +333,10 @@
 * Q: What about a password generator?
 
   A: The consensus is that the requirements for password generators are too
-     variable for it to be a good match for the standard library [28]_. No
+     variable for it to be a good match for the standard library [29]_. No
      password generator will be included in the initial release of the
      module, instead it will be given in the documentation as a recipe (? la
-     the recipes in the ``itertools`` module) [29]_.
+     the recipes in the ``itertools`` module) [30]_.
 
 
 References
@@ -367,46 +371,46 @@
 
 .. [12] https://github.com/pyca/cryptography/issues/2347
 
-.. [xx] See discussion thread starting with
-        https://mail.python.org/pipermail/python-ideas/2015-September/036509.html
+.. [13] https://mail.python.org/pipermail/python-ideas/2015-September/036517.html
+        https://mail.python.org/pipermail/python-ideas/2015-September/036515.html
 
-.. [13] https://mail.python.org/pipermail/python-ideas/2015-September/036474.html
+.. [14] https://mail.python.org/pipermail/python-ideas/2015-September/036474.html
 
-.. [14] Link needed.
+.. [15] Link needed.
 
-.. [15] By default PHP seeds the MT PRNG with the time (citation needed),
+.. [16] By default PHP seeds the MT PRNG with the time (citation needed),
         which is exploitable by attackers, while Python seeds the PRNG with
         output from the system CSPRNG, which is believed to be much harder to
         exploit.
 
-.. [16] http://legacy.python.org/dev/peps/pep-0504/
+.. [17] http://legacy.python.org/dev/peps/pep-0504/
 
-.. [17] http://php.net/manual/en/function.uniqid.php
+.. [18] http://php.net/manual/en/function.uniqid.php
 
-.. [18] http://php.net/manual/en/function.openssl-random-pseudo-bytes.php
+.. [19] http://php.net/manual/en/function.openssl-random-pseudo-bytes.php
 
-.. [19] Volunteers and patches are welcome.
+.. [20] Volunteers and patches are welcome.
 
-.. [20] http://ifsec.blogspot.fr/2012/05/cross-domain-mathrandom-prediction.html
+.. [21] http://ifsec.blogspot.fr/2012/05/cross-domain-mathrandom-prediction.html
 
-.. [21] http://ruby-doc.org/stdlib-2.1.2/libdoc/securerandom/rdoc/SecureRandom.html
+.. [22] http://ruby-doc.org/stdlib-2.1.2/libdoc/securerandom/rdoc/SecureRandom.html
 
-.. [22] https://mail.python.org/pipermail/python-ideas/2015-September/036254.html
+.. [23] https://mail.python.org/pipermail/python-ideas/2015-September/036254.html
 
-.. [23] https://pypi.python.org/pypi/secrets
+.. [24] https://pypi.python.org/pypi/secrets
 
-.. [24] https://jazzy.id.au/2010/09/22/cracking_random_number_generators_part_3.html
+.. [25] https://jazzy.id.au/2010/09/22/cracking_random_number_generators_part_3.html
 
-.. [25] https://mail.python.org/pipermail/python-ideas/2015-September/036077.html
+.. [26] https://mail.python.org/pipermail/python-ideas/2015-September/036077.html
 
-.. [26] https://media.blackhat.com/bh-us-12/Briefings/Argyros/BH_US_12_Argyros_PRNG_WP.pdf
+.. [27] https://media.blackhat.com/bh-us-12/Briefings/Argyros/BH_US_12_Argyros_PRNG_WP.pdf
 
-.. [27] https://mail.python.org/pipermail/python-ideas/2015-September/036157.html
+.. [28] https://mail.python.org/pipermail/python-ideas/2015-September/036157.html
 
-.. [28] https://mail.python.org/pipermail/python-ideas/2015-September/036476.html
+.. [29] https://mail.python.org/pipermail/python-ideas/2015-September/036476.html
         https://mail.python.org/pipermail/python-ideas/2015-September/036478.html
 
-.. [29] https://mail.python.org/pipermail/python-ideas/2015-September/036488.html
+.. [30] https://mail.python.org/pipermail/python-ideas/2015-September/036488.html
 
 
 Copyright

-- 
Repository URL: https://hg.python.org/peps

From python-checkins at python.org  Tue Oct  6 14:11:23 2015
From: python-checkins at python.org (steven.daprano)
Date: Tue, 06 Oct 2015 12:11:23 +0000
Subject: [Python-checkins] =?utf-8?q?peps=3A_Add_sample_code=2C_default_ar?=
 =?utf-8?q?guments=2C_naming_conventions=2E?=
Message-ID: <20151006121120.457.51643@psf.io>

https://hg.python.org/peps/rev/6de7d0d8d201
changeset:   6106:6de7d0d8d201
parent:      6102:c7712f920a8f
user:        Steven D'Aprano <steve+python at pearwood.info>
date:        Sat Oct 03 21:42:05 2015 +1000
summary:
  Add sample code, default arguments, naming conventions.

files:
  pep-0506.txt |  136 +++++++++++++++++++++++++++++---------
  1 files changed, 103 insertions(+), 33 deletions(-)


diff --git a/pep-0506.txt b/pep-0506.txt
--- a/pep-0506.txt
+++ b/pep-0506.txt
@@ -50,7 +50,7 @@
 and expressed some concern [1]_ about the use of MT for generating sensitive
 information such as passwords, secure tokens, session keys and similar.
 
-Although the documentation for the random module explicitly states that
+Although the documentation for the ``random`` module explicitly states that
 the default is not suitable for security purposes [2]_, it is strongly
 believed that this warning may be missed, ignored or misunderstood by
 many Python developers.  In particular:
@@ -58,7 +58,7 @@
 * developers may not have read the documentation and consequently
   not seen the warning;
 
-* they may not realise that their specific use of it has security
+* they may not realise that their specific use of the module has security
   implications; or
 
 * not realising that there could be a problem, they have copied code
@@ -140,20 +140,67 @@
 the ``random`` module to support these uses, ``SystemRandom`` will be
 sufficient.
 
-Some illustrative implementations have been given by Nick Coghlan [10]_.
-This idea has also been discussed on the issue tracker for the
-"cryptography" module [11]_.
+Some illustrative implementations have been given by Nick Coghlan [10]_
+and a minimalist API by Tim Peters [11]_. This idea has also been discussed
+on the issue tracker for the "cryptography" module [12]_.  The following
+pseudo-code can be taken as a possible starting point for the real
+implementation::
+
+    from random import SystemRandom
+    from hmac import compare_digest as equal
+
+    _sysrand = SystemRandom()
+
+    randrange = _sysrand.randrange
+    randint = _sysrand.randint
+    randbits = _sysrand.getrandbits
+    choice = _sysrand.choice
+
+    def randbelow(exclusive_upper_bound):
+        return _sysrand._randbelow(exclusive_upper_bound)
+
+    def token_bytes(nbytes=32):
+        return os.urandom(nbytes)
+
+    def token_hex(nbytes=32):
+        return binascii.hexlify(token_bytes(nbytes)).decode('ascii')
+
+    def token_url(nbytes=32):
+        return base64.urlsafe_b64encode(token_bytes(nbytes)).decode('ascii')
+
 
 The ``secrets`` module itself will be pure Python, and other Python
 implementations can easily make use of it unchanged, or adapt it as
 necessary.
 
+Default arguments
+~~~~~~~~~~~~~~~~~
+
+One difficult question is "How many bytes should my token be?" We can help
+with this question by giving the "token_*" functions a sensible default for
+the ``nbytes`` argument. This default value should be large enough to be
+expected to be secure for medium-security uses [xxx]_.
+
+It is expected that future versions will need to increase those default
+values, possibly even during 
+
+Naming conventions
+~~~~~~~~~~~~~~~~~~
+
+One question is the naming conventions used in the module [13]_, whether to
+use C-like naming conventions such as "randrange" or more Pythonic names
+such as "random_range".
+
+Functions which are simply bound methods of the private ``SystemRandom``
+instance (e.g. ``randrange``), or a thin wrapper around such, should keep
+the familiar names. Those which are something new (such as the various
+``token_*`` functions) will use more Pythonic names.
 
 Alternatives
 ============
 
 One alternative is to change the default PRNG provided by the ``random``
-module [12]_.  This received considerable scepticism and outright opposition:
+module [14]_.  This received considerable scepticism and outright opposition:
 
 * There is fear that a CSPRNG may be slower than the current PRNG (which
   in the case of MT is already quite slow).
@@ -172,13 +219,13 @@
 
 * Demonstrated attacks against MT are typically against PHP applications.
   It is believed that PHP's version of MT is a significantly softer target
-  than Python's version, due to a poor seeding technique [13]_.  Consequently,
+  than Python's version, due to a poor seeding technique [15]_.  Consequently,
   without a proven attack against Python applications, many people object
   to a backwards-incompatible change.
 
 Nick Coghlan made an earlier suggestion for a globally configurable PRNG
-which uses the system CSPRNG by default [14]_, but has since hinted that he
-may withdraw it in favour of this proposal [15]_.
+which uses the system CSPRNG by default [16]_, but has since withdrawn it
+in favour of this proposal.
 
 
 Comparison To Other Languages
@@ -186,7 +233,7 @@
 
 * PHP
 
-  PHP includes a function ``uniqid`` [16]_ which by default returns a
+  PHP includes a function ``uniqid`` [17]_ which by default returns a
   thirteen character string based on the current time in microseconds.
   Translated into Python syntax, it has the following signature::
 
@@ -197,7 +244,7 @@
   applications use it for that purpose (citation needed).
 
   PHP 5.3 and better also includes a function ``openssl_random_pseudo_bytes``
-  [17]_.  Translated into Python syntax, it has roughly the following
+  [18]_.  Translated into Python syntax, it has roughly the following
   signature::
 
     def openssl_random_pseudo_bytes(length:int)->Tuple[str, bool]
@@ -209,16 +256,16 @@
 
 * Javascript
 
-  Based on a rather cursory search [18]_, there doesn't appear to be any
+  Based on a rather cursory search [19]_, there do not appear to be any
   well-known standard functions for producing strong random values in
   Javascript, although there may be good quality third-party libraries.
   Standard Javascript doesn't seem to include an interface to the
   system CSPRNG either, and people have extensively written about the
-  weaknesses of Javascript's ``Math.random`` [19]_.
+  weaknesses of Javascript's ``Math.random`` [20]_.
 
 * Ruby
 
-  The Ruby standard library includes a module ``SecureRandom`` [20]_
+  The Ruby standard library includes a module ``SecureRandom`` [21]_
   which includes the following methods:
 
   * base64 - returns a Base64 encoded random string.
@@ -240,12 +287,15 @@
 
 There was a proposal to add a "random.safe" submodule, quoting the Zen
 of Python "Namespaces are one honking great idea" koan.  However, the
-author of the Zen, Tim Peters, has come out against this idea [21]_, and
+author of the Zen, Tim Peters, has come out against this idea [22]_, and
 recommends a top-level module.
 
 In discussion on the python-ideas mailing list so far, the name "secrets"
 has received some approval, and no strong opposition.
 
+There is already an existing third-party module with the same name [23]_,
+but it appears to be unused and abandoned.
+
 
 Frequently Asked Questions
 ==========================
@@ -255,9 +305,9 @@
 
   A: The consensus among security professionals is that MT is not safe
   in security contexts.  It is not difficult to reconstruct the internal
-  state of MT [22]_ [23]_ and so predict all past and future values.  There
+  state of MT [24]_ [25]_ and so predict all past and future values.  There
   are a number of known, practical attacks on systems using MT for
-  randomness [24]_.
+  randomness [26]_.
 
   While there are currently no known direct attacks on applications
   written in Python due to the use of MT, there is widespread agreement
@@ -268,7 +318,7 @@
   A: No. This is a "batteries included" solution, not a full-featured
   "nuclear reactor".  It is intended to mitigate against some basic
   security errors, not be a solution to all security-related issues. To
-  quote Nick Coghlan referring to his earlier proposal [25]_::
+  quote Nick Coghlan referring to his earlier proposal [27]_::
 
       "...folks really are better off learning to use things like
       cryptography.io for security sensitive software, so this change
@@ -276,6 +326,14 @@
       non-trivial proportion of the millions of current and future
       Python developers won't do that."
 
+* Q: What about a password generator?
+
+  A: The consensus is that the requirements for password generators are too
+     variable for it to be a good match for the standard library [28]_. No
+     password generator will be included in the initial release of the
+     module, instead it will be given in the documentation as a recipe (? la
+     the recipes in the ``itertools`` module) [29]_.
+
 
 References
 ==========
@@ -305,38 +363,50 @@
 
 .. [10] https://mail.python.org/pipermail/python-ideas/2015-September/036271.html
 
-.. [11] https://github.com/pyca/cryptography/issues/2347
+.. [11] https://mail.python.org/pipermail/python-ideas/2015-September/036350.html
 
-.. [12] Link needed.
+.. [12] https://github.com/pyca/cryptography/issues/2347
 
-.. [13] By default PHP seeds the MT PRNG with the time (citation needed),
+.. [xx] See discussion thread starting with
+        https://mail.python.org/pipermail/python-ideas/2015-September/036509.html
+
+.. [13] https://mail.python.org/pipermail/python-ideas/2015-September/036474.html
+
+.. [14] Link needed.
+
+.. [15] By default PHP seeds the MT PRNG with the time (citation needed),
         which is exploitable by attackers, while Python seeds the PRNG with
         output from the system CSPRNG, which is believed to be much harder to
         exploit.
 
-.. [14] http://legacy.python.org/dev/peps/pep-0504/
+.. [16] http://legacy.python.org/dev/peps/pep-0504/
 
-.. [15] https://mail.python.org/pipermail/python-ideas/2015-September/036243.html
+.. [17] http://php.net/manual/en/function.uniqid.php
 
-.. [16] http://php.net/manual/en/function.uniqid.php
+.. [18] http://php.net/manual/en/function.openssl-random-pseudo-bytes.php
 
-.. [17] http://php.net/manual/en/function.openssl-random-pseudo-bytes.php
+.. [19] Volunteers and patches are welcome.
 
-.. [18] Volunteers and patches are welcome.
+.. [20] http://ifsec.blogspot.fr/2012/05/cross-domain-mathrandom-prediction.html
 
-.. [19] http://ifsec.blogspot.fr/2012/05/cross-domain-mathrandom-prediction.html
+.. [21] http://ruby-doc.org/stdlib-2.1.2/libdoc/securerandom/rdoc/SecureRandom.html
 
-.. [20] http://ruby-doc.org/stdlib-2.1.2/libdoc/securerandom/rdoc/SecureRandom.html
+.. [22] https://mail.python.org/pipermail/python-ideas/2015-September/036254.html
 
-.. [21] https://mail.python.org/pipermail/python-ideas/2015-September/036254.html
+.. [23] https://pypi.python.org/pypi/secrets
 
-.. [22] https://jazzy.id.au/2010/09/22/cracking_random_number_generators_part_3.html
+.. [24] https://jazzy.id.au/2010/09/22/cracking_random_number_generators_part_3.html
 
-.. [23] https://mail.python.org/pipermail/python-ideas/2015-September/036077.html
+.. [25] https://mail.python.org/pipermail/python-ideas/2015-September/036077.html
 
-.. [24] https://media.blackhat.com/bh-us-12/Briefings/Argyros/BH_US_12_Argyros_PRNG_WP.pdf
+.. [26] https://media.blackhat.com/bh-us-12/Briefings/Argyros/BH_US_12_Argyros_PRNG_WP.pdf
 
-.. [25] https://mail.python.org/pipermail/python-ideas/2015-September/036157.html
+.. [27] https://mail.python.org/pipermail/python-ideas/2015-September/036157.html
+
+.. [28] https://mail.python.org/pipermail/python-ideas/2015-September/036476.html
+        https://mail.python.org/pipermail/python-ideas/2015-September/036478.html
+
+.. [29] https://mail.python.org/pipermail/python-ideas/2015-September/036488.html
 
 
 Copyright

-- 
Repository URL: https://hg.python.org/peps

From lp_benchmark_robot at intel.com  Tue Oct  6 17:14:55 2015
From: lp_benchmark_robot at intel.com (lp_benchmark_robot at intel.com)
Date: Tue, 6 Oct 2015 16:14:55 +0100
Subject: [Python-checkins] Benchmark Results for Python 2.7 2015-10-06
Message-ID: <1a9c8028-b466-4540-830c-53f3ddb078b0@irsmsx152.ger.corp.intel.com>

No new revisions. Here are the previous results:

Results for project python_2.7-nightly, build date 2015-10-06 03:44:26 
commit:		45a04eadefd6ed13c110059375d2932f6b0d7490
revision date:	2015-10-04 10:52:40 +0000
environment:	Haswell-EP
	cpu:	Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz 2x18 cores, stepping 2, LLC 45 MB
	mem:	128 GB
	os:	CentOS 7.1
	kernel:	Linux 3.10.0-229.4.2.el7.x86_64

Baseline results were generated using release v2.7.10, with hash
15c95b7d81dcf821daade360741e00714667653f from 2015-05-23 16:02:14+00:00

------------------------------------------------------------------------------------------
               benchmark    relative      change since      change since  current rev with
                            std_dev*          last run           v2.7.10      regrtest PGO
------------------------------------------------------------------------------------------
:-)            django_v2    0.11335%          0.94969%          5.43955%          8.53025%
:-)              pybench    0.16785%          0.69147%          6.77273%          6.45979%
:-|             regex_v8    1.07392%          0.01169%         -1.82451%          7.81632%
:-)                nbody    0.15352%          0.15454%          8.67803%          3.87488%
:-)         json_dump_v2    0.21320%         -0.26509%          3.37121%         12.71810%
:-(       normal_startup    1.91283%         -0.47298%         -2.14007%          3.41653%
:-|              ssbench    0.10483%          0.27749%          1.31698%          1.08129%
------------------------------------------------------------------------------------------
Note: Benchmark results for ssbench are measured in requests/second while all
other are measured in seconds.
* Relative Standard Deviation (Standard Deviation/Average)

Our lab does a nightly source pull and build of the Python project and measures
performance changes against the previous stable version and the previous nightly
measurement. This is provided as a service to the community so that quality
issues with current hardware can be identified quickly.

Intel technologies' features and benefits depend on system configuration and may
require enabled hardware, software or service activation. Performance varies
depending on system configuration.


From lp_benchmark_robot at intel.com  Tue Oct  6 17:14:29 2015
From: lp_benchmark_robot at intel.com (lp_benchmark_robot at intel.com)
Date: Tue, 6 Oct 2015 16:14:29 +0100
Subject: [Python-checkins] Benchmark Results for Python Default 2015-10-06
Message-ID: <7e5ab4d6-44a2-4bb4-aa87-747d5dc3813a@irsmsx152.ger.corp.intel.com>

Results for project python_default-nightly, build date 2015-10-06 03:02:01 
commit:		d00c0544880a15dd134cdd268accc1a7e32dfc71
revision date:	2015-10-06 02:52:37 +0000
environment:	Haswell-EP
	cpu:	Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz 2x18 cores, stepping 2, LLC 45 MB
	mem:	128 GB
	os:	CentOS 7.1
	kernel:	Linux 3.10.0-229.4.2.el7.x86_64

Baseline results were generated using release v3.4.3, with hash
b4cbecbc0781e89a309d03b60a1f75f8499250e6 from 2015-02-25 12:15:33+00:00

------------------------------------------------------------------------------------------
               benchmark    relative      change since      change since  current rev with
                            std_dev*          last run            v3.4.3      regrtest PGO
------------------------------------------------------------------------------------------
:-)            django_v2    0.27960%          1.26479%          8.95277%         14.81902%
:-(              pybench    0.10000%         -0.08965%         -2.01005%          7.82206%
:-(             regex_v8    2.64818%         -0.03498%         -5.23658%          8.22164%
:-|                nbody    0.38417%         -0.90068%         -1.17148%          9.92874%
:-|         json_dump_v2    0.29148%          0.88046%         -0.20192%         11.46522%
:-|       normal_startup    0.97431%         -0.18504%          0.36361%          5.51143%
------------------------------------------------------------------------------------------
Note: Benchmark results are measured in seconds.
* Relative Standard Deviation (Standard Deviation/Average)

Our lab does a nightly source pull and build of the Python project and measures
performance changes against the previous stable version and the previous nightly
measurement. This is provided as a service to the community so that quality
issues with current hardware can be identified quickly.

Intel technologies' features and benefits depend on system configuration and may
require enabled hardware, software or service activation. Performance varies
depending on system configuration.


From python-checkins at python.org  Tue Oct  6 17:25:53 2015
From: python-checkins at python.org (guido.van.rossum)
Date: Tue, 06 Oct 2015 15:25:53 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzIzOTcy?=
 =?utf-8?q?=3A_Fix_tests_for_Windows_and_Debian=2E?=
Message-ID: <20151006152550.20771.48320@psf.io>

https://hg.python.org/cpython/rev/aebbf205ef6f
changeset:   98552:aebbf205ef6f
branch:      3.4
parent:      98549:6108d30dde21
user:        Guido van Rossum <guido at python.org>
date:        Tue Oct 06 08:24:10 2015 -0700
summary:
  Issue #23972: Fix tests for Windows and Debian.

files:
  Lib/test/test_asyncio/test_base_events.py |  5 +----
  1 files changed, 1 insertions(+), 4 deletions(-)


diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py
--- a/Lib/test/test_asyncio/test_base_events.py
+++ b/Lib/test/test_asyncio/test_base_events.py
@@ -1215,6 +1215,7 @@
 
     def test_create_datagram_endpoint_sock(self):
         sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        sock.bind(('127.0.0.1', 0))
         fut = self.loop.create_datagram_endpoint(
             lambda: MyDatagramProto(create_future=True, loop=self.loop),
             sock=sock)
@@ -1307,10 +1308,6 @@
             self.assertTrue(
                 sock.getsockopt(
                     socket.SOL_SOCKET, socket.SO_REUSEPORT))
-        else:
-            self.assertFalse(
-                sock.getsockopt(
-                    socket.SOL_SOCKET, socket.SO_REUSEPORT))
         self.assertTrue(
             sock.getsockopt(
                 socket.SOL_SOCKET, socket.SO_BROADCAST))

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Tue Oct  6 17:25:53 2015
From: python-checkins at python.org (guido.van.rossum)
Date: Tue, 06 Oct 2015 15:25:53 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2323972=3A_Fix_tests_for_Windows_and_Debian=2E_?=
 =?utf-8?b?KE1lcmdlIDMuNS0+My42KQ==?=
Message-ID: <20151006152550.3289.90769@psf.io>

https://hg.python.org/cpython/rev/3e2218a4e629
changeset:   98554:3e2218a4e629
parent:      98551:41f29bbf520d
parent:      98553:4d643c5df2a5
user:        Guido van Rossum <guido at dropbox.com>
date:        Tue Oct 06 08:25:21 2015 -0700
summary:
  Issue #23972: Fix tests for Windows and Debian. (Merge 3.5->3.6)

files:
  Lib/test/test_asyncio/test_base_events.py |  5 +----
  1 files changed, 1 insertions(+), 4 deletions(-)


diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py
--- a/Lib/test/test_asyncio/test_base_events.py
+++ b/Lib/test/test_asyncio/test_base_events.py
@@ -1215,6 +1215,7 @@
 
     def test_create_datagram_endpoint_sock(self):
         sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        sock.bind(('127.0.0.1', 0))
         fut = self.loop.create_datagram_endpoint(
             lambda: MyDatagramProto(create_future=True, loop=self.loop),
             sock=sock)
@@ -1307,10 +1308,6 @@
             self.assertTrue(
                 sock.getsockopt(
                     socket.SOL_SOCKET, socket.SO_REUSEPORT))
-        else:
-            self.assertFalse(
-                sock.getsockopt(
-                    socket.SOL_SOCKET, socket.SO_REUSEPORT))
         self.assertTrue(
             sock.getsockopt(
                 socket.SOL_SOCKET, socket.SO_BROADCAST))

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Tue Oct  6 17:25:52 2015
From: python-checkins at python.org (guido.van.rossum)
Date: Tue, 06 Oct 2015 15:25:52 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
 =?utf-8?q?_Issue_=2323972=3A_Fix_tests_for_Windows_and_Debian=2E_=28Merge?=
 =?utf-8?b?IDMuNC0+My41KQ==?=
Message-ID: <20151006152550.18370.19374@psf.io>

https://hg.python.org/cpython/rev/4d643c5df2a5
changeset:   98553:4d643c5df2a5
branch:      3.5
parent:      98550:3719e842a7b1
parent:      98552:aebbf205ef6f
user:        Guido van Rossum <guido at python.org>
date:        Tue Oct 06 08:24:44 2015 -0700
summary:
  Issue #23972: Fix tests for Windows and Debian. (Merge 3.4->3.5)

files:
  Lib/test/test_asyncio/test_base_events.py |  5 +----
  1 files changed, 1 insertions(+), 4 deletions(-)


diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py
--- a/Lib/test/test_asyncio/test_base_events.py
+++ b/Lib/test/test_asyncio/test_base_events.py
@@ -1215,6 +1215,7 @@
 
     def test_create_datagram_endpoint_sock(self):
         sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        sock.bind(('127.0.0.1', 0))
         fut = self.loop.create_datagram_endpoint(
             lambda: MyDatagramProto(create_future=True, loop=self.loop),
             sock=sock)
@@ -1307,10 +1308,6 @@
             self.assertTrue(
                 sock.getsockopt(
                     socket.SOL_SOCKET, socket.SO_REUSEPORT))
-        else:
-            self.assertFalse(
-                sock.getsockopt(
-                    socket.SOL_SOCKET, socket.SO_REUSEPORT))
         self.assertTrue(
             sock.getsockopt(
                 socket.SOL_SOCKET, socket.SO_BROADCAST))

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Tue Oct  6 17:53:47 2015
From: python-checkins at python.org (serhiy.storchaka)
Date: Tue, 06 Oct 2015 15:53:47 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzI1MzE3?=
 =?utf-8?q?=3A_Converted_doctests_in_test=5Ftokenize_to_unittests=2E?=
Message-ID: <20151006155346.70990.58390@psf.io>

https://hg.python.org/cpython/rev/7b2af8ee6dfa
changeset:   98555:7b2af8ee6dfa
branch:      2.7
parent:      98548:69a26f0800b3
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Tue Oct 06 18:13:38 2015 +0300
summary:
  Issue #25317: Converted doctests in test_tokenize to unittests.

files:
  Lib/test/test_tokenize.py |  544 ++++++++++++++-----------
  1 files changed, 294 insertions(+), 250 deletions(-)


diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -1,20 +1,42 @@
-doctests = """
-Tests for the tokenize module.
+from test import test_support
+from tokenize import (untokenize, generate_tokens, NUMBER, NAME, OP,
+                     STRING, ENDMARKER, tok_name, Untokenizer, tokenize)
+from StringIO import StringIO
+import os
+from unittest import TestCase
 
-    >>> import glob, random, sys
 
-The tests can be really simple. Given a small fragment of source
-code, print out a table with tokens. The ENDMARKER is omitted for
-brevity.
+class TokenizeTest(TestCase):
+    # Tests for the tokenize module.
 
-    >>> dump_tokens("1 + 1")
+    # The tests can be really simple. Given a small fragment of source
+    # code, print out a table with tokens. The ENDMARKER is omitted for
+    # brevity.
+
+    def check_tokenize(self, s, expected):
+        # Format the tokens in s in a table format.
+        # The ENDMARKER is omitted.
+        result = []
+        f = StringIO(s)
+        for type, token, start, end, line in generate_tokens(f.readline):
+            if type == ENDMARKER:
+                break
+            type = tok_name[type]
+            result.append("    %(type)-10.10s %(token)-13.13r %(start)s %(end)s" %
+                          locals())
+        self.assertEqual(result,
+                         expected.rstrip().splitlines())
+
+
+    def test_basic(self):
+        self.check_tokenize("1 + 1", """\
     NUMBER     '1'           (1, 0) (1, 1)
     OP         '+'           (1, 2) (1, 3)
     NUMBER     '1'           (1, 4) (1, 5)
-
-    >>> dump_tokens("if False:\\n"
-    ...             "    # NL\\n"
-    ...             "    True = False # NEWLINE\\n")
+    """)
+        self.check_tokenize("if False:\n"
+                            "    # NL\n"
+                            "    True = False # NEWLINE\n", """\
     NAME       'if'          (1, 0) (1, 2)
     NAME       'False'       (1, 3) (1, 8)
     OP         ':'           (1, 8) (1, 9)
@@ -28,122 +50,48 @@
     COMMENT    '# NEWLINE'   (3, 17) (3, 26)
     NEWLINE    '\\n'          (3, 26) (3, 27)
     DEDENT     ''            (4, 0) (4, 0)
+    """)
 
-    >>> indent_error_file = \"""
-    ... def k(x):
-    ...     x += 2
-    ...   x += 5
-    ... \"""
+        indent_error_file = """\
+def k(x):
+    x += 2
+  x += 5
+"""
+        with self.assertRaisesRegexp(IndentationError,
+                                     "unindent does not match any "
+                                     "outer indentation level"):
+            for tok in generate_tokens(StringIO(indent_error_file).readline):
+                pass
 
-    >>> for tok in generate_tokens(StringIO(indent_error_file).readline): pass
-    Traceback (most recent call last):
-        ...
-    IndentationError: unindent does not match any outer indentation level
-
-Test roundtrip for `untokenize`. `f` is an open file or a string. The source
-code in f is tokenized, converted back to source code via tokenize.untokenize(),
-and tokenized again from the latter. The test fails if the second tokenization
-doesn't match the first.
-
-    >>> def roundtrip(f):
-    ...     if isinstance(f, str): f = StringIO(f)
-    ...     token_list = list(generate_tokens(f.readline))
-    ...     f.close()
-    ...     tokens1 = [tok[:2] for tok in token_list]
-    ...     new_text = untokenize(tokens1)
-    ...     readline = iter(new_text.splitlines(1)).next
-    ...     tokens2 = [tok[:2] for tok in generate_tokens(readline)]
-    ...     return tokens1 == tokens2
-    ...
-
-There are some standard formatting practices that are easy to get right.
-
-    >>> roundtrip("if x == 1:\\n"
-    ...           "    print x\\n")
-    True
-
-    >>> roundtrip("# This is a comment\\n# This also")
-    True
-
-Some people use different formatting conventions, which makes
-untokenize a little trickier. Note that this test involves trailing
-whitespace after the colon. Note that we use hex escapes to make the
-two trailing blanks apperant in the expected output.
-
-    >>> roundtrip("if x == 1 : \\n"
-    ...           "  print x\\n")
-    True
-
-    >>> f = test_support.findfile("tokenize_tests" + os.extsep + "txt")
-    >>> roundtrip(open(f))
-    True
-
-    >>> roundtrip("if x == 1:\\n"
-    ...           "    # A comment by itself.\\n"
-    ...           "    print x # Comment here, too.\\n"
-    ...           "    # Another comment.\\n"
-    ...           "after_if = True\\n")
-    True
-
-    >>> roundtrip("if (x # The comments need to go in the right place\\n"
-    ...           "    == 1):\\n"
-    ...           "    print 'x==1'\\n")
-    True
-
-    >>> roundtrip("class Test: # A comment here\\n"
-    ...           "  # A comment with weird indent\\n"
-    ...           "  after_com = 5\\n"
-    ...           "  def x(m): return m*5 # a one liner\\n"
-    ...           "  def y(m): # A whitespace after the colon\\n"
-    ...           "     return y*4 # 3-space indent\\n")
-    True
-
-Some error-handling code
-
-    >>> roundtrip("try: import somemodule\\n"
-    ...           "except ImportError: # comment\\n"
-    ...           "    print 'Can not import' # comment2\\n"
-    ...           "else:   print 'Loaded'\\n")
-    True
-
-Balancing continuation
-
-    >>> roundtrip("a = (3,4, \\n"
-    ...           "5,6)\\n"
-    ...           "y = [3, 4,\\n"
-    ...           "5]\\n"
-    ...           "z = {'a': 5,\\n"
-    ...           "'b':15, 'c':True}\\n"
-    ...           "x = len(y) + 5 - a[\\n"
-    ...           "3] - a[2]\\n"
-    ...           "+ len(z) - z[\\n"
-    ...           "'b']\\n")
-    True
-
-Ordinary integers and binary operators
-
-    >>> dump_tokens("0xff <= 255")
+    def test_int(self):
+        # Ordinary integers and binary operators
+        self.check_tokenize("0xff <= 255", """\
     NUMBER     '0xff'        (1, 0) (1, 4)
     OP         '<='          (1, 5) (1, 7)
     NUMBER     '255'         (1, 8) (1, 11)
-    >>> dump_tokens("0b10 <= 255")
+    """)
+        self.check_tokenize("0b10 <= 255", """\
     NUMBER     '0b10'        (1, 0) (1, 4)
     OP         '<='          (1, 5) (1, 7)
     NUMBER     '255'         (1, 8) (1, 11)
-    >>> dump_tokens("0o123 <= 0123")
+    """)
+        self.check_tokenize("0o123 <= 0123", """\
     NUMBER     '0o123'       (1, 0) (1, 5)
     OP         '<='          (1, 6) (1, 8)
     NUMBER     '0123'        (1, 9) (1, 13)
-    >>> dump_tokens("01234567 > ~0x15")
+    """)
+        self.check_tokenize("01234567 > ~0x15", """\
     NUMBER     '01234567'    (1, 0) (1, 8)
     OP         '>'           (1, 9) (1, 10)
     OP         '~'           (1, 11) (1, 12)
     NUMBER     '0x15'        (1, 12) (1, 16)
-    >>> dump_tokens("2134568 != 01231515")
+    """)
+        self.check_tokenize("2134568 != 01231515", """\
     NUMBER     '2134568'     (1, 0) (1, 7)
     OP         '!='          (1, 8) (1, 10)
     NUMBER     '01231515'    (1, 11) (1, 19)
-    >>> dump_tokens("(-124561-1) & 0200000000")
+    """)
+        self.check_tokenize("(-124561-1) & 0200000000", """\
     OP         '('           (1, 0) (1, 1)
     OP         '-'           (1, 1) (1, 2)
     NUMBER     '124561'      (1, 2) (1, 8)
@@ -152,78 +100,93 @@
     OP         ')'           (1, 10) (1, 11)
     OP         '&'           (1, 12) (1, 13)
     NUMBER     '0200000000'  (1, 14) (1, 24)
-    >>> dump_tokens("0xdeadbeef != -1")
+    """)
+        self.check_tokenize("0xdeadbeef != -1", """\
     NUMBER     '0xdeadbeef'  (1, 0) (1, 10)
     OP         '!='          (1, 11) (1, 13)
     OP         '-'           (1, 14) (1, 15)
     NUMBER     '1'           (1, 15) (1, 16)
-    >>> dump_tokens("0xdeadc0de & 012345")
+    """)
+        self.check_tokenize("0xdeadc0de & 012345", """\
     NUMBER     '0xdeadc0de'  (1, 0) (1, 10)
     OP         '&'           (1, 11) (1, 12)
     NUMBER     '012345'      (1, 13) (1, 19)
-    >>> dump_tokens("0xFF & 0x15 | 1234")
+    """)
+        self.check_tokenize("0xFF & 0x15 | 1234", """\
     NUMBER     '0xFF'        (1, 0) (1, 4)
     OP         '&'           (1, 5) (1, 6)
     NUMBER     '0x15'        (1, 7) (1, 11)
     OP         '|'           (1, 12) (1, 13)
     NUMBER     '1234'        (1, 14) (1, 18)
+    """)
 
-Long integers
-
-    >>> dump_tokens("x = 0L")
+    def test_long(self):
+        # Long integers
+        self.check_tokenize("x = 0L", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '0L'          (1, 4) (1, 6)
-    >>> dump_tokens("x = 0xfffffffffff")
+    """)
+        self.check_tokenize("x = 0xfffffffffff", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '0xffffffffff (1, 4) (1, 17)
-    >>> dump_tokens("x = 123141242151251616110l")
+    """)
+        self.check_tokenize("x = 123141242151251616110l", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '123141242151 (1, 4) (1, 26)
-    >>> dump_tokens("x = -15921590215012591L")
+    """)
+        self.check_tokenize("x = -15921590215012591L", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     OP         '-'           (1, 4) (1, 5)
     NUMBER     '159215902150 (1, 5) (1, 23)
+    """)
 
-Floating point numbers
-
-    >>> dump_tokens("x = 3.14159")
+    def test_float(self):
+        # Floating point numbers
+        self.check_tokenize("x = 3.14159", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '3.14159'     (1, 4) (1, 11)
-    >>> dump_tokens("x = 314159.")
+    """)
+        self.check_tokenize("x = 314159.", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '314159.'     (1, 4) (1, 11)
-    >>> dump_tokens("x = .314159")
+    """)
+        self.check_tokenize("x = .314159", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '.314159'     (1, 4) (1, 11)
-    >>> dump_tokens("x = 3e14159")
+    """)
+        self.check_tokenize("x = 3e14159", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '3e14159'     (1, 4) (1, 11)
-    >>> dump_tokens("x = 3E123")
+    """)
+        self.check_tokenize("x = 3E123", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '3E123'       (1, 4) (1, 9)
-    >>> dump_tokens("x+y = 3e-1230")
+    """)
+        self.check_tokenize("x+y = 3e-1230", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '+'           (1, 1) (1, 2)
     NAME       'y'           (1, 2) (1, 3)
     OP         '='           (1, 4) (1, 5)
     NUMBER     '3e-1230'     (1, 6) (1, 13)
-    >>> dump_tokens("x = 3.14e159")
+    """)
+        self.check_tokenize("x = 3.14e159", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '3.14e159'    (1, 4) (1, 12)
+    """)
 
-String literals
-
-    >>> dump_tokens("x = ''; y = \\\"\\\"")
+    def test_string(self):
+        # String literals
+        self.check_tokenize("x = ''; y = \"\"", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     "''"          (1, 4) (1, 6)
@@ -231,7 +194,8 @@
     NAME       'y'           (1, 8) (1, 9)
     OP         '='           (1, 10) (1, 11)
     STRING     '""'          (1, 12) (1, 14)
-    >>> dump_tokens("x = '\\\"'; y = \\\"'\\\"")
+    """)
+        self.check_tokenize("x = '\"'; y = \"'\"", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     '\\'"\\''       (1, 4) (1, 7)
@@ -239,25 +203,29 @@
     NAME       'y'           (1, 9) (1, 10)
     OP         '='           (1, 11) (1, 12)
     STRING     '"\\'"'        (1, 13) (1, 16)
-    >>> dump_tokens("x = \\\"doesn't \\\"shrink\\\", does it\\\"")
+    """)
+        self.check_tokenize("x = \"doesn't \"shrink\", does it\"", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     '"doesn\\'t "' (1, 4) (1, 14)
     NAME       'shrink'      (1, 14) (1, 20)
     STRING     '", does it"' (1, 20) (1, 31)
-    >>> dump_tokens("x = u'abc' + U'ABC'")
+    """)
+        self.check_tokenize("x = u'abc' + U'ABC'", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     "u'abc'"      (1, 4) (1, 10)
     OP         '+'           (1, 11) (1, 12)
     STRING     "U'ABC'"      (1, 13) (1, 19)
-    >>> dump_tokens('y = u"ABC" + U"ABC"')
+    """)
+        self.check_tokenize('y = u"ABC" + U"ABC"', """\
     NAME       'y'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     'u"ABC"'      (1, 4) (1, 10)
     OP         '+'           (1, 11) (1, 12)
     STRING     'U"ABC"'      (1, 13) (1, 19)
-    >>> dump_tokens("x = ur'abc' + Ur'ABC' + uR'ABC' + UR'ABC'")
+    """)
+        self.check_tokenize("x = ur'abc' + Ur'ABC' + uR'ABC' + UR'ABC'", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     "ur'abc'"     (1, 4) (1, 11)
@@ -267,7 +235,8 @@
     STRING     "uR'ABC'"     (1, 24) (1, 31)
     OP         '+'           (1, 32) (1, 33)
     STRING     "UR'ABC'"     (1, 34) (1, 41)
-    >>> dump_tokens('y = ur"abc" + Ur"ABC" + uR"ABC" + UR"ABC"')
+    """)
+        self.check_tokenize('y = ur"abc" + Ur"ABC" + uR"ABC" + UR"ABC"', """\
     NAME       'y'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     'ur"abc"'     (1, 4) (1, 11)
@@ -278,15 +247,18 @@
     OP         '+'           (1, 32) (1, 33)
     STRING     'UR"ABC"'     (1, 34) (1, 41)
 
-    >>> dump_tokens("b'abc' + B'abc'")
+    """)
+        self.check_tokenize("b'abc' + B'abc'", """\
     STRING     "b'abc'"      (1, 0) (1, 6)
     OP         '+'           (1, 7) (1, 8)
     STRING     "B'abc'"      (1, 9) (1, 15)
-    >>> dump_tokens('b"abc" + B"abc"')
+    """)
+        self.check_tokenize('b"abc" + B"abc"', """\
     STRING     'b"abc"'      (1, 0) (1, 6)
     OP         '+'           (1, 7) (1, 8)
     STRING     'B"abc"'      (1, 9) (1, 15)
-    >>> dump_tokens("br'abc' + bR'abc' + Br'abc' + BR'abc'")
+    """)
+        self.check_tokenize("br'abc' + bR'abc' + Br'abc' + BR'abc'", """\
     STRING     "br'abc'"     (1, 0) (1, 7)
     OP         '+'           (1, 8) (1, 9)
     STRING     "bR'abc'"     (1, 10) (1, 17)
@@ -294,7 +266,8 @@
     STRING     "Br'abc'"     (1, 20) (1, 27)
     OP         '+'           (1, 28) (1, 29)
     STRING     "BR'abc'"     (1, 30) (1, 37)
-    >>> dump_tokens('br"abc" + bR"abc" + Br"abc" + BR"abc"')
+    """)
+        self.check_tokenize('br"abc" + bR"abc" + Br"abc" + BR"abc"', """\
     STRING     'br"abc"'     (1, 0) (1, 7)
     OP         '+'           (1, 8) (1, 9)
     STRING     'bR"abc"'     (1, 10) (1, 17)
@@ -302,10 +275,10 @@
     STRING     'Br"abc"'     (1, 20) (1, 27)
     OP         '+'           (1, 28) (1, 29)
     STRING     'BR"abc"'     (1, 30) (1, 37)
+    """)
 
-Operators
-
-    >>> dump_tokens("def d22(a, b, c=2, d=2, *k): pass")
+    def test_function(self):
+        self.check_tokenize("def d22(a, b, c=2, d=2, *k): pass", """\
     NAME       'def'         (1, 0) (1, 3)
     NAME       'd22'         (1, 4) (1, 7)
     OP         '('           (1, 7) (1, 8)
@@ -326,7 +299,8 @@
     OP         ')'           (1, 26) (1, 27)
     OP         ':'           (1, 27) (1, 28)
     NAME       'pass'        (1, 29) (1, 33)
-    >>> dump_tokens("def d01v_(a=1, *k, **w): pass")
+    """)
+        self.check_tokenize("def d01v_(a=1, *k, **w): pass", """\
     NAME       'def'         (1, 0) (1, 3)
     NAME       'd01v_'       (1, 4) (1, 9)
     OP         '('           (1, 9) (1, 10)
@@ -342,11 +316,12 @@
     OP         ')'           (1, 22) (1, 23)
     OP         ':'           (1, 23) (1, 24)
     NAME       'pass'        (1, 25) (1, 29)
+    """)
 
-Comparison
-
-    >>> dump_tokens("if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != " +
-    ...             "1 and 5 in 1 not in 1 is 1 or 5 is not 1: pass")
+    def test_comparison(self):
+        # Comparison
+        self.check_tokenize("if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != " +
+                            "1 and 5 in 1 not in 1 is 1 or 5 is not 1: pass", """\
     NAME       'if'          (1, 0) (1, 2)
     NUMBER     '1'           (1, 3) (1, 4)
     OP         '<'           (1, 5) (1, 6)
@@ -379,10 +354,11 @@
     NUMBER     '1'           (1, 81) (1, 82)
     OP         ':'           (1, 82) (1, 83)
     NAME       'pass'        (1, 84) (1, 88)
+    """)
 
-Shift
-
-    >>> dump_tokens("x = 1 << 1 >> 5")
+    def test_shift(self):
+        # Shift
+        self.check_tokenize("x = 1 << 1 >> 5", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '1'           (1, 4) (1, 5)
@@ -390,10 +366,11 @@
     NUMBER     '1'           (1, 9) (1, 10)
     OP         '>>'          (1, 11) (1, 13)
     NUMBER     '5'           (1, 14) (1, 15)
+    """)
 
-Additive
-
-    >>> dump_tokens("x = 1 - y + 15 - 01 + 0x124 + z + a[5]")
+    def test_additive(self):
+        # Additive
+        self.check_tokenize("x = 1 - y + 15 - 01 + 0x124 + z + a[5]", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '1'           (1, 4) (1, 5)
@@ -412,10 +389,11 @@
     OP         '['           (1, 35) (1, 36)
     NUMBER     '5'           (1, 36) (1, 37)
     OP         ']'           (1, 37) (1, 38)
+    """)
 
-Multiplicative
-
-    >>> dump_tokens("x = 1//1*1/5*12%0x12")
+    def test_multiplicative(self):
+        # Multiplicative
+        self.check_tokenize("x = 1//1*1/5*12%0x12", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '1'           (1, 4) (1, 5)
@@ -429,10 +407,11 @@
     NUMBER     '12'          (1, 13) (1, 15)
     OP         '%'           (1, 15) (1, 16)
     NUMBER     '0x12'        (1, 16) (1, 20)
+    """)
 
-Unary
-
-    >>> dump_tokens("~1 ^ 1 & 1 |1 ^ -1")
+    def test_unary(self):
+        # Unary
+        self.check_tokenize("~1 ^ 1 & 1 |1 ^ -1", """\
     OP         '~'           (1, 0) (1, 1)
     NUMBER     '1'           (1, 1) (1, 2)
     OP         '^'           (1, 3) (1, 4)
@@ -444,7 +423,8 @@
     OP         '^'           (1, 14) (1, 15)
     OP         '-'           (1, 16) (1, 17)
     NUMBER     '1'           (1, 17) (1, 18)
-    >>> dump_tokens("-1*1/1+1*1//1 - ---1**1")
+    """)
+        self.check_tokenize("-1*1/1+1*1//1 - ---1**1", """\
     OP         '-'           (1, 0) (1, 1)
     NUMBER     '1'           (1, 1) (1, 2)
     OP         '*'           (1, 2) (1, 3)
@@ -464,10 +444,12 @@
     NUMBER     '1'           (1, 19) (1, 20)
     OP         '**'          (1, 20) (1, 22)
     NUMBER     '1'           (1, 22) (1, 23)
+    """)
 
-Selector
-
-    >>> dump_tokens("import sys, time\\nx = sys.modules['time'].time()")
+    def test_selector(self):
+        # Selector
+        self.check_tokenize("import sys, time\n"
+                            "x = sys.modules['time'].time()", """\
     NAME       'import'      (1, 0) (1, 6)
     NAME       'sys'         (1, 7) (1, 10)
     OP         ','           (1, 10) (1, 11)
@@ -485,10 +467,12 @@
     NAME       'time'        (2, 24) (2, 28)
     OP         '('           (2, 28) (2, 29)
     OP         ')'           (2, 29) (2, 30)
+    """)
 
-Methods
-
-    >>> dump_tokens("@staticmethod\\ndef foo(x,y): pass")
+    def test_method(self):
+        # Methods
+        self.check_tokenize("@staticmethod\n"
+                            "def foo(x,y): pass", """\
     OP         '@'           (1, 0) (1, 1)
     NAME       'staticmethod (1, 1) (1, 13)
     NEWLINE    '\\n'          (1, 13) (1, 14)
@@ -501,41 +485,13 @@
     OP         ')'           (2, 11) (2, 12)
     OP         ':'           (2, 12) (2, 13)
     NAME       'pass'        (2, 14) (2, 18)
+    """)
 
-Backslash means line continuation, except for comments
-
-    >>> roundtrip("x=1+\\\\n"
-    ...           "1\\n"
-    ...           "# This is a comment\\\\n"
-    ...           "# This also\\n")
-    True
-    >>> roundtrip("# Comment \\\\nx = 0")
-    True
-
-Two string literals on the same line
-
-    >>> roundtrip("'' ''")
-    True
-
-Test roundtrip on random python modules.
-pass the '-ucpu' option to process the full directory.
-
-    >>>
-    >>> tempdir = os.path.dirname(f) or os.curdir
-    >>> testfiles = glob.glob(os.path.join(tempdir, "test*.py"))
-
-    >>> if not test_support.is_resource_enabled("cpu"):
-    ...     testfiles = random.sample(testfiles, 10)
-    ...
-    >>> for testfile in testfiles:
-    ...     if not roundtrip(open(testfile)):
-    ...         print "Roundtrip failed for file %s" % testfile
-    ...         break
-    ... else: True
-    True
-
-Evil tabs
-    >>> dump_tokens("def f():\\n\\tif x\\n        \\tpass")
+    def test_tabs(self):
+        # Evil tabs
+        self.check_tokenize("def f():\n"
+                            "\tif x\n"
+                            "        \tpass", """\
     NAME       'def'         (1, 0) (1, 3)
     NAME       'f'           (1, 4) (1, 5)
     OP         '('           (1, 5) (1, 6)
@@ -550,56 +506,16 @@
     NAME       'pass'        (3, 9) (3, 13)
     DEDENT     ''            (4, 0) (4, 0)
     DEDENT     ''            (4, 0) (4, 0)
+    """)
 
-Pathological whitespace (http://bugs.python.org/issue16152)
-    >>> dump_tokens("@          ")
+    def test_pathological_trailing_whitespace(self):
+        # Pathological whitespace (http://bugs.python.org/issue16152)
+        self.check_tokenize("@          ", """\
     OP         '@'           (1, 0) (1, 1)
-"""
+    """)
 
 
-from test import test_support
-from tokenize import (untokenize, generate_tokens, NUMBER, NAME, OP,
-                     STRING, ENDMARKER, tok_name, Untokenizer, tokenize)
-from StringIO import StringIO
-import os
-from unittest import TestCase
-
-def dump_tokens(s):
-    """Print out the tokens in s in a table format.
-
-    The ENDMARKER is omitted.
-    """
-    f = StringIO(s)
-    for type, token, start, end, line in generate_tokens(f.readline):
-        if type == ENDMARKER:
-            break
-        type = tok_name[type]
-        print("%(type)-10.10s %(token)-13.13r %(start)s %(end)s" % locals())
-
-# This is an example from the docs, set up as a doctest.
 def decistmt(s):
-    """Substitute Decimals for floats in a string of statements.
-
-    >>> from decimal import Decimal
-    >>> s = 'print +21.3e-5*-.1234/81.7'
-    >>> decistmt(s)
-    "print +Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')"
-
-    The format of the exponent is inherited from the platform C library.
-    Known cases are "e-007" (Windows) and "e-07" (not Windows).  Since
-    we're only showing 12 digits, and the 13th isn't close to 5, the
-    rest of the output should be platform-independent.
-
-    >>> exec(s) #doctest: +ELLIPSIS
-    -3.21716034272e-0...7
-
-    Output from calculations with Decimal should be identical across all
-    platforms.
-
-    >>> exec(decistmt(s))
-    -3.217160342717258261933904529E-7
-    """
-
     result = []
     g = generate_tokens(StringIO(s).readline)   # tokenize the string
     for toknum, tokval, _, _, _  in g:
@@ -614,6 +530,27 @@
             result.append((toknum, tokval))
     return untokenize(result)
 
+class TestMisc(TestCase):
+
+    def test_decistmt(self):
+        # Substitute Decimals for floats in a string of statements.
+        # This is an example from the docs.
+
+        from decimal import Decimal
+        s = '+21.3e-5*-.1234/81.7'
+        self.assertEqual(decistmt(s),
+                         "+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')")
+
+        # The format of the exponent is inherited from the platform C library.
+        # Known cases are "e-007" (Windows) and "e-07" (not Windows).  Since
+        # we're only showing 12 digits, and the 13th isn't close to 5, the
+        # rest of the output should be platform-independent.
+        self.assertRegexpMatches(str(eval(s)), '-3.21716034272e-0+7')
+
+        # Output from calculations with Decimal should be identical across all
+        # platforms.
+        self.assertEqual(eval(decistmt(s)), Decimal('-3.217160342717258261933904529E-7'))
+
 
 class UntokenizeTest(TestCase):
 
@@ -651,6 +588,115 @@
 
 
 class TestRoundtrip(TestCase):
+
+    def check_roundtrip(self, f):
+        """
+        Test roundtrip for `untokenize`. `f` is an open file or a string.
+        The source code in f is tokenized, converted back to source code
+        via tokenize.untokenize(), and tokenized again from the latter.
+        The test fails if the second tokenization doesn't match the first.
+        """
+        if isinstance(f, str): f = StringIO(f)
+        token_list = list(generate_tokens(f.readline))
+        f.close()
+        tokens1 = [tok[:2] for tok in token_list]
+        new_text = untokenize(tokens1)
+        readline = iter(new_text.splitlines(1)).next
+        tokens2 = [tok[:2] for tok in generate_tokens(readline)]
+        self.assertEqual(tokens2, tokens1)
+
+    def test_roundtrip(self):
+        # There are some standard formatting practices that are easy to get right.
+
+        self.check_roundtrip("if x == 1:\n"
+                             "    print(x)\n")
+
+        # There are some standard formatting practices that are easy to get right.
+
+        self.check_roundtrip("if x == 1:\n"
+                             "    print x\n")
+        self.check_roundtrip("# This is a comment\n"
+                             "# This also")
+
+        # Some people use different formatting conventions, which makes
+        # untokenize a little trickier. Note that this test involves trailing
+        # whitespace after the colon. Note that we use hex escapes to make the
+        # two trailing blanks apperant in the expected output.
+
+        self.check_roundtrip("if x == 1 : \n"
+                             "  print x\n")
+        fn = test_support.findfile("tokenize_tests" + os.extsep + "txt")
+        with open(fn) as f:
+            self.check_roundtrip(f)
+        self.check_roundtrip("if x == 1:\n"
+                             "    # A comment by itself.\n"
+                             "    print x # Comment here, too.\n"
+                             "    # Another comment.\n"
+                             "after_if = True\n")
+        self.check_roundtrip("if (x # The comments need to go in the right place\n"
+                             "    == 1):\n"
+                             "    print 'x==1'\n")
+        self.check_roundtrip("class Test: # A comment here\n"
+                             "  # A comment with weird indent\n"
+                             "  after_com = 5\n"
+                             "  def x(m): return m*5 # a one liner\n"
+                             "  def y(m): # A whitespace after the colon\n"
+                             "     return y*4 # 3-space indent\n")
+
+        # Some error-handling code
+
+        self.check_roundtrip("try: import somemodule\n"
+                             "except ImportError: # comment\n"
+                             "    print 'Can not import' # comment2\n"
+                             "else:   print 'Loaded'\n")
+
+    def test_continuation(self):
+        # Balancing continuation
+        self.check_roundtrip("a = (3,4, \n"
+                             "5,6)\n"
+                             "y = [3, 4,\n"
+                             "5]\n"
+                             "z = {'a': 5,\n"
+                             "'b':15, 'c':True}\n"
+                             "x = len(y) + 5 - a[\n"
+                             "3] - a[2]\n"
+                             "+ len(z) - z[\n"
+                             "'b']\n")
+
+    def test_backslash_continuation(self):
+        # Backslash means line continuation, except for comments
+        self.check_roundtrip("x=1+\\\n"
+                             "1\n"
+                             "# This is a comment\\\n"
+                             "# This also\n")
+        self.check_roundtrip("# Comment \\\n"
+                             "x = 0")
+
+    def test_string_concatenation(self):
+        # Two string literals on the same line
+        self.check_roundtrip("'' ''")
+
+    def test_random_files(self):
+        # Test roundtrip on random python modules.
+        # pass the '-ucpu' option to process the full directory.
+
+        import glob, random
+        fn = test_support.findfile("tokenize_tests" + os.extsep + "txt")
+        tempdir = os.path.dirname(fn) or os.curdir
+        testfiles = glob.glob(os.path.join(tempdir, "test*.py"))
+
+        if not test_support.is_resource_enabled("cpu"):
+            testfiles = random.sample(testfiles, 10)
+
+        for testfile in testfiles:
+            try:
+                with open(testfile, 'rb') as f:
+                    self.check_roundtrip(f)
+            except:
+                print "Roundtrip failed for file %s" % testfile
+                raise
+
+
     def roundtrip(self, code):
         if isinstance(code, str):
             code = code.encode('utf-8')
@@ -667,13 +713,11 @@
         self.assertEqual(codelines[1], codelines[2])
 
 
-__test__ = {"doctests" : doctests, 'decistmt': decistmt}
-
 def test_main():
-    from test import test_tokenize
-    test_support.run_doctest(test_tokenize, True)
+    test_support.run_unittest(TokenizeTest)
     test_support.run_unittest(UntokenizeTest)
     test_support.run_unittest(TestRoundtrip)
+    test_support.run_unittest(TestMisc)
 
 if __name__ == "__main__":
     test_main()

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Tue Oct  6 17:54:19 2015
From: python-checkins at python.org (serhiy.storchaka)
Date: Tue, 06 Oct 2015 15:54:19 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
	=?utf-8?q?_Null_merge?=
Message-ID: <20151006155418.3285.23768@psf.io>

https://hg.python.org/cpython/rev/b0ce3ef2ea21
changeset:   98562:b0ce3ef2ea21
branch:      3.5
parent:      98560:df8ccac22006
parent:      98559:91f36d2b097a
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Tue Oct 06 18:47:26 2015 +0300
summary:
  Null merge

files:



-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Tue Oct  6 17:54:19 2015
From: python-checkins at python.org (serhiy.storchaka)
Date: Tue, 06 Oct 2015 15:54:19 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_default_-=3E_default?=
	=?utf-8?q?=29=3A_Merge_heads?=
Message-ID: <20151006155418.20763.71889@psf.io>

https://hg.python.org/cpython/rev/727d72b05ff5
changeset:   98561:727d72b05ff5
parent:      98558:66d239660997
parent:      98554:3e2218a4e629
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Tue Oct 06 18:40:09 2015 +0300
summary:
  Merge heads

files:
  Lib/test/test_asyncio/test_base_events.py |  5 +----
  1 files changed, 1 insertions(+), 4 deletions(-)


diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py
--- a/Lib/test/test_asyncio/test_base_events.py
+++ b/Lib/test/test_asyncio/test_base_events.py
@@ -1215,6 +1215,7 @@
 
     def test_create_datagram_endpoint_sock(self):
         sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        sock.bind(('127.0.0.1', 0))
         fut = self.loop.create_datagram_endpoint(
             lambda: MyDatagramProto(create_future=True, loop=self.loop),
             sock=sock)
@@ -1307,10 +1308,6 @@
             self.assertTrue(
                 sock.getsockopt(
                     socket.SOL_SOCKET, socket.SO_REUSEPORT))
-        else:
-            self.assertFalse(
-                sock.getsockopt(
-                    socket.SOL_SOCKET, socket.SO_REUSEPORT))
         self.assertTrue(
             sock.getsockopt(
                 socket.SOL_SOCKET, socket.SO_BROADCAST))

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Tue Oct  6 17:54:19 2015
From: python-checkins at python.org (serhiy.storchaka)
Date: Tue, 06 Oct 2015 15:54:19 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy41IC0+IDMuNSk6?=
	=?utf-8?q?_Merge_heads?=
Message-ID: <20151006155417.2685.80829@psf.io>

https://hg.python.org/cpython/rev/df8ccac22006
changeset:   98560:df8ccac22006
branch:      3.5
parent:      98553:4d643c5df2a5
parent:      98557:bff40616d2a5
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Tue Oct 06 18:39:58 2015 +0300
summary:
  Merge heads

files:
  Lib/test/test_tokenize.py |  811 ++++++++++++-------------
  1 files changed, 395 insertions(+), 416 deletions(-)


diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -1,22 +1,44 @@
-doctests = """
-Tests for the tokenize module.
+from test import support
+from tokenize import (tokenize, _tokenize, untokenize, NUMBER, NAME, OP,
+                     STRING, ENDMARKER, ENCODING, tok_name, detect_encoding,
+                     open as tokenize_open, Untokenizer)
+from io import BytesIO
+from unittest import TestCase, mock
+import os
+import token
 
-The tests can be really simple. Given a small fragment of source
-code, print out a table with tokens. The ENDMARKER is omitted for
-brevity.
 
-    >>> import glob
+class TokenizeTest(TestCase):
+    # Tests for the tokenize module.
 
-    >>> dump_tokens("1 + 1")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    # The tests can be really simple. Given a small fragment of source
+    # code, print out a table with tokens. The ENDMARKER is omitted for
+    # brevity.
+
+    def check_tokenize(self, s, expected):
+        # Format the tokens in s in a table format.
+        # The ENDMARKER is omitted.
+        result = []
+        f = BytesIO(s.encode('utf-8'))
+        for type, token, start, end, line in tokenize(f.readline):
+            if type == ENDMARKER:
+                break
+            type = tok_name[type]
+            result.append("    %(type)-10.10s %(token)-13.13r %(start)s %(end)s" %
+                          locals())
+        self.assertEqual(result,
+                         ["    ENCODING   'utf-8'       (0, 0) (0, 0)"] +
+                         expected.rstrip().splitlines())
+
+    def test_basic(self):
+        self.check_tokenize("1 + 1", """\
     NUMBER     '1'           (1, 0) (1, 1)
     OP         '+'           (1, 2) (1, 3)
     NUMBER     '1'           (1, 4) (1, 5)
-
-    >>> dump_tokens("if False:\\n"
-    ...             "    # NL\\n"
-    ...             "    True = False # NEWLINE\\n")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("if False:\n"
+                            "    # NL\n"
+                            "    True = False # NEWLINE\n", """\
     NAME       'if'          (1, 0) (1, 2)
     NAME       'False'       (1, 3) (1, 8)
     OP         ':'           (1, 8) (1, 9)
@@ -30,112 +52,48 @@
     COMMENT    '# NEWLINE'   (3, 17) (3, 26)
     NEWLINE    '\\n'          (3, 26) (3, 27)
     DEDENT     ''            (4, 0) (4, 0)
+    """)
+        indent_error_file = b"""\
+def k(x):
+    x += 2
+  x += 5
+"""
+        readline = BytesIO(indent_error_file).readline
+        with self.assertRaisesRegex(IndentationError,
+                                    "unindent does not match any "
+                                    "outer indentation level"):
+            for tok in tokenize(readline):
+                pass
 
-    >>> indent_error_file = \"""
-    ... def k(x):
-    ...     x += 2
-    ...   x += 5
-    ... \"""
-    >>> readline = BytesIO(indent_error_file.encode('utf-8')).readline
-    >>> for tok in tokenize(readline): pass
-    Traceback (most recent call last):
-        ...
-    IndentationError: unindent does not match any outer indentation level
-
-There are some standard formatting practices that are easy to get right.
-
-    >>> roundtrip("if x == 1:\\n"
-    ...           "    print(x)\\n")
-    True
-
-    >>> roundtrip("# This is a comment\\n# This also")
-    True
-
-Some people use different formatting conventions, which makes
-untokenize a little trickier. Note that this test involves trailing
-whitespace after the colon. Note that we use hex escapes to make the
-two trailing blanks apparent in the expected output.
-
-    >>> roundtrip("if x == 1 : \\n"
-    ...           "  print(x)\\n")
-    True
-
-    >>> f = support.findfile("tokenize_tests.txt")
-    >>> roundtrip(open(f, 'rb'))
-    True
-
-    >>> roundtrip("if x == 1:\\n"
-    ...           "    # A comment by itself.\\n"
-    ...           "    print(x) # Comment here, too.\\n"
-    ...           "    # Another comment.\\n"
-    ...           "after_if = True\\n")
-    True
-
-    >>> roundtrip("if (x # The comments need to go in the right place\\n"
-    ...           "    == 1):\\n"
-    ...           "    print('x==1')\\n")
-    True
-
-    >>> roundtrip("class Test: # A comment here\\n"
-    ...           "  # A comment with weird indent\\n"
-    ...           "  after_com = 5\\n"
-    ...           "  def x(m): return m*5 # a one liner\\n"
-    ...           "  def y(m): # A whitespace after the colon\\n"
-    ...           "     return y*4 # 3-space indent\\n")
-    True
-
-Some error-handling code
-
-    >>> roundtrip("try: import somemodule\\n"
-    ...           "except ImportError: # comment\\n"
-    ...           "    print('Can not import' # comment2\\n)"
-    ...           "else:   print('Loaded')\\n")
-    True
-
-Balancing continuation
-
-    >>> roundtrip("a = (3,4, \\n"
-    ...           "5,6)\\n"
-    ...           "y = [3, 4,\\n"
-    ...           "5]\\n"
-    ...           "z = {'a': 5,\\n"
-    ...           "'b':15, 'c':True}\\n"
-    ...           "x = len(y) + 5 - a[\\n"
-    ...           "3] - a[2]\\n"
-    ...           "+ len(z) - z[\\n"
-    ...           "'b']\\n")
-    True
-
-Ordinary integers and binary operators
-
-    >>> dump_tokens("0xff <= 255")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_int(self):
+        # Ordinary integers and binary operators
+        self.check_tokenize("0xff <= 255", """\
     NUMBER     '0xff'        (1, 0) (1, 4)
     OP         '<='          (1, 5) (1, 7)
     NUMBER     '255'         (1, 8) (1, 11)
-    >>> dump_tokens("0b10 <= 255")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("0b10 <= 255", """\
     NUMBER     '0b10'        (1, 0) (1, 4)
     OP         '<='          (1, 5) (1, 7)
     NUMBER     '255'         (1, 8) (1, 11)
-    >>> dump_tokens("0o123 <= 0O123")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("0o123 <= 0O123", """\
     NUMBER     '0o123'       (1, 0) (1, 5)
     OP         '<='          (1, 6) (1, 8)
     NUMBER     '0O123'       (1, 9) (1, 14)
-    >>> dump_tokens("1234567 > ~0x15")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("1234567 > ~0x15", """\
     NUMBER     '1234567'     (1, 0) (1, 7)
     OP         '>'           (1, 8) (1, 9)
     OP         '~'           (1, 10) (1, 11)
     NUMBER     '0x15'        (1, 11) (1, 15)
-    >>> dump_tokens("2134568 != 1231515")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("2134568 != 1231515", """\
     NUMBER     '2134568'     (1, 0) (1, 7)
     OP         '!='          (1, 8) (1, 10)
     NUMBER     '1231515'     (1, 11) (1, 18)
-    >>> dump_tokens("(-124561-1) & 200000000")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("(-124561-1) & 200000000", """\
     OP         '('           (1, 0) (1, 1)
     OP         '-'           (1, 1) (1, 2)
     NUMBER     '124561'      (1, 2) (1, 8)
@@ -144,93 +102,93 @@
     OP         ')'           (1, 10) (1, 11)
     OP         '&'           (1, 12) (1, 13)
     NUMBER     '200000000'   (1, 14) (1, 23)
-    >>> dump_tokens("0xdeadbeef != -1")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("0xdeadbeef != -1", """\
     NUMBER     '0xdeadbeef'  (1, 0) (1, 10)
     OP         '!='          (1, 11) (1, 13)
     OP         '-'           (1, 14) (1, 15)
     NUMBER     '1'           (1, 15) (1, 16)
-    >>> dump_tokens("0xdeadc0de & 12345")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("0xdeadc0de & 12345", """\
     NUMBER     '0xdeadc0de'  (1, 0) (1, 10)
     OP         '&'           (1, 11) (1, 12)
     NUMBER     '12345'       (1, 13) (1, 18)
-    >>> dump_tokens("0xFF & 0x15 | 1234")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("0xFF & 0x15 | 1234", """\
     NUMBER     '0xFF'        (1, 0) (1, 4)
     OP         '&'           (1, 5) (1, 6)
     NUMBER     '0x15'        (1, 7) (1, 11)
     OP         '|'           (1, 12) (1, 13)
     NUMBER     '1234'        (1, 14) (1, 18)
+    """)
 
-Long integers
-
-    >>> dump_tokens("x = 0")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_long(self):
+        # Long integers
+        self.check_tokenize("x = 0", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '0'           (1, 4) (1, 5)
-    >>> dump_tokens("x = 0xfffffffffff")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = 0xfffffffffff", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '0xffffffffff (1, 4) (1, 17)
-    >>> dump_tokens("x = 123141242151251616110")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = 123141242151251616110", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '123141242151 (1, 4) (1, 25)
-    >>> dump_tokens("x = -15921590215012591")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = -15921590215012591", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     OP         '-'           (1, 4) (1, 5)
     NUMBER     '159215902150 (1, 5) (1, 22)
+    """)
 
-Floating point numbers
-
-    >>> dump_tokens("x = 3.14159")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_float(self):
+        # Floating point numbers
+        self.check_tokenize("x = 3.14159", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '3.14159'     (1, 4) (1, 11)
-    >>> dump_tokens("x = 314159.")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = 314159.", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '314159.'     (1, 4) (1, 11)
-    >>> dump_tokens("x = .314159")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = .314159", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '.314159'     (1, 4) (1, 11)
-    >>> dump_tokens("x = 3e14159")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = 3e14159", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '3e14159'     (1, 4) (1, 11)
-    >>> dump_tokens("x = 3E123")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = 3E123", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '3E123'       (1, 4) (1, 9)
-    >>> dump_tokens("x+y = 3e-1230")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x+y = 3e-1230", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '+'           (1, 1) (1, 2)
     NAME       'y'           (1, 2) (1, 3)
     OP         '='           (1, 4) (1, 5)
     NUMBER     '3e-1230'     (1, 6) (1, 13)
-    >>> dump_tokens("x = 3.14e159")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = 3.14e159", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '3.14e159'    (1, 4) (1, 12)
+    """)
 
-String literals
-
-    >>> dump_tokens("x = ''; y = \\\"\\\"")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_string(self):
+        # String literals
+        self.check_tokenize("x = ''; y = \"\"", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     "''"          (1, 4) (1, 6)
@@ -238,8 +196,8 @@
     NAME       'y'           (1, 8) (1, 9)
     OP         '='           (1, 10) (1, 11)
     STRING     '""'          (1, 12) (1, 14)
-    >>> dump_tokens("x = '\\\"'; y = \\\"'\\\"")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = '\"'; y = \"'\"", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     '\\'"\\''       (1, 4) (1, 7)
@@ -247,29 +205,29 @@
     NAME       'y'           (1, 9) (1, 10)
     OP         '='           (1, 11) (1, 12)
     STRING     '"\\'"'        (1, 13) (1, 16)
-    >>> dump_tokens("x = \\\"doesn't \\\"shrink\\\", does it\\\"")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = \"doesn't \"shrink\", does it\"", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     '"doesn\\'t "' (1, 4) (1, 14)
     NAME       'shrink'      (1, 14) (1, 20)
     STRING     '", does it"' (1, 20) (1, 31)
-    >>> dump_tokens("x = 'abc' + 'ABC'")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = 'abc' + 'ABC'", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     "'abc'"       (1, 4) (1, 9)
     OP         '+'           (1, 10) (1, 11)
     STRING     "'ABC'"       (1, 12) (1, 17)
-    >>> dump_tokens('y = "ABC" + "ABC"')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize('y = "ABC" + "ABC"', """\
     NAME       'y'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     '"ABC"'       (1, 4) (1, 9)
     OP         '+'           (1, 10) (1, 11)
     STRING     '"ABC"'       (1, 12) (1, 17)
-    >>> dump_tokens("x = r'abc' + r'ABC' + R'ABC' + R'ABC'")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = r'abc' + r'ABC' + R'ABC' + R'ABC'", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     "r'abc'"      (1, 4) (1, 10)
@@ -279,8 +237,8 @@
     STRING     "R'ABC'"      (1, 22) (1, 28)
     OP         '+'           (1, 29) (1, 30)
     STRING     "R'ABC'"      (1, 31) (1, 37)
-    >>> dump_tokens('y = r"abc" + r"ABC" + R"ABC" + R"ABC"')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize('y = r"abc" + r"ABC" + R"ABC" + R"ABC"', """\
     NAME       'y'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     'r"abc"'      (1, 4) (1, 10)
@@ -290,30 +248,30 @@
     STRING     'R"ABC"'      (1, 22) (1, 28)
     OP         '+'           (1, 29) (1, 30)
     STRING     'R"ABC"'      (1, 31) (1, 37)
+    """)
 
-    >>> dump_tokens("u'abc' + U'abc'")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("u'abc' + U'abc'", """\
     STRING     "u'abc'"      (1, 0) (1, 6)
     OP         '+'           (1, 7) (1, 8)
     STRING     "U'abc'"      (1, 9) (1, 15)
-    >>> dump_tokens('u"abc" + U"abc"')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize('u"abc" + U"abc"', """\
     STRING     'u"abc"'      (1, 0) (1, 6)
     OP         '+'           (1, 7) (1, 8)
     STRING     'U"abc"'      (1, 9) (1, 15)
+    """)
 
-    >>> dump_tokens("b'abc' + B'abc'")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("b'abc' + B'abc'", """\
     STRING     "b'abc'"      (1, 0) (1, 6)
     OP         '+'           (1, 7) (1, 8)
     STRING     "B'abc'"      (1, 9) (1, 15)
-    >>> dump_tokens('b"abc" + B"abc"')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize('b"abc" + B"abc"', """\
     STRING     'b"abc"'      (1, 0) (1, 6)
     OP         '+'           (1, 7) (1, 8)
     STRING     'B"abc"'      (1, 9) (1, 15)
-    >>> dump_tokens("br'abc' + bR'abc' + Br'abc' + BR'abc'")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("br'abc' + bR'abc' + Br'abc' + BR'abc'", """\
     STRING     "br'abc'"     (1, 0) (1, 7)
     OP         '+'           (1, 8) (1, 9)
     STRING     "bR'abc'"     (1, 10) (1, 17)
@@ -321,8 +279,8 @@
     STRING     "Br'abc'"     (1, 20) (1, 27)
     OP         '+'           (1, 28) (1, 29)
     STRING     "BR'abc'"     (1, 30) (1, 37)
-    >>> dump_tokens('br"abc" + bR"abc" + Br"abc" + BR"abc"')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize('br"abc" + bR"abc" + Br"abc" + BR"abc"', """\
     STRING     'br"abc"'     (1, 0) (1, 7)
     OP         '+'           (1, 8) (1, 9)
     STRING     'bR"abc"'     (1, 10) (1, 17)
@@ -330,8 +288,8 @@
     STRING     'Br"abc"'     (1, 20) (1, 27)
     OP         '+'           (1, 28) (1, 29)
     STRING     'BR"abc"'     (1, 30) (1, 37)
-    >>> dump_tokens("rb'abc' + rB'abc' + Rb'abc' + RB'abc'")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("rb'abc' + rB'abc' + Rb'abc' + RB'abc'", """\
     STRING     "rb'abc'"     (1, 0) (1, 7)
     OP         '+'           (1, 8) (1, 9)
     STRING     "rB'abc'"     (1, 10) (1, 17)
@@ -339,8 +297,8 @@
     STRING     "Rb'abc'"     (1, 20) (1, 27)
     OP         '+'           (1, 28) (1, 29)
     STRING     "RB'abc'"     (1, 30) (1, 37)
-    >>> dump_tokens('rb"abc" + rB"abc" + Rb"abc" + RB"abc"')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize('rb"abc" + rB"abc" + Rb"abc" + RB"abc"', """\
     STRING     'rb"abc"'     (1, 0) (1, 7)
     OP         '+'           (1, 8) (1, 9)
     STRING     'rB"abc"'     (1, 10) (1, 17)
@@ -348,11 +306,10 @@
     STRING     'Rb"abc"'     (1, 20) (1, 27)
     OP         '+'           (1, 28) (1, 29)
     STRING     'RB"abc"'     (1, 30) (1, 37)
+    """)
 
-Operators
-
-    >>> dump_tokens("def d22(a, b, c=2, d=2, *k): pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_function(self):
+        self.check_tokenize("def d22(a, b, c=2, d=2, *k): pass", """\
     NAME       'def'         (1, 0) (1, 3)
     NAME       'd22'         (1, 4) (1, 7)
     OP         '('           (1, 7) (1, 8)
@@ -373,8 +330,8 @@
     OP         ')'           (1, 26) (1, 27)
     OP         ':'           (1, 27) (1, 28)
     NAME       'pass'        (1, 29) (1, 33)
-    >>> dump_tokens("def d01v_(a=1, *k, **w): pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("def d01v_(a=1, *k, **w): pass", """\
     NAME       'def'         (1, 0) (1, 3)
     NAME       'd01v_'       (1, 4) (1, 9)
     OP         '('           (1, 9) (1, 10)
@@ -390,12 +347,12 @@
     OP         ')'           (1, 22) (1, 23)
     OP         ':'           (1, 23) (1, 24)
     NAME       'pass'        (1, 25) (1, 29)
+    """)
 
-Comparison
-
-    >>> dump_tokens("if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != " +
-    ...             "1 and 5 in 1 not in 1 is 1 or 5 is not 1: pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_comparison(self):
+        # Comparison
+        self.check_tokenize("if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != "
+                            "1 and 5 in 1 not in 1 is 1 or 5 is not 1: pass", """\
     NAME       'if'          (1, 0) (1, 2)
     NUMBER     '1'           (1, 3) (1, 4)
     OP         '<'           (1, 5) (1, 6)
@@ -428,11 +385,11 @@
     NUMBER     '1'           (1, 81) (1, 82)
     OP         ':'           (1, 82) (1, 83)
     NAME       'pass'        (1, 84) (1, 88)
+    """)
 
-Shift
-
-    >>> dump_tokens("x = 1 << 1 >> 5")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_shift(self):
+        # Shift
+        self.check_tokenize("x = 1 << 1 >> 5", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '1'           (1, 4) (1, 5)
@@ -440,11 +397,11 @@
     NUMBER     '1'           (1, 9) (1, 10)
     OP         '>>'          (1, 11) (1, 13)
     NUMBER     '5'           (1, 14) (1, 15)
+    """)
 
-Additive
-
-    >>> dump_tokens("x = 1 - y + 15 - 1 + 0x124 + z + a[5]")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_additive(self):
+        # Additive
+        self.check_tokenize("x = 1 - y + 15 - 1 + 0x124 + z + a[5]", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '1'           (1, 4) (1, 5)
@@ -463,11 +420,11 @@
     OP         '['           (1, 34) (1, 35)
     NUMBER     '5'           (1, 35) (1, 36)
     OP         ']'           (1, 36) (1, 37)
+    """)
 
-Multiplicative
-
-    >>> dump_tokens("x = 1//1*1/5*12%0x12 at 42")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_multiplicative(self):
+        # Multiplicative
+        self.check_tokenize("x = 1//1*1/5*12%0x12 at 42", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '1'           (1, 4) (1, 5)
@@ -483,11 +440,11 @@
     NUMBER     '0x12'        (1, 16) (1, 20)
     OP         '@'           (1, 20) (1, 21)
     NUMBER     '42'          (1, 21) (1, 23)
+    """)
 
-Unary
-
-    >>> dump_tokens("~1 ^ 1 & 1 |1 ^ -1")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_unary(self):
+        # Unary
+        self.check_tokenize("~1 ^ 1 & 1 |1 ^ -1", """\
     OP         '~'           (1, 0) (1, 1)
     NUMBER     '1'           (1, 1) (1, 2)
     OP         '^'           (1, 3) (1, 4)
@@ -499,8 +456,8 @@
     OP         '^'           (1, 14) (1, 15)
     OP         '-'           (1, 16) (1, 17)
     NUMBER     '1'           (1, 17) (1, 18)
-    >>> dump_tokens("-1*1/1+1*1//1 - ---1**1")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("-1*1/1+1*1//1 - ---1**1", """\
     OP         '-'           (1, 0) (1, 1)
     NUMBER     '1'           (1, 1) (1, 2)
     OP         '*'           (1, 2) (1, 3)
@@ -520,11 +477,11 @@
     NUMBER     '1'           (1, 19) (1, 20)
     OP         '**'          (1, 20) (1, 22)
     NUMBER     '1'           (1, 22) (1, 23)
+    """)
 
-Selector
-
-    >>> dump_tokens("import sys, time\\nx = sys.modules['time'].time()")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_selector(self):
+        # Selector
+        self.check_tokenize("import sys, time\nx = sys.modules['time'].time()", """\
     NAME       'import'      (1, 0) (1, 6)
     NAME       'sys'         (1, 7) (1, 10)
     OP         ','           (1, 10) (1, 11)
@@ -542,11 +499,11 @@
     NAME       'time'        (2, 24) (2, 28)
     OP         '('           (2, 28) (2, 29)
     OP         ')'           (2, 29) (2, 30)
+    """)
 
-Methods
-
-    >>> dump_tokens("@staticmethod\\ndef foo(x,y): pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_method(self):
+        # Methods
+        self.check_tokenize("@staticmethod\ndef foo(x,y): pass", """\
     OP         '@'           (1, 0) (1, 1)
     NAME       'staticmethod (1, 1) (1, 13)
     NEWLINE    '\\n'          (1, 13) (1, 14)
@@ -559,52 +516,13 @@
     OP         ')'           (2, 11) (2, 12)
     OP         ':'           (2, 12) (2, 13)
     NAME       'pass'        (2, 14) (2, 18)
+    """)
 
-Backslash means line continuation, except for comments
-
-    >>> roundtrip("x=1+\\\\n"
-    ...           "1\\n"
-    ...           "# This is a comment\\\\n"
-    ...           "# This also\\n")
-    True
-    >>> roundtrip("# Comment \\\\nx = 0")
-    True
-
-Two string literals on the same line
-
-    >>> roundtrip("'' ''")
-    True
-
-Test roundtrip on random python modules.
-pass the '-ucpu' option to process the full directory.
-
-    >>> import random
-    >>> tempdir = os.path.dirname(f) or os.curdir
-    >>> testfiles = glob.glob(os.path.join(tempdir, "test*.py"))
-
-Tokenize is broken on test_pep3131.py because regular expressions are
-broken on the obscure unicode identifiers in it. *sigh*
-With roundtrip extended to test the 5-tuple mode of  untokenize,
-7 more testfiles fail.  Remove them also until the failure is diagnosed.
-
-    >>> testfiles.remove(os.path.join(tempdir, "test_pep3131.py"))
-    >>> for f in ('buffer', 'builtin', 'fileio', 'inspect', 'os', 'platform', 'sys'):
-    ...     testfiles.remove(os.path.join(tempdir, "test_%s.py") % f)
-    ...
-    >>> if not support.is_resource_enabled("cpu"):
-    ...     testfiles = random.sample(testfiles, 10)
-    ...
-    >>> for testfile in testfiles:
-    ...     if not roundtrip(open(testfile, 'rb')):
-    ...         print("Roundtrip failed for file %s" % testfile)
-    ...         break
-    ... else: True
-    True
-
-Evil tabs
-
-    >>> dump_tokens("def f():\\n\\tif x\\n        \\tpass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_tabs(self):
+        # Evil tabs
+        self.check_tokenize("def f():\n"
+                            "\tif x\n"
+                            "        \tpass", """\
     NAME       'def'         (1, 0) (1, 3)
     NAME       'f'           (1, 4) (1, 5)
     OP         '('           (1, 5) (1, 6)
@@ -619,11 +537,11 @@
     NAME       'pass'        (3, 9) (3, 13)
     DEDENT     ''            (4, 0) (4, 0)
     DEDENT     ''            (4, 0) (4, 0)
+    """)
 
-Non-ascii identifiers
-
-    >>> dump_tokens("?rter = 'places'\\ngr?n = 'green'")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_non_ascii_identifiers(self):
+        # Non-ascii identifiers
+        self.check_tokenize("?rter = 'places'\ngr?n = 'green'", """\
     NAME       '?rter'       (1, 0) (1, 5)
     OP         '='           (1, 6) (1, 7)
     STRING     "'places'"    (1, 8) (1, 16)
@@ -631,11 +549,11 @@
     NAME       'gr?n'        (2, 0) (2, 4)
     OP         '='           (2, 5) (2, 6)
     STRING     "'green'"     (2, 7) (2, 14)
+    """)
 
-Legacy unicode literals:
-
-    >>> dump_tokens("?rter = u'places'\\ngr?n = U'green'")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_unicode(self):
+        # Legacy unicode literals:
+        self.check_tokenize("?rter = u'places'\ngr?n = U'green'", """\
     NAME       '?rter'       (1, 0) (1, 5)
     OP         '='           (1, 6) (1, 7)
     STRING     "u'places'"   (1, 8) (1, 17)
@@ -643,17 +561,17 @@
     NAME       'gr?n'        (2, 0) (2, 4)
     OP         '='           (2, 5) (2, 6)
     STRING     "U'green'"    (2, 7) (2, 15)
+    """)
 
-Async/await extension:
-
-    >>> dump_tokens("async = 1")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_async(self):
+        # Async/await extension:
+        self.check_tokenize("async = 1", """\
     NAME       'async'       (1, 0) (1, 5)
     OP         '='           (1, 6) (1, 7)
     NUMBER     '1'           (1, 8) (1, 9)
+    """)
 
-    >>> dump_tokens("a = (async = 1)")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("a = (async = 1)", """\
     NAME       'a'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     OP         '('           (1, 4) (1, 5)
@@ -661,15 +579,15 @@
     OP         '='           (1, 11) (1, 12)
     NUMBER     '1'           (1, 13) (1, 14)
     OP         ')'           (1, 14) (1, 15)
+    """)
 
-    >>> dump_tokens("async()")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async()", """\
     NAME       'async'       (1, 0) (1, 5)
     OP         '('           (1, 5) (1, 6)
     OP         ')'           (1, 6) (1, 7)
+    """)
 
-    >>> dump_tokens("class async(Bar):pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("class async(Bar):pass", """\
     NAME       'class'       (1, 0) (1, 5)
     NAME       'async'       (1, 6) (1, 11)
     OP         '('           (1, 11) (1, 12)
@@ -677,28 +595,28 @@
     OP         ')'           (1, 15) (1, 16)
     OP         ':'           (1, 16) (1, 17)
     NAME       'pass'        (1, 17) (1, 21)
+    """)
 
-    >>> dump_tokens("class async:pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("class async:pass", """\
     NAME       'class'       (1, 0) (1, 5)
     NAME       'async'       (1, 6) (1, 11)
     OP         ':'           (1, 11) (1, 12)
     NAME       'pass'        (1, 12) (1, 16)
+    """)
 
-    >>> dump_tokens("await = 1")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("await = 1", """\
     NAME       'await'       (1, 0) (1, 5)
     OP         '='           (1, 6) (1, 7)
     NUMBER     '1'           (1, 8) (1, 9)
+    """)
 
-    >>> dump_tokens("foo.async")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("foo.async", """\
     NAME       'foo'         (1, 0) (1, 3)
     OP         '.'           (1, 3) (1, 4)
     NAME       'async'       (1, 4) (1, 9)
+    """)
 
-    >>> dump_tokens("async for a in b: pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async for a in b: pass", """\
     NAME       'async'       (1, 0) (1, 5)
     NAME       'for'         (1, 6) (1, 9)
     NAME       'a'           (1, 10) (1, 11)
@@ -706,9 +624,9 @@
     NAME       'b'           (1, 15) (1, 16)
     OP         ':'           (1, 16) (1, 17)
     NAME       'pass'        (1, 18) (1, 22)
+    """)
 
-    >>> dump_tokens("async with a as b: pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async with a as b: pass", """\
     NAME       'async'       (1, 0) (1, 5)
     NAME       'with'        (1, 6) (1, 10)
     NAME       'a'           (1, 11) (1, 12)
@@ -716,49 +634,49 @@
     NAME       'b'           (1, 16) (1, 17)
     OP         ':'           (1, 17) (1, 18)
     NAME       'pass'        (1, 19) (1, 23)
+    """)
 
-    >>> dump_tokens("async.foo")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async.foo", """\
     NAME       'async'       (1, 0) (1, 5)
     OP         '.'           (1, 5) (1, 6)
     NAME       'foo'         (1, 6) (1, 9)
+    """)
 
-    >>> dump_tokens("async")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async", """\
     NAME       'async'       (1, 0) (1, 5)
+    """)
 
-    >>> dump_tokens("async\\n#comment\\nawait")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async\n#comment\nawait", """\
     NAME       'async'       (1, 0) (1, 5)
     NEWLINE    '\\n'          (1, 5) (1, 6)
     COMMENT    '#comment'    (2, 0) (2, 8)
     NL         '\\n'          (2, 8) (2, 9)
     NAME       'await'       (3, 0) (3, 5)
+    """)
 
-    >>> dump_tokens("async\\n...\\nawait")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async\n...\nawait", """\
     NAME       'async'       (1, 0) (1, 5)
     NEWLINE    '\\n'          (1, 5) (1, 6)
     OP         '...'         (2, 0) (2, 3)
     NEWLINE    '\\n'          (2, 3) (2, 4)
     NAME       'await'       (3, 0) (3, 5)
+    """)
 
-    >>> dump_tokens("async\\nawait")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async\nawait", """\
     NAME       'async'       (1, 0) (1, 5)
     NEWLINE    '\\n'          (1, 5) (1, 6)
     NAME       'await'       (2, 0) (2, 5)
+    """)
 
-    >>> dump_tokens("foo.async + 1")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("foo.async + 1", """\
     NAME       'foo'         (1, 0) (1, 3)
     OP         '.'           (1, 3) (1, 4)
     NAME       'async'       (1, 4) (1, 9)
     OP         '+'           (1, 10) (1, 11)
     NUMBER     '1'           (1, 12) (1, 13)
+    """)
 
-    >>> dump_tokens("async def foo(): pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async def foo(): pass", """\
     ASYNC      'async'       (1, 0) (1, 5)
     NAME       'def'         (1, 6) (1, 9)
     NAME       'foo'         (1, 10) (1, 13)
@@ -766,15 +684,16 @@
     OP         ')'           (1, 14) (1, 15)
     OP         ':'           (1, 15) (1, 16)
     NAME       'pass'        (1, 17) (1, 21)
+    """)
 
-    >>> dump_tokens('''async def foo():
-    ...   def foo(await):
-    ...     await = 1
-    ...   if 1:
-    ...     await
-    ... async += 1
-    ... ''')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize('''\
+async def foo():
+  def foo(await):
+    await = 1
+  if 1:
+    await
+async += 1
+''', """\
     ASYNC      'async'       (1, 0) (1, 5)
     NAME       'def'         (1, 6) (1, 9)
     NAME       'foo'         (1, 10) (1, 13)
@@ -809,10 +728,11 @@
     OP         '+='          (6, 6) (6, 8)
     NUMBER     '1'           (6, 9) (6, 10)
     NEWLINE    '\\n'          (6, 10) (6, 11)
+    """)
 
-    >>> dump_tokens('''async def foo():
-    ...   async for i in 1: pass''')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize('''\
+async def foo():
+  async for i in 1: pass''', """\
     ASYNC      'async'       (1, 0) (1, 5)
     NAME       'def'         (1, 6) (1, 9)
     NAME       'foo'         (1, 10) (1, 13)
@@ -829,9 +749,9 @@
     OP         ':'           (2, 18) (2, 19)
     NAME       'pass'        (2, 20) (2, 24)
     DEDENT     ''            (3, 0) (3, 0)
+    """)
 
-    >>> dump_tokens('''async def foo(async): await''')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize('''async def foo(async): await''', """\
     ASYNC      'async'       (1, 0) (1, 5)
     NAME       'def'         (1, 6) (1, 9)
     NAME       'foo'         (1, 10) (1, 13)
@@ -840,14 +760,15 @@
     OP         ')'           (1, 19) (1, 20)
     OP         ':'           (1, 20) (1, 21)
     AWAIT      'await'       (1, 22) (1, 27)
+    """)
 
-    >>> dump_tokens('''def f():
-    ...
-    ...   def baz(): pass
-    ...   async def bar(): pass
-    ...
-    ...   await = 2''')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize('''\
+def f():
+
+  def baz(): pass
+  async def bar(): pass
+
+  await = 2''', """\
     NAME       'def'         (1, 0) (1, 3)
     NAME       'f'           (1, 4) (1, 5)
     OP         '('           (1, 5) (1, 6)
@@ -876,14 +797,15 @@
     OP         '='           (6, 8) (6, 9)
     NUMBER     '2'           (6, 10) (6, 11)
     DEDENT     ''            (7, 0) (7, 0)
+    """)
 
-    >>> dump_tokens('''async def f():
-    ...
-    ...   def baz(): pass
-    ...   async def bar(): pass
-    ...
-    ...   await = 2''')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize('''\
+async def f():
+
+  def baz(): pass
+  async def bar(): pass
+
+  await = 2''', """\
     ASYNC      'async'       (1, 0) (1, 5)
     NAME       'def'         (1, 6) (1, 9)
     NAME       'f'           (1, 10) (1, 11)
@@ -913,89 +835,10 @@
     OP         '='           (6, 8) (6, 9)
     NUMBER     '2'           (6, 10) (6, 11)
     DEDENT     ''            (7, 0) (7, 0)
-"""
+    """)
 
-from test import support
-from tokenize import (tokenize, _tokenize, untokenize, NUMBER, NAME, OP,
-                     STRING, ENDMARKER, ENCODING, tok_name, detect_encoding,
-                     open as tokenize_open, Untokenizer)
-from io import BytesIO
-from unittest import TestCase, mock
-import os
-import token
 
-def dump_tokens(s):
-    """Print out the tokens in s in a table format.
-
-    The ENDMARKER is omitted.
-    """
-    f = BytesIO(s.encode('utf-8'))
-    for type, token, start, end, line in tokenize(f.readline):
-        if type == ENDMARKER:
-            break
-        type = tok_name[type]
-        print("%(type)-10.10s %(token)-13.13r %(start)s %(end)s" % locals())
-
-def roundtrip(f):
-    """
-    Test roundtrip for `untokenize`. `f` is an open file or a string.
-    The source code in f is tokenized to both 5- and 2-tuples.
-    Both sequences are converted back to source code via
-    tokenize.untokenize(), and the latter tokenized again to 2-tuples.
-    The test fails if the 3 pair tokenizations do not match.
-
-    When untokenize bugs are fixed, untokenize with 5-tuples should
-    reproduce code that does not contain a backslash continuation
-    following spaces.  A proper test should test this.
-
-    This function would be more useful for correcting bugs if it reported
-    the first point of failure, like assertEqual, rather than just
-    returning False -- or if it were only used in unittests and not
-    doctest and actually used assertEqual.
-    """
-    # Get source code and original tokenizations
-    if isinstance(f, str):
-        code = f.encode('utf-8')
-    else:
-        code = f.read()
-        f.close()
-    readline = iter(code.splitlines(keepends=True)).__next__
-    tokens5 = list(tokenize(readline))
-    tokens2 = [tok[:2] for tok in tokens5]
-    # Reproduce tokens2 from pairs
-    bytes_from2 = untokenize(tokens2)
-    readline2 = iter(bytes_from2.splitlines(keepends=True)).__next__
-    tokens2_from2 = [tok[:2] for tok in tokenize(readline2)]
-    # Reproduce tokens2 from 5-tuples
-    bytes_from5 = untokenize(tokens5)
-    readline5 = iter(bytes_from5.splitlines(keepends=True)).__next__
-    tokens2_from5 = [tok[:2] for tok in tokenize(readline5)]
-    # Compare 3 versions
-    return tokens2 == tokens2_from2 == tokens2_from5
-
-# This is an example from the docs, set up as a doctest.
 def decistmt(s):
-    """Substitute Decimals for floats in a string of statements.
-
-    >>> from decimal import Decimal
-    >>> s = 'print(+21.3e-5*-.1234/81.7)'
-    >>> decistmt(s)
-    "print (+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7'))"
-
-    The format of the exponent is inherited from the platform C library.
-    Known cases are "e-007" (Windows) and "e-07" (not Windows).  Since
-    we're only showing 11 digits, and the 12th isn't close to 5, the
-    rest of the output should be platform-independent.
-
-    >>> exec(s) #doctest: +ELLIPSIS
-    -3.2171603427...e-0...7
-
-    Output from calculations with Decimal should be identical across all
-    platforms.
-
-    >>> exec(decistmt(s))
-    -3.217160342717258261933904529E-7
-    """
     result = []
     g = tokenize(BytesIO(s.encode('utf-8')).readline)   # tokenize the string
     for toknum, tokval, _, _, _  in g:
@@ -1010,6 +853,28 @@
             result.append((toknum, tokval))
     return untokenize(result).decode('utf-8')
 
+class TestMisc(TestCase):
+
+    def test_decistmt(self):
+        # Substitute Decimals for floats in a string of statements.
+        # This is an example from the docs.
+
+        from decimal import Decimal
+        s = '+21.3e-5*-.1234/81.7'
+        self.assertEqual(decistmt(s),
+                         "+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')")
+
+        # The format of the exponent is inherited from the platform C library.
+        # Known cases are "e-007" (Windows) and "e-07" (not Windows).  Since
+        # we're only showing 11 digits, and the 12th isn't close to 5, the
+        # rest of the output should be platform-independent.
+        self.assertRegex(repr(eval(s)), '-3.2171603427[0-9]*e-0+7')
+
+        # Output from calculations with Decimal should be identical across all
+        # platforms.
+        self.assertEqual(eval(decistmt(s)),
+                         Decimal('-3.217160342717258261933904529E-7'))
+
 
 class TestTokenizerAdheresToPep0263(TestCase):
     """
@@ -1018,11 +883,11 @@
 
     def _testFile(self, filename):
         path = os.path.join(os.path.dirname(__file__), filename)
-        return roundtrip(open(path, 'rb'))
+        TestRoundtrip.check_roundtrip(self, open(path, 'rb'))
 
     def test_utf8_coding_cookie_and_no_utf8_bom(self):
         f = 'tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt'
-        self.assertTrue(self._testFile(f))
+        self._testFile(f)
 
     def test_latin1_coding_cookie_and_utf8_bom(self):
         """
@@ -1037,11 +902,11 @@
 
     def test_no_coding_cookie_and_utf8_bom(self):
         f = 'tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txt'
-        self.assertTrue(self._testFile(f))
+        self._testFile(f)
 
     def test_utf8_coding_cookie_and_utf8_bom(self):
         f = 'tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt'
-        self.assertTrue(self._testFile(f))
+        self._testFile(f)
 
     def test_bad_coding_cookie(self):
         self.assertRaises(SyntaxError, self._testFile, 'bad_coding.py')
@@ -1340,7 +1205,6 @@
         self.assertTrue(m.closed)
 
 
-
 class TestTokenize(TestCase):
 
     def test_tokenize(self):
@@ -1472,6 +1336,7 @@
         # See http://bugs.python.org/issue16152
         self.assertExactTypeEqual('@          ', token.AT)
 
+
 class UntokenizeTest(TestCase):
 
     def test_bad_input_order(self):
@@ -1497,7 +1362,7 @@
         u.prev_row = 2
         u.add_whitespace((4, 4))
         self.assertEqual(u.tokens, ['\\\n', '\\\n\\\n', '    '])
-        self.assertTrue(roundtrip('a\n  b\n    c\n  \\\n  c\n'))
+        TestRoundtrip.check_roundtrip(self, 'a\n  b\n    c\n  \\\n  c\n')
 
     def test_iter_compat(self):
         u = Untokenizer()
@@ -1514,6 +1379,131 @@
 
 
 class TestRoundtrip(TestCase):
+
+    def check_roundtrip(self, f):
+        """
+        Test roundtrip for `untokenize`. `f` is an open file or a string.
+        The source code in f is tokenized to both 5- and 2-tuples.
+        Both sequences are converted back to source code via
+        tokenize.untokenize(), and the latter tokenized again to 2-tuples.
+        The test fails if the 3 pair tokenizations do not match.
+
+        When untokenize bugs are fixed, untokenize with 5-tuples should
+        reproduce code that does not contain a backslash continuation
+        following spaces.  A proper test should test this.
+        """
+        # Get source code and original tokenizations
+        if isinstance(f, str):
+            code = f.encode('utf-8')
+        else:
+            code = f.read()
+            f.close()
+        readline = iter(code.splitlines(keepends=True)).__next__
+        tokens5 = list(tokenize(readline))
+        tokens2 = [tok[:2] for tok in tokens5]
+        # Reproduce tokens2 from pairs
+        bytes_from2 = untokenize(tokens2)
+        readline2 = iter(bytes_from2.splitlines(keepends=True)).__next__
+        tokens2_from2 = [tok[:2] for tok in tokenize(readline2)]
+        self.assertEqual(tokens2_from2, tokens2)
+        # Reproduce tokens2 from 5-tuples
+        bytes_from5 = untokenize(tokens5)
+        readline5 = iter(bytes_from5.splitlines(keepends=True)).__next__
+        tokens2_from5 = [tok[:2] for tok in tokenize(readline5)]
+        self.assertEqual(tokens2_from5, tokens2)
+
+    def test_roundtrip(self):
+        # There are some standard formatting practices that are easy to get right.
+
+        self.check_roundtrip("if x == 1:\n"
+                             "    print(x)\n")
+        self.check_roundtrip("# This is a comment\n"
+                             "# This also")
+
+        # Some people use different formatting conventions, which makes
+        # untokenize a little trickier. Note that this test involves trailing
+        # whitespace after the colon. Note that we use hex escapes to make the
+        # two trailing blanks apparent in the expected output.
+
+        self.check_roundtrip("if x == 1 : \n"
+                             "  print(x)\n")
+        fn = support.findfile("tokenize_tests.txt")
+        with open(fn, 'rb') as f:
+            self.check_roundtrip(f)
+        self.check_roundtrip("if x == 1:\n"
+                             "    # A comment by itself.\n"
+                             "    print(x) # Comment here, too.\n"
+                             "    # Another comment.\n"
+                             "after_if = True\n")
+        self.check_roundtrip("if (x # The comments need to go in the right place\n"
+                             "    == 1):\n"
+                             "    print('x==1')\n")
+        self.check_roundtrip("class Test: # A comment here\n"
+                             "  # A comment with weird indent\n"
+                             "  after_com = 5\n"
+                             "  def x(m): return m*5 # a one liner\n"
+                             "  def y(m): # A whitespace after the colon\n"
+                             "     return y*4 # 3-space indent\n")
+
+        # Some error-handling code
+        self.check_roundtrip("try: import somemodule\n"
+                             "except ImportError: # comment\n"
+                             "    print('Can not import' # comment2\n)"
+                             "else:   print('Loaded')\n")
+
+    def test_continuation(self):
+        # Balancing continuation
+        self.check_roundtrip("a = (3,4, \n"
+                             "5,6)\n"
+                             "y = [3, 4,\n"
+                             "5]\n"
+                             "z = {'a': 5,\n"
+                             "'b':15, 'c':True}\n"
+                             "x = len(y) + 5 - a[\n"
+                             "3] - a[2]\n"
+                             "+ len(z) - z[\n"
+                             "'b']\n")
+
+    def test_backslash_continuation(self):
+        # Backslash means line continuation, except for comments
+        self.check_roundtrip("x=1+\\\n"
+                             "1\n"
+                             "# This is a comment\\\n"
+                             "# This also\n")
+        self.check_roundtrip("# Comment \\\n"
+                             "x = 0")
+
+    def test_string_concatenation(self):
+        # Two string literals on the same line
+        self.check_roundtrip("'' ''")
+
+    def test_random_files(self):
+        # Test roundtrip on random python modules.
+        # pass the '-ucpu' option to process the full directory.
+
+        import glob, random
+        fn = support.findfile("tokenize_tests.txt")
+        tempdir = os.path.dirname(fn) or os.curdir
+        testfiles = glob.glob(os.path.join(tempdir, "test*.py"))
+
+        # Tokenize is broken on test_pep3131.py because regular expressions are
+        # broken on the obscure unicode identifiers in it. *sigh*
+        # With roundtrip extended to test the 5-tuple mode of  untokenize,
+        # 7 more testfiles fail.  Remove them also until the failure is diagnosed.
+
+        testfiles.remove(os.path.join(tempdir, "test_pep3131.py"))
+        for f in ('buffer', 'builtin', 'fileio', 'inspect', 'os', 'platform', 'sys'):
+            testfiles.remove(os.path.join(tempdir, "test_%s.py") % f)
+
+        if not support.is_resource_enabled("cpu"):
+            testfiles = random.sample(testfiles, 10)
+
+        for testfile in testfiles:
+            with open(testfile, 'rb') as f:
+                with self.subTest(file=testfile):
+                    self.check_roundtrip(f)
+
+
     def roundtrip(self, code):
         if isinstance(code, str):
             code = code.encode('utf-8')
@@ -1527,19 +1517,8 @@
         code = "if False:\n\tx=3\n\tx=3\n"
         codelines = self.roundtrip(code).split('\n')
         self.assertEqual(codelines[1], codelines[2])
+        self.check_roundtrip(code)
 
 
-__test__ = {"doctests" : doctests, 'decistmt': decistmt}
-
-def test_main():
-    from test import test_tokenize
-    support.run_doctest(test_tokenize, True)
-    support.run_unittest(TestTokenizerAdheresToPep0263)
-    support.run_unittest(Test_Tokenize)
-    support.run_unittest(TestDetectEncoding)
-    support.run_unittest(TestTokenize)
-    support.run_unittest(UntokenizeTest)
-    support.run_unittest(TestRoundtrip)
-
 if __name__ == "__main__":
-    test_main()
+    unittest.main()

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Tue Oct  6 17:54:19 2015
From: python-checkins at python.org (serhiy.storchaka)
Date: Tue, 06 Oct 2015 15:54:19 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
	=?utf-8?b?KTogTWVyZ2Ugd2l0aCAzLjUu?=
Message-ID: <20151006155418.2667.13806@psf.io>

https://hg.python.org/cpython/rev/39a24ce364dc
changeset:   98563:39a24ce364dc
parent:      98561:727d72b05ff5
parent:      98562:b0ce3ef2ea21
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Tue Oct 06 18:52:52 2015 +0300
summary:
  Merge with 3.5.

files:
  Lib/pydoc.py                      |  0 
  Lib/site-packages/README          |  2 ++
  Lib/test/regrtest.py              |  0 
  Lib/timeit.py                     |  0 
  Parser/asdl_c.py                  |  0 
  Tools/buildbot/build-amd64.bat    |  5 +++++
  Tools/buildbot/clean-amd64.bat    |  5 +++++
  Tools/buildbot/external-amd64.bat |  3 +++
  Tools/buildbot/external.bat       |  3 +++
  Tools/buildbot/test-amd64.bat     |  6 ++++++
  configure                         |  0 
  11 files changed, 24 insertions(+), 0 deletions(-)


diff --git a/Lib/pydoc.py b/Lib/pydoc.py
old mode 100755
new mode 100644
diff --git a/Lib/site-packages/README b/Lib/site-packages/README
new file mode 100644
--- /dev/null
+++ b/Lib/site-packages/README
@@ -0,0 +1,2 @@
+This directory exists so that 3rd party packages can be installed
+here.  Read the source for site.py for more details.
diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py
old mode 100755
new mode 100644
diff --git a/Lib/timeit.py b/Lib/timeit.py
old mode 100755
new mode 100644
diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
old mode 100755
new mode 100644
diff --git a/Tools/buildbot/build-amd64.bat b/Tools/buildbot/build-amd64.bat
new file mode 100644
--- /dev/null
+++ b/Tools/buildbot/build-amd64.bat
@@ -0,0 +1,5 @@
+ at rem Formerly used by the buildbot "compile" step.
+ at echo This script is no longer used and may be removed in the future.
+ at echo To get the same effect as this script, use
+ at echo     PCbuild\build.bat -d -e -k -p x64
+call "%~dp0build.bat" -p x64 %*
diff --git a/Tools/buildbot/clean-amd64.bat b/Tools/buildbot/clean-amd64.bat
new file mode 100644
--- /dev/null
+++ b/Tools/buildbot/clean-amd64.bat
@@ -0,0 +1,5 @@
+ at rem Formerly used by the buildbot "clean" step.
+ at echo This script is no longer used and may be removed in the future.
+ at echo To get the same effect as this script, use `clean.bat` from this
+ at echo directory and pass `-p x64` as two arguments.
+call "%~dp0clean.bat" -p x64 %*
diff --git a/Tools/buildbot/external-amd64.bat b/Tools/buildbot/external-amd64.bat
new file mode 100644
--- /dev/null
+++ b/Tools/buildbot/external-amd64.bat
@@ -0,0 +1,3 @@
+ at echo This script is no longer used and may be removed in the future.
+ at echo Please use PCbuild\get_externals.bat instead.
+@"%~dp0..\..\PCbuild\get_externals.bat" %*
diff --git a/Tools/buildbot/external.bat b/Tools/buildbot/external.bat
new file mode 100644
--- /dev/null
+++ b/Tools/buildbot/external.bat
@@ -0,0 +1,3 @@
+ at echo This script is no longer used and may be removed in the future.
+ at echo Please use PCbuild\get_externals.bat instead.
+@"%~dp0..\..\PCbuild\get_externals.bat" %*
diff --git a/Tools/buildbot/test-amd64.bat b/Tools/buildbot/test-amd64.bat
new file mode 100644
--- /dev/null
+++ b/Tools/buildbot/test-amd64.bat
@@ -0,0 +1,6 @@
+ at rem Formerly used by the buildbot "test" step.
+ at echo This script is no longer used and may be removed in the future.
+ at echo To get the same effect as this script, use
+ at echo     PCbuild\rt.bat -q -d -x64 -uall -rwW
+ at echo or use `test.bat` in this directory and pass `-x64` as an argument.
+call "%~dp0test.bat" -x64 %*
diff --git a/configure b/configure
old mode 100755
new mode 100644

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Tue Oct  6 17:54:19 2015
From: python-checkins at python.org (serhiy.storchaka)
Date: Tue, 06 Oct 2015 15:54:19 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNCk6?=
	=?utf-8?q?_Merge_heads?=
Message-ID: <20151006155417.97710.94766@psf.io>

https://hg.python.org/cpython/rev/91f36d2b097a
changeset:   98559:91f36d2b097a
branch:      3.4
parent:      98556:d272f3cbae05
parent:      98552:aebbf205ef6f
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Tue Oct 06 18:38:25 2015 +0300
summary:
  Merge heads

files:
  Lib/test/test_asyncio/test_base_events.py |  5 +----
  1 files changed, 1 insertions(+), 4 deletions(-)


diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py
--- a/Lib/test/test_asyncio/test_base_events.py
+++ b/Lib/test/test_asyncio/test_base_events.py
@@ -1215,6 +1215,7 @@
 
     def test_create_datagram_endpoint_sock(self):
         sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        sock.bind(('127.0.0.1', 0))
         fut = self.loop.create_datagram_endpoint(
             lambda: MyDatagramProto(create_future=True, loop=self.loop),
             sock=sock)
@@ -1307,10 +1308,6 @@
             self.assertTrue(
                 sock.getsockopt(
                     socket.SOL_SOCKET, socket.SO_REUSEPORT))
-        else:
-            self.assertFalse(
-                sock.getsockopt(
-                    socket.SOL_SOCKET, socket.SO_REUSEPORT))
         self.assertTrue(
             sock.getsockopt(
                 socket.SOL_SOCKET, socket.SO_BROADCAST))

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Tue Oct  6 17:54:19 2015
From: python-checkins at python.org (serhiy.storchaka)
Date: Tue, 06 Oct 2015 15:54:19 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2325317=3A_Converted_doctests_in_test=5Ftokenize_?=
 =?utf-8?q?to_unittests=2E?=
Message-ID: <20151006155417.97700.68973@psf.io>

https://hg.python.org/cpython/rev/66d239660997
changeset:   98558:66d239660997
parent:      98551:41f29bbf520d
parent:      98557:bff40616d2a5
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Tue Oct 06 18:24:46 2015 +0300
summary:
  Issue #25317: Converted doctests in test_tokenize to unittests.
Made test_tokenize discoverable.

files:
  Lib/test/test_tokenize.py |  811 ++++++++++++-------------
  1 files changed, 395 insertions(+), 416 deletions(-)


diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -1,22 +1,44 @@
-doctests = """
-Tests for the tokenize module.
+from test import support
+from tokenize import (tokenize, _tokenize, untokenize, NUMBER, NAME, OP,
+                     STRING, ENDMARKER, ENCODING, tok_name, detect_encoding,
+                     open as tokenize_open, Untokenizer)
+from io import BytesIO
+from unittest import TestCase, mock
+import os
+import token
 
-The tests can be really simple. Given a small fragment of source
-code, print out a table with tokens. The ENDMARKER is omitted for
-brevity.
 
-    >>> import glob
+class TokenizeTest(TestCase):
+    # Tests for the tokenize module.
 
-    >>> dump_tokens("1 + 1")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    # The tests can be really simple. Given a small fragment of source
+    # code, print out a table with tokens. The ENDMARKER is omitted for
+    # brevity.
+
+    def check_tokenize(self, s, expected):
+        # Format the tokens in s in a table format.
+        # The ENDMARKER is omitted.
+        result = []
+        f = BytesIO(s.encode('utf-8'))
+        for type, token, start, end, line in tokenize(f.readline):
+            if type == ENDMARKER:
+                break
+            type = tok_name[type]
+            result.append("    %(type)-10.10s %(token)-13.13r %(start)s %(end)s" %
+                          locals())
+        self.assertEqual(result,
+                         ["    ENCODING   'utf-8'       (0, 0) (0, 0)"] +
+                         expected.rstrip().splitlines())
+
+    def test_basic(self):
+        self.check_tokenize("1 + 1", """\
     NUMBER     '1'           (1, 0) (1, 1)
     OP         '+'           (1, 2) (1, 3)
     NUMBER     '1'           (1, 4) (1, 5)
-
-    >>> dump_tokens("if False:\\n"
-    ...             "    # NL\\n"
-    ...             "    True = False # NEWLINE\\n")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("if False:\n"
+                            "    # NL\n"
+                            "    True = False # NEWLINE\n", """\
     NAME       'if'          (1, 0) (1, 2)
     NAME       'False'       (1, 3) (1, 8)
     OP         ':'           (1, 8) (1, 9)
@@ -30,112 +52,48 @@
     COMMENT    '# NEWLINE'   (3, 17) (3, 26)
     NEWLINE    '\\n'          (3, 26) (3, 27)
     DEDENT     ''            (4, 0) (4, 0)
+    """)
+        indent_error_file = b"""\
+def k(x):
+    x += 2
+  x += 5
+"""
+        readline = BytesIO(indent_error_file).readline
+        with self.assertRaisesRegex(IndentationError,
+                                    "unindent does not match any "
+                                    "outer indentation level"):
+            for tok in tokenize(readline):
+                pass
 
-    >>> indent_error_file = \"""
-    ... def k(x):
-    ...     x += 2
-    ...   x += 5
-    ... \"""
-    >>> readline = BytesIO(indent_error_file.encode('utf-8')).readline
-    >>> for tok in tokenize(readline): pass
-    Traceback (most recent call last):
-        ...
-    IndentationError: unindent does not match any outer indentation level
-
-There are some standard formatting practices that are easy to get right.
-
-    >>> roundtrip("if x == 1:\\n"
-    ...           "    print(x)\\n")
-    True
-
-    >>> roundtrip("# This is a comment\\n# This also")
-    True
-
-Some people use different formatting conventions, which makes
-untokenize a little trickier. Note that this test involves trailing
-whitespace after the colon. Note that we use hex escapes to make the
-two trailing blanks apparent in the expected output.
-
-    >>> roundtrip("if x == 1 : \\n"
-    ...           "  print(x)\\n")
-    True
-
-    >>> f = support.findfile("tokenize_tests.txt")
-    >>> roundtrip(open(f, 'rb'))
-    True
-
-    >>> roundtrip("if x == 1:\\n"
-    ...           "    # A comment by itself.\\n"
-    ...           "    print(x) # Comment here, too.\\n"
-    ...           "    # Another comment.\\n"
-    ...           "after_if = True\\n")
-    True
-
-    >>> roundtrip("if (x # The comments need to go in the right place\\n"
-    ...           "    == 1):\\n"
-    ...           "    print('x==1')\\n")
-    True
-
-    >>> roundtrip("class Test: # A comment here\\n"
-    ...           "  # A comment with weird indent\\n"
-    ...           "  after_com = 5\\n"
-    ...           "  def x(m): return m*5 # a one liner\\n"
-    ...           "  def y(m): # A whitespace after the colon\\n"
-    ...           "     return y*4 # 3-space indent\\n")
-    True
-
-Some error-handling code
-
-    >>> roundtrip("try: import somemodule\\n"
-    ...           "except ImportError: # comment\\n"
-    ...           "    print('Can not import' # comment2\\n)"
-    ...           "else:   print('Loaded')\\n")
-    True
-
-Balancing continuation
-
-    >>> roundtrip("a = (3,4, \\n"
-    ...           "5,6)\\n"
-    ...           "y = [3, 4,\\n"
-    ...           "5]\\n"
-    ...           "z = {'a': 5,\\n"
-    ...           "'b':15, 'c':True}\\n"
-    ...           "x = len(y) + 5 - a[\\n"
-    ...           "3] - a[2]\\n"
-    ...           "+ len(z) - z[\\n"
-    ...           "'b']\\n")
-    True
-
-Ordinary integers and binary operators
-
-    >>> dump_tokens("0xff <= 255")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_int(self):
+        # Ordinary integers and binary operators
+        self.check_tokenize("0xff <= 255", """\
     NUMBER     '0xff'        (1, 0) (1, 4)
     OP         '<='          (1, 5) (1, 7)
     NUMBER     '255'         (1, 8) (1, 11)
-    >>> dump_tokens("0b10 <= 255")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("0b10 <= 255", """\
     NUMBER     '0b10'        (1, 0) (1, 4)
     OP         '<='          (1, 5) (1, 7)
     NUMBER     '255'         (1, 8) (1, 11)
-    >>> dump_tokens("0o123 <= 0O123")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("0o123 <= 0O123", """\
     NUMBER     '0o123'       (1, 0) (1, 5)
     OP         '<='          (1, 6) (1, 8)
     NUMBER     '0O123'       (1, 9) (1, 14)
-    >>> dump_tokens("1234567 > ~0x15")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("1234567 > ~0x15", """\
     NUMBER     '1234567'     (1, 0) (1, 7)
     OP         '>'           (1, 8) (1, 9)
     OP         '~'           (1, 10) (1, 11)
     NUMBER     '0x15'        (1, 11) (1, 15)
-    >>> dump_tokens("2134568 != 1231515")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("2134568 != 1231515", """\
     NUMBER     '2134568'     (1, 0) (1, 7)
     OP         '!='          (1, 8) (1, 10)
     NUMBER     '1231515'     (1, 11) (1, 18)
-    >>> dump_tokens("(-124561-1) & 200000000")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("(-124561-1) & 200000000", """\
     OP         '('           (1, 0) (1, 1)
     OP         '-'           (1, 1) (1, 2)
     NUMBER     '124561'      (1, 2) (1, 8)
@@ -144,93 +102,93 @@
     OP         ')'           (1, 10) (1, 11)
     OP         '&'           (1, 12) (1, 13)
     NUMBER     '200000000'   (1, 14) (1, 23)
-    >>> dump_tokens("0xdeadbeef != -1")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("0xdeadbeef != -1", """\
     NUMBER     '0xdeadbeef'  (1, 0) (1, 10)
     OP         '!='          (1, 11) (1, 13)
     OP         '-'           (1, 14) (1, 15)
     NUMBER     '1'           (1, 15) (1, 16)
-    >>> dump_tokens("0xdeadc0de & 12345")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("0xdeadc0de & 12345", """\
     NUMBER     '0xdeadc0de'  (1, 0) (1, 10)
     OP         '&'           (1, 11) (1, 12)
     NUMBER     '12345'       (1, 13) (1, 18)
-    >>> dump_tokens("0xFF & 0x15 | 1234")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("0xFF & 0x15 | 1234", """\
     NUMBER     '0xFF'        (1, 0) (1, 4)
     OP         '&'           (1, 5) (1, 6)
     NUMBER     '0x15'        (1, 7) (1, 11)
     OP         '|'           (1, 12) (1, 13)
     NUMBER     '1234'        (1, 14) (1, 18)
+    """)
 
-Long integers
-
-    >>> dump_tokens("x = 0")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_long(self):
+        # Long integers
+        self.check_tokenize("x = 0", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '0'           (1, 4) (1, 5)
-    >>> dump_tokens("x = 0xfffffffffff")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = 0xfffffffffff", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '0xffffffffff (1, 4) (1, 17)
-    >>> dump_tokens("x = 123141242151251616110")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = 123141242151251616110", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '123141242151 (1, 4) (1, 25)
-    >>> dump_tokens("x = -15921590215012591")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = -15921590215012591", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     OP         '-'           (1, 4) (1, 5)
     NUMBER     '159215902150 (1, 5) (1, 22)
+    """)
 
-Floating point numbers
-
-    >>> dump_tokens("x = 3.14159")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_float(self):
+        # Floating point numbers
+        self.check_tokenize("x = 3.14159", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '3.14159'     (1, 4) (1, 11)
-    >>> dump_tokens("x = 314159.")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = 314159.", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '314159.'     (1, 4) (1, 11)
-    >>> dump_tokens("x = .314159")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = .314159", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '.314159'     (1, 4) (1, 11)
-    >>> dump_tokens("x = 3e14159")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = 3e14159", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '3e14159'     (1, 4) (1, 11)
-    >>> dump_tokens("x = 3E123")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = 3E123", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '3E123'       (1, 4) (1, 9)
-    >>> dump_tokens("x+y = 3e-1230")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x+y = 3e-1230", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '+'           (1, 1) (1, 2)
     NAME       'y'           (1, 2) (1, 3)
     OP         '='           (1, 4) (1, 5)
     NUMBER     '3e-1230'     (1, 6) (1, 13)
-    >>> dump_tokens("x = 3.14e159")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = 3.14e159", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '3.14e159'    (1, 4) (1, 12)
+    """)
 
-String literals
-
-    >>> dump_tokens("x = ''; y = \\\"\\\"")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_string(self):
+        # String literals
+        self.check_tokenize("x = ''; y = \"\"", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     "''"          (1, 4) (1, 6)
@@ -238,8 +196,8 @@
     NAME       'y'           (1, 8) (1, 9)
     OP         '='           (1, 10) (1, 11)
     STRING     '""'          (1, 12) (1, 14)
-    >>> dump_tokens("x = '\\\"'; y = \\\"'\\\"")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = '\"'; y = \"'\"", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     '\\'"\\''       (1, 4) (1, 7)
@@ -247,29 +205,29 @@
     NAME       'y'           (1, 9) (1, 10)
     OP         '='           (1, 11) (1, 12)
     STRING     '"\\'"'        (1, 13) (1, 16)
-    >>> dump_tokens("x = \\\"doesn't \\\"shrink\\\", does it\\\"")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = \"doesn't \"shrink\", does it\"", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     '"doesn\\'t "' (1, 4) (1, 14)
     NAME       'shrink'      (1, 14) (1, 20)
     STRING     '", does it"' (1, 20) (1, 31)
-    >>> dump_tokens("x = 'abc' + 'ABC'")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = 'abc' + 'ABC'", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     "'abc'"       (1, 4) (1, 9)
     OP         '+'           (1, 10) (1, 11)
     STRING     "'ABC'"       (1, 12) (1, 17)
-    >>> dump_tokens('y = "ABC" + "ABC"')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize('y = "ABC" + "ABC"', """\
     NAME       'y'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     '"ABC"'       (1, 4) (1, 9)
     OP         '+'           (1, 10) (1, 11)
     STRING     '"ABC"'       (1, 12) (1, 17)
-    >>> dump_tokens("x = r'abc' + r'ABC' + R'ABC' + R'ABC'")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = r'abc' + r'ABC' + R'ABC' + R'ABC'", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     "r'abc'"      (1, 4) (1, 10)
@@ -279,8 +237,8 @@
     STRING     "R'ABC'"      (1, 22) (1, 28)
     OP         '+'           (1, 29) (1, 30)
     STRING     "R'ABC'"      (1, 31) (1, 37)
-    >>> dump_tokens('y = r"abc" + r"ABC" + R"ABC" + R"ABC"')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize('y = r"abc" + r"ABC" + R"ABC" + R"ABC"', """\
     NAME       'y'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     'r"abc"'      (1, 4) (1, 10)
@@ -290,30 +248,30 @@
     STRING     'R"ABC"'      (1, 22) (1, 28)
     OP         '+'           (1, 29) (1, 30)
     STRING     'R"ABC"'      (1, 31) (1, 37)
+    """)
 
-    >>> dump_tokens("u'abc' + U'abc'")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("u'abc' + U'abc'", """\
     STRING     "u'abc'"      (1, 0) (1, 6)
     OP         '+'           (1, 7) (1, 8)
     STRING     "U'abc'"      (1, 9) (1, 15)
-    >>> dump_tokens('u"abc" + U"abc"')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize('u"abc" + U"abc"', """\
     STRING     'u"abc"'      (1, 0) (1, 6)
     OP         '+'           (1, 7) (1, 8)
     STRING     'U"abc"'      (1, 9) (1, 15)
+    """)
 
-    >>> dump_tokens("b'abc' + B'abc'")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("b'abc' + B'abc'", """\
     STRING     "b'abc'"      (1, 0) (1, 6)
     OP         '+'           (1, 7) (1, 8)
     STRING     "B'abc'"      (1, 9) (1, 15)
-    >>> dump_tokens('b"abc" + B"abc"')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize('b"abc" + B"abc"', """\
     STRING     'b"abc"'      (1, 0) (1, 6)
     OP         '+'           (1, 7) (1, 8)
     STRING     'B"abc"'      (1, 9) (1, 15)
-    >>> dump_tokens("br'abc' + bR'abc' + Br'abc' + BR'abc'")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("br'abc' + bR'abc' + Br'abc' + BR'abc'", """\
     STRING     "br'abc'"     (1, 0) (1, 7)
     OP         '+'           (1, 8) (1, 9)
     STRING     "bR'abc'"     (1, 10) (1, 17)
@@ -321,8 +279,8 @@
     STRING     "Br'abc'"     (1, 20) (1, 27)
     OP         '+'           (1, 28) (1, 29)
     STRING     "BR'abc'"     (1, 30) (1, 37)
-    >>> dump_tokens('br"abc" + bR"abc" + Br"abc" + BR"abc"')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize('br"abc" + bR"abc" + Br"abc" + BR"abc"', """\
     STRING     'br"abc"'     (1, 0) (1, 7)
     OP         '+'           (1, 8) (1, 9)
     STRING     'bR"abc"'     (1, 10) (1, 17)
@@ -330,8 +288,8 @@
     STRING     'Br"abc"'     (1, 20) (1, 27)
     OP         '+'           (1, 28) (1, 29)
     STRING     'BR"abc"'     (1, 30) (1, 37)
-    >>> dump_tokens("rb'abc' + rB'abc' + Rb'abc' + RB'abc'")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("rb'abc' + rB'abc' + Rb'abc' + RB'abc'", """\
     STRING     "rb'abc'"     (1, 0) (1, 7)
     OP         '+'           (1, 8) (1, 9)
     STRING     "rB'abc'"     (1, 10) (1, 17)
@@ -339,8 +297,8 @@
     STRING     "Rb'abc'"     (1, 20) (1, 27)
     OP         '+'           (1, 28) (1, 29)
     STRING     "RB'abc'"     (1, 30) (1, 37)
-    >>> dump_tokens('rb"abc" + rB"abc" + Rb"abc" + RB"abc"')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize('rb"abc" + rB"abc" + Rb"abc" + RB"abc"', """\
     STRING     'rb"abc"'     (1, 0) (1, 7)
     OP         '+'           (1, 8) (1, 9)
     STRING     'rB"abc"'     (1, 10) (1, 17)
@@ -348,11 +306,10 @@
     STRING     'Rb"abc"'     (1, 20) (1, 27)
     OP         '+'           (1, 28) (1, 29)
     STRING     'RB"abc"'     (1, 30) (1, 37)
+    """)
 
-Operators
-
-    >>> dump_tokens("def d22(a, b, c=2, d=2, *k): pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_function(self):
+        self.check_tokenize("def d22(a, b, c=2, d=2, *k): pass", """\
     NAME       'def'         (1, 0) (1, 3)
     NAME       'd22'         (1, 4) (1, 7)
     OP         '('           (1, 7) (1, 8)
@@ -373,8 +330,8 @@
     OP         ')'           (1, 26) (1, 27)
     OP         ':'           (1, 27) (1, 28)
     NAME       'pass'        (1, 29) (1, 33)
-    >>> dump_tokens("def d01v_(a=1, *k, **w): pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("def d01v_(a=1, *k, **w): pass", """\
     NAME       'def'         (1, 0) (1, 3)
     NAME       'd01v_'       (1, 4) (1, 9)
     OP         '('           (1, 9) (1, 10)
@@ -390,12 +347,12 @@
     OP         ')'           (1, 22) (1, 23)
     OP         ':'           (1, 23) (1, 24)
     NAME       'pass'        (1, 25) (1, 29)
+    """)
 
-Comparison
-
-    >>> dump_tokens("if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != " +
-    ...             "1 and 5 in 1 not in 1 is 1 or 5 is not 1: pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_comparison(self):
+        # Comparison
+        self.check_tokenize("if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != "
+                            "1 and 5 in 1 not in 1 is 1 or 5 is not 1: pass", """\
     NAME       'if'          (1, 0) (1, 2)
     NUMBER     '1'           (1, 3) (1, 4)
     OP         '<'           (1, 5) (1, 6)
@@ -428,11 +385,11 @@
     NUMBER     '1'           (1, 81) (1, 82)
     OP         ':'           (1, 82) (1, 83)
     NAME       'pass'        (1, 84) (1, 88)
+    """)
 
-Shift
-
-    >>> dump_tokens("x = 1 << 1 >> 5")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_shift(self):
+        # Shift
+        self.check_tokenize("x = 1 << 1 >> 5", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '1'           (1, 4) (1, 5)
@@ -440,11 +397,11 @@
     NUMBER     '1'           (1, 9) (1, 10)
     OP         '>>'          (1, 11) (1, 13)
     NUMBER     '5'           (1, 14) (1, 15)
+    """)
 
-Additive
-
-    >>> dump_tokens("x = 1 - y + 15 - 1 + 0x124 + z + a[5]")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_additive(self):
+        # Additive
+        self.check_tokenize("x = 1 - y + 15 - 1 + 0x124 + z + a[5]", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '1'           (1, 4) (1, 5)
@@ -463,11 +420,11 @@
     OP         '['           (1, 34) (1, 35)
     NUMBER     '5'           (1, 35) (1, 36)
     OP         ']'           (1, 36) (1, 37)
+    """)
 
-Multiplicative
-
-    >>> dump_tokens("x = 1//1*1/5*12%0x12 at 42")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_multiplicative(self):
+        # Multiplicative
+        self.check_tokenize("x = 1//1*1/5*12%0x12 at 42", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '1'           (1, 4) (1, 5)
@@ -483,11 +440,11 @@
     NUMBER     '0x12'        (1, 16) (1, 20)
     OP         '@'           (1, 20) (1, 21)
     NUMBER     '42'          (1, 21) (1, 23)
+    """)
 
-Unary
-
-    >>> dump_tokens("~1 ^ 1 & 1 |1 ^ -1")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_unary(self):
+        # Unary
+        self.check_tokenize("~1 ^ 1 & 1 |1 ^ -1", """\
     OP         '~'           (1, 0) (1, 1)
     NUMBER     '1'           (1, 1) (1, 2)
     OP         '^'           (1, 3) (1, 4)
@@ -499,8 +456,8 @@
     OP         '^'           (1, 14) (1, 15)
     OP         '-'           (1, 16) (1, 17)
     NUMBER     '1'           (1, 17) (1, 18)
-    >>> dump_tokens("-1*1/1+1*1//1 - ---1**1")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("-1*1/1+1*1//1 - ---1**1", """\
     OP         '-'           (1, 0) (1, 1)
     NUMBER     '1'           (1, 1) (1, 2)
     OP         '*'           (1, 2) (1, 3)
@@ -520,11 +477,11 @@
     NUMBER     '1'           (1, 19) (1, 20)
     OP         '**'          (1, 20) (1, 22)
     NUMBER     '1'           (1, 22) (1, 23)
+    """)
 
-Selector
-
-    >>> dump_tokens("import sys, time\\nx = sys.modules['time'].time()")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_selector(self):
+        # Selector
+        self.check_tokenize("import sys, time\nx = sys.modules['time'].time()", """\
     NAME       'import'      (1, 0) (1, 6)
     NAME       'sys'         (1, 7) (1, 10)
     OP         ','           (1, 10) (1, 11)
@@ -542,11 +499,11 @@
     NAME       'time'        (2, 24) (2, 28)
     OP         '('           (2, 28) (2, 29)
     OP         ')'           (2, 29) (2, 30)
+    """)
 
-Methods
-
-    >>> dump_tokens("@staticmethod\\ndef foo(x,y): pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_method(self):
+        # Methods
+        self.check_tokenize("@staticmethod\ndef foo(x,y): pass", """\
     OP         '@'           (1, 0) (1, 1)
     NAME       'staticmethod (1, 1) (1, 13)
     NEWLINE    '\\n'          (1, 13) (1, 14)
@@ -559,52 +516,13 @@
     OP         ')'           (2, 11) (2, 12)
     OP         ':'           (2, 12) (2, 13)
     NAME       'pass'        (2, 14) (2, 18)
+    """)
 
-Backslash means line continuation, except for comments
-
-    >>> roundtrip("x=1+\\\\n"
-    ...           "1\\n"
-    ...           "# This is a comment\\\\n"
-    ...           "# This also\\n")
-    True
-    >>> roundtrip("# Comment \\\\nx = 0")
-    True
-
-Two string literals on the same line
-
-    >>> roundtrip("'' ''")
-    True
-
-Test roundtrip on random python modules.
-pass the '-ucpu' option to process the full directory.
-
-    >>> import random
-    >>> tempdir = os.path.dirname(f) or os.curdir
-    >>> testfiles = glob.glob(os.path.join(tempdir, "test*.py"))
-
-Tokenize is broken on test_pep3131.py because regular expressions are
-broken on the obscure unicode identifiers in it. *sigh*
-With roundtrip extended to test the 5-tuple mode of  untokenize,
-7 more testfiles fail.  Remove them also until the failure is diagnosed.
-
-    >>> testfiles.remove(os.path.join(tempdir, "test_pep3131.py"))
-    >>> for f in ('buffer', 'builtin', 'fileio', 'inspect', 'os', 'platform', 'sys'):
-    ...     testfiles.remove(os.path.join(tempdir, "test_%s.py") % f)
-    ...
-    >>> if not support.is_resource_enabled("cpu"):
-    ...     testfiles = random.sample(testfiles, 10)
-    ...
-    >>> for testfile in testfiles:
-    ...     if not roundtrip(open(testfile, 'rb')):
-    ...         print("Roundtrip failed for file %s" % testfile)
-    ...         break
-    ... else: True
-    True
-
-Evil tabs
-
-    >>> dump_tokens("def f():\\n\\tif x\\n        \\tpass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_tabs(self):
+        # Evil tabs
+        self.check_tokenize("def f():\n"
+                            "\tif x\n"
+                            "        \tpass", """\
     NAME       'def'         (1, 0) (1, 3)
     NAME       'f'           (1, 4) (1, 5)
     OP         '('           (1, 5) (1, 6)
@@ -619,11 +537,11 @@
     NAME       'pass'        (3, 9) (3, 13)
     DEDENT     ''            (4, 0) (4, 0)
     DEDENT     ''            (4, 0) (4, 0)
+    """)
 
-Non-ascii identifiers
-
-    >>> dump_tokens("?rter = 'places'\\ngr?n = 'green'")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_non_ascii_identifiers(self):
+        # Non-ascii identifiers
+        self.check_tokenize("?rter = 'places'\ngr?n = 'green'", """\
     NAME       '?rter'       (1, 0) (1, 5)
     OP         '='           (1, 6) (1, 7)
     STRING     "'places'"    (1, 8) (1, 16)
@@ -631,11 +549,11 @@
     NAME       'gr?n'        (2, 0) (2, 4)
     OP         '='           (2, 5) (2, 6)
     STRING     "'green'"     (2, 7) (2, 14)
+    """)
 
-Legacy unicode literals:
-
-    >>> dump_tokens("?rter = u'places'\\ngr?n = U'green'")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_unicode(self):
+        # Legacy unicode literals:
+        self.check_tokenize("?rter = u'places'\ngr?n = U'green'", """\
     NAME       '?rter'       (1, 0) (1, 5)
     OP         '='           (1, 6) (1, 7)
     STRING     "u'places'"   (1, 8) (1, 17)
@@ -643,17 +561,17 @@
     NAME       'gr?n'        (2, 0) (2, 4)
     OP         '='           (2, 5) (2, 6)
     STRING     "U'green'"    (2, 7) (2, 15)
+    """)
 
-Async/await extension:
-
-    >>> dump_tokens("async = 1")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_async(self):
+        # Async/await extension:
+        self.check_tokenize("async = 1", """\
     NAME       'async'       (1, 0) (1, 5)
     OP         '='           (1, 6) (1, 7)
     NUMBER     '1'           (1, 8) (1, 9)
+    """)
 
-    >>> dump_tokens("a = (async = 1)")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("a = (async = 1)", """\
     NAME       'a'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     OP         '('           (1, 4) (1, 5)
@@ -661,15 +579,15 @@
     OP         '='           (1, 11) (1, 12)
     NUMBER     '1'           (1, 13) (1, 14)
     OP         ')'           (1, 14) (1, 15)
+    """)
 
-    >>> dump_tokens("async()")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async()", """\
     NAME       'async'       (1, 0) (1, 5)
     OP         '('           (1, 5) (1, 6)
     OP         ')'           (1, 6) (1, 7)
+    """)
 
-    >>> dump_tokens("class async(Bar):pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("class async(Bar):pass", """\
     NAME       'class'       (1, 0) (1, 5)
     NAME       'async'       (1, 6) (1, 11)
     OP         '('           (1, 11) (1, 12)
@@ -677,28 +595,28 @@
     OP         ')'           (1, 15) (1, 16)
     OP         ':'           (1, 16) (1, 17)
     NAME       'pass'        (1, 17) (1, 21)
+    """)
 
-    >>> dump_tokens("class async:pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("class async:pass", """\
     NAME       'class'       (1, 0) (1, 5)
     NAME       'async'       (1, 6) (1, 11)
     OP         ':'           (1, 11) (1, 12)
     NAME       'pass'        (1, 12) (1, 16)
+    """)
 
-    >>> dump_tokens("await = 1")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("await = 1", """\
     NAME       'await'       (1, 0) (1, 5)
     OP         '='           (1, 6) (1, 7)
     NUMBER     '1'           (1, 8) (1, 9)
+    """)
 
-    >>> dump_tokens("foo.async")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("foo.async", """\
     NAME       'foo'         (1, 0) (1, 3)
     OP         '.'           (1, 3) (1, 4)
     NAME       'async'       (1, 4) (1, 9)
+    """)
 
-    >>> dump_tokens("async for a in b: pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async for a in b: pass", """\
     NAME       'async'       (1, 0) (1, 5)
     NAME       'for'         (1, 6) (1, 9)
     NAME       'a'           (1, 10) (1, 11)
@@ -706,9 +624,9 @@
     NAME       'b'           (1, 15) (1, 16)
     OP         ':'           (1, 16) (1, 17)
     NAME       'pass'        (1, 18) (1, 22)
+    """)
 
-    >>> dump_tokens("async with a as b: pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async with a as b: pass", """\
     NAME       'async'       (1, 0) (1, 5)
     NAME       'with'        (1, 6) (1, 10)
     NAME       'a'           (1, 11) (1, 12)
@@ -716,49 +634,49 @@
     NAME       'b'           (1, 16) (1, 17)
     OP         ':'           (1, 17) (1, 18)
     NAME       'pass'        (1, 19) (1, 23)
+    """)
 
-    >>> dump_tokens("async.foo")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async.foo", """\
     NAME       'async'       (1, 0) (1, 5)
     OP         '.'           (1, 5) (1, 6)
     NAME       'foo'         (1, 6) (1, 9)
+    """)
 
-    >>> dump_tokens("async")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async", """\
     NAME       'async'       (1, 0) (1, 5)
+    """)
 
-    >>> dump_tokens("async\\n#comment\\nawait")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async\n#comment\nawait", """\
     NAME       'async'       (1, 0) (1, 5)
     NEWLINE    '\\n'          (1, 5) (1, 6)
     COMMENT    '#comment'    (2, 0) (2, 8)
     NL         '\\n'          (2, 8) (2, 9)
     NAME       'await'       (3, 0) (3, 5)
+    """)
 
-    >>> dump_tokens("async\\n...\\nawait")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async\n...\nawait", """\
     NAME       'async'       (1, 0) (1, 5)
     NEWLINE    '\\n'          (1, 5) (1, 6)
     OP         '...'         (2, 0) (2, 3)
     NEWLINE    '\\n'          (2, 3) (2, 4)
     NAME       'await'       (3, 0) (3, 5)
+    """)
 
-    >>> dump_tokens("async\\nawait")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async\nawait", """\
     NAME       'async'       (1, 0) (1, 5)
     NEWLINE    '\\n'          (1, 5) (1, 6)
     NAME       'await'       (2, 0) (2, 5)
+    """)
 
-    >>> dump_tokens("foo.async + 1")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("foo.async + 1", """\
     NAME       'foo'         (1, 0) (1, 3)
     OP         '.'           (1, 3) (1, 4)
     NAME       'async'       (1, 4) (1, 9)
     OP         '+'           (1, 10) (1, 11)
     NUMBER     '1'           (1, 12) (1, 13)
+    """)
 
-    >>> dump_tokens("async def foo(): pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async def foo(): pass", """\
     ASYNC      'async'       (1, 0) (1, 5)
     NAME       'def'         (1, 6) (1, 9)
     NAME       'foo'         (1, 10) (1, 13)
@@ -766,15 +684,16 @@
     OP         ')'           (1, 14) (1, 15)
     OP         ':'           (1, 15) (1, 16)
     NAME       'pass'        (1, 17) (1, 21)
+    """)
 
-    >>> dump_tokens('''async def foo():
-    ...   def foo(await):
-    ...     await = 1
-    ...   if 1:
-    ...     await
-    ... async += 1
-    ... ''')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize('''\
+async def foo():
+  def foo(await):
+    await = 1
+  if 1:
+    await
+async += 1
+''', """\
     ASYNC      'async'       (1, 0) (1, 5)
     NAME       'def'         (1, 6) (1, 9)
     NAME       'foo'         (1, 10) (1, 13)
@@ -809,10 +728,11 @@
     OP         '+='          (6, 6) (6, 8)
     NUMBER     '1'           (6, 9) (6, 10)
     NEWLINE    '\\n'          (6, 10) (6, 11)
+    """)
 
-    >>> dump_tokens('''async def foo():
-    ...   async for i in 1: pass''')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize('''\
+async def foo():
+  async for i in 1: pass''', """\
     ASYNC      'async'       (1, 0) (1, 5)
     NAME       'def'         (1, 6) (1, 9)
     NAME       'foo'         (1, 10) (1, 13)
@@ -829,9 +749,9 @@
     OP         ':'           (2, 18) (2, 19)
     NAME       'pass'        (2, 20) (2, 24)
     DEDENT     ''            (3, 0) (3, 0)
+    """)
 
-    >>> dump_tokens('''async def foo(async): await''')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize('''async def foo(async): await''', """\
     ASYNC      'async'       (1, 0) (1, 5)
     NAME       'def'         (1, 6) (1, 9)
     NAME       'foo'         (1, 10) (1, 13)
@@ -840,14 +760,15 @@
     OP         ')'           (1, 19) (1, 20)
     OP         ':'           (1, 20) (1, 21)
     AWAIT      'await'       (1, 22) (1, 27)
+    """)
 
-    >>> dump_tokens('''def f():
-    ...
-    ...   def baz(): pass
-    ...   async def bar(): pass
-    ...
-    ...   await = 2''')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize('''\
+def f():
+
+  def baz(): pass
+  async def bar(): pass
+
+  await = 2''', """\
     NAME       'def'         (1, 0) (1, 3)
     NAME       'f'           (1, 4) (1, 5)
     OP         '('           (1, 5) (1, 6)
@@ -876,14 +797,15 @@
     OP         '='           (6, 8) (6, 9)
     NUMBER     '2'           (6, 10) (6, 11)
     DEDENT     ''            (7, 0) (7, 0)
+    """)
 
-    >>> dump_tokens('''async def f():
-    ...
-    ...   def baz(): pass
-    ...   async def bar(): pass
-    ...
-    ...   await = 2''')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize('''\
+async def f():
+
+  def baz(): pass
+  async def bar(): pass
+
+  await = 2''', """\
     ASYNC      'async'       (1, 0) (1, 5)
     NAME       'def'         (1, 6) (1, 9)
     NAME       'f'           (1, 10) (1, 11)
@@ -913,89 +835,10 @@
     OP         '='           (6, 8) (6, 9)
     NUMBER     '2'           (6, 10) (6, 11)
     DEDENT     ''            (7, 0) (7, 0)
-"""
+    """)
 
-from test import support
-from tokenize import (tokenize, _tokenize, untokenize, NUMBER, NAME, OP,
-                     STRING, ENDMARKER, ENCODING, tok_name, detect_encoding,
-                     open as tokenize_open, Untokenizer)
-from io import BytesIO
-from unittest import TestCase, mock
-import os
-import token
 
-def dump_tokens(s):
-    """Print out the tokens in s in a table format.
-
-    The ENDMARKER is omitted.
-    """
-    f = BytesIO(s.encode('utf-8'))
-    for type, token, start, end, line in tokenize(f.readline):
-        if type == ENDMARKER:
-            break
-        type = tok_name[type]
-        print("%(type)-10.10s %(token)-13.13r %(start)s %(end)s" % locals())
-
-def roundtrip(f):
-    """
-    Test roundtrip for `untokenize`. `f` is an open file or a string.
-    The source code in f is tokenized to both 5- and 2-tuples.
-    Both sequences are converted back to source code via
-    tokenize.untokenize(), and the latter tokenized again to 2-tuples.
-    The test fails if the 3 pair tokenizations do not match.
-
-    When untokenize bugs are fixed, untokenize with 5-tuples should
-    reproduce code that does not contain a backslash continuation
-    following spaces.  A proper test should test this.
-
-    This function would be more useful for correcting bugs if it reported
-    the first point of failure, like assertEqual, rather than just
-    returning False -- or if it were only used in unittests and not
-    doctest and actually used assertEqual.
-    """
-    # Get source code and original tokenizations
-    if isinstance(f, str):
-        code = f.encode('utf-8')
-    else:
-        code = f.read()
-        f.close()
-    readline = iter(code.splitlines(keepends=True)).__next__
-    tokens5 = list(tokenize(readline))
-    tokens2 = [tok[:2] for tok in tokens5]
-    # Reproduce tokens2 from pairs
-    bytes_from2 = untokenize(tokens2)
-    readline2 = iter(bytes_from2.splitlines(keepends=True)).__next__
-    tokens2_from2 = [tok[:2] for tok in tokenize(readline2)]
-    # Reproduce tokens2 from 5-tuples
-    bytes_from5 = untokenize(tokens5)
-    readline5 = iter(bytes_from5.splitlines(keepends=True)).__next__
-    tokens2_from5 = [tok[:2] for tok in tokenize(readline5)]
-    # Compare 3 versions
-    return tokens2 == tokens2_from2 == tokens2_from5
-
-# This is an example from the docs, set up as a doctest.
 def decistmt(s):
-    """Substitute Decimals for floats in a string of statements.
-
-    >>> from decimal import Decimal
-    >>> s = 'print(+21.3e-5*-.1234/81.7)'
-    >>> decistmt(s)
-    "print (+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7'))"
-
-    The format of the exponent is inherited from the platform C library.
-    Known cases are "e-007" (Windows) and "e-07" (not Windows).  Since
-    we're only showing 11 digits, and the 12th isn't close to 5, the
-    rest of the output should be platform-independent.
-
-    >>> exec(s) #doctest: +ELLIPSIS
-    -3.2171603427...e-0...7
-
-    Output from calculations with Decimal should be identical across all
-    platforms.
-
-    >>> exec(decistmt(s))
-    -3.217160342717258261933904529E-7
-    """
     result = []
     g = tokenize(BytesIO(s.encode('utf-8')).readline)   # tokenize the string
     for toknum, tokval, _, _, _  in g:
@@ -1010,6 +853,28 @@
             result.append((toknum, tokval))
     return untokenize(result).decode('utf-8')
 
+class TestMisc(TestCase):
+
+    def test_decistmt(self):
+        # Substitute Decimals for floats in a string of statements.
+        # This is an example from the docs.
+
+        from decimal import Decimal
+        s = '+21.3e-5*-.1234/81.7'
+        self.assertEqual(decistmt(s),
+                         "+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')")
+
+        # The format of the exponent is inherited from the platform C library.
+        # Known cases are "e-007" (Windows) and "e-07" (not Windows).  Since
+        # we're only showing 11 digits, and the 12th isn't close to 5, the
+        # rest of the output should be platform-independent.
+        self.assertRegex(repr(eval(s)), '-3.2171603427[0-9]*e-0+7')
+
+        # Output from calculations with Decimal should be identical across all
+        # platforms.
+        self.assertEqual(eval(decistmt(s)),
+                         Decimal('-3.217160342717258261933904529E-7'))
+
 
 class TestTokenizerAdheresToPep0263(TestCase):
     """
@@ -1018,11 +883,11 @@
 
     def _testFile(self, filename):
         path = os.path.join(os.path.dirname(__file__), filename)
-        return roundtrip(open(path, 'rb'))
+        TestRoundtrip.check_roundtrip(self, open(path, 'rb'))
 
     def test_utf8_coding_cookie_and_no_utf8_bom(self):
         f = 'tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt'
-        self.assertTrue(self._testFile(f))
+        self._testFile(f)
 
     def test_latin1_coding_cookie_and_utf8_bom(self):
         """
@@ -1037,11 +902,11 @@
 
     def test_no_coding_cookie_and_utf8_bom(self):
         f = 'tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txt'
-        self.assertTrue(self._testFile(f))
+        self._testFile(f)
 
     def test_utf8_coding_cookie_and_utf8_bom(self):
         f = 'tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt'
-        self.assertTrue(self._testFile(f))
+        self._testFile(f)
 
     def test_bad_coding_cookie(self):
         self.assertRaises(SyntaxError, self._testFile, 'bad_coding.py')
@@ -1340,7 +1205,6 @@
         self.assertTrue(m.closed)
 
 
-
 class TestTokenize(TestCase):
 
     def test_tokenize(self):
@@ -1472,6 +1336,7 @@
         # See http://bugs.python.org/issue16152
         self.assertExactTypeEqual('@          ', token.AT)
 
+
 class UntokenizeTest(TestCase):
 
     def test_bad_input_order(self):
@@ -1497,7 +1362,7 @@
         u.prev_row = 2
         u.add_whitespace((4, 4))
         self.assertEqual(u.tokens, ['\\\n', '\\\n\\\n', '    '])
-        self.assertTrue(roundtrip('a\n  b\n    c\n  \\\n  c\n'))
+        TestRoundtrip.check_roundtrip(self, 'a\n  b\n    c\n  \\\n  c\n')
 
     def test_iter_compat(self):
         u = Untokenizer()
@@ -1514,6 +1379,131 @@
 
 
 class TestRoundtrip(TestCase):
+
+    def check_roundtrip(self, f):
+        """
+        Test roundtrip for `untokenize`. `f` is an open file or a string.
+        The source code in f is tokenized to both 5- and 2-tuples.
+        Both sequences are converted back to source code via
+        tokenize.untokenize(), and the latter tokenized again to 2-tuples.
+        The test fails if the 3 pair tokenizations do not match.
+
+        When untokenize bugs are fixed, untokenize with 5-tuples should
+        reproduce code that does not contain a backslash continuation
+        following spaces.  A proper test should test this.
+        """
+        # Get source code and original tokenizations
+        if isinstance(f, str):
+            code = f.encode('utf-8')
+        else:
+            code = f.read()
+            f.close()
+        readline = iter(code.splitlines(keepends=True)).__next__
+        tokens5 = list(tokenize(readline))
+        tokens2 = [tok[:2] for tok in tokens5]
+        # Reproduce tokens2 from pairs
+        bytes_from2 = untokenize(tokens2)
+        readline2 = iter(bytes_from2.splitlines(keepends=True)).__next__
+        tokens2_from2 = [tok[:2] for tok in tokenize(readline2)]
+        self.assertEqual(tokens2_from2, tokens2)
+        # Reproduce tokens2 from 5-tuples
+        bytes_from5 = untokenize(tokens5)
+        readline5 = iter(bytes_from5.splitlines(keepends=True)).__next__
+        tokens2_from5 = [tok[:2] for tok in tokenize(readline5)]
+        self.assertEqual(tokens2_from5, tokens2)
+
+    def test_roundtrip(self):
+        # There are some standard formatting practices that are easy to get right.
+
+        self.check_roundtrip("if x == 1:\n"
+                             "    print(x)\n")
+        self.check_roundtrip("# This is a comment\n"
+                             "# This also")
+
+        # Some people use different formatting conventions, which makes
+        # untokenize a little trickier. Note that this test involves trailing
+        # whitespace after the colon. Note that we use hex escapes to make the
+        # two trailing blanks apparent in the expected output.
+
+        self.check_roundtrip("if x == 1 : \n"
+                             "  print(x)\n")
+        fn = support.findfile("tokenize_tests.txt")
+        with open(fn, 'rb') as f:
+            self.check_roundtrip(f)
+        self.check_roundtrip("if x == 1:\n"
+                             "    # A comment by itself.\n"
+                             "    print(x) # Comment here, too.\n"
+                             "    # Another comment.\n"
+                             "after_if = True\n")
+        self.check_roundtrip("if (x # The comments need to go in the right place\n"
+                             "    == 1):\n"
+                             "    print('x==1')\n")
+        self.check_roundtrip("class Test: # A comment here\n"
+                             "  # A comment with weird indent\n"
+                             "  after_com = 5\n"
+                             "  def x(m): return m*5 # a one liner\n"
+                             "  def y(m): # A whitespace after the colon\n"
+                             "     return y*4 # 3-space indent\n")
+
+        # Some error-handling code
+        self.check_roundtrip("try: import somemodule\n"
+                             "except ImportError: # comment\n"
+                             "    print('Can not import' # comment2\n)"
+                             "else:   print('Loaded')\n")
+
+    def test_continuation(self):
+        # Balancing continuation
+        self.check_roundtrip("a = (3,4, \n"
+                             "5,6)\n"
+                             "y = [3, 4,\n"
+                             "5]\n"
+                             "z = {'a': 5,\n"
+                             "'b':15, 'c':True}\n"
+                             "x = len(y) + 5 - a[\n"
+                             "3] - a[2]\n"
+                             "+ len(z) - z[\n"
+                             "'b']\n")
+
+    def test_backslash_continuation(self):
+        # Backslash means line continuation, except for comments
+        self.check_roundtrip("x=1+\\\n"
+                             "1\n"
+                             "# This is a comment\\\n"
+                             "# This also\n")
+        self.check_roundtrip("# Comment \\\n"
+                             "x = 0")
+
+    def test_string_concatenation(self):
+        # Two string literals on the same line
+        self.check_roundtrip("'' ''")
+
+    def test_random_files(self):
+        # Test roundtrip on random python modules.
+        # pass the '-ucpu' option to process the full directory.
+
+        import glob, random
+        fn = support.findfile("tokenize_tests.txt")
+        tempdir = os.path.dirname(fn) or os.curdir
+        testfiles = glob.glob(os.path.join(tempdir, "test*.py"))
+
+        # Tokenize is broken on test_pep3131.py because regular expressions are
+        # broken on the obscure unicode identifiers in it. *sigh*
+        # With roundtrip extended to test the 5-tuple mode of  untokenize,
+        # 7 more testfiles fail.  Remove them also until the failure is diagnosed.
+
+        testfiles.remove(os.path.join(tempdir, "test_pep3131.py"))
+        for f in ('buffer', 'builtin', 'fileio', 'inspect', 'os', 'platform', 'sys'):
+            testfiles.remove(os.path.join(tempdir, "test_%s.py") % f)
+
+        if not support.is_resource_enabled("cpu"):
+            testfiles = random.sample(testfiles, 10)
+
+        for testfile in testfiles:
+            with open(testfile, 'rb') as f:
+                with self.subTest(file=testfile):
+                    self.check_roundtrip(f)
+
+
     def roundtrip(self, code):
         if isinstance(code, str):
             code = code.encode('utf-8')
@@ -1527,19 +1517,8 @@
         code = "if False:\n\tx=3\n\tx=3\n"
         codelines = self.roundtrip(code).split('\n')
         self.assertEqual(codelines[1], codelines[2])
+        self.check_roundtrip(code)
 
 
-__test__ = {"doctests" : doctests, 'decistmt': decistmt}
-
-def test_main():
-    from test import test_tokenize
-    support.run_doctest(test_tokenize, True)
-    support.run_unittest(TestTokenizerAdheresToPep0263)
-    support.run_unittest(Test_Tokenize)
-    support.run_unittest(TestDetectEncoding)
-    support.run_unittest(TestTokenize)
-    support.run_unittest(UntokenizeTest)
-    support.run_unittest(TestRoundtrip)
-
 if __name__ == "__main__":
-    test_main()
+    unittest.main()

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Tue Oct  6 17:54:19 2015
From: python-checkins at python.org (serhiy.storchaka)
Date: Tue, 06 Oct 2015 15:54:19 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
 =?utf-8?q?_Issue_=2325317=3A_Converted_doctests_in_test=5Ftokenize_to_uni?=
 =?utf-8?q?ttests=2E?=
Message-ID: <20151006155416.2685.27346@psf.io>

https://hg.python.org/cpython/rev/bff40616d2a5
changeset:   98557:bff40616d2a5
branch:      3.5
parent:      98550:3719e842a7b1
parent:      98556:d272f3cbae05
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Tue Oct 06 18:23:12 2015 +0300
summary:
  Issue #25317: Converted doctests in test_tokenize to unittests.
Made test_tokenize discoverable.

files:
  Lib/test/test_tokenize.py |  811 ++++++++++++-------------
  1 files changed, 395 insertions(+), 416 deletions(-)


diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -1,22 +1,44 @@
-doctests = """
-Tests for the tokenize module.
+from test import support
+from tokenize import (tokenize, _tokenize, untokenize, NUMBER, NAME, OP,
+                     STRING, ENDMARKER, ENCODING, tok_name, detect_encoding,
+                     open as tokenize_open, Untokenizer)
+from io import BytesIO
+from unittest import TestCase, mock
+import os
+import token
 
-The tests can be really simple. Given a small fragment of source
-code, print out a table with tokens. The ENDMARKER is omitted for
-brevity.
 
-    >>> import glob
+class TokenizeTest(TestCase):
+    # Tests for the tokenize module.
 
-    >>> dump_tokens("1 + 1")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    # The tests can be really simple. Given a small fragment of source
+    # code, print out a table with tokens. The ENDMARKER is omitted for
+    # brevity.
+
+    def check_tokenize(self, s, expected):
+        # Format the tokens in s in a table format.
+        # The ENDMARKER is omitted.
+        result = []
+        f = BytesIO(s.encode('utf-8'))
+        for type, token, start, end, line in tokenize(f.readline):
+            if type == ENDMARKER:
+                break
+            type = tok_name[type]
+            result.append("    %(type)-10.10s %(token)-13.13r %(start)s %(end)s" %
+                          locals())
+        self.assertEqual(result,
+                         ["    ENCODING   'utf-8'       (0, 0) (0, 0)"] +
+                         expected.rstrip().splitlines())
+
+    def test_basic(self):
+        self.check_tokenize("1 + 1", """\
     NUMBER     '1'           (1, 0) (1, 1)
     OP         '+'           (1, 2) (1, 3)
     NUMBER     '1'           (1, 4) (1, 5)
-
-    >>> dump_tokens("if False:\\n"
-    ...             "    # NL\\n"
-    ...             "    True = False # NEWLINE\\n")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("if False:\n"
+                            "    # NL\n"
+                            "    True = False # NEWLINE\n", """\
     NAME       'if'          (1, 0) (1, 2)
     NAME       'False'       (1, 3) (1, 8)
     OP         ':'           (1, 8) (1, 9)
@@ -30,112 +52,48 @@
     COMMENT    '# NEWLINE'   (3, 17) (3, 26)
     NEWLINE    '\\n'          (3, 26) (3, 27)
     DEDENT     ''            (4, 0) (4, 0)
+    """)
+        indent_error_file = b"""\
+def k(x):
+    x += 2
+  x += 5
+"""
+        readline = BytesIO(indent_error_file).readline
+        with self.assertRaisesRegex(IndentationError,
+                                    "unindent does not match any "
+                                    "outer indentation level"):
+            for tok in tokenize(readline):
+                pass
 
-    >>> indent_error_file = \"""
-    ... def k(x):
-    ...     x += 2
-    ...   x += 5
-    ... \"""
-    >>> readline = BytesIO(indent_error_file.encode('utf-8')).readline
-    >>> for tok in tokenize(readline): pass
-    Traceback (most recent call last):
-        ...
-    IndentationError: unindent does not match any outer indentation level
-
-There are some standard formatting practices that are easy to get right.
-
-    >>> roundtrip("if x == 1:\\n"
-    ...           "    print(x)\\n")
-    True
-
-    >>> roundtrip("# This is a comment\\n# This also")
-    True
-
-Some people use different formatting conventions, which makes
-untokenize a little trickier. Note that this test involves trailing
-whitespace after the colon. Note that we use hex escapes to make the
-two trailing blanks apparent in the expected output.
-
-    >>> roundtrip("if x == 1 : \\n"
-    ...           "  print(x)\\n")
-    True
-
-    >>> f = support.findfile("tokenize_tests.txt")
-    >>> roundtrip(open(f, 'rb'))
-    True
-
-    >>> roundtrip("if x == 1:\\n"
-    ...           "    # A comment by itself.\\n"
-    ...           "    print(x) # Comment here, too.\\n"
-    ...           "    # Another comment.\\n"
-    ...           "after_if = True\\n")
-    True
-
-    >>> roundtrip("if (x # The comments need to go in the right place\\n"
-    ...           "    == 1):\\n"
-    ...           "    print('x==1')\\n")
-    True
-
-    >>> roundtrip("class Test: # A comment here\\n"
-    ...           "  # A comment with weird indent\\n"
-    ...           "  after_com = 5\\n"
-    ...           "  def x(m): return m*5 # a one liner\\n"
-    ...           "  def y(m): # A whitespace after the colon\\n"
-    ...           "     return y*4 # 3-space indent\\n")
-    True
-
-Some error-handling code
-
-    >>> roundtrip("try: import somemodule\\n"
-    ...           "except ImportError: # comment\\n"
-    ...           "    print('Can not import' # comment2\\n)"
-    ...           "else:   print('Loaded')\\n")
-    True
-
-Balancing continuation
-
-    >>> roundtrip("a = (3,4, \\n"
-    ...           "5,6)\\n"
-    ...           "y = [3, 4,\\n"
-    ...           "5]\\n"
-    ...           "z = {'a': 5,\\n"
-    ...           "'b':15, 'c':True}\\n"
-    ...           "x = len(y) + 5 - a[\\n"
-    ...           "3] - a[2]\\n"
-    ...           "+ len(z) - z[\\n"
-    ...           "'b']\\n")
-    True
-
-Ordinary integers and binary operators
-
-    >>> dump_tokens("0xff <= 255")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_int(self):
+        # Ordinary integers and binary operators
+        self.check_tokenize("0xff <= 255", """\
     NUMBER     '0xff'        (1, 0) (1, 4)
     OP         '<='          (1, 5) (1, 7)
     NUMBER     '255'         (1, 8) (1, 11)
-    >>> dump_tokens("0b10 <= 255")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("0b10 <= 255", """\
     NUMBER     '0b10'        (1, 0) (1, 4)
     OP         '<='          (1, 5) (1, 7)
     NUMBER     '255'         (1, 8) (1, 11)
-    >>> dump_tokens("0o123 <= 0O123")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("0o123 <= 0O123", """\
     NUMBER     '0o123'       (1, 0) (1, 5)
     OP         '<='          (1, 6) (1, 8)
     NUMBER     '0O123'       (1, 9) (1, 14)
-    >>> dump_tokens("1234567 > ~0x15")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("1234567 > ~0x15", """\
     NUMBER     '1234567'     (1, 0) (1, 7)
     OP         '>'           (1, 8) (1, 9)
     OP         '~'           (1, 10) (1, 11)
     NUMBER     '0x15'        (1, 11) (1, 15)
-    >>> dump_tokens("2134568 != 1231515")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("2134568 != 1231515", """\
     NUMBER     '2134568'     (1, 0) (1, 7)
     OP         '!='          (1, 8) (1, 10)
     NUMBER     '1231515'     (1, 11) (1, 18)
-    >>> dump_tokens("(-124561-1) & 200000000")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("(-124561-1) & 200000000", """\
     OP         '('           (1, 0) (1, 1)
     OP         '-'           (1, 1) (1, 2)
     NUMBER     '124561'      (1, 2) (1, 8)
@@ -144,93 +102,93 @@
     OP         ')'           (1, 10) (1, 11)
     OP         '&'           (1, 12) (1, 13)
     NUMBER     '200000000'   (1, 14) (1, 23)
-    >>> dump_tokens("0xdeadbeef != -1")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("0xdeadbeef != -1", """\
     NUMBER     '0xdeadbeef'  (1, 0) (1, 10)
     OP         '!='          (1, 11) (1, 13)
     OP         '-'           (1, 14) (1, 15)
     NUMBER     '1'           (1, 15) (1, 16)
-    >>> dump_tokens("0xdeadc0de & 12345")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("0xdeadc0de & 12345", """\
     NUMBER     '0xdeadc0de'  (1, 0) (1, 10)
     OP         '&'           (1, 11) (1, 12)
     NUMBER     '12345'       (1, 13) (1, 18)
-    >>> dump_tokens("0xFF & 0x15 | 1234")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("0xFF & 0x15 | 1234", """\
     NUMBER     '0xFF'        (1, 0) (1, 4)
     OP         '&'           (1, 5) (1, 6)
     NUMBER     '0x15'        (1, 7) (1, 11)
     OP         '|'           (1, 12) (1, 13)
     NUMBER     '1234'        (1, 14) (1, 18)
+    """)
 
-Long integers
-
-    >>> dump_tokens("x = 0")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_long(self):
+        # Long integers
+        self.check_tokenize("x = 0", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '0'           (1, 4) (1, 5)
-    >>> dump_tokens("x = 0xfffffffffff")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = 0xfffffffffff", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '0xffffffffff (1, 4) (1, 17)
-    >>> dump_tokens("x = 123141242151251616110")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = 123141242151251616110", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '123141242151 (1, 4) (1, 25)
-    >>> dump_tokens("x = -15921590215012591")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = -15921590215012591", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     OP         '-'           (1, 4) (1, 5)
     NUMBER     '159215902150 (1, 5) (1, 22)
+    """)
 
-Floating point numbers
-
-    >>> dump_tokens("x = 3.14159")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_float(self):
+        # Floating point numbers
+        self.check_tokenize("x = 3.14159", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '3.14159'     (1, 4) (1, 11)
-    >>> dump_tokens("x = 314159.")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = 314159.", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '314159.'     (1, 4) (1, 11)
-    >>> dump_tokens("x = .314159")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = .314159", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '.314159'     (1, 4) (1, 11)
-    >>> dump_tokens("x = 3e14159")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = 3e14159", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '3e14159'     (1, 4) (1, 11)
-    >>> dump_tokens("x = 3E123")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = 3E123", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '3E123'       (1, 4) (1, 9)
-    >>> dump_tokens("x+y = 3e-1230")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x+y = 3e-1230", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '+'           (1, 1) (1, 2)
     NAME       'y'           (1, 2) (1, 3)
     OP         '='           (1, 4) (1, 5)
     NUMBER     '3e-1230'     (1, 6) (1, 13)
-    >>> dump_tokens("x = 3.14e159")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = 3.14e159", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '3.14e159'    (1, 4) (1, 12)
+    """)
 
-String literals
-
-    >>> dump_tokens("x = ''; y = \\\"\\\"")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_string(self):
+        # String literals
+        self.check_tokenize("x = ''; y = \"\"", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     "''"          (1, 4) (1, 6)
@@ -238,8 +196,8 @@
     NAME       'y'           (1, 8) (1, 9)
     OP         '='           (1, 10) (1, 11)
     STRING     '""'          (1, 12) (1, 14)
-    >>> dump_tokens("x = '\\\"'; y = \\\"'\\\"")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = '\"'; y = \"'\"", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     '\\'"\\''       (1, 4) (1, 7)
@@ -247,29 +205,29 @@
     NAME       'y'           (1, 9) (1, 10)
     OP         '='           (1, 11) (1, 12)
     STRING     '"\\'"'        (1, 13) (1, 16)
-    >>> dump_tokens("x = \\\"doesn't \\\"shrink\\\", does it\\\"")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = \"doesn't \"shrink\", does it\"", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     '"doesn\\'t "' (1, 4) (1, 14)
     NAME       'shrink'      (1, 14) (1, 20)
     STRING     '", does it"' (1, 20) (1, 31)
-    >>> dump_tokens("x = 'abc' + 'ABC'")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = 'abc' + 'ABC'", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     "'abc'"       (1, 4) (1, 9)
     OP         '+'           (1, 10) (1, 11)
     STRING     "'ABC'"       (1, 12) (1, 17)
-    >>> dump_tokens('y = "ABC" + "ABC"')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize('y = "ABC" + "ABC"', """\
     NAME       'y'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     '"ABC"'       (1, 4) (1, 9)
     OP         '+'           (1, 10) (1, 11)
     STRING     '"ABC"'       (1, 12) (1, 17)
-    >>> dump_tokens("x = r'abc' + r'ABC' + R'ABC' + R'ABC'")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("x = r'abc' + r'ABC' + R'ABC' + R'ABC'", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     "r'abc'"      (1, 4) (1, 10)
@@ -279,8 +237,8 @@
     STRING     "R'ABC'"      (1, 22) (1, 28)
     OP         '+'           (1, 29) (1, 30)
     STRING     "R'ABC'"      (1, 31) (1, 37)
-    >>> dump_tokens('y = r"abc" + r"ABC" + R"ABC" + R"ABC"')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize('y = r"abc" + r"ABC" + R"ABC" + R"ABC"', """\
     NAME       'y'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     'r"abc"'      (1, 4) (1, 10)
@@ -290,30 +248,30 @@
     STRING     'R"ABC"'      (1, 22) (1, 28)
     OP         '+'           (1, 29) (1, 30)
     STRING     'R"ABC"'      (1, 31) (1, 37)
+    """)
 
-    >>> dump_tokens("u'abc' + U'abc'")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("u'abc' + U'abc'", """\
     STRING     "u'abc'"      (1, 0) (1, 6)
     OP         '+'           (1, 7) (1, 8)
     STRING     "U'abc'"      (1, 9) (1, 15)
-    >>> dump_tokens('u"abc" + U"abc"')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize('u"abc" + U"abc"', """\
     STRING     'u"abc"'      (1, 0) (1, 6)
     OP         '+'           (1, 7) (1, 8)
     STRING     'U"abc"'      (1, 9) (1, 15)
+    """)
 
-    >>> dump_tokens("b'abc' + B'abc'")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("b'abc' + B'abc'", """\
     STRING     "b'abc'"      (1, 0) (1, 6)
     OP         '+'           (1, 7) (1, 8)
     STRING     "B'abc'"      (1, 9) (1, 15)
-    >>> dump_tokens('b"abc" + B"abc"')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize('b"abc" + B"abc"', """\
     STRING     'b"abc"'      (1, 0) (1, 6)
     OP         '+'           (1, 7) (1, 8)
     STRING     'B"abc"'      (1, 9) (1, 15)
-    >>> dump_tokens("br'abc' + bR'abc' + Br'abc' + BR'abc'")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("br'abc' + bR'abc' + Br'abc' + BR'abc'", """\
     STRING     "br'abc'"     (1, 0) (1, 7)
     OP         '+'           (1, 8) (1, 9)
     STRING     "bR'abc'"     (1, 10) (1, 17)
@@ -321,8 +279,8 @@
     STRING     "Br'abc'"     (1, 20) (1, 27)
     OP         '+'           (1, 28) (1, 29)
     STRING     "BR'abc'"     (1, 30) (1, 37)
-    >>> dump_tokens('br"abc" + bR"abc" + Br"abc" + BR"abc"')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize('br"abc" + bR"abc" + Br"abc" + BR"abc"', """\
     STRING     'br"abc"'     (1, 0) (1, 7)
     OP         '+'           (1, 8) (1, 9)
     STRING     'bR"abc"'     (1, 10) (1, 17)
@@ -330,8 +288,8 @@
     STRING     'Br"abc"'     (1, 20) (1, 27)
     OP         '+'           (1, 28) (1, 29)
     STRING     'BR"abc"'     (1, 30) (1, 37)
-    >>> dump_tokens("rb'abc' + rB'abc' + Rb'abc' + RB'abc'")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("rb'abc' + rB'abc' + Rb'abc' + RB'abc'", """\
     STRING     "rb'abc'"     (1, 0) (1, 7)
     OP         '+'           (1, 8) (1, 9)
     STRING     "rB'abc'"     (1, 10) (1, 17)
@@ -339,8 +297,8 @@
     STRING     "Rb'abc'"     (1, 20) (1, 27)
     OP         '+'           (1, 28) (1, 29)
     STRING     "RB'abc'"     (1, 30) (1, 37)
-    >>> dump_tokens('rb"abc" + rB"abc" + Rb"abc" + RB"abc"')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize('rb"abc" + rB"abc" + Rb"abc" + RB"abc"', """\
     STRING     'rb"abc"'     (1, 0) (1, 7)
     OP         '+'           (1, 8) (1, 9)
     STRING     'rB"abc"'     (1, 10) (1, 17)
@@ -348,11 +306,10 @@
     STRING     'Rb"abc"'     (1, 20) (1, 27)
     OP         '+'           (1, 28) (1, 29)
     STRING     'RB"abc"'     (1, 30) (1, 37)
+    """)
 
-Operators
-
-    >>> dump_tokens("def d22(a, b, c=2, d=2, *k): pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_function(self):
+        self.check_tokenize("def d22(a, b, c=2, d=2, *k): pass", """\
     NAME       'def'         (1, 0) (1, 3)
     NAME       'd22'         (1, 4) (1, 7)
     OP         '('           (1, 7) (1, 8)
@@ -373,8 +330,8 @@
     OP         ')'           (1, 26) (1, 27)
     OP         ':'           (1, 27) (1, 28)
     NAME       'pass'        (1, 29) (1, 33)
-    >>> dump_tokens("def d01v_(a=1, *k, **w): pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("def d01v_(a=1, *k, **w): pass", """\
     NAME       'def'         (1, 0) (1, 3)
     NAME       'd01v_'       (1, 4) (1, 9)
     OP         '('           (1, 9) (1, 10)
@@ -390,12 +347,12 @@
     OP         ')'           (1, 22) (1, 23)
     OP         ':'           (1, 23) (1, 24)
     NAME       'pass'        (1, 25) (1, 29)
+    """)
 
-Comparison
-
-    >>> dump_tokens("if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != " +
-    ...             "1 and 5 in 1 not in 1 is 1 or 5 is not 1: pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_comparison(self):
+        # Comparison
+        self.check_tokenize("if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != "
+                            "1 and 5 in 1 not in 1 is 1 or 5 is not 1: pass", """\
     NAME       'if'          (1, 0) (1, 2)
     NUMBER     '1'           (1, 3) (1, 4)
     OP         '<'           (1, 5) (1, 6)
@@ -428,11 +385,11 @@
     NUMBER     '1'           (1, 81) (1, 82)
     OP         ':'           (1, 82) (1, 83)
     NAME       'pass'        (1, 84) (1, 88)
+    """)
 
-Shift
-
-    >>> dump_tokens("x = 1 << 1 >> 5")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_shift(self):
+        # Shift
+        self.check_tokenize("x = 1 << 1 >> 5", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '1'           (1, 4) (1, 5)
@@ -440,11 +397,11 @@
     NUMBER     '1'           (1, 9) (1, 10)
     OP         '>>'          (1, 11) (1, 13)
     NUMBER     '5'           (1, 14) (1, 15)
+    """)
 
-Additive
-
-    >>> dump_tokens("x = 1 - y + 15 - 1 + 0x124 + z + a[5]")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_additive(self):
+        # Additive
+        self.check_tokenize("x = 1 - y + 15 - 1 + 0x124 + z + a[5]", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '1'           (1, 4) (1, 5)
@@ -463,11 +420,11 @@
     OP         '['           (1, 34) (1, 35)
     NUMBER     '5'           (1, 35) (1, 36)
     OP         ']'           (1, 36) (1, 37)
+    """)
 
-Multiplicative
-
-    >>> dump_tokens("x = 1//1*1/5*12%0x12 at 42")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_multiplicative(self):
+        # Multiplicative
+        self.check_tokenize("x = 1//1*1/5*12%0x12 at 42", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '1'           (1, 4) (1, 5)
@@ -483,11 +440,11 @@
     NUMBER     '0x12'        (1, 16) (1, 20)
     OP         '@'           (1, 20) (1, 21)
     NUMBER     '42'          (1, 21) (1, 23)
+    """)
 
-Unary
-
-    >>> dump_tokens("~1 ^ 1 & 1 |1 ^ -1")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_unary(self):
+        # Unary
+        self.check_tokenize("~1 ^ 1 & 1 |1 ^ -1", """\
     OP         '~'           (1, 0) (1, 1)
     NUMBER     '1'           (1, 1) (1, 2)
     OP         '^'           (1, 3) (1, 4)
@@ -499,8 +456,8 @@
     OP         '^'           (1, 14) (1, 15)
     OP         '-'           (1, 16) (1, 17)
     NUMBER     '1'           (1, 17) (1, 18)
-    >>> dump_tokens("-1*1/1+1*1//1 - ---1**1")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    """)
+        self.check_tokenize("-1*1/1+1*1//1 - ---1**1", """\
     OP         '-'           (1, 0) (1, 1)
     NUMBER     '1'           (1, 1) (1, 2)
     OP         '*'           (1, 2) (1, 3)
@@ -520,11 +477,11 @@
     NUMBER     '1'           (1, 19) (1, 20)
     OP         '**'          (1, 20) (1, 22)
     NUMBER     '1'           (1, 22) (1, 23)
+    """)
 
-Selector
-
-    >>> dump_tokens("import sys, time\\nx = sys.modules['time'].time()")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_selector(self):
+        # Selector
+        self.check_tokenize("import sys, time\nx = sys.modules['time'].time()", """\
     NAME       'import'      (1, 0) (1, 6)
     NAME       'sys'         (1, 7) (1, 10)
     OP         ','           (1, 10) (1, 11)
@@ -542,11 +499,11 @@
     NAME       'time'        (2, 24) (2, 28)
     OP         '('           (2, 28) (2, 29)
     OP         ')'           (2, 29) (2, 30)
+    """)
 
-Methods
-
-    >>> dump_tokens("@staticmethod\\ndef foo(x,y): pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_method(self):
+        # Methods
+        self.check_tokenize("@staticmethod\ndef foo(x,y): pass", """\
     OP         '@'           (1, 0) (1, 1)
     NAME       'staticmethod (1, 1) (1, 13)
     NEWLINE    '\\n'          (1, 13) (1, 14)
@@ -559,52 +516,13 @@
     OP         ')'           (2, 11) (2, 12)
     OP         ':'           (2, 12) (2, 13)
     NAME       'pass'        (2, 14) (2, 18)
+    """)
 
-Backslash means line continuation, except for comments
-
-    >>> roundtrip("x=1+\\\\n"
-    ...           "1\\n"
-    ...           "# This is a comment\\\\n"
-    ...           "# This also\\n")
-    True
-    >>> roundtrip("# Comment \\\\nx = 0")
-    True
-
-Two string literals on the same line
-
-    >>> roundtrip("'' ''")
-    True
-
-Test roundtrip on random python modules.
-pass the '-ucpu' option to process the full directory.
-
-    >>> import random
-    >>> tempdir = os.path.dirname(f) or os.curdir
-    >>> testfiles = glob.glob(os.path.join(tempdir, "test*.py"))
-
-Tokenize is broken on test_pep3131.py because regular expressions are
-broken on the obscure unicode identifiers in it. *sigh*
-With roundtrip extended to test the 5-tuple mode of  untokenize,
-7 more testfiles fail.  Remove them also until the failure is diagnosed.
-
-    >>> testfiles.remove(os.path.join(tempdir, "test_pep3131.py"))
-    >>> for f in ('buffer', 'builtin', 'fileio', 'inspect', 'os', 'platform', 'sys'):
-    ...     testfiles.remove(os.path.join(tempdir, "test_%s.py") % f)
-    ...
-    >>> if not support.is_resource_enabled("cpu"):
-    ...     testfiles = random.sample(testfiles, 10)
-    ...
-    >>> for testfile in testfiles:
-    ...     if not roundtrip(open(testfile, 'rb')):
-    ...         print("Roundtrip failed for file %s" % testfile)
-    ...         break
-    ... else: True
-    True
-
-Evil tabs
-
-    >>> dump_tokens("def f():\\n\\tif x\\n        \\tpass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_tabs(self):
+        # Evil tabs
+        self.check_tokenize("def f():\n"
+                            "\tif x\n"
+                            "        \tpass", """\
     NAME       'def'         (1, 0) (1, 3)
     NAME       'f'           (1, 4) (1, 5)
     OP         '('           (1, 5) (1, 6)
@@ -619,11 +537,11 @@
     NAME       'pass'        (3, 9) (3, 13)
     DEDENT     ''            (4, 0) (4, 0)
     DEDENT     ''            (4, 0) (4, 0)
+    """)
 
-Non-ascii identifiers
-
-    >>> dump_tokens("?rter = 'places'\\ngr?n = 'green'")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_non_ascii_identifiers(self):
+        # Non-ascii identifiers
+        self.check_tokenize("?rter = 'places'\ngr?n = 'green'", """\
     NAME       '?rter'       (1, 0) (1, 5)
     OP         '='           (1, 6) (1, 7)
     STRING     "'places'"    (1, 8) (1, 16)
@@ -631,11 +549,11 @@
     NAME       'gr?n'        (2, 0) (2, 4)
     OP         '='           (2, 5) (2, 6)
     STRING     "'green'"     (2, 7) (2, 14)
+    """)
 
-Legacy unicode literals:
-
-    >>> dump_tokens("?rter = u'places'\\ngr?n = U'green'")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_unicode(self):
+        # Legacy unicode literals:
+        self.check_tokenize("?rter = u'places'\ngr?n = U'green'", """\
     NAME       '?rter'       (1, 0) (1, 5)
     OP         '='           (1, 6) (1, 7)
     STRING     "u'places'"   (1, 8) (1, 17)
@@ -643,17 +561,17 @@
     NAME       'gr?n'        (2, 0) (2, 4)
     OP         '='           (2, 5) (2, 6)
     STRING     "U'green'"    (2, 7) (2, 15)
+    """)
 
-Async/await extension:
-
-    >>> dump_tokens("async = 1")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+    def test_async(self):
+        # Async/await extension:
+        self.check_tokenize("async = 1", """\
     NAME       'async'       (1, 0) (1, 5)
     OP         '='           (1, 6) (1, 7)
     NUMBER     '1'           (1, 8) (1, 9)
+    """)
 
-    >>> dump_tokens("a = (async = 1)")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("a = (async = 1)", """\
     NAME       'a'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     OP         '('           (1, 4) (1, 5)
@@ -661,15 +579,15 @@
     OP         '='           (1, 11) (1, 12)
     NUMBER     '1'           (1, 13) (1, 14)
     OP         ')'           (1, 14) (1, 15)
+    """)
 
-    >>> dump_tokens("async()")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async()", """\
     NAME       'async'       (1, 0) (1, 5)
     OP         '('           (1, 5) (1, 6)
     OP         ')'           (1, 6) (1, 7)
+    """)
 
-    >>> dump_tokens("class async(Bar):pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("class async(Bar):pass", """\
     NAME       'class'       (1, 0) (1, 5)
     NAME       'async'       (1, 6) (1, 11)
     OP         '('           (1, 11) (1, 12)
@@ -677,28 +595,28 @@
     OP         ')'           (1, 15) (1, 16)
     OP         ':'           (1, 16) (1, 17)
     NAME       'pass'        (1, 17) (1, 21)
+    """)
 
-    >>> dump_tokens("class async:pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("class async:pass", """\
     NAME       'class'       (1, 0) (1, 5)
     NAME       'async'       (1, 6) (1, 11)
     OP         ':'           (1, 11) (1, 12)
     NAME       'pass'        (1, 12) (1, 16)
+    """)
 
-    >>> dump_tokens("await = 1")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("await = 1", """\
     NAME       'await'       (1, 0) (1, 5)
     OP         '='           (1, 6) (1, 7)
     NUMBER     '1'           (1, 8) (1, 9)
+    """)
 
-    >>> dump_tokens("foo.async")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("foo.async", """\
     NAME       'foo'         (1, 0) (1, 3)
     OP         '.'           (1, 3) (1, 4)
     NAME       'async'       (1, 4) (1, 9)
+    """)
 
-    >>> dump_tokens("async for a in b: pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async for a in b: pass", """\
     NAME       'async'       (1, 0) (1, 5)
     NAME       'for'         (1, 6) (1, 9)
     NAME       'a'           (1, 10) (1, 11)
@@ -706,9 +624,9 @@
     NAME       'b'           (1, 15) (1, 16)
     OP         ':'           (1, 16) (1, 17)
     NAME       'pass'        (1, 18) (1, 22)
+    """)
 
-    >>> dump_tokens("async with a as b: pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async with a as b: pass", """\
     NAME       'async'       (1, 0) (1, 5)
     NAME       'with'        (1, 6) (1, 10)
     NAME       'a'           (1, 11) (1, 12)
@@ -716,49 +634,49 @@
     NAME       'b'           (1, 16) (1, 17)
     OP         ':'           (1, 17) (1, 18)
     NAME       'pass'        (1, 19) (1, 23)
+    """)
 
-    >>> dump_tokens("async.foo")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async.foo", """\
     NAME       'async'       (1, 0) (1, 5)
     OP         '.'           (1, 5) (1, 6)
     NAME       'foo'         (1, 6) (1, 9)
+    """)
 
-    >>> dump_tokens("async")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async", """\
     NAME       'async'       (1, 0) (1, 5)
+    """)
 
-    >>> dump_tokens("async\\n#comment\\nawait")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async\n#comment\nawait", """\
     NAME       'async'       (1, 0) (1, 5)
     NEWLINE    '\\n'          (1, 5) (1, 6)
     COMMENT    '#comment'    (2, 0) (2, 8)
     NL         '\\n'          (2, 8) (2, 9)
     NAME       'await'       (3, 0) (3, 5)
+    """)
 
-    >>> dump_tokens("async\\n...\\nawait")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async\n...\nawait", """\
     NAME       'async'       (1, 0) (1, 5)
     NEWLINE    '\\n'          (1, 5) (1, 6)
     OP         '...'         (2, 0) (2, 3)
     NEWLINE    '\\n'          (2, 3) (2, 4)
     NAME       'await'       (3, 0) (3, 5)
+    """)
 
-    >>> dump_tokens("async\\nawait")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async\nawait", """\
     NAME       'async'       (1, 0) (1, 5)
     NEWLINE    '\\n'          (1, 5) (1, 6)
     NAME       'await'       (2, 0) (2, 5)
+    """)
 
-    >>> dump_tokens("foo.async + 1")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("foo.async + 1", """\
     NAME       'foo'         (1, 0) (1, 3)
     OP         '.'           (1, 3) (1, 4)
     NAME       'async'       (1, 4) (1, 9)
     OP         '+'           (1, 10) (1, 11)
     NUMBER     '1'           (1, 12) (1, 13)
+    """)
 
-    >>> dump_tokens("async def foo(): pass")
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize("async def foo(): pass", """\
     ASYNC      'async'       (1, 0) (1, 5)
     NAME       'def'         (1, 6) (1, 9)
     NAME       'foo'         (1, 10) (1, 13)
@@ -766,15 +684,16 @@
     OP         ')'           (1, 14) (1, 15)
     OP         ':'           (1, 15) (1, 16)
     NAME       'pass'        (1, 17) (1, 21)
+    """)
 
-    >>> dump_tokens('''async def foo():
-    ...   def foo(await):
-    ...     await = 1
-    ...   if 1:
-    ...     await
-    ... async += 1
-    ... ''')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize('''\
+async def foo():
+  def foo(await):
+    await = 1
+  if 1:
+    await
+async += 1
+''', """\
     ASYNC      'async'       (1, 0) (1, 5)
     NAME       'def'         (1, 6) (1, 9)
     NAME       'foo'         (1, 10) (1, 13)
@@ -809,10 +728,11 @@
     OP         '+='          (6, 6) (6, 8)
     NUMBER     '1'           (6, 9) (6, 10)
     NEWLINE    '\\n'          (6, 10) (6, 11)
+    """)
 
-    >>> dump_tokens('''async def foo():
-    ...   async for i in 1: pass''')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize('''\
+async def foo():
+  async for i in 1: pass''', """\
     ASYNC      'async'       (1, 0) (1, 5)
     NAME       'def'         (1, 6) (1, 9)
     NAME       'foo'         (1, 10) (1, 13)
@@ -829,9 +749,9 @@
     OP         ':'           (2, 18) (2, 19)
     NAME       'pass'        (2, 20) (2, 24)
     DEDENT     ''            (3, 0) (3, 0)
+    """)
 
-    >>> dump_tokens('''async def foo(async): await''')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize('''async def foo(async): await''', """\
     ASYNC      'async'       (1, 0) (1, 5)
     NAME       'def'         (1, 6) (1, 9)
     NAME       'foo'         (1, 10) (1, 13)
@@ -840,14 +760,15 @@
     OP         ')'           (1, 19) (1, 20)
     OP         ':'           (1, 20) (1, 21)
     AWAIT      'await'       (1, 22) (1, 27)
+    """)
 
-    >>> dump_tokens('''def f():
-    ...
-    ...   def baz(): pass
-    ...   async def bar(): pass
-    ...
-    ...   await = 2''')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize('''\
+def f():
+
+  def baz(): pass
+  async def bar(): pass
+
+  await = 2''', """\
     NAME       'def'         (1, 0) (1, 3)
     NAME       'f'           (1, 4) (1, 5)
     OP         '('           (1, 5) (1, 6)
@@ -876,14 +797,15 @@
     OP         '='           (6, 8) (6, 9)
     NUMBER     '2'           (6, 10) (6, 11)
     DEDENT     ''            (7, 0) (7, 0)
+    """)
 
-    >>> dump_tokens('''async def f():
-    ...
-    ...   def baz(): pass
-    ...   async def bar(): pass
-    ...
-    ...   await = 2''')
-    ENCODING   'utf-8'       (0, 0) (0, 0)
+        self.check_tokenize('''\
+async def f():
+
+  def baz(): pass
+  async def bar(): pass
+
+  await = 2''', """\
     ASYNC      'async'       (1, 0) (1, 5)
     NAME       'def'         (1, 6) (1, 9)
     NAME       'f'           (1, 10) (1, 11)
@@ -913,89 +835,10 @@
     OP         '='           (6, 8) (6, 9)
     NUMBER     '2'           (6, 10) (6, 11)
     DEDENT     ''            (7, 0) (7, 0)
-"""
+    """)
 
-from test import support
-from tokenize import (tokenize, _tokenize, untokenize, NUMBER, NAME, OP,
-                     STRING, ENDMARKER, ENCODING, tok_name, detect_encoding,
-                     open as tokenize_open, Untokenizer)
-from io import BytesIO
-from unittest import TestCase, mock
-import os
-import token
 
-def dump_tokens(s):
-    """Print out the tokens in s in a table format.
-
-    The ENDMARKER is omitted.
-    """
-    f = BytesIO(s.encode('utf-8'))
-    for type, token, start, end, line in tokenize(f.readline):
-        if type == ENDMARKER:
-            break
-        type = tok_name[type]
-        print("%(type)-10.10s %(token)-13.13r %(start)s %(end)s" % locals())
-
-def roundtrip(f):
-    """
-    Test roundtrip for `untokenize`. `f` is an open file or a string.
-    The source code in f is tokenized to both 5- and 2-tuples.
-    Both sequences are converted back to source code via
-    tokenize.untokenize(), and the latter tokenized again to 2-tuples.
-    The test fails if the 3 pair tokenizations do not match.
-
-    When untokenize bugs are fixed, untokenize with 5-tuples should
-    reproduce code that does not contain a backslash continuation
-    following spaces.  A proper test should test this.
-
-    This function would be more useful for correcting bugs if it reported
-    the first point of failure, like assertEqual, rather than just
-    returning False -- or if it were only used in unittests and not
-    doctest and actually used assertEqual.
-    """
-    # Get source code and original tokenizations
-    if isinstance(f, str):
-        code = f.encode('utf-8')
-    else:
-        code = f.read()
-        f.close()
-    readline = iter(code.splitlines(keepends=True)).__next__
-    tokens5 = list(tokenize(readline))
-    tokens2 = [tok[:2] for tok in tokens5]
-    # Reproduce tokens2 from pairs
-    bytes_from2 = untokenize(tokens2)
-    readline2 = iter(bytes_from2.splitlines(keepends=True)).__next__
-    tokens2_from2 = [tok[:2] for tok in tokenize(readline2)]
-    # Reproduce tokens2 from 5-tuples
-    bytes_from5 = untokenize(tokens5)
-    readline5 = iter(bytes_from5.splitlines(keepends=True)).__next__
-    tokens2_from5 = [tok[:2] for tok in tokenize(readline5)]
-    # Compare 3 versions
-    return tokens2 == tokens2_from2 == tokens2_from5
-
-# This is an example from the docs, set up as a doctest.
 def decistmt(s):
-    """Substitute Decimals for floats in a string of statements.
-
-    >>> from decimal import Decimal
-    >>> s = 'print(+21.3e-5*-.1234/81.7)'
-    >>> decistmt(s)
-    "print (+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7'))"
-
-    The format of the exponent is inherited from the platform C library.
-    Known cases are "e-007" (Windows) and "e-07" (not Windows).  Since
-    we're only showing 11 digits, and the 12th isn't close to 5, the
-    rest of the output should be platform-independent.
-
-    >>> exec(s) #doctest: +ELLIPSIS
-    -3.2171603427...e-0...7
-
-    Output from calculations with Decimal should be identical across all
-    platforms.
-
-    >>> exec(decistmt(s))
-    -3.217160342717258261933904529E-7
-    """
     result = []
     g = tokenize(BytesIO(s.encode('utf-8')).readline)   # tokenize the string
     for toknum, tokval, _, _, _  in g:
@@ -1010,6 +853,28 @@
             result.append((toknum, tokval))
     return untokenize(result).decode('utf-8')
 
+class TestMisc(TestCase):
+
+    def test_decistmt(self):
+        # Substitute Decimals for floats in a string of statements.
+        # This is an example from the docs.
+
+        from decimal import Decimal
+        s = '+21.3e-5*-.1234/81.7'
+        self.assertEqual(decistmt(s),
+                         "+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')")
+
+        # The format of the exponent is inherited from the platform C library.
+        # Known cases are "e-007" (Windows) and "e-07" (not Windows).  Since
+        # we're only showing 11 digits, and the 12th isn't close to 5, the
+        # rest of the output should be platform-independent.
+        self.assertRegex(repr(eval(s)), '-3.2171603427[0-9]*e-0+7')
+
+        # Output from calculations with Decimal should be identical across all
+        # platforms.
+        self.assertEqual(eval(decistmt(s)),
+                         Decimal('-3.217160342717258261933904529E-7'))
+
 
 class TestTokenizerAdheresToPep0263(TestCase):
     """
@@ -1018,11 +883,11 @@
 
     def _testFile(self, filename):
         path = os.path.join(os.path.dirname(__file__), filename)
-        return roundtrip(open(path, 'rb'))
+        TestRoundtrip.check_roundtrip(self, open(path, 'rb'))
 
     def test_utf8_coding_cookie_and_no_utf8_bom(self):
         f = 'tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt'
-        self.assertTrue(self._testFile(f))
+        self._testFile(f)
 
     def test_latin1_coding_cookie_and_utf8_bom(self):
         """
@@ -1037,11 +902,11 @@
 
     def test_no_coding_cookie_and_utf8_bom(self):
         f = 'tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txt'
-        self.assertTrue(self._testFile(f))
+        self._testFile(f)
 
     def test_utf8_coding_cookie_and_utf8_bom(self):
         f = 'tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt'
-        self.assertTrue(self._testFile(f))
+        self._testFile(f)
 
     def test_bad_coding_cookie(self):
         self.assertRaises(SyntaxError, self._testFile, 'bad_coding.py')
@@ -1340,7 +1205,6 @@
         self.assertTrue(m.closed)
 
 
-
 class TestTokenize(TestCase):
 
     def test_tokenize(self):
@@ -1472,6 +1336,7 @@
         # See http://bugs.python.org/issue16152
         self.assertExactTypeEqual('@          ', token.AT)
 
+
 class UntokenizeTest(TestCase):
 
     def test_bad_input_order(self):
@@ -1497,7 +1362,7 @@
         u.prev_row = 2
         u.add_whitespace((4, 4))
         self.assertEqual(u.tokens, ['\\\n', '\\\n\\\n', '    '])
-        self.assertTrue(roundtrip('a\n  b\n    c\n  \\\n  c\n'))
+        TestRoundtrip.check_roundtrip(self, 'a\n  b\n    c\n  \\\n  c\n')
 
     def test_iter_compat(self):
         u = Untokenizer()
@@ -1514,6 +1379,131 @@
 
 
 class TestRoundtrip(TestCase):
+
+    def check_roundtrip(self, f):
+        """
+        Test roundtrip for `untokenize`. `f` is an open file or a string.
+        The source code in f is tokenized to both 5- and 2-tuples.
+        Both sequences are converted back to source code via
+        tokenize.untokenize(), and the latter tokenized again to 2-tuples.
+        The test fails if the 3 pair tokenizations do not match.
+
+        When untokenize bugs are fixed, untokenize with 5-tuples should
+        reproduce code that does not contain a backslash continuation
+        following spaces.  A proper test should test this.
+        """
+        # Get source code and original tokenizations
+        if isinstance(f, str):
+            code = f.encode('utf-8')
+        else:
+            code = f.read()
+            f.close()
+        readline = iter(code.splitlines(keepends=True)).__next__
+        tokens5 = list(tokenize(readline))
+        tokens2 = [tok[:2] for tok in tokens5]
+        # Reproduce tokens2 from pairs
+        bytes_from2 = untokenize(tokens2)
+        readline2 = iter(bytes_from2.splitlines(keepends=True)).__next__
+        tokens2_from2 = [tok[:2] for tok in tokenize(readline2)]
+        self.assertEqual(tokens2_from2, tokens2)
+        # Reproduce tokens2 from 5-tuples
+        bytes_from5 = untokenize(tokens5)
+        readline5 = iter(bytes_from5.splitlines(keepends=True)).__next__
+        tokens2_from5 = [tok[:2] for tok in tokenize(readline5)]
+        self.assertEqual(tokens2_from5, tokens2)
+
+    def test_roundtrip(self):
+        # There are some standard formatting practices that are easy to get right.
+
+        self.check_roundtrip("if x == 1:\n"
+                             "    print(x)\n")
+        self.check_roundtrip("# This is a comment\n"
+                             "# This also")
+
+        # Some people use different formatting conventions, which makes
+        # untokenize a little trickier. Note that this test involves trailing
+        # whitespace after the colon. Note that we use hex escapes to make the
+        # two trailing blanks apparent in the expected output.
+
+        self.check_roundtrip("if x == 1 : \n"
+                             "  print(x)\n")
+        fn = support.findfile("tokenize_tests.txt")
+        with open(fn, 'rb') as f:
+            self.check_roundtrip(f)
+        self.check_roundtrip("if x == 1:\n"
+                             "    # A comment by itself.\n"
+                             "    print(x) # Comment here, too.\n"
+                             "    # Another comment.\n"
+                             "after_if = True\n")
+        self.check_roundtrip("if (x # The comments need to go in the right place\n"
+                             "    == 1):\n"
+                             "    print('x==1')\n")
+        self.check_roundtrip("class Test: # A comment here\n"
+                             "  # A comment with weird indent\n"
+                             "  after_com = 5\n"
+                             "  def x(m): return m*5 # a one liner\n"
+                             "  def y(m): # A whitespace after the colon\n"
+                             "     return y*4 # 3-space indent\n")
+
+        # Some error-handling code
+        self.check_roundtrip("try: import somemodule\n"
+                             "except ImportError: # comment\n"
+                             "    print('Can not import' # comment2\n)"
+                             "else:   print('Loaded')\n")
+
+    def test_continuation(self):
+        # Balancing continuation
+        self.check_roundtrip("a = (3,4, \n"
+                             "5,6)\n"
+                             "y = [3, 4,\n"
+                             "5]\n"
+                             "z = {'a': 5,\n"
+                             "'b':15, 'c':True}\n"
+                             "x = len(y) + 5 - a[\n"
+                             "3] - a[2]\n"
+                             "+ len(z) - z[\n"
+                             "'b']\n")
+
+    def test_backslash_continuation(self):
+        # Backslash means line continuation, except for comments
+        self.check_roundtrip("x=1+\\\n"
+                             "1\n"
+                             "# This is a comment\\\n"
+                             "# This also\n")
+        self.check_roundtrip("# Comment \\\n"
+                             "x = 0")
+
+    def test_string_concatenation(self):
+        # Two string literals on the same line
+        self.check_roundtrip("'' ''")
+
+    def test_random_files(self):
+        # Test roundtrip on random python modules.
+        # pass the '-ucpu' option to process the full directory.
+
+        import glob, random
+        fn = support.findfile("tokenize_tests.txt")
+        tempdir = os.path.dirname(fn) or os.curdir
+        testfiles = glob.glob(os.path.join(tempdir, "test*.py"))
+
+        # Tokenize is broken on test_pep3131.py because regular expressions are
+        # broken on the obscure unicode identifiers in it. *sigh*
+        # With roundtrip extended to test the 5-tuple mode of  untokenize,
+        # 7 more testfiles fail.  Remove them also until the failure is diagnosed.
+
+        testfiles.remove(os.path.join(tempdir, "test_pep3131.py"))
+        for f in ('buffer', 'builtin', 'fileio', 'inspect', 'os', 'platform', 'sys'):
+            testfiles.remove(os.path.join(tempdir, "test_%s.py") % f)
+
+        if not support.is_resource_enabled("cpu"):
+            testfiles = random.sample(testfiles, 10)
+
+        for testfile in testfiles:
+            with open(testfile, 'rb') as f:
+                with self.subTest(file=testfile):
+                    self.check_roundtrip(f)
+
+
     def roundtrip(self, code):
         if isinstance(code, str):
             code = code.encode('utf-8')
@@ -1527,19 +1517,8 @@
         code = "if False:\n\tx=3\n\tx=3\n"
         codelines = self.roundtrip(code).split('\n')
         self.assertEqual(codelines[1], codelines[2])
+        self.check_roundtrip(code)
 
 
-__test__ = {"doctests" : doctests, 'decistmt': decistmt}
-
-def test_main():
-    from test import test_tokenize
-    support.run_doctest(test_tokenize, True)
-    support.run_unittest(TestTokenizerAdheresToPep0263)
-    support.run_unittest(Test_Tokenize)
-    support.run_unittest(TestDetectEncoding)
-    support.run_unittest(TestTokenize)
-    support.run_unittest(UntokenizeTest)
-    support.run_unittest(TestRoundtrip)
-
 if __name__ == "__main__":
-    test_main()
+    unittest.main()

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Tue Oct  6 19:30:04 2015
From: python-checkins at python.org (alexander.belopolsky)
Date: Tue, 06 Oct 2015 17:30:04 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Closes_issue_=2312006=3A_A?=
 =?utf-8?q?dd_ISO_8601_year=2C_week=2C_and_day_directives_to_strptime=2E?=
Message-ID: <20151006173002.7236.2403@psf.io>

https://hg.python.org/cpython/rev/acdebfbfbdcf
changeset:   98564:acdebfbfbdcf
user:        Alexander Belopolsky <alexander.belopolsky at gmail.com>
date:        Tue Oct 06 13:29:56 2015 -0400
summary:
  Closes issue #12006: Add ISO 8601 year, week, and day directives to strptime.

This commit adds %G, %V, and %u directives to strptime.  Thanks Ashley Anderson
for the implementation.

files:
  Doc/library/datetime.rst  |  37 ++++++++++++-
  Doc/whatsnew/3.6.rst      |   8 ++
  Lib/_strptime.py          |  81 ++++++++++++++++++++++----
  Lib/test/test_strptime.py |  57 ++++++++++++++----
  Misc/NEWS                 |   3 +
  5 files changed, 159 insertions(+), 27 deletions(-)


diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst
--- a/Doc/library/datetime.rst
+++ b/Doc/library/datetime.rst
@@ -1909,6 +1909,34 @@
 | ``%%``    | A literal ``'%'`` character.   | %                      |       |
 +-----------+--------------------------------+------------------------+-------+
 
+Several additional directives not required by the C89 standard are included for
+convenience. These parameters all correspond to ISO 8601 date values. These
+may not be available on all platforms when used with the :meth:`strftime`
+method. The ISO 8601 year and ISO 8601 week directives are not interchangeable
+with the year and week number directives above. Calling :meth:`strptime` with
+incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`.
+
++-----------+--------------------------------+------------------------+-------+
+| Directive | Meaning                        | Example                | Notes |
++===========+================================+========================+=======+
+| ``%G``    | ISO 8601 year with century     | 0001, 0002, ..., 2013, | \(8)  |
+|           | representing the year that     | 2014, ..., 9998, 9999  |       |
+|           | contains the greater part of   |                        |       |
+|           | the ISO week (``%V``).         |                        |       |
++-----------+--------------------------------+------------------------+-------+
+| ``%u``    | ISO 8601 weekday as a decimal  | 1, 2, ..., 7           |       |
+|           | number where 1 is Monday.      |                        |       |
++-----------+--------------------------------+------------------------+-------+
+| ``%V``    | ISO 8601 week as a decimal     | 01, 02, ..., 53        | \(8)  |
+|           | number with Monday as          |                        |       |
+|           | the first day of the week.     |                        |       |
+|           | Week 01 is the week containing |                        |       |
+|           | Jan 4.                         |                        |       |
++-----------+--------------------------------+------------------------+-------+
+
+.. versionadded:: 3.6
+   ``%G``, ``%u`` and ``%V`` were added.
+
 Notes:
 
 (1)
@@ -1973,7 +2001,14 @@
 
 (7)
    When used with the :meth:`strptime` method, ``%U`` and ``%W`` are only used
-   in calculations when the day of the week and the year are specified.
+   in calculations when the day of the week and the calendar year (``%Y``)
+   are specified.
+
+(8)
+   Similar to ``%U`` and ``%W``, ``%V`` is only used in calculations when the
+   day of the week and the ISO year (``%G``) are specified in a
+   :meth:`strptime` format string. Also note that ``%G`` and ``%Y`` are not
+   interchangable.
 
 .. rubric:: Footnotes
 
diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst
--- a/Doc/whatsnew/3.6.rst
+++ b/Doc/whatsnew/3.6.rst
@@ -110,6 +110,14 @@
 with underscores.  A space or a colon can be added after completed keyword.
 (Contributed by Serhiy Storchaka in :issue:`25011` and :issue:`25209`.)
 
+datetime
+--------
+
+* :meth:`datetime.stftime <datetime.datetime.stftime>`  and
+  :meth:`date.stftime <datetime.date.stftime>` methods now support ISO 8601
+  date directives ``%G``, ``%u`` and ``%V``.
+  (Contributed by Ashley Anderson in :issue:`12006`.)
+
 
 Optimizations
 =============
diff --git a/Lib/_strptime.py b/Lib/_strptime.py
--- a/Lib/_strptime.py
+++ b/Lib/_strptime.py
@@ -195,12 +195,15 @@
             'f': r"(?P<f>[0-9]{1,6})",
             'H': r"(?P<H>2[0-3]|[0-1]\d|\d)",
             'I': r"(?P<I>1[0-2]|0[1-9]|[1-9])",
+            'G': r"(?P<G>\d\d\d\d)",
             'j': r"(?P<j>36[0-6]|3[0-5]\d|[1-2]\d\d|0[1-9]\d|00[1-9]|[1-9]\d|0[1-9]|[1-9])",
             'm': r"(?P<m>1[0-2]|0[1-9]|[1-9])",
             'M': r"(?P<M>[0-5]\d|\d)",
             'S': r"(?P<S>6[0-1]|[0-5]\d|\d)",
             'U': r"(?P<U>5[0-3]|[0-4]\d|\d)",
             'w': r"(?P<w>[0-6])",
+            'u': r"(?P<u>[1-7])",
+            'V': r"(?P<V>5[0-3]|0[1-9]|[1-4]\d|\d)",
             # W is set below by using 'U'
             'y': r"(?P<y>\d\d)",
             #XXX: Does 'Y' need to worry about having less or more than
@@ -295,6 +298,22 @@
         return 1 + days_to_week + day_of_week
 
 
+def _calc_julian_from_V(iso_year, iso_week, iso_weekday):
+    """Calculate the Julian day based on the ISO 8601 year, week, and weekday.
+    ISO weeks start on Mondays, with week 01 being the week containing 4 Jan.
+    ISO week days range from 1 (Monday) to 7 (Sunday).
+    """
+    correction = datetime_date(iso_year, 1, 4).isoweekday() + 3
+    ordinal = (iso_week * 7) + iso_weekday - correction
+    # ordinal may be negative or 0 now, which means the date is in the previous
+    # calendar year
+    if ordinal < 1:
+        ordinal += datetime_date(iso_year, 1, 1).toordinal()
+        iso_year -= 1
+        ordinal -= datetime_date(iso_year, 1, 1).toordinal()
+    return iso_year, ordinal
+
+
 def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"):
     """Return a 2-tuple consisting of a time struct and an int containing
     the number of microseconds based on the input string and the
@@ -339,15 +358,15 @@
         raise ValueError("unconverted data remains: %s" %
                           data_string[found.end():])
 
-    year = None
+    iso_year = year = None
     month = day = 1
     hour = minute = second = fraction = 0
     tz = -1
     tzoffset = None
     # Default to -1 to signify that values not known; not critical to have,
     # though
-    week_of_year = -1
-    week_of_year_start = -1
+    iso_week = week_of_year = None
+    week_of_year_start = None
     # weekday and julian defaulted to None so as to signal need to calculate
     # values
     weekday = julian = None
@@ -369,6 +388,8 @@
                 year += 1900
         elif group_key == 'Y':
             year = int(found_dict['Y'])
+        elif group_key == 'G':
+            iso_year = int(found_dict['G'])
         elif group_key == 'm':
             month = int(found_dict['m'])
         elif group_key == 'B':
@@ -414,6 +435,9 @@
                 weekday = 6
             else:
                 weekday -= 1
+        elif group_key == 'u':
+            weekday = int(found_dict['u'])
+            weekday -= 1
         elif group_key == 'j':
             julian = int(found_dict['j'])
         elif group_key in ('U', 'W'):
@@ -424,6 +448,8 @@
             else:
                 # W starts week on Monday.
                 week_of_year_start = 0
+        elif group_key == 'V':
+            iso_week = int(found_dict['V'])
         elif group_key == 'z':
             z = found_dict['z']
             tzoffset = int(z[1:3]) * 60 + int(z[3:5])
@@ -444,28 +470,57 @@
                     else:
                         tz = value
                         break
+    # Deal with the cases where ambiguities arize
+    # don't assume default values for ISO week/year
+    if year is None and iso_year is not None:
+        if iso_week is None or weekday is None:
+            raise ValueError("ISO year directive '%G' must be used with "
+                             "the ISO week directive '%V' and a weekday "
+                             "directive ('%A', '%a', '%w', or '%u').")
+        if julian is not None:
+            raise ValueError("Day of the year directive '%j' is not "
+                             "compatible with ISO year directive '%G'. "
+                             "Use '%Y' instead.")
+    elif week_of_year is None and iso_week is not None:
+        if weekday is None:
+            raise ValueError("ISO week directive '%V' must be used with "
+                             "the ISO year directive '%G' and a weekday "
+                             "directive ('%A', '%a', '%w', or '%u').")
+        else:
+            raise ValueError("ISO week directive '%V' is incompatible with "
+                             "the year directive '%Y'. Use the ISO year '%G' "
+                             "instead.")
+
     leap_year_fix = False
     if year is None and month == 2 and day == 29:
         year = 1904  # 1904 is first leap year of 20th century
         leap_year_fix = True
     elif year is None:
         year = 1900
+
+
     # If we know the week of the year and what day of that week, we can figure
     # out the Julian day of the year.
-    if julian is None and week_of_year != -1 and weekday is not None:
-        week_starts_Mon = True if week_of_year_start == 0 else False
-        julian = _calc_julian_from_U_or_W(year, week_of_year, weekday,
-                                            week_starts_Mon)
-    # Cannot pre-calculate datetime_date() since can change in Julian
-    # calculation and thus could have different value for the day of the week
-    # calculation.
+    if julian is None and weekday is not None:
+        if week_of_year is not None:
+            week_starts_Mon = True if week_of_year_start == 0 else False
+            julian = _calc_julian_from_U_or_W(year, week_of_year, weekday,
+                                                week_starts_Mon)
+        elif iso_year is not None and iso_week is not None:
+            year, julian = _calc_julian_from_V(iso_year, iso_week, weekday + 1)
+
     if julian is None:
+        # Cannot pre-calculate datetime_date() since can change in Julian
+        # calculation and thus could have different value for the day of
+        # the week calculation.
         # Need to add 1 to result since first day of the year is 1, not 0.
         julian = datetime_date(year, month, day).toordinal() - \
                   datetime_date(year, 1, 1).toordinal() + 1
-    else:  # Assume that if they bothered to include Julian day it will
-           # be accurate.
-        datetime_result = datetime_date.fromordinal((julian - 1) + datetime_date(year, 1, 1).toordinal())
+    else:  # Assume that if they bothered to include Julian day (or if it was
+           # calculated above with year/week/weekday) it will be accurate.
+        datetime_result = datetime_date.fromordinal(
+                            (julian - 1) +
+                            datetime_date(year, 1, 1).toordinal())
         year = datetime_result.year
         month = datetime_result.month
         day = datetime_result.day
diff --git a/Lib/test/test_strptime.py b/Lib/test/test_strptime.py
--- a/Lib/test/test_strptime.py
+++ b/Lib/test/test_strptime.py
@@ -152,8 +152,8 @@
                          "'%s' using '%s'; group 'a' = '%s', group 'b' = %s'" %
                          (found.string, found.re.pattern, found.group('a'),
                           found.group('b')))
-        for directive in ('a','A','b','B','c','d','H','I','j','m','M','p','S',
-                          'U','w','W','x','X','y','Y','Z','%'):
+        for directive in ('a','A','b','B','c','d','G','H','I','j','m','M','p',
+                          'S','u','U','V','w','W','x','X','y','Y','Z','%'):
             compiled = self.time_re.compile("%" + directive)
             found = compiled.match(time.strftime("%" + directive))
             self.assertTrue(found, "Matching failed on '%s' using '%s' regex" %
@@ -218,6 +218,26 @@
             else:
                 self.fail("'%s' did not raise ValueError" % bad_format)
 
+        # Ambiguous or incomplete cases using ISO year/week/weekday directives
+        # 1. ISO week (%V) is specified, but the year is specified with %Y
+        # instead of %G
+        with self.assertRaises(ValueError):
+            _strptime._strptime("1999 50", "%Y %V")
+        # 2. ISO year (%G) and ISO week (%V) are specified, but weekday is not
+        with self.assertRaises(ValueError):
+            _strptime._strptime("1999 51", "%G %V")
+        # 3. ISO year (%G) and weekday are specified, but ISO week (%V) is not
+        for w in ('A', 'a', 'w', 'u'):
+            with self.assertRaises(ValueError):
+                _strptime._strptime("1999 51","%G %{}".format(w))
+        # 4. ISO year is specified alone (e.g. time.strptime('2015', '%G'))
+        with self.assertRaises(ValueError):
+            _strptime._strptime("2015", "%G")
+        # 5. Julian/ordinal day (%j) is specified with %G, but not %Y
+        with self.assertRaises(ValueError):
+            _strptime._strptime("1999 256", "%G %j")
+
+
     def test_strptime_exception_context(self):
         # check that this doesn't chain exceptions needlessly (see #17572)
         with self.assertRaises(ValueError) as e:
@@ -289,7 +309,7 @@
 
     def test_weekday(self):
         # Test weekday directives
-        for directive in ('A', 'a', 'w'):
+        for directive in ('A', 'a', 'w', 'u'):
             self.helper(directive,6)
 
     def test_julian(self):
@@ -458,16 +478,20 @@
         # Should be able to infer date if given year, week of year (%U or %W)
         # and day of the week
         def test_helper(ymd_tuple, test_reason):
-            for directive in ('W', 'U'):
-                format_string = "%%Y %%%s %%w" % directive
-                dt_date = datetime_date(*ymd_tuple)
-                strp_input = dt_date.strftime(format_string)
-                strp_output = _strptime._strptime_time(strp_input, format_string)
-                self.assertTrue(strp_output[:3] == ymd_tuple,
-                        "%s(%s) test failed w/ '%s': %s != %s (%s != %s)" %
-                            (test_reason, directive, strp_input,
-                                strp_output[:3], ymd_tuple,
-                                strp_output[7], dt_date.timetuple()[7]))
+            for year_week_format in ('%Y %W', '%Y %U', '%G %V'):
+                for weekday_format in ('%w', '%u', '%a', '%A'):
+                    format_string = year_week_format + ' ' + weekday_format
+                    with self.subTest(test_reason,
+                                      date=ymd_tuple,
+                                      format=format_string):
+                        dt_date = datetime_date(*ymd_tuple)
+                        strp_input = dt_date.strftime(format_string)
+                        strp_output = _strptime._strptime_time(strp_input,
+                                                               format_string)
+                        msg = "%r: %s != %s" % (strp_input,
+                                                strp_output[7],
+                                                dt_date.timetuple()[7])
+                        self.assertEqual(strp_output[:3], ymd_tuple, msg)
         test_helper((1901, 1, 3), "week 0")
         test_helper((1901, 1, 8), "common case")
         test_helper((1901, 1, 13), "day on Sunday")
@@ -499,18 +523,25 @@
             self.assertEqual(_strptime._strptime_time(value, format)[:-1], expected)
         check('2015 0 0', '%Y %U %w', 2014, 12, 28, 0, 0, 0, 6, -3)
         check('2015 0 0', '%Y %W %w', 2015, 1, 4, 0, 0, 0, 6, 4)
+        check('2015 1 1', '%G %V %u', 2014, 12, 29, 0, 0, 0, 0, 363)
         check('2015 0 1', '%Y %U %w', 2014, 12, 29, 0, 0, 0, 0, -2)
         check('2015 0 1', '%Y %W %w', 2014, 12, 29, 0, 0, 0, 0, -2)
+        check('2015 1 2', '%G %V %u', 2014, 12, 30, 0, 0, 0, 1, 364)
         check('2015 0 2', '%Y %U %w', 2014, 12, 30, 0, 0, 0, 1, -1)
         check('2015 0 2', '%Y %W %w', 2014, 12, 30, 0, 0, 0, 1, -1)
+        check('2015 1 3', '%G %V %u', 2014, 12, 31, 0, 0, 0, 2, 365)
         check('2015 0 3', '%Y %U %w', 2014, 12, 31, 0, 0, 0, 2, 0)
         check('2015 0 3', '%Y %W %w', 2014, 12, 31, 0, 0, 0, 2, 0)
+        check('2015 1 4', '%G %V %u', 2015, 1, 1, 0, 0, 0, 3, 1)
         check('2015 0 4', '%Y %U %w', 2015, 1, 1, 0, 0, 0, 3, 1)
         check('2015 0 4', '%Y %W %w', 2015, 1, 1, 0, 0, 0, 3, 1)
+        check('2015 1 5', '%G %V %u', 2015, 1, 2, 0, 0, 0, 4, 2)
         check('2015 0 5', '%Y %U %w', 2015, 1, 2, 0, 0, 0, 4, 2)
         check('2015 0 5', '%Y %W %w', 2015, 1, 2, 0, 0, 0, 4, 2)
+        check('2015 1 6', '%G %V %u', 2015, 1, 3, 0, 0, 0, 5, 3)
         check('2015 0 6', '%Y %U %w', 2015, 1, 3, 0, 0, 0, 5, 3)
         check('2015 0 6', '%Y %W %w', 2015, 1, 3, 0, 0, 0, 5, 3)
+        check('2015 1 7', '%G %V %u', 2015, 1, 4, 0, 0, 0, 6, 4)
 
 
 class CacheTests(unittest.TestCase):
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -383,6 +383,9 @@
 - Issue #23572: Fixed functools.singledispatch on classes with falsy
   metaclasses.  Patch by Ethan Furman.
 
+- Issue #12006: Add ISO 8601 year, week, and day directives (%G, %V, %u) to
+  strptime.
+
 Documentation
 -------------
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Tue Oct  6 22:23:27 2015
From: python-checkins at python.org (zach.ware)
Date: Tue, 06 Oct 2015 20:23:27 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzI1MDk3?=
 =?utf-8?q?=3A_fix_Windows_error_number_access?=
Message-ID: <20151006202326.464.17724@psf.io>

https://hg.python.org/cpython/rev/4e7697ccceeb
changeset:   98565:4e7697ccceeb
branch:      3.4
parent:      98559:91f36d2b097a
user:        Zachary Ware <zachary.ware at gmail.com>
date:        Tue Oct 06 15:22:13 2015 -0500
summary:
  Issue #25097: fix Windows error number access

files:
  Lib/test/test_logging.py |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py
--- a/Lib/test/test_logging.py
+++ b/Lib/test/test_logging.py
@@ -4104,7 +4104,7 @@
         try:
             h = logging.handlers.NTEventLogHandler('test_logging')
         except pywintypes.error as e:
-            if e[0] == 5:  # access denied
+            if e.winerror == 5:  # access denied
                 raise unittest.SkipTest('Insufficient privileges to run test')
 
         r = logging.makeLogRecord({'msg': 'Test Log Message'})

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Tue Oct  6 22:23:27 2015
From: python-checkins at python.org (zach.ware)
Date: Tue, 06 Oct 2015 20:23:27 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
 =?utf-8?q?_Issue_=2325097=3A_Merge_with_3=2E4?=
Message-ID: <20151006202326.485.26636@psf.io>

https://hg.python.org/cpython/rev/440d4da352fa
changeset:   98566:440d4da352fa
branch:      3.5
parent:      98562:b0ce3ef2ea21
parent:      98565:4e7697ccceeb
user:        Zachary Ware <zachary.ware at gmail.com>
date:        Tue Oct 06 15:22:41 2015 -0500
summary:
  Issue #25097: Merge with 3.4

files:
  Lib/test/test_logging.py |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py
--- a/Lib/test/test_logging.py
+++ b/Lib/test/test_logging.py
@@ -4134,7 +4134,7 @@
         try:
             h = logging.handlers.NTEventLogHandler('test_logging')
         except pywintypes.error as e:
-            if e[0] == 5:  # access denied
+            if e.winerror == 5:  # access denied
                 raise unittest.SkipTest('Insufficient privileges to run test')
 
         r = logging.makeLogRecord({'msg': 'Test Log Message'})

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Tue Oct  6 22:23:32 2015
From: python-checkins at python.org (zach.ware)
Date: Tue, 06 Oct 2015 20:23:32 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2325097=3A_Merge_with_3=2E5?=
Message-ID: <20151006202331.18370.70039@psf.io>

https://hg.python.org/cpython/rev/d91f9fc7b85d
changeset:   98567:d91f9fc7b85d
parent:      98564:acdebfbfbdcf
parent:      98566:440d4da352fa
user:        Zachary Ware <zachary.ware at gmail.com>
date:        Tue Oct 06 15:23:16 2015 -0500
summary:
  Issue #25097: Merge with 3.5

files:
  Lib/test/test_logging.py |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py
--- a/Lib/test/test_logging.py
+++ b/Lib/test/test_logging.py
@@ -4134,7 +4134,7 @@
         try:
             h = logging.handlers.NTEventLogHandler('test_logging')
         except pywintypes.error as e:
-            if e[0] == 5:  # access denied
+            if e.winerror == 5:  # access denied
                 raise unittest.SkipTest('Insufficient privileges to run test')
 
         r = logging.makeLogRecord({'msg': 'Test Log Message'})

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Tue Oct  6 22:29:16 2015
From: python-checkins at python.org (zach.ware)
Date: Tue, 06 Oct 2015 20:29:16 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
 =?utf-8?q?_Issue_=2325097=3A_Merge_with_3=2E4?=
Message-ID: <20151006202916.20763.99249@psf.io>

https://hg.python.org/cpython/rev/95a26798819b
changeset:   98569:95a26798819b
branch:      3.5
parent:      98566:440d4da352fa
parent:      98568:03a569eb0e0e
user:        Zachary Ware <zachary.ware at gmail.com>
date:        Tue Oct 06 15:28:56 2015 -0500
summary:
  Issue #25097: Merge with 3.4

files:
  Lib/test/test_logging.py |  1 +
  1 files changed, 1 insertions(+), 0 deletions(-)


diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py
--- a/Lib/test/test_logging.py
+++ b/Lib/test/test_logging.py
@@ -4136,6 +4136,7 @@
         except pywintypes.error as e:
             if e.winerror == 5:  # access denied
                 raise unittest.SkipTest('Insufficient privileges to run test')
+            raise
 
         r = logging.makeLogRecord({'msg': 'Test Log Message'})
         h.handle(r)

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Tue Oct  6 22:29:17 2015
From: python-checkins at python.org (zach.ware)
Date: Tue, 06 Oct 2015 20:29:17 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2325097=3A_Merge_with_3=2E5?=
Message-ID: <20151006202916.18370.4720@psf.io>

https://hg.python.org/cpython/rev/db782c81bba9
changeset:   98570:db782c81bba9
parent:      98567:d91f9fc7b85d
parent:      98569:95a26798819b
user:        Zachary Ware <zachary.ware at gmail.com>
date:        Tue Oct 06 15:29:09 2015 -0500
summary:
  Issue #25097: Merge with 3.5

files:
  Lib/test/test_logging.py |  1 +
  1 files changed, 1 insertions(+), 0 deletions(-)


diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py
--- a/Lib/test/test_logging.py
+++ b/Lib/test/test_logging.py
@@ -4136,6 +4136,7 @@
         except pywintypes.error as e:
             if e.winerror == 5:  # access denied
                 raise unittest.SkipTest('Insufficient privileges to run test')
+            raise
 
         r = logging.makeLogRecord({'msg': 'Test Log Message'})
         h.handle(r)

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Tue Oct  6 22:29:20 2015
From: python-checkins at python.org (zach.ware)
Date: Tue, 06 Oct 2015 20:29:20 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzI1MDk3?=
 =?utf-8?q?=3A_Re-raise_any_other_pywin32_error?=
Message-ID: <20151006202916.70994.33433@psf.io>

https://hg.python.org/cpython/rev/03a569eb0e0e
changeset:   98568:03a569eb0e0e
branch:      3.4
parent:      98565:4e7697ccceeb
user:        Zachary Ware <zachary.ware at gmail.com>
date:        Tue Oct 06 15:28:43 2015 -0500
summary:
  Issue #25097: Re-raise any other pywin32 error

files:
  Lib/test/test_logging.py |  1 +
  1 files changed, 1 insertions(+), 0 deletions(-)


diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py
--- a/Lib/test/test_logging.py
+++ b/Lib/test/test_logging.py
@@ -4106,6 +4106,7 @@
         except pywintypes.error as e:
             if e.winerror == 5:  # access denied
                 raise unittest.SkipTest('Insufficient privileges to run test')
+            raise
 
         r = logging.makeLogRecord({'msg': 'Test Log Message'})
         h.handle(r)

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Wed Oct  7 04:42:56 2015
From: python-checkins at python.org (benjamin.peterson)
Date: Wed, 07 Oct 2015 02:42:56 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E4=29=3A_prevent_unacce?=
 =?utf-8?q?ptable_bases_from_becoming_bases_through_multiple_inheritance?=
Message-ID: <20151007024256.18362.14028@psf.io>

https://hg.python.org/cpython/rev/e670b37e7b14
changeset:   98573:e670b37e7b14
branch:      3.4
parent:      98568:03a569eb0e0e
user:        Benjamin Peterson <benjamin at python.org>
date:        Tue Oct 06 19:36:54 2015 -0700
summary:
  prevent unacceptable bases from becoming bases through multiple inheritance (#24806)

files:
  Lib/test/test_descr.py |  31 ++++++++++++++++++++++++++++++
  Misc/NEWS              |   3 ++
  Objects/typeobject.c   |  12 +++++-----
  3 files changed, 40 insertions(+), 6 deletions(-)


diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py
--- a/Lib/test/test_descr.py
+++ b/Lib/test/test_descr.py
@@ -3735,6 +3735,37 @@
         else:
             assert 0, "best_base calculation found wanting"
 
+    def test_unsubclassable_types(self):
+        with self.assertRaises(TypeError):
+            class X(type(None)):
+                pass
+        with self.assertRaises(TypeError):
+            class X(object, type(None)):
+                pass
+        with self.assertRaises(TypeError):
+            class X(type(None), object):
+                pass
+        class O(object):
+            pass
+        with self.assertRaises(TypeError):
+            class X(O, type(None)):
+                pass
+        with self.assertRaises(TypeError):
+            class X(type(None), O):
+                pass
+
+        class X(object):
+            pass
+        with self.assertRaises(TypeError):
+            X.__bases__ = type(None),
+        with self.assertRaises(TypeError):
+            X.__bases__ = object, type(None)
+        with self.assertRaises(TypeError):
+            X.__bases__ = type(None), object
+        with self.assertRaises(TypeError):
+            X.__bases__ = O, type(None)
+        with self.assertRaises(TypeError):
+            X.__bases__ = type(None), O
 
     def test_mutable_bases_with_failing_mro(self):
         # Testing mutable bases with failing mro...
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,9 @@
 Core and Builtins
 -----------------
 
+- Issue #24806: Prevent builtin types that are not allowed to be subclassed from
+  being subclassed through multiple inheritance.
+
 - Issue #24848: Fixed a number of bugs in UTF-7 decoding of misformed data.
 
 - Issue #25280: Import trace messages emitted in verbose (-v) mode are no
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -1937,6 +1937,12 @@
             if (PyType_Ready(base_i) < 0)
                 return NULL;
         }
+        if (!PyType_HasFeature(base_i, Py_TPFLAGS_BASETYPE)) {
+            PyErr_Format(PyExc_TypeError,
+                         "type '%.100s' is not an acceptable base type",
+                         base_i->tp_name);
+            return NULL;
+        }
         candidate = solid_base(base_i);
         if (winner == NULL) {
             winner = candidate;
@@ -2317,12 +2323,6 @@
     if (base == NULL) {
         goto error;
     }
-    if (!PyType_HasFeature(base, Py_TPFLAGS_BASETYPE)) {
-        PyErr_Format(PyExc_TypeError,
-                     "type '%.100s' is not an acceptable base type",
-                     base->tp_name);
-        goto error;
-    }
 
     dict = PyDict_Copy(orig_dict);
     if (dict == NULL)

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Wed Oct  7 04:42:56 2015
From: python-checkins at python.org (benjamin.peterson)
Date: Wed, 07 Oct 2015 02:42:56 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
 =?utf-8?q?_merge_3=2E4_=28=2324806=29?=
Message-ID: <20151007024256.3297.13349@psf.io>

https://hg.python.org/cpython/rev/e02e4afcce6a
changeset:   98574:e02e4afcce6a
branch:      3.5
parent:      98569:95a26798819b
parent:      98573:e670b37e7b14
user:        Benjamin Peterson <benjamin at python.org>
date:        Tue Oct 06 19:42:02 2015 -0700
summary:
  merge 3.4 (#24806)

files:
  Lib/test/test_descr.py |  31 ++++++++++++++++++++++++++++++
  Misc/NEWS              |   3 ++
  Objects/typeobject.c   |  12 +++++-----
  3 files changed, 40 insertions(+), 6 deletions(-)


diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py
--- a/Lib/test/test_descr.py
+++ b/Lib/test/test_descr.py
@@ -3798,6 +3798,37 @@
         else:
             assert 0, "best_base calculation found wanting"
 
+    def test_unsubclassable_types(self):
+        with self.assertRaises(TypeError):
+            class X(type(None)):
+                pass
+        with self.assertRaises(TypeError):
+            class X(object, type(None)):
+                pass
+        with self.assertRaises(TypeError):
+            class X(type(None), object):
+                pass
+        class O(object):
+            pass
+        with self.assertRaises(TypeError):
+            class X(O, type(None)):
+                pass
+        with self.assertRaises(TypeError):
+            class X(type(None), O):
+                pass
+
+        class X(object):
+            pass
+        with self.assertRaises(TypeError):
+            X.__bases__ = type(None),
+        with self.assertRaises(TypeError):
+            X.__bases__ = object, type(None)
+        with self.assertRaises(TypeError):
+            X.__bases__ = type(None), object
+        with self.assertRaises(TypeError):
+            X.__bases__ = O, type(None)
+        with self.assertRaises(TypeError):
+            X.__bases__ = type(None), O
 
     def test_mutable_bases_with_failing_mro(self):
         # Testing mutable bases with failing mro...
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -11,6 +11,9 @@
 Core and Builtins
 -----------------
 
+- Issue #24806: Prevent builtin types that are not allowed to be subclassed from
+  being subclassed through multiple inheritance.
+
 - Issue #24848: Fixed a number of bugs in UTF-7 decoding of misformed data.
 
 - Issue #25280: Import trace messages emitted in verbose (-v) mode are no
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -1965,6 +1965,12 @@
             if (PyType_Ready(base_i) < 0)
                 return NULL;
         }
+        if (!PyType_HasFeature(base_i, Py_TPFLAGS_BASETYPE)) {
+            PyErr_Format(PyExc_TypeError,
+                         "type '%.100s' is not an acceptable base type",
+                         base_i->tp_name);
+            return NULL;
+        }
         candidate = solid_base(base_i);
         if (winner == NULL) {
             winner = candidate;
@@ -2345,12 +2351,6 @@
     if (base == NULL) {
         goto error;
     }
-    if (!PyType_HasFeature(base, Py_TPFLAGS_BASETYPE)) {
-        PyErr_Format(PyExc_TypeError,
-                     "type '%.100s' is not an acceptable base type",
-                     base->tp_name);
-        goto error;
-    }
 
     dict = PyDict_Copy(orig_dict);
     if (dict == NULL)

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Wed Oct  7 04:42:57 2015
From: python-checkins at python.org (benjamin.peterson)
Date: Wed, 07 Oct 2015 02:42:57 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_prevent_unacce?=
 =?utf-8?q?ptable_bases_from_becoming_bases_through_multiple_inheritance?=
Message-ID: <20151007024256.20763.54858@psf.io>

https://hg.python.org/cpython/rev/c46ccfac8763
changeset:   98571:c46ccfac8763
branch:      2.7
parent:      98548:69a26f0800b3
user:        Benjamin Peterson <benjamin at python.org>
date:        Tue Oct 06 19:36:54 2015 -0700
summary:
  prevent unacceptable bases from becoming bases through multiple inheritance (#24806)

files:
  Lib/test/test_descr.py |  31 ++++++++++++++++++++++++++++++
  Misc/NEWS              |   3 ++
  Objects/typeobject.c   |  13 +++++------
  3 files changed, 40 insertions(+), 7 deletions(-)


diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py
--- a/Lib/test/test_descr.py
+++ b/Lib/test/test_descr.py
@@ -4065,6 +4065,37 @@
         else:
             assert 0, "best_base calculation found wanting"
 
+    def test_unsubclassable_types(self):
+        with self.assertRaises(TypeError):
+            class X(types.NoneType):
+                pass
+        with self.assertRaises(TypeError):
+            class X(object, types.NoneType):
+                pass
+        with self.assertRaises(TypeError):
+            class X(types.NoneType, object):
+                pass
+        class O(object):
+            pass
+        with self.assertRaises(TypeError):
+            class X(O, types.NoneType):
+                pass
+        with self.assertRaises(TypeError):
+            class X(types.NoneType, O):
+                pass
+
+        class X(object):
+            pass
+        with self.assertRaises(TypeError):
+            X.__bases__ = types.NoneType,
+        with self.assertRaises(TypeError):
+            X.__bases__ = object, types.NoneType
+        with self.assertRaises(TypeError):
+            X.__bases__ = types.NoneType, object
+        with self.assertRaises(TypeError):
+            X.__bases__ = O, types.NoneType
+        with self.assertRaises(TypeError):
+            X.__bases__ = types.NoneType, O
 
     def test_mutable_bases_with_failing_mro(self):
         # Testing mutable bases with failing mro...
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,9 @@
 Core and Builtins
 -----------------
 
+- Issue #24806: Prevent builtin types that are not allowed to be subclassed from
+  being subclassed through multiple inheritance.
+
 - Issue #24848: Fixed a number of bugs in UTF-7 decoding of misformed data.
 
 - Issue #25003: os.urandom() doesn't use getentropy() on Solaris because
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -1724,6 +1724,12 @@
             if (PyType_Ready(base_i) < 0)
                 return NULL;
         }
+        if (!PyType_HasFeature(base_i, Py_TPFLAGS_BASETYPE)) {
+            PyErr_Format(PyExc_TypeError,
+                         "type '%.100s' is not an acceptable base type",
+                         base_i->tp_name);
+            return NULL;
+        }
         candidate = solid_base(base_i);
         if (winner == NULL) {
             winner = candidate;
@@ -2148,13 +2154,6 @@
         Py_DECREF(bases);
         return NULL;
     }
-    if (!PyType_HasFeature(base, Py_TPFLAGS_BASETYPE)) {
-        PyErr_Format(PyExc_TypeError,
-                     "type '%.100s' is not an acceptable base type",
-                     base->tp_name);
-        Py_DECREF(bases);
-        return NULL;
-    }
 
     /* Check for a __slots__ sequence variable in dict, and count it */
     slots = PyDict_GetItemString(dict, "__slots__");

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Wed Oct  7 04:42:56 2015
From: python-checkins at python.org (benjamin.peterson)
Date: Wed, 07 Oct 2015 02:42:56 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMi43IC0+IDIuNyk6?=
	=?utf-8?q?_merge_heads?=
Message-ID: <20151007024256.18388.80772@psf.io>

https://hg.python.org/cpython/rev/60c44a09c5fc
changeset:   98572:60c44a09c5fc
branch:      2.7
parent:      98571:c46ccfac8763
parent:      98555:7b2af8ee6dfa
user:        Benjamin Peterson <benjamin at python.org>
date:        Tue Oct 06 19:37:15 2015 -0700
summary:
  merge heads

files:
  Lib/test/test_tokenize.py |  544 ++++++++++++++-----------
  1 files changed, 294 insertions(+), 250 deletions(-)


diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -1,20 +1,42 @@
-doctests = """
-Tests for the tokenize module.
+from test import test_support
+from tokenize import (untokenize, generate_tokens, NUMBER, NAME, OP,
+                     STRING, ENDMARKER, tok_name, Untokenizer, tokenize)
+from StringIO import StringIO
+import os
+from unittest import TestCase
 
-    >>> import glob, random, sys
 
-The tests can be really simple. Given a small fragment of source
-code, print out a table with tokens. The ENDMARKER is omitted for
-brevity.
+class TokenizeTest(TestCase):
+    # Tests for the tokenize module.
 
-    >>> dump_tokens("1 + 1")
+    # The tests can be really simple. Given a small fragment of source
+    # code, print out a table with tokens. The ENDMARKER is omitted for
+    # brevity.
+
+    def check_tokenize(self, s, expected):
+        # Format the tokens in s in a table format.
+        # The ENDMARKER is omitted.
+        result = []
+        f = StringIO(s)
+        for type, token, start, end, line in generate_tokens(f.readline):
+            if type == ENDMARKER:
+                break
+            type = tok_name[type]
+            result.append("    %(type)-10.10s %(token)-13.13r %(start)s %(end)s" %
+                          locals())
+        self.assertEqual(result,
+                         expected.rstrip().splitlines())
+
+
+    def test_basic(self):
+        self.check_tokenize("1 + 1", """\
     NUMBER     '1'           (1, 0) (1, 1)
     OP         '+'           (1, 2) (1, 3)
     NUMBER     '1'           (1, 4) (1, 5)
-
-    >>> dump_tokens("if False:\\n"
-    ...             "    # NL\\n"
-    ...             "    True = False # NEWLINE\\n")
+    """)
+        self.check_tokenize("if False:\n"
+                            "    # NL\n"
+                            "    True = False # NEWLINE\n", """\
     NAME       'if'          (1, 0) (1, 2)
     NAME       'False'       (1, 3) (1, 8)
     OP         ':'           (1, 8) (1, 9)
@@ -28,122 +50,48 @@
     COMMENT    '# NEWLINE'   (3, 17) (3, 26)
     NEWLINE    '\\n'          (3, 26) (3, 27)
     DEDENT     ''            (4, 0) (4, 0)
+    """)
 
-    >>> indent_error_file = \"""
-    ... def k(x):
-    ...     x += 2
-    ...   x += 5
-    ... \"""
+        indent_error_file = """\
+def k(x):
+    x += 2
+  x += 5
+"""
+        with self.assertRaisesRegexp(IndentationError,
+                                     "unindent does not match any "
+                                     "outer indentation level"):
+            for tok in generate_tokens(StringIO(indent_error_file).readline):
+                pass
 
-    >>> for tok in generate_tokens(StringIO(indent_error_file).readline): pass
-    Traceback (most recent call last):
-        ...
-    IndentationError: unindent does not match any outer indentation level
-
-Test roundtrip for `untokenize`. `f` is an open file or a string. The source
-code in f is tokenized, converted back to source code via tokenize.untokenize(),
-and tokenized again from the latter. The test fails if the second tokenization
-doesn't match the first.
-
-    >>> def roundtrip(f):
-    ...     if isinstance(f, str): f = StringIO(f)
-    ...     token_list = list(generate_tokens(f.readline))
-    ...     f.close()
-    ...     tokens1 = [tok[:2] for tok in token_list]
-    ...     new_text = untokenize(tokens1)
-    ...     readline = iter(new_text.splitlines(1)).next
-    ...     tokens2 = [tok[:2] for tok in generate_tokens(readline)]
-    ...     return tokens1 == tokens2
-    ...
-
-There are some standard formatting practices that are easy to get right.
-
-    >>> roundtrip("if x == 1:\\n"
-    ...           "    print x\\n")
-    True
-
-    >>> roundtrip("# This is a comment\\n# This also")
-    True
-
-Some people use different formatting conventions, which makes
-untokenize a little trickier. Note that this test involves trailing
-whitespace after the colon. Note that we use hex escapes to make the
-two trailing blanks apperant in the expected output.
-
-    >>> roundtrip("if x == 1 : \\n"
-    ...           "  print x\\n")
-    True
-
-    >>> f = test_support.findfile("tokenize_tests" + os.extsep + "txt")
-    >>> roundtrip(open(f))
-    True
-
-    >>> roundtrip("if x == 1:\\n"
-    ...           "    # A comment by itself.\\n"
-    ...           "    print x # Comment here, too.\\n"
-    ...           "    # Another comment.\\n"
-    ...           "after_if = True\\n")
-    True
-
-    >>> roundtrip("if (x # The comments need to go in the right place\\n"
-    ...           "    == 1):\\n"
-    ...           "    print 'x==1'\\n")
-    True
-
-    >>> roundtrip("class Test: # A comment here\\n"
-    ...           "  # A comment with weird indent\\n"
-    ...           "  after_com = 5\\n"
-    ...           "  def x(m): return m*5 # a one liner\\n"
-    ...           "  def y(m): # A whitespace after the colon\\n"
-    ...           "     return y*4 # 3-space indent\\n")
-    True
-
-Some error-handling code
-
-    >>> roundtrip("try: import somemodule\\n"
-    ...           "except ImportError: # comment\\n"
-    ...           "    print 'Can not import' # comment2\\n"
-    ...           "else:   print 'Loaded'\\n")
-    True
-
-Balancing continuation
-
-    >>> roundtrip("a = (3,4, \\n"
-    ...           "5,6)\\n"
-    ...           "y = [3, 4,\\n"
-    ...           "5]\\n"
-    ...           "z = {'a': 5,\\n"
-    ...           "'b':15, 'c':True}\\n"
-    ...           "x = len(y) + 5 - a[\\n"
-    ...           "3] - a[2]\\n"
-    ...           "+ len(z) - z[\\n"
-    ...           "'b']\\n")
-    True
-
-Ordinary integers and binary operators
-
-    >>> dump_tokens("0xff <= 255")
+    def test_int(self):
+        # Ordinary integers and binary operators
+        self.check_tokenize("0xff <= 255", """\
     NUMBER     '0xff'        (1, 0) (1, 4)
     OP         '<='          (1, 5) (1, 7)
     NUMBER     '255'         (1, 8) (1, 11)
-    >>> dump_tokens("0b10 <= 255")
+    """)
+        self.check_tokenize("0b10 <= 255", """\
     NUMBER     '0b10'        (1, 0) (1, 4)
     OP         '<='          (1, 5) (1, 7)
     NUMBER     '255'         (1, 8) (1, 11)
-    >>> dump_tokens("0o123 <= 0123")
+    """)
+        self.check_tokenize("0o123 <= 0123", """\
     NUMBER     '0o123'       (1, 0) (1, 5)
     OP         '<='          (1, 6) (1, 8)
     NUMBER     '0123'        (1, 9) (1, 13)
-    >>> dump_tokens("01234567 > ~0x15")
+    """)
+        self.check_tokenize("01234567 > ~0x15", """\
     NUMBER     '01234567'    (1, 0) (1, 8)
     OP         '>'           (1, 9) (1, 10)
     OP         '~'           (1, 11) (1, 12)
     NUMBER     '0x15'        (1, 12) (1, 16)
-    >>> dump_tokens("2134568 != 01231515")
+    """)
+        self.check_tokenize("2134568 != 01231515", """\
     NUMBER     '2134568'     (1, 0) (1, 7)
     OP         '!='          (1, 8) (1, 10)
     NUMBER     '01231515'    (1, 11) (1, 19)
-    >>> dump_tokens("(-124561-1) & 0200000000")
+    """)
+        self.check_tokenize("(-124561-1) & 0200000000", """\
     OP         '('           (1, 0) (1, 1)
     OP         '-'           (1, 1) (1, 2)
     NUMBER     '124561'      (1, 2) (1, 8)
@@ -152,78 +100,93 @@
     OP         ')'           (1, 10) (1, 11)
     OP         '&'           (1, 12) (1, 13)
     NUMBER     '0200000000'  (1, 14) (1, 24)
-    >>> dump_tokens("0xdeadbeef != -1")
+    """)
+        self.check_tokenize("0xdeadbeef != -1", """\
     NUMBER     '0xdeadbeef'  (1, 0) (1, 10)
     OP         '!='          (1, 11) (1, 13)
     OP         '-'           (1, 14) (1, 15)
     NUMBER     '1'           (1, 15) (1, 16)
-    >>> dump_tokens("0xdeadc0de & 012345")
+    """)
+        self.check_tokenize("0xdeadc0de & 012345", """\
     NUMBER     '0xdeadc0de'  (1, 0) (1, 10)
     OP         '&'           (1, 11) (1, 12)
     NUMBER     '012345'      (1, 13) (1, 19)
-    >>> dump_tokens("0xFF & 0x15 | 1234")
+    """)
+        self.check_tokenize("0xFF & 0x15 | 1234", """\
     NUMBER     '0xFF'        (1, 0) (1, 4)
     OP         '&'           (1, 5) (1, 6)
     NUMBER     '0x15'        (1, 7) (1, 11)
     OP         '|'           (1, 12) (1, 13)
     NUMBER     '1234'        (1, 14) (1, 18)
+    """)
 
-Long integers
-
-    >>> dump_tokens("x = 0L")
+    def test_long(self):
+        # Long integers
+        self.check_tokenize("x = 0L", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '0L'          (1, 4) (1, 6)
-    >>> dump_tokens("x = 0xfffffffffff")
+    """)
+        self.check_tokenize("x = 0xfffffffffff", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '0xffffffffff (1, 4) (1, 17)
-    >>> dump_tokens("x = 123141242151251616110l")
+    """)
+        self.check_tokenize("x = 123141242151251616110l", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '123141242151 (1, 4) (1, 26)
-    >>> dump_tokens("x = -15921590215012591L")
+    """)
+        self.check_tokenize("x = -15921590215012591L", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     OP         '-'           (1, 4) (1, 5)
     NUMBER     '159215902150 (1, 5) (1, 23)
+    """)
 
-Floating point numbers
-
-    >>> dump_tokens("x = 3.14159")
+    def test_float(self):
+        # Floating point numbers
+        self.check_tokenize("x = 3.14159", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '3.14159'     (1, 4) (1, 11)
-    >>> dump_tokens("x = 314159.")
+    """)
+        self.check_tokenize("x = 314159.", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '314159.'     (1, 4) (1, 11)
-    >>> dump_tokens("x = .314159")
+    """)
+        self.check_tokenize("x = .314159", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '.314159'     (1, 4) (1, 11)
-    >>> dump_tokens("x = 3e14159")
+    """)
+        self.check_tokenize("x = 3e14159", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '3e14159'     (1, 4) (1, 11)
-    >>> dump_tokens("x = 3E123")
+    """)
+        self.check_tokenize("x = 3E123", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '3E123'       (1, 4) (1, 9)
-    >>> dump_tokens("x+y = 3e-1230")
+    """)
+        self.check_tokenize("x+y = 3e-1230", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '+'           (1, 1) (1, 2)
     NAME       'y'           (1, 2) (1, 3)
     OP         '='           (1, 4) (1, 5)
     NUMBER     '3e-1230'     (1, 6) (1, 13)
-    >>> dump_tokens("x = 3.14e159")
+    """)
+        self.check_tokenize("x = 3.14e159", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '3.14e159'    (1, 4) (1, 12)
+    """)
 
-String literals
-
-    >>> dump_tokens("x = ''; y = \\\"\\\"")
+    def test_string(self):
+        # String literals
+        self.check_tokenize("x = ''; y = \"\"", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     "''"          (1, 4) (1, 6)
@@ -231,7 +194,8 @@
     NAME       'y'           (1, 8) (1, 9)
     OP         '='           (1, 10) (1, 11)
     STRING     '""'          (1, 12) (1, 14)
-    >>> dump_tokens("x = '\\\"'; y = \\\"'\\\"")
+    """)
+        self.check_tokenize("x = '\"'; y = \"'\"", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     '\\'"\\''       (1, 4) (1, 7)
@@ -239,25 +203,29 @@
     NAME       'y'           (1, 9) (1, 10)
     OP         '='           (1, 11) (1, 12)
     STRING     '"\\'"'        (1, 13) (1, 16)
-    >>> dump_tokens("x = \\\"doesn't \\\"shrink\\\", does it\\\"")
+    """)
+        self.check_tokenize("x = \"doesn't \"shrink\", does it\"", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     '"doesn\\'t "' (1, 4) (1, 14)
     NAME       'shrink'      (1, 14) (1, 20)
     STRING     '", does it"' (1, 20) (1, 31)
-    >>> dump_tokens("x = u'abc' + U'ABC'")
+    """)
+        self.check_tokenize("x = u'abc' + U'ABC'", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     "u'abc'"      (1, 4) (1, 10)
     OP         '+'           (1, 11) (1, 12)
     STRING     "U'ABC'"      (1, 13) (1, 19)
-    >>> dump_tokens('y = u"ABC" + U"ABC"')
+    """)
+        self.check_tokenize('y = u"ABC" + U"ABC"', """\
     NAME       'y'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     'u"ABC"'      (1, 4) (1, 10)
     OP         '+'           (1, 11) (1, 12)
     STRING     'U"ABC"'      (1, 13) (1, 19)
-    >>> dump_tokens("x = ur'abc' + Ur'ABC' + uR'ABC' + UR'ABC'")
+    """)
+        self.check_tokenize("x = ur'abc' + Ur'ABC' + uR'ABC' + UR'ABC'", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     "ur'abc'"     (1, 4) (1, 11)
@@ -267,7 +235,8 @@
     STRING     "uR'ABC'"     (1, 24) (1, 31)
     OP         '+'           (1, 32) (1, 33)
     STRING     "UR'ABC'"     (1, 34) (1, 41)
-    >>> dump_tokens('y = ur"abc" + Ur"ABC" + uR"ABC" + UR"ABC"')
+    """)
+        self.check_tokenize('y = ur"abc" + Ur"ABC" + uR"ABC" + UR"ABC"', """\
     NAME       'y'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     'ur"abc"'     (1, 4) (1, 11)
@@ -278,15 +247,18 @@
     OP         '+'           (1, 32) (1, 33)
     STRING     'UR"ABC"'     (1, 34) (1, 41)
 
-    >>> dump_tokens("b'abc' + B'abc'")
+    """)
+        self.check_tokenize("b'abc' + B'abc'", """\
     STRING     "b'abc'"      (1, 0) (1, 6)
     OP         '+'           (1, 7) (1, 8)
     STRING     "B'abc'"      (1, 9) (1, 15)
-    >>> dump_tokens('b"abc" + B"abc"')
+    """)
+        self.check_tokenize('b"abc" + B"abc"', """\
     STRING     'b"abc"'      (1, 0) (1, 6)
     OP         '+'           (1, 7) (1, 8)
     STRING     'B"abc"'      (1, 9) (1, 15)
-    >>> dump_tokens("br'abc' + bR'abc' + Br'abc' + BR'abc'")
+    """)
+        self.check_tokenize("br'abc' + bR'abc' + Br'abc' + BR'abc'", """\
     STRING     "br'abc'"     (1, 0) (1, 7)
     OP         '+'           (1, 8) (1, 9)
     STRING     "bR'abc'"     (1, 10) (1, 17)
@@ -294,7 +266,8 @@
     STRING     "Br'abc'"     (1, 20) (1, 27)
     OP         '+'           (1, 28) (1, 29)
     STRING     "BR'abc'"     (1, 30) (1, 37)
-    >>> dump_tokens('br"abc" + bR"abc" + Br"abc" + BR"abc"')
+    """)
+        self.check_tokenize('br"abc" + bR"abc" + Br"abc" + BR"abc"', """\
     STRING     'br"abc"'     (1, 0) (1, 7)
     OP         '+'           (1, 8) (1, 9)
     STRING     'bR"abc"'     (1, 10) (1, 17)
@@ -302,10 +275,10 @@
     STRING     'Br"abc"'     (1, 20) (1, 27)
     OP         '+'           (1, 28) (1, 29)
     STRING     'BR"abc"'     (1, 30) (1, 37)
+    """)
 
-Operators
-
-    >>> dump_tokens("def d22(a, b, c=2, d=2, *k): pass")
+    def test_function(self):
+        self.check_tokenize("def d22(a, b, c=2, d=2, *k): pass", """\
     NAME       'def'         (1, 0) (1, 3)
     NAME       'd22'         (1, 4) (1, 7)
     OP         '('           (1, 7) (1, 8)
@@ -326,7 +299,8 @@
     OP         ')'           (1, 26) (1, 27)
     OP         ':'           (1, 27) (1, 28)
     NAME       'pass'        (1, 29) (1, 33)
-    >>> dump_tokens("def d01v_(a=1, *k, **w): pass")
+    """)
+        self.check_tokenize("def d01v_(a=1, *k, **w): pass", """\
     NAME       'def'         (1, 0) (1, 3)
     NAME       'd01v_'       (1, 4) (1, 9)
     OP         '('           (1, 9) (1, 10)
@@ -342,11 +316,12 @@
     OP         ')'           (1, 22) (1, 23)
     OP         ':'           (1, 23) (1, 24)
     NAME       'pass'        (1, 25) (1, 29)
+    """)
 
-Comparison
-
-    >>> dump_tokens("if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != " +
-    ...             "1 and 5 in 1 not in 1 is 1 or 5 is not 1: pass")
+    def test_comparison(self):
+        # Comparison
+        self.check_tokenize("if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != " +
+                            "1 and 5 in 1 not in 1 is 1 or 5 is not 1: pass", """\
     NAME       'if'          (1, 0) (1, 2)
     NUMBER     '1'           (1, 3) (1, 4)
     OP         '<'           (1, 5) (1, 6)
@@ -379,10 +354,11 @@
     NUMBER     '1'           (1, 81) (1, 82)
     OP         ':'           (1, 82) (1, 83)
     NAME       'pass'        (1, 84) (1, 88)
+    """)
 
-Shift
-
-    >>> dump_tokens("x = 1 << 1 >> 5")
+    def test_shift(self):
+        # Shift
+        self.check_tokenize("x = 1 << 1 >> 5", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '1'           (1, 4) (1, 5)
@@ -390,10 +366,11 @@
     NUMBER     '1'           (1, 9) (1, 10)
     OP         '>>'          (1, 11) (1, 13)
     NUMBER     '5'           (1, 14) (1, 15)
+    """)
 
-Additive
-
-    >>> dump_tokens("x = 1 - y + 15 - 01 + 0x124 + z + a[5]")
+    def test_additive(self):
+        # Additive
+        self.check_tokenize("x = 1 - y + 15 - 01 + 0x124 + z + a[5]", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '1'           (1, 4) (1, 5)
@@ -412,10 +389,11 @@
     OP         '['           (1, 35) (1, 36)
     NUMBER     '5'           (1, 36) (1, 37)
     OP         ']'           (1, 37) (1, 38)
+    """)
 
-Multiplicative
-
-    >>> dump_tokens("x = 1//1*1/5*12%0x12")
+    def test_multiplicative(self):
+        # Multiplicative
+        self.check_tokenize("x = 1//1*1/5*12%0x12", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '1'           (1, 4) (1, 5)
@@ -429,10 +407,11 @@
     NUMBER     '12'          (1, 13) (1, 15)
     OP         '%'           (1, 15) (1, 16)
     NUMBER     '0x12'        (1, 16) (1, 20)
+    """)
 
-Unary
-
-    >>> dump_tokens("~1 ^ 1 & 1 |1 ^ -1")
+    def test_unary(self):
+        # Unary
+        self.check_tokenize("~1 ^ 1 & 1 |1 ^ -1", """\
     OP         '~'           (1, 0) (1, 1)
     NUMBER     '1'           (1, 1) (1, 2)
     OP         '^'           (1, 3) (1, 4)
@@ -444,7 +423,8 @@
     OP         '^'           (1, 14) (1, 15)
     OP         '-'           (1, 16) (1, 17)
     NUMBER     '1'           (1, 17) (1, 18)
-    >>> dump_tokens("-1*1/1+1*1//1 - ---1**1")
+    """)
+        self.check_tokenize("-1*1/1+1*1//1 - ---1**1", """\
     OP         '-'           (1, 0) (1, 1)
     NUMBER     '1'           (1, 1) (1, 2)
     OP         '*'           (1, 2) (1, 3)
@@ -464,10 +444,12 @@
     NUMBER     '1'           (1, 19) (1, 20)
     OP         '**'          (1, 20) (1, 22)
     NUMBER     '1'           (1, 22) (1, 23)
+    """)
 
-Selector
-
-    >>> dump_tokens("import sys, time\\nx = sys.modules['time'].time()")
+    def test_selector(self):
+        # Selector
+        self.check_tokenize("import sys, time\n"
+                            "x = sys.modules['time'].time()", """\
     NAME       'import'      (1, 0) (1, 6)
     NAME       'sys'         (1, 7) (1, 10)
     OP         ','           (1, 10) (1, 11)
@@ -485,10 +467,12 @@
     NAME       'time'        (2, 24) (2, 28)
     OP         '('           (2, 28) (2, 29)
     OP         ')'           (2, 29) (2, 30)
+    """)
 
-Methods
-
-    >>> dump_tokens("@staticmethod\\ndef foo(x,y): pass")
+    def test_method(self):
+        # Methods
+        self.check_tokenize("@staticmethod\n"
+                            "def foo(x,y): pass", """\
     OP         '@'           (1, 0) (1, 1)
     NAME       'staticmethod (1, 1) (1, 13)
     NEWLINE    '\\n'          (1, 13) (1, 14)
@@ -501,41 +485,13 @@
     OP         ')'           (2, 11) (2, 12)
     OP         ':'           (2, 12) (2, 13)
     NAME       'pass'        (2, 14) (2, 18)
+    """)
 
-Backslash means line continuation, except for comments
-
-    >>> roundtrip("x=1+\\\\n"
-    ...           "1\\n"
-    ...           "# This is a comment\\\\n"
-    ...           "# This also\\n")
-    True
-    >>> roundtrip("# Comment \\\\nx = 0")
-    True
-
-Two string literals on the same line
-
-    >>> roundtrip("'' ''")
-    True
-
-Test roundtrip on random python modules.
-pass the '-ucpu' option to process the full directory.
-
-    >>>
-    >>> tempdir = os.path.dirname(f) or os.curdir
-    >>> testfiles = glob.glob(os.path.join(tempdir, "test*.py"))
-
-    >>> if not test_support.is_resource_enabled("cpu"):
-    ...     testfiles = random.sample(testfiles, 10)
-    ...
-    >>> for testfile in testfiles:
-    ...     if not roundtrip(open(testfile)):
-    ...         print "Roundtrip failed for file %s" % testfile
-    ...         break
-    ... else: True
-    True
-
-Evil tabs
-    >>> dump_tokens("def f():\\n\\tif x\\n        \\tpass")
+    def test_tabs(self):
+        # Evil tabs
+        self.check_tokenize("def f():\n"
+                            "\tif x\n"
+                            "        \tpass", """\
     NAME       'def'         (1, 0) (1, 3)
     NAME       'f'           (1, 4) (1, 5)
     OP         '('           (1, 5) (1, 6)
@@ -550,56 +506,16 @@
     NAME       'pass'        (3, 9) (3, 13)
     DEDENT     ''            (4, 0) (4, 0)
     DEDENT     ''            (4, 0) (4, 0)
+    """)
 
-Pathological whitespace (http://bugs.python.org/issue16152)
-    >>> dump_tokens("@          ")
+    def test_pathological_trailing_whitespace(self):
+        # Pathological whitespace (http://bugs.python.org/issue16152)
+        self.check_tokenize("@          ", """\
     OP         '@'           (1, 0) (1, 1)
-"""
+    """)
 
 
-from test import test_support
-from tokenize import (untokenize, generate_tokens, NUMBER, NAME, OP,
-                     STRING, ENDMARKER, tok_name, Untokenizer, tokenize)
-from StringIO import StringIO
-import os
-from unittest import TestCase
-
-def dump_tokens(s):
-    """Print out the tokens in s in a table format.
-
-    The ENDMARKER is omitted.
-    """
-    f = StringIO(s)
-    for type, token, start, end, line in generate_tokens(f.readline):
-        if type == ENDMARKER:
-            break
-        type = tok_name[type]
-        print("%(type)-10.10s %(token)-13.13r %(start)s %(end)s" % locals())
-
-# This is an example from the docs, set up as a doctest.
 def decistmt(s):
-    """Substitute Decimals for floats in a string of statements.
-
-    >>> from decimal import Decimal
-    >>> s = 'print +21.3e-5*-.1234/81.7'
-    >>> decistmt(s)
-    "print +Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')"
-
-    The format of the exponent is inherited from the platform C library.
-    Known cases are "e-007" (Windows) and "e-07" (not Windows).  Since
-    we're only showing 12 digits, and the 13th isn't close to 5, the
-    rest of the output should be platform-independent.
-
-    >>> exec(s) #doctest: +ELLIPSIS
-    -3.21716034272e-0...7
-
-    Output from calculations with Decimal should be identical across all
-    platforms.
-
-    >>> exec(decistmt(s))
-    -3.217160342717258261933904529E-7
-    """
-
     result = []
     g = generate_tokens(StringIO(s).readline)   # tokenize the string
     for toknum, tokval, _, _, _  in g:
@@ -614,6 +530,27 @@
             result.append((toknum, tokval))
     return untokenize(result)
 
+class TestMisc(TestCase):
+
+    def test_decistmt(self):
+        # Substitute Decimals for floats in a string of statements.
+        # This is an example from the docs.
+
+        from decimal import Decimal
+        s = '+21.3e-5*-.1234/81.7'
+        self.assertEqual(decistmt(s),
+                         "+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')")
+
+        # The format of the exponent is inherited from the platform C library.
+        # Known cases are "e-007" (Windows) and "e-07" (not Windows).  Since
+        # we're only showing 12 digits, and the 13th isn't close to 5, the
+        # rest of the output should be platform-independent.
+        self.assertRegexpMatches(str(eval(s)), '-3.21716034272e-0+7')
+
+        # Output from calculations with Decimal should be identical across all
+        # platforms.
+        self.assertEqual(eval(decistmt(s)), Decimal('-3.217160342717258261933904529E-7'))
+
 
 class UntokenizeTest(TestCase):
 
@@ -651,6 +588,115 @@
 
 
 class TestRoundtrip(TestCase):
+
+    def check_roundtrip(self, f):
+        """
+        Test roundtrip for `untokenize`. `f` is an open file or a string.
+        The source code in f is tokenized, converted back to source code
+        via tokenize.untokenize(), and tokenized again from the latter.
+        The test fails if the second tokenization doesn't match the first.
+        """
+        if isinstance(f, str): f = StringIO(f)
+        token_list = list(generate_tokens(f.readline))
+        f.close()
+        tokens1 = [tok[:2] for tok in token_list]
+        new_text = untokenize(tokens1)
+        readline = iter(new_text.splitlines(1)).next
+        tokens2 = [tok[:2] for tok in generate_tokens(readline)]
+        self.assertEqual(tokens2, tokens1)
+
+    def test_roundtrip(self):
+        # There are some standard formatting practices that are easy to get right.
+
+        self.check_roundtrip("if x == 1:\n"
+                             "    print(x)\n")
+
+        # There are some standard formatting practices that are easy to get right.
+
+        self.check_roundtrip("if x == 1:\n"
+                             "    print x\n")
+        self.check_roundtrip("# This is a comment\n"
+                             "# This also")
+
+        # Some people use different formatting conventions, which makes
+        # untokenize a little trickier. Note that this test involves trailing
+        # whitespace after the colon. Note that we use hex escapes to make the
+        # two trailing blanks apperant in the expected output.
+
+        self.check_roundtrip("if x == 1 : \n"
+                             "  print x\n")
+        fn = test_support.findfile("tokenize_tests" + os.extsep + "txt")
+        with open(fn) as f:
+            self.check_roundtrip(f)
+        self.check_roundtrip("if x == 1:\n"
+                             "    # A comment by itself.\n"
+                             "    print x # Comment here, too.\n"
+                             "    # Another comment.\n"
+                             "after_if = True\n")
+        self.check_roundtrip("if (x # The comments need to go in the right place\n"
+                             "    == 1):\n"
+                             "    print 'x==1'\n")
+        self.check_roundtrip("class Test: # A comment here\n"
+                             "  # A comment with weird indent\n"
+                             "  after_com = 5\n"
+                             "  def x(m): return m*5 # a one liner\n"
+                             "  def y(m): # A whitespace after the colon\n"
+                             "     return y*4 # 3-space indent\n")
+
+        # Some error-handling code
+
+        self.check_roundtrip("try: import somemodule\n"
+                             "except ImportError: # comment\n"
+                             "    print 'Can not import' # comment2\n"
+                             "else:   print 'Loaded'\n")
+
+    def test_continuation(self):
+        # Balancing continuation
+        self.check_roundtrip("a = (3,4, \n"
+                             "5,6)\n"
+                             "y = [3, 4,\n"
+                             "5]\n"
+                             "z = {'a': 5,\n"
+                             "'b':15, 'c':True}\n"
+                             "x = len(y) + 5 - a[\n"
+                             "3] - a[2]\n"
+                             "+ len(z) - z[\n"
+                             "'b']\n")
+
+    def test_backslash_continuation(self):
+        # Backslash means line continuation, except for comments
+        self.check_roundtrip("x=1+\\\n"
+                             "1\n"
+                             "# This is a comment\\\n"
+                             "# This also\n")
+        self.check_roundtrip("# Comment \\\n"
+                             "x = 0")
+
+    def test_string_concatenation(self):
+        # Two string literals on the same line
+        self.check_roundtrip("'' ''")
+
+    def test_random_files(self):
+        # Test roundtrip on random python modules.
+        # pass the '-ucpu' option to process the full directory.
+
+        import glob, random
+        fn = test_support.findfile("tokenize_tests" + os.extsep + "txt")
+        tempdir = os.path.dirname(fn) or os.curdir
+        testfiles = glob.glob(os.path.join(tempdir, "test*.py"))
+
+        if not test_support.is_resource_enabled("cpu"):
+            testfiles = random.sample(testfiles, 10)
+
+        for testfile in testfiles:
+            try:
+                with open(testfile, 'rb') as f:
+                    self.check_roundtrip(f)
+            except:
+                print "Roundtrip failed for file %s" % testfile
+                raise
+
+
     def roundtrip(self, code):
         if isinstance(code, str):
             code = code.encode('utf-8')
@@ -667,13 +713,11 @@
         self.assertEqual(codelines[1], codelines[2])
 
 
-__test__ = {"doctests" : doctests, 'decistmt': decistmt}
-
 def test_main():
-    from test import test_tokenize
-    test_support.run_doctest(test_tokenize, True)
+    test_support.run_unittest(TokenizeTest)
     test_support.run_unittest(UntokenizeTest)
     test_support.run_unittest(TestRoundtrip)
+    test_support.run_unittest(TestMisc)
 
 if __name__ == "__main__":
     test_main()

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Wed Oct  7 04:42:57 2015
From: python-checkins at python.org (benjamin.peterson)
Date: Wed, 07 Oct 2015 02:42:57 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
 =?utf-8?q?=29=3A_merge_3=2E5_=28closes_=2324806=29?=
Message-ID: <20151007024256.2685.95193@psf.io>

https://hg.python.org/cpython/rev/4b2a2688d2ad
changeset:   98575:4b2a2688d2ad
parent:      98570:db782c81bba9
parent:      98574:e02e4afcce6a
user:        Benjamin Peterson <benjamin at python.org>
date:        Tue Oct 06 19:42:46 2015 -0700
summary:
  merge 3.5 (closes #24806)

files:
  Lib/test/test_descr.py |  31 ++++++++++++++++++++++++++++++
  Misc/NEWS              |   3 ++
  Objects/typeobject.c   |  12 +++++-----
  3 files changed, 40 insertions(+), 6 deletions(-)


diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py
--- a/Lib/test/test_descr.py
+++ b/Lib/test/test_descr.py
@@ -3798,6 +3798,37 @@
         else:
             assert 0, "best_base calculation found wanting"
 
+    def test_unsubclassable_types(self):
+        with self.assertRaises(TypeError):
+            class X(type(None)):
+                pass
+        with self.assertRaises(TypeError):
+            class X(object, type(None)):
+                pass
+        with self.assertRaises(TypeError):
+            class X(type(None), object):
+                pass
+        class O(object):
+            pass
+        with self.assertRaises(TypeError):
+            class X(O, type(None)):
+                pass
+        with self.assertRaises(TypeError):
+            class X(type(None), O):
+                pass
+
+        class X(object):
+            pass
+        with self.assertRaises(TypeError):
+            X.__bases__ = type(None),
+        with self.assertRaises(TypeError):
+            X.__bases__ = object, type(None)
+        with self.assertRaises(TypeError):
+            X.__bases__ = type(None), object
+        with self.assertRaises(TypeError):
+            X.__bases__ = O, type(None)
+        with self.assertRaises(TypeError):
+            X.__bases__ = type(None), O
 
     def test_mutable_bases_with_failing_mro(self):
         # Testing mutable bases with failing mro...
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,9 @@
 Core and Builtins
 -----------------
 
+- Issue #24806: Prevent builtin types that are not allowed to be subclassed from
+  being subclassed through multiple inheritance.
+
 * Issue #25301: The UTF-8 decoder is now up to 15 times as fast for error
   handlers: ``ignore``, ``replace`` and ``surrogateescape``.
 
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -1973,6 +1973,12 @@
             if (PyType_Ready(base_i) < 0)
                 return NULL;
         }
+        if (!PyType_HasFeature(base_i, Py_TPFLAGS_BASETYPE)) {
+            PyErr_Format(PyExc_TypeError,
+                         "type '%.100s' is not an acceptable base type",
+                         base_i->tp_name);
+            return NULL;
+        }
         candidate = solid_base(base_i);
         if (winner == NULL) {
             winner = candidate;
@@ -2353,12 +2359,6 @@
     if (base == NULL) {
         goto error;
     }
-    if (!PyType_HasFeature(base, Py_TPFLAGS_BASETYPE)) {
-        PyErr_Format(PyExc_TypeError,
-                     "type '%.100s' is not an acceptable base type",
-                     base->tp_name);
-        goto error;
-    }
 
     dict = PyDict_Copy(orig_dict);
     if (dict == NULL)

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Wed Oct  7 05:08:17 2015
From: python-checkins at python.org (raymond.hettinger)
Date: Wed, 07 Oct 2015 03:08:17 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E5=29=3A_Backport_early?=
 =?utf-8?q?-out_91259f061cfb_to_reduce_the_cost_of_bb1a2944bcb6?=
Message-ID: <20151007030817.97720.65964@psf.io>

https://hg.python.org/cpython/rev/6fb0f26ed858
changeset:   98576:6fb0f26ed858
branch:      3.5
parent:      98574:e02e4afcce6a
user:        Raymond Hettinger <python at rcn.com>
date:        Tue Oct 06 23:06:17 2015 -0400
summary:
  Backport early-out 91259f061cfb to reduce the cost of bb1a2944bcb6

files:
  Modules/_collectionsmodule.c |  6 +++++-
  1 files changed, 5 insertions(+), 1 deletions(-)


diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c
--- a/Modules/_collectionsmodule.c
+++ b/Modules/_collectionsmodule.c
@@ -1045,6 +1045,9 @@
     Py_ssize_t n;
     PyObject *item;
 
+    if (Py_SIZE(deque) == 0)
+        return;
+
     /* During the process of clearing a deque, decrefs can cause the
        deque to mutate.  To avoid fatal confusion, we have to make the
        deque empty before clearing the blocks and never refer to
@@ -1423,7 +1426,8 @@
         }
     }
     deque->maxlen = maxlen;
-    deque_clear(deque);
+    if (Py_SIZE(deque) > 0)
+        deque_clear(deque);
     if (iterable != NULL) {
         PyObject *rv = deque_extend(deque, iterable);
         if (rv == NULL)

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Wed Oct  7 05:08:18 2015
From: python-checkins at python.org (raymond.hettinger)
Date: Wed, 07 Oct 2015 03:08:18 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
	=?utf-8?q?=29=3A_merge?=
Message-ID: <20151007030817.55482.17044@psf.io>

https://hg.python.org/cpython/rev/bf803e582cb4
changeset:   98577:bf803e582cb4
parent:      98575:4b2a2688d2ad
parent:      98576:6fb0f26ed858
user:        Raymond Hettinger <python at rcn.com>
date:        Tue Oct 06 23:08:11 2015 -0400
summary:
  merge

files:



-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Wed Oct  7 05:12:07 2015
From: python-checkins at python.org (raymond.hettinger)
Date: Wed, 07 Oct 2015 03:12:07 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_Backport_early?=
 =?utf-8?q?-out_91259f061cfb_to_reduce_the_cost_of_bb1a2944bcb6?=
Message-ID: <20151007031207.18390.20414@psf.io>

https://hg.python.org/cpython/rev/37aee118e1a3
changeset:   98578:37aee118e1a3
branch:      2.7
parent:      98572:60c44a09c5fc
user:        Raymond Hettinger <python at rcn.com>
date:        Tue Oct 06 23:12:02 2015 -0400
summary:
  Backport early-out 91259f061cfb to reduce the cost of bb1a2944bcb6

files:
  Modules/_collectionsmodule.c |  6 +++++-
  1 files changed, 5 insertions(+), 1 deletions(-)


diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c
--- a/Modules/_collectionsmodule.c
+++ b/Modules/_collectionsmodule.c
@@ -651,6 +651,9 @@
     Py_ssize_t n;
     PyObject *item;
 
+    if (Py_SIZE(deque) == 0)
+        return;
+
     /* During the process of clearing a deque, decrefs can cause the
        deque to mutate.  To avoid fatal confusion, we have to make the
        deque empty before clearing the blocks and never refer to
@@ -1083,7 +1086,8 @@
         }
     }
     deque->maxlen = maxlen;
-    deque_clear(deque);
+    if (Py_SIZE(deque) > 0)
+        deque_clear(deque);
     if (iterable != NULL) {
         PyObject *rv = deque_extend(deque, iterable);
         if (rv == NULL)

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Wed Oct  7 05:37:07 2015
From: python-checkins at python.org (benjamin.peterson)
Date: Wed, 07 Oct 2015 03:37:07 +0000
Subject: [Python-checkins] =?utf-8?q?devguide=3A_no_DSA_keys_allowed?=
Message-ID: <20151007033707.18390.50804@psf.io>

https://hg.python.org/devguide/rev/a55c0ddbd6cb
changeset:   766:a55c0ddbd6cb
user:        Benjamin Peterson <benjamin at python.org>
date:        Tue Oct 06 20:37:03 2015 -0700
summary:
  no DSA keys allowed

files:
  faq.rst |  4 ++--
  1 files changed, 2 insertions(+), 2 deletions(-)


diff --git a/faq.rst b/faq.rst
--- a/faq.rst
+++ b/faq.rst
@@ -875,8 +875,8 @@
 How do I generate an SSH-2 public key?
 --------------------------------------
 
-All generated SSH keys should be sent to hgaccounts at python.org for
-adding to the list of keys.
+All generated SSH keys should be sent to hgaccounts at python.org for adding to the
+list of keys. DSA keys are unacceptable.
 
 UNIX
 ''''

-- 
Repository URL: https://hg.python.org/devguide

From python-checkins at python.org  Wed Oct  7 05:37:07 2015
From: python-checkins at python.org (benjamin.peterson)
Date: Wed, 07 Oct 2015 03:37:07 +0000
Subject: [Python-checkins] =?utf-8?q?devguide=3A_tell_people_to_generate_r?=
	=?utf-8?q?sa_keys_with_puttygen?=
Message-ID: <20151007033707.3291.67205@psf.io>

https://hg.python.org/devguide/rev/847ccac0e0eb
changeset:   765:847ccac0e0eb
user:        Benjamin Peterson <benjamin at python.org>
date:        Tue Oct 06 20:36:36 2015 -0700
summary:
  tell people to generate rsa keys with puttygen

files:
  faq.rst |  8 ++++----
  1 files changed, 4 insertions(+), 4 deletions(-)


diff --git a/faq.rst b/faq.rst
--- a/faq.rst
+++ b/faq.rst
@@ -891,10 +891,10 @@
 Windows
 '''''''
 
-Use PuTTYgen_ to generate your public key.  Choose the "SSH2 DSA" radio button,
-have it create an OpenSSH formatted key, choose a password, and save the private
-key to a file.  Copy the section with the public key (using Alt-P) to a file;
-that file now has your public key.
+Use PuTTYgen_ to generate your public key.  Choose the "SSH-2 RSA" radio button,
+set 4096 as the key size, choose a password, and save the private key to a file.
+Copy the section with the public key (using Alt-P) to a file; that file now has
+your public key.
 
 .. _PuTTYgen: http://www.chiark.greenend.org.uk/~sgtatham/putty/download.html
 

-- 
Repository URL: https://hg.python.org/devguide

From python-checkins at python.org  Wed Oct  7 06:17:07 2015
From: python-checkins at python.org (benjamin.peterson)
Date: Wed, 07 Oct 2015 04:17:07 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_make_configure_executable?=
Message-ID: <20151007041706.3275.19090@psf.io>

https://hg.python.org/cpython/rev/9322f381ddb7
changeset:   98579:9322f381ddb7
parent:      98577:bf803e582cb4
user:        Benjamin Peterson <benjamin at python.org>
date:        Tue Oct 06 21:17:02 2015 -0700
summary:
  make configure executable

files:
  configure |  0 
  1 files changed, 0 insertions(+), 0 deletions(-)


diff --git a/configure b/configure
old mode 100644
new mode 100755

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Wed Oct  7 06:53:40 2015
From: python-checkins at python.org (berker.peksag)
Date: Wed, 07 Oct 2015 04:53:40 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Add_a_versionadded_directi?=
 =?utf-8?q?ve_for_reopenIfNeeded=28=29?=
Message-ID: <20151007045339.20779.43794@psf.io>

https://hg.python.org/cpython/rev/b6bfc2dad87f
changeset:   98580:b6bfc2dad87f
user:        Berker Peksag <berker.peksag at gmail.com>
date:        Wed Oct 07 07:54:23 2015 +0300
summary:
  Add a versionadded directive for reopenIfNeeded()

files:
  Doc/library/logging.handlers.rst |  2 ++
  1 files changed, 2 insertions(+), 0 deletions(-)


diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst
--- a/Doc/library/logging.handlers.rst
+++ b/Doc/library/logging.handlers.rst
@@ -168,6 +168,8 @@
       flushed and closed and the file opened again, typically as a precursor to
       outputting the record to the file.
 
+      .. versionadded:: 3.6
+
 
    .. method:: emit(record)
 

-- 
Repository URL: https://hg.python.org/cpython

From solipsis at pitrou.net  Wed Oct  7 10:44:06 2015
From: solipsis at pitrou.net (solipsis at pitrou.net)
Date: Wed, 07 Oct 2015 08:44:06 +0000
Subject: [Python-checkins] Daily reference leaks (b6bfc2dad87f): sum=17880
Message-ID: <20151007084404.97718.90590@psf.io>

results for b6bfc2dad87f on branch "default"
--------------------------------------------

test_asyncio leaked [3, 0, 0] memory blocks, sum=3
test_capi leaked [1598, 1598, 1598] references, sum=4794
test_capi leaked [387, 389, 389] memory blocks, sum=1165
test_functools leaked [0, 2, 2] memory blocks, sum=4
test_threading leaked [3196, 3196, 3196] references, sum=9588
test_threading leaked [774, 776, 776] memory blocks, sum=2326


Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogyFh9ty', '--timeout', '7200']

From python-checkins at python.org  Wed Oct  7 12:09:29 2015
From: python-checkins at python.org (martin.panter)
Date: Wed, 07 Oct 2015 10:09:29 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzI1Mjg2?=
 =?utf-8?q?=3A_Dictionary_views_are_not_sequences?=
Message-ID: <20151007100929.7260.15300@psf.io>

https://hg.python.org/cpython/rev/92429e01f444
changeset:   98581:92429e01f444
branch:      3.4
parent:      98573:e670b37e7b14
user:        Martin Panter <vadmium+py at gmail.com>
date:        Wed Oct 07 09:56:46 2015 +0000
summary:
  Issue #25286: Dictionary views are not sequences

Also change glossary heading from view

files:
  Doc/glossary.rst                |  15 ++++++++-------
  Doc/library/collections.abc.rst |   2 +-
  2 files changed, 9 insertions(+), 8 deletions(-)


diff --git a/Doc/glossary.rst b/Doc/glossary.rst
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -183,6 +183,14 @@
       keys can be any object with :meth:`__hash__` and :meth:`__eq__` methods.
       Called a hash in Perl.
 
+   dictionary view
+      The objects returned from :meth:`dict.keys`, :meth:`dict.values`, and
+      :meth:`dict.items` are called dictionary views. They provide a dynamic
+      view on the dictionary?s entries, which means that when the dictionary
+      changes, the view reflects these changes. To force the
+      dictionary view to become a full list use ``list(dictview)``.  See
+      :ref:`dict-views`.
+
    docstring
       A string literal which appears as the first expression in a class,
       function or module.  While ignored when the suite is executed, it is
@@ -853,13 +861,6 @@
       ``'\r'``.  See :pep:`278` and :pep:`3116`, as well as
       :func:`bytes.splitlines` for an additional use.
 
-   view
-      The objects returned from :meth:`dict.keys`, :meth:`dict.values`, and
-      :meth:`dict.items` are called dictionary views.  They are lazy sequences
-      that will see changes in the underlying dictionary.  To force the
-      dictionary view to become a full list use ``list(dictview)``.  See
-      :ref:`dict-views`.
-
    virtual environment
       A cooperatively isolated runtime environment that allows Python users
       and applications to install and upgrade Python distribution packages
diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst
--- a/Doc/library/collections.abc.rst
+++ b/Doc/library/collections.abc.rst
@@ -122,7 +122,7 @@
            KeysView
            ValuesView
 
-   ABCs for mapping, items, keys, and values :term:`views <view>`.
+   ABCs for mapping, items, keys, and values :term:`views <dictionary view>`.
 
 
 These ABCs allow us to ask classes or instances if they provide

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Wed Oct  7 12:09:29 2015
From: python-checkins at python.org (martin.panter)
Date: Wed, 07 Oct 2015 10:09:29 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
 =?utf-8?q?_Issue_=2325286=3A_Merge_dictionary_view_glossary_from_3=2E4_in?=
 =?utf-8?q?to_3=2E5?=
Message-ID: <20151007100929.3293.38926@psf.io>

https://hg.python.org/cpython/rev/c29f1114ef65
changeset:   98582:c29f1114ef65
branch:      3.5
parent:      98576:6fb0f26ed858
parent:      98581:92429e01f444
user:        Martin Panter <vadmium+py at gmail.com>
date:        Wed Oct 07 10:01:04 2015 +0000
summary:
  Issue #25286: Merge dictionary view glossary from 3.4 into 3.5

files:
  Doc/glossary.rst                |  15 ++++++++-------
  Doc/library/collections.abc.rst |   2 +-
  2 files changed, 9 insertions(+), 8 deletions(-)


diff --git a/Doc/glossary.rst b/Doc/glossary.rst
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -241,6 +241,14 @@
       keys can be any object with :meth:`__hash__` and :meth:`__eq__` methods.
       Called a hash in Perl.
 
+   dictionary view
+      The objects returned from :meth:`dict.keys`, :meth:`dict.values`, and
+      :meth:`dict.items` are called dictionary views. They provide a dynamic
+      view on the dictionary?s entries, which means that when the dictionary
+      changes, the view reflects these changes. To force the
+      dictionary view to become a full list use ``list(dictview)``.  See
+      :ref:`dict-views`.
+
    docstring
       A string literal which appears as the first expression in a class,
       function or module.  While ignored when the suite is executed, it is
@@ -934,13 +942,6 @@
       ``'\r'``.  See :pep:`278` and :pep:`3116`, as well as
       :func:`bytes.splitlines` for an additional use.
 
-   view
-      The objects returned from :meth:`dict.keys`, :meth:`dict.values`, and
-      :meth:`dict.items` are called dictionary views.  They are lazy sequences
-      that will see changes in the underlying dictionary.  To force the
-      dictionary view to become a full list use ``list(dictview)``.  See
-      :ref:`dict-views`.
-
    virtual environment
       A cooperatively isolated runtime environment that allows Python users
       and applications to install and upgrade Python distribution packages
diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst
--- a/Doc/library/collections.abc.rst
+++ b/Doc/library/collections.abc.rst
@@ -150,7 +150,7 @@
            KeysView
            ValuesView
 
-   ABCs for mapping, items, keys, and values :term:`views <view>`.
+   ABCs for mapping, items, keys, and values :term:`views <dictionary view>`.
 
 .. class:: Awaitable
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Wed Oct  7 12:09:29 2015
From: python-checkins at python.org (martin.panter)
Date: Wed, 07 Oct 2015 10:09:29 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy41KTogSXNzdWUgIzI1Mjg2?=
 =?utf-8?q?=3A_Update_dictionary_view_link=3B_patch_by_Akira_Li?=
Message-ID: <20151007100929.2687.97090@psf.io>

https://hg.python.org/cpython/rev/d43c33f032a2
changeset:   98583:d43c33f032a2
branch:      3.5
user:        Martin Panter <vadmium+py at gmail.com>
date:        Wed Oct 07 10:03:20 2015 +0000
summary:
  Issue #25286: Update dictionary view link; patch by Akira Li

files:
  Doc/library/collections.rst |  4 ++--
  1 files changed, 2 insertions(+), 2 deletions(-)


diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst
--- a/Doc/library/collections.rst
+++ b/Doc/library/collections.rst
@@ -1018,8 +1018,8 @@
 semantics pass-in keyword arguments using a regular unordered dictionary.
 
 .. versionchanged:: 3.5
-   The items, keys, and values :term:`views <view>` of :class:`OrderedDict` now
-   support reverse iteration using :func:`reversed`.
+   The items, keys, and values :term:`views <dictionary view>`
+   of :class:`OrderedDict` now support reverse iteration using :func:`reversed`.
 
 :class:`OrderedDict` Examples and Recipes
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Wed Oct  7 12:09:29 2015
From: python-checkins at python.org (martin.panter)
Date: Wed, 07 Oct 2015 10:09:29 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2325286=3A_Merge_dictionary_view_glossary_from_3?=
 =?utf-8?q?=2E5?=
Message-ID: <20151007100929.70972.18589@psf.io>

https://hg.python.org/cpython/rev/7067420c3e72
changeset:   98584:7067420c3e72
parent:      98580:b6bfc2dad87f
parent:      98583:d43c33f032a2
user:        Martin Panter <vadmium+py at gmail.com>
date:        Wed Oct 07 10:04:49 2015 +0000
summary:
  Issue #25286: Merge dictionary view glossary from 3.5

files:
  Doc/glossary.rst                |  15 ++++++++-------
  Doc/library/collections.abc.rst |   2 +-
  Doc/library/collections.rst     |   4 ++--
  3 files changed, 11 insertions(+), 10 deletions(-)


diff --git a/Doc/glossary.rst b/Doc/glossary.rst
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -241,6 +241,14 @@
       keys can be any object with :meth:`__hash__` and :meth:`__eq__` methods.
       Called a hash in Perl.
 
+   dictionary view
+      The objects returned from :meth:`dict.keys`, :meth:`dict.values`, and
+      :meth:`dict.items` are called dictionary views. They provide a dynamic
+      view on the dictionary?s entries, which means that when the dictionary
+      changes, the view reflects these changes. To force the
+      dictionary view to become a full list use ``list(dictview)``.  See
+      :ref:`dict-views`.
+
    docstring
       A string literal which appears as the first expression in a class,
       function or module.  While ignored when the suite is executed, it is
@@ -934,13 +942,6 @@
       ``'\r'``.  See :pep:`278` and :pep:`3116`, as well as
       :func:`bytes.splitlines` for an additional use.
 
-   view
-      The objects returned from :meth:`dict.keys`, :meth:`dict.values`, and
-      :meth:`dict.items` are called dictionary views.  They are lazy sequences
-      that will see changes in the underlying dictionary.  To force the
-      dictionary view to become a full list use ``list(dictview)``.  See
-      :ref:`dict-views`.
-
    virtual environment
       A cooperatively isolated runtime environment that allows Python users
       and applications to install and upgrade Python distribution packages
diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst
--- a/Doc/library/collections.abc.rst
+++ b/Doc/library/collections.abc.rst
@@ -150,7 +150,7 @@
            KeysView
            ValuesView
 
-   ABCs for mapping, items, keys, and values :term:`views <view>`.
+   ABCs for mapping, items, keys, and values :term:`views <dictionary view>`.
 
 .. class:: Awaitable
 
diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst
--- a/Doc/library/collections.rst
+++ b/Doc/library/collections.rst
@@ -1021,8 +1021,8 @@
 semantics pass-in keyword arguments using a regular unordered dictionary.
 
 .. versionchanged:: 3.5
-   The items, keys, and values :term:`views <view>` of :class:`OrderedDict` now
-   support reverse iteration using :func:`reversed`.
+   The items, keys, and values :term:`views <dictionary view>`
+   of :class:`OrderedDict` now support reverse iteration using :func:`reversed`.
 
 :class:`OrderedDict` Examples and Recipes
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Wed Oct  7 12:23:58 2015
From: python-checkins at python.org (martin.panter)
Date: Wed, 07 Oct 2015 10:23:58 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzI1Mjg2?=
 =?utf-8?q?=3A_Dictionary_views_are_not_sequences?=
Message-ID: <20151007102357.128852.76937@psf.io>

https://hg.python.org/cpython/rev/41e1f2500047
changeset:   98585:41e1f2500047
branch:      2.7
parent:      98578:37aee118e1a3
user:        Martin Panter <vadmium+py at gmail.com>
date:        Wed Oct 07 10:19:39 2015 +0000
summary:
  Issue #25286: Dictionary views are not sequences

Also change glossary heading from "view" to "dictionary view". Patch by Akira
Li.

files:
  Doc/glossary.rst            |  15 ++++++++-------
  Doc/library/collections.rst |   2 +-
  2 files changed, 9 insertions(+), 8 deletions(-)


diff --git a/Doc/glossary.rst b/Doc/glossary.rst
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -182,6 +182,14 @@
       keys can be any object with :meth:`__hash__`  and :meth:`__eq__` methods.
       Called a hash in Perl.
 
+   dictionary view
+      The objects returned from :meth:`dict.viewkeys`, :meth:`dict.viewvalues`,
+      and :meth:`dict.viewitems` are called dictionary views. They provide a dynamic
+      view on the dictionary?s entries, which means that when the dictionary
+      changes, the view reflects these changes. To force
+      dictionary view to become a full list use ``list(dictview)``.  See
+      :ref:`dict-views`.
+
    docstring
       A string literal which appears as the first expression in a class,
       function or module.  While ignored when the suite is executed, it is
@@ -713,13 +721,6 @@
       ``'\r'``.  See :pep:`278` and :pep:`3116`, as well as
       :func:`str.splitlines` for an additional use.
 
-   view
-      The objects returned from :meth:`dict.viewkeys`, :meth:`dict.viewvalues`,
-      and :meth:`dict.viewitems` are called dictionary views.  They are lazy
-      sequences that will see changes in the underlying dictionary.  To force
-      the dictionary view to become a full list use ``list(dictview)``.  See
-      :ref:`dict-views`.
-
    virtual environment
       A cooperatively isolated runtime environment that allows Python users
       and applications to install and upgrade Python distribution packages
diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst
--- a/Doc/library/collections.rst
+++ b/Doc/library/collections.rst
@@ -978,7 +978,7 @@
            KeysView
            ValuesView
 
-   ABCs for mapping, items, keys, and values :term:`views <view>`.
+   ABCs for mapping, items, keys, and values :term:`views <dictionary view>`.
 
 
 These ABCs allow us to ask classes or instances if they provide

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Wed Oct  7 12:40:52 2015
From: python-checkins at python.org (martin.panter)
Date: Wed, 07 Oct 2015 10:40:52 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzI1Mjg2?=
 =?utf-8?q?=3A_Accidentally_dropped_=22the=22?=
Message-ID: <20151007104052.7240.71260@psf.io>

https://hg.python.org/cpython/rev/04815b55227f
changeset:   98586:04815b55227f
branch:      2.7
user:        Martin Panter <vadmium+py at gmail.com>
date:        Wed Oct 07 10:39:13 2015 +0000
summary:
  Issue #25286: Accidentally dropped "the"

files:
  Doc/glossary.rst |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Doc/glossary.rst b/Doc/glossary.rst
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -186,7 +186,7 @@
       The objects returned from :meth:`dict.viewkeys`, :meth:`dict.viewvalues`,
       and :meth:`dict.viewitems` are called dictionary views. They provide a dynamic
       view on the dictionary?s entries, which means that when the dictionary
-      changes, the view reflects these changes. To force
+      changes, the view reflects these changes. To force the
       dictionary view to become a full list use ``list(dictview)``.  See
       :ref:`dict-views`.
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Wed Oct  7 13:28:45 2015
From: python-checkins at python.org (martin.panter)
Date: Wed, 07 Oct 2015 11:28:45 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
 =?utf-8?q?_Merge_typo_fixes_from_3=2E4_into_3=2E5?=
Message-ID: <20151007112845.7262.85052@psf.io>

https://hg.python.org/cpython/rev/ab7601859c99
changeset:   98588:ab7601859c99
branch:      3.5
parent:      98583:d43c33f032a2
parent:      98587:a9cbf3becfb7
user:        Martin Panter <vadmium+py at gmail.com>
date:        Wed Oct 07 11:01:47 2015 +0000
summary:
  Merge typo fixes from 3.4 into 3.5

files:
  Doc/library/collections.rst |  2 +-
  Doc/library/test.rst        |  2 +-
  Lib/http/server.py          |  2 +-
  Misc/HISTORY                |  2 +-
  Misc/NEWS                   |  2 +-
  Modules/_io/stringio.c      |  2 +-
  Objects/object.c            |  2 +-
  Objects/typeobject.c        |  2 +-
  Python/ceval.c              |  2 +-
  9 files changed, 9 insertions(+), 9 deletions(-)


diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst
--- a/Doc/library/collections.rst
+++ b/Doc/library/collections.rst
@@ -1015,7 +1015,7 @@
 
 The :class:`OrderedDict` constructor and :meth:`update` method both accept
 keyword arguments, but their order is lost because Python's function call
-semantics pass-in keyword arguments using a regular unordered dictionary.
+semantics pass in keyword arguments using a regular unordered dictionary.
 
 .. versionchanged:: 3.5
    The items, keys, and values :term:`views <dictionary view>`
diff --git a/Doc/library/test.rst b/Doc/library/test.rst
--- a/Doc/library/test.rst
+++ b/Doc/library/test.rst
@@ -160,7 +160,7 @@
 The :mod:`test` package can be run as a script to drive Python's regression
 test suite, thanks to the :option:`-m` option: :program:`python -m test`. Under
 the hood, it uses :mod:`test.regrtest`; the call :program:`python -m
-test.regrtest` used in previous Python versions still works).  Running the
+test.regrtest` used in previous Python versions still works.  Running the
 script by itself automatically starts running all regression tests in the
 :mod:`test` package. It does this by finding all modules in the package whose
 name starts with ``test_``, importing them, and executing the function
diff --git a/Lib/http/server.py b/Lib/http/server.py
--- a/Lib/http/server.py
+++ b/Lib/http/server.py
@@ -831,7 +831,7 @@
 def _url_collapse_path(path):
     """
     Given a URL path, remove extra '/'s and '.' path elements and collapse
-    any '..' references and returns a colllapsed path.
+    any '..' references and returns a collapsed path.
 
     Implements something akin to RFC-2396 5.2 step 6 to parse relative paths.
     The utility of this function is limited to is_cgi method and helps
diff --git a/Misc/HISTORY b/Misc/HISTORY
--- a/Misc/HISTORY
+++ b/Misc/HISTORY
@@ -6751,7 +6751,7 @@
 
 - Issue #7895: platform.mac_ver() no longer crashes after calling os.fork().
 
-- Issue #9323: Fixed a bug in trace.py that resulted in loosing the name of the
+- Issue #9323: Fixed a bug in trace.py that resulted in losing the name of the
   script being traced.  Patch by Eli Bendersky.
 
 - Issue #9282: Fixed --listfuncs option of trace.py.  Thanks Eli Bendersky for
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -1717,7 +1717,7 @@
   engine friendly) error messages when "exec" and "print" are used as
   statements.
 
-- Issue #21642: If the conditional if-else expression, allow an integer written
+- Issue #21642: In the conditional if-else expression, allow an integer written
   with no space between itself and the ``else`` keyword (e.g. ``True if 42else
   False``) to be valid syntax.
 
diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c
--- a/Modules/_io/stringio.c
+++ b/Modules/_io/stringio.c
@@ -750,7 +750,7 @@
     /* If newline == "", we don't translate anything.
        If newline == "\n" or newline == None, we translate to "\n", which is
        a no-op.
-       (for newline == None, TextIOWrapper translates to os.sepline, but it
+       (for newline == None, TextIOWrapper translates to os.linesep, but it
        is pointless for StringIO)
     */
     if (newline != NULL && newline[0] == '\r') {
diff --git a/Objects/object.c b/Objects/object.c
--- a/Objects/object.c
+++ b/Objects/object.c
@@ -475,7 +475,7 @@
 #ifdef Py_DEBUG
     /* PyObject_Repr() must not be called with an exception set,
        because it may clear it (directly or indirectly) and so the
-       caller looses its exception */
+       caller loses its exception */
     assert(!PyErr_Occurred());
 #endif
 
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -901,7 +901,7 @@
 #ifdef Py_DEBUG
     /* type_call() must not be called with an exception set,
        because it may clear it (directly or indirectly) and so the
-       caller looses its exception */
+       caller loses its exception */
     assert(!PyErr_Occurred());
 #endif
 
diff --git a/Python/ceval.c b/Python/ceval.c
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -1214,7 +1214,7 @@
 #ifdef Py_DEBUG
     /* PyEval_EvalFrameEx() must not be called with an exception set,
        because it may clear it (directly or indirectly) and so the
-       caller looses its exception */
+       caller loses its exception */
     assert(!PyErr_Occurred());
 #endif
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Wed Oct  7 13:28:45 2015
From: python-checkins at python.org (martin.panter)
Date: Wed, 07 Oct 2015 11:28:45 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
 =?utf-8?q?=29=3A_Merge_typo_fixes_from_3=2E5?=
Message-ID: <20151007112845.20783.83659@psf.io>

https://hg.python.org/cpython/rev/2333efe17938
changeset:   98590:2333efe17938
parent:      98584:7067420c3e72
parent:      98589:e75e8aeb8ac7
user:        Martin Panter <vadmium+py at gmail.com>
date:        Wed Oct 07 11:13:55 2015 +0000
summary:
  Merge typo fixes from 3.5

files:
  Doc/library/collections.rst |  2 +-
  Doc/library/test.rst        |  2 +-
  Lib/http/server.py          |  2 +-
  Misc/HISTORY                |  2 +-
  Misc/NEWS                   |  6 +++---
  Modules/_io/stringio.c      |  2 +-
  Objects/methodobject.c      |  2 +-
  Objects/object.c            |  2 +-
  Objects/typeobject.c        |  2 +-
  Python/ceval.c              |  2 +-
  10 files changed, 12 insertions(+), 12 deletions(-)


diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst
--- a/Doc/library/collections.rst
+++ b/Doc/library/collections.rst
@@ -1018,7 +1018,7 @@
 
 The :class:`OrderedDict` constructor and :meth:`update` method both accept
 keyword arguments, but their order is lost because Python's function call
-semantics pass-in keyword arguments using a regular unordered dictionary.
+semantics pass in keyword arguments using a regular unordered dictionary.
 
 .. versionchanged:: 3.5
    The items, keys, and values :term:`views <dictionary view>`
diff --git a/Doc/library/test.rst b/Doc/library/test.rst
--- a/Doc/library/test.rst
+++ b/Doc/library/test.rst
@@ -160,7 +160,7 @@
 The :mod:`test` package can be run as a script to drive Python's regression
 test suite, thanks to the :option:`-m` option: :program:`python -m test`. Under
 the hood, it uses :mod:`test.regrtest`; the call :program:`python -m
-test.regrtest` used in previous Python versions still works).  Running the
+test.regrtest` used in previous Python versions still works.  Running the
 script by itself automatically starts running all regression tests in the
 :mod:`test` package. It does this by finding all modules in the package whose
 name starts with ``test_``, importing them, and executing the function
diff --git a/Lib/http/server.py b/Lib/http/server.py
--- a/Lib/http/server.py
+++ b/Lib/http/server.py
@@ -831,7 +831,7 @@
 def _url_collapse_path(path):
     """
     Given a URL path, remove extra '/'s and '.' path elements and collapse
-    any '..' references and returns a colllapsed path.
+    any '..' references and returns a collapsed path.
 
     Implements something akin to RFC-2396 5.2 step 6 to parse relative paths.
     The utility of this function is limited to is_cgi method and helps
diff --git a/Misc/HISTORY b/Misc/HISTORY
--- a/Misc/HISTORY
+++ b/Misc/HISTORY
@@ -6751,7 +6751,7 @@
 
 - Issue #7895: platform.mac_ver() no longer crashes after calling os.fork().
 
-- Issue #9323: Fixed a bug in trace.py that resulted in loosing the name of the
+- Issue #9323: Fixed a bug in trace.py that resulted in losing the name of the
   script being traced.  Patch by Eli Bendersky.
 
 - Issue #9282: Fixed --listfuncs option of trace.py.  Thanks Eli Bendersky for
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -421,7 +421,7 @@
 -----
 
 - Issue #24915: Add LLVM support for PGO builds and use the test suite to
-  generate the profile data. Initiial patch by Alecsandru Patrascu of Intel.
+  generate the profile data. Initial patch by Alecsandru Patrascu of Intel.
 
 - Issue #24910: Windows MSIs now have unique display names.
 
@@ -1830,7 +1830,7 @@
   type) can now be weakref'ed.  Patch by Wei Wu.
 
 - Issue #22077: Improve index error messages for bytearrays, bytes, lists,
-  and tuples by adding 'or slices'. Added ', not <typename' for bytearrays.
+  and tuples by adding 'or slices'. Added ', not <typename>' for bytearrays.
   Original patch by Claudiu Popa.
 
 - Issue #20179: Apply Argument Clinic to bytes and bytearray.
@@ -1854,7 +1854,7 @@
   engine friendly) error messages when "exec" and "print" are used as
   statements.
 
-- Issue #21642: If the conditional if-else expression, allow an integer written
+- Issue #21642: In the conditional if-else expression, allow an integer written
   with no space between itself and the ``else`` keyword (e.g. ``True if 42else
   False``) to be valid syntax.
 
diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c
--- a/Modules/_io/stringio.c
+++ b/Modules/_io/stringio.c
@@ -750,7 +750,7 @@
     /* If newline == "", we don't translate anything.
        If newline == "\n" or newline == None, we translate to "\n", which is
        a no-op.
-       (for newline == None, TextIOWrapper translates to os.sepline, but it
+       (for newline == None, TextIOWrapper translates to os.linesep, but it
        is pointless for StringIO)
     */
     if (newline != NULL && newline[0] == '\r') {
diff --git a/Objects/methodobject.c b/Objects/methodobject.c
--- a/Objects/methodobject.c
+++ b/Objects/methodobject.c
@@ -89,7 +89,7 @@
 
     /* PyCFunction_Call() must not be called with an exception set,
        because it may clear it (directly or indirectly) and so the
-       caller looses its exception */
+       caller loses its exception */
     assert(!PyErr_Occurred());
 
     flags = PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST);
diff --git a/Objects/object.c b/Objects/object.c
--- a/Objects/object.c
+++ b/Objects/object.c
@@ -475,7 +475,7 @@
 #ifdef Py_DEBUG
     /* PyObject_Repr() must not be called with an exception set,
        because it may clear it (directly or indirectly) and so the
-       caller looses its exception */
+       caller loses its exception */
     assert(!PyErr_Occurred());
 #endif
 
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -901,7 +901,7 @@
 #ifdef Py_DEBUG
     /* type_call() must not be called with an exception set,
        because it may clear it (directly or indirectly) and so the
-       caller looses its exception */
+       caller loses its exception */
     assert(!PyErr_Occurred());
 #endif
 
diff --git a/Python/ceval.c b/Python/ceval.c
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -1214,7 +1214,7 @@
 #ifdef Py_DEBUG
     /* PyEval_EvalFrameEx() must not be called with an exception set,
        because it may clear it (directly or indirectly) and so the
-       caller looses its exception */
+       caller loses its exception */
     assert(!PyErr_Occurred());
 #endif
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Wed Oct  7 13:28:46 2015
From: python-checkins at python.org (martin.panter)
Date: Wed, 07 Oct 2015 11:28:46 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_One_more_typo_in_a_comment?=
Message-ID: <20151007112845.18360.33091@psf.io>

https://hg.python.org/cpython/rev/3291e6132a67
changeset:   98591:3291e6132a67
user:        Martin Panter <vadmium+py at gmail.com>
date:        Wed Oct 07 11:15:15 2015 +0000
summary:
  One more typo in a comment

files:
  Python/ast.c |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Python/ast.c b/Python/ast.c
--- a/Python/ast.c
+++ b/Python/ast.c
@@ -4035,7 +4035,7 @@
     assert(expr_end >= 0 && expr_end < PyUnicode_GET_LENGTH(str));
     assert(expr_end >= expr_start);
 
-    /* There has to be at least on character on each side of the
+    /* There has to be at least one character on each side of the
        expression inside this str. This will have been caught before
        we're called. */
     assert(expr_start >= 1);

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Wed Oct  7 13:28:47 2015
From: python-checkins at python.org (martin.panter)
Date: Wed, 07 Oct 2015 11:28:47 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E5=29=3A_More_typos_in_?=
 =?utf-8?q?3=2E5_documentation_and_comments?=
Message-ID: <20151007112845.97700.15264@psf.io>

https://hg.python.org/cpython/rev/e75e8aeb8ac7
changeset:   98589:e75e8aeb8ac7
branch:      3.5
user:        Martin Panter <vadmium+py at gmail.com>
date:        Wed Oct 07 11:03:53 2015 +0000
summary:
  More typos in 3.5 documentation and comments

files:
  Lib/test/test_time.py  |  4 ++--
  Misc/NEWS              |  4 ++--
  Objects/abstract.c     |  2 +-
  Objects/methodobject.c |  2 +-
  4 files changed, 6 insertions(+), 6 deletions(-)


diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py
--- a/Lib/test/test_time.py
+++ b/Lib/test/test_time.py
@@ -773,7 +773,7 @@
                 (2**23 - 1e-9,  8388607999999999),
                 (2**23,         8388608000000000),
 
-                # start loosing precision for value > 2^23 seconds
+                # start losing precision for value > 2^23 seconds
                 (2**23 + 1e-9,  8388608000000002),
 
                 # nanoseconds are lost for value > 2^23 seconds
@@ -848,7 +848,7 @@
             (4194304000000000, 2**22),
             (4194304000000001, 2**22 + 1e-9),
 
-            # start loosing precision for value > 2^23 seconds
+            # start losing precision for value > 2^23 seconds
             (8388608000000002, 2**23 + 1e-9),
 
             # nanoseconds are lost for value > 2^23 seconds
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -285,7 +285,7 @@
 -----
 
 - Issue #24915: Add LLVM support for PGO builds and use the test suite to
-  generate the profile data. Initiial patch by Alecsandru Patrascu of Intel.
+  generate the profile data. Initial patch by Alecsandru Patrascu of Intel.
 
 - Issue #24910: Windows MSIs now have unique display names.
 
@@ -1693,7 +1693,7 @@
   type) can now be weakref'ed.  Patch by Wei Wu.
 
 - Issue #22077: Improve index error messages for bytearrays, bytes, lists,
-  and tuples by adding 'or slices'. Added ', not <typename' for bytearrays.
+  and tuples by adding 'or slices'. Added ', not <typename>' for bytearrays.
   Original patch by Claudiu Popa.
 
 - Issue #20179: Apply Argument Clinic to bytes and bytearray.
diff --git a/Objects/abstract.c b/Objects/abstract.c
--- a/Objects/abstract.c
+++ b/Objects/abstract.c
@@ -2131,7 +2131,7 @@
 
     /* PyObject_Call() must not be called with an exception set,
        because it may clear it (directly or indirectly) and so the
-       caller looses its exception */
+       caller loses its exception */
     assert(!PyErr_Occurred());
 
     call = func->ob_type->tp_call;
diff --git a/Objects/methodobject.c b/Objects/methodobject.c
--- a/Objects/methodobject.c
+++ b/Objects/methodobject.c
@@ -89,7 +89,7 @@
 
     /* PyCFunction_Call() must not be called with an exception set,
        because it may clear it (directly or indirectly) and so the
-       caller looses its exception */
+       caller loses its exception */
     assert(!PyErr_Occurred());
 
     flags = PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST);

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Wed Oct  7 13:28:47 2015
From: python-checkins at python.org (martin.panter)
Date: Wed, 07 Oct 2015 11:28:47 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E4=29=3A_Various_minor_?=
 =?utf-8?q?typos_in_documentation_and_comments?=
Message-ID: <20151007112845.97720.29719@psf.io>

https://hg.python.org/cpython/rev/a9cbf3becfb7
changeset:   98587:a9cbf3becfb7
branch:      3.4
parent:      98581:92429e01f444
user:        Martin Panter <vadmium+py at gmail.com>
date:        Wed Oct 07 10:26:23 2015 +0000
summary:
  Various minor typos in documentation and comments

files:
  Doc/library/collections.rst |  2 +-
  Doc/library/test.rst        |  2 +-
  Lib/http/server.py          |  2 +-
  Misc/HISTORY                |  2 +-
  Misc/NEWS                   |  2 +-
  Modules/_io/stringio.c      |  2 +-
  Objects/object.c            |  2 +-
  Objects/typeobject.c        |  2 +-
  PC/VS9.0/kill_python.c      |  2 +-
  PCbuild/kill_python.c       |  2 +-
  Python/ceval.c              |  4 ++--
  11 files changed, 12 insertions(+), 12 deletions(-)


diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst
--- a/Doc/library/collections.rst
+++ b/Doc/library/collections.rst
@@ -980,7 +980,7 @@
 
 The :class:`OrderedDict` constructor and :meth:`update` method both accept
 keyword arguments, but their order is lost because Python's function call
-semantics pass-in keyword arguments using a regular unordered dictionary.
+semantics pass in keyword arguments using a regular unordered dictionary.
 
 
 :class:`OrderedDict` Examples and Recipes
diff --git a/Doc/library/test.rst b/Doc/library/test.rst
--- a/Doc/library/test.rst
+++ b/Doc/library/test.rst
@@ -160,7 +160,7 @@
 The :mod:`test` package can be run as a script to drive Python's regression
 test suite, thanks to the :option:`-m` option: :program:`python -m test`. Under
 the hood, it uses :mod:`test.regrtest`; the call :program:`python -m
-test.regrtest` used in previous Python versions still works).  Running the
+test.regrtest` used in previous Python versions still works.  Running the
 script by itself automatically starts running all regression tests in the
 :mod:`test` package. It does this by finding all modules in the package whose
 name starts with ``test_``, importing them, and executing the function
diff --git a/Lib/http/server.py b/Lib/http/server.py
--- a/Lib/http/server.py
+++ b/Lib/http/server.py
@@ -881,7 +881,7 @@
 def _url_collapse_path(path):
     """
     Given a URL path, remove extra '/'s and '.' path elements and collapse
-    any '..' references and returns a colllapsed path.
+    any '..' references and returns a collapsed path.
 
     Implements something akin to RFC-2396 5.2 step 6 to parse relative paths.
     The utility of this function is limited to is_cgi method and helps
diff --git a/Misc/HISTORY b/Misc/HISTORY
--- a/Misc/HISTORY
+++ b/Misc/HISTORY
@@ -6718,7 +6718,7 @@
 
 - Issue #7895: platform.mac_ver() no longer crashes after calling os.fork().
 
-- Issue #9323: Fixed a bug in trace.py that resulted in loosing the name of the
+- Issue #9323: Fixed a bug in trace.py that resulted in losing the name of the
   script being traced.  Patch by Eli Bendersky.
 
 - Issue #9282: Fixed --listfuncs option of trace.py.  Thanks Eli Bendersky for
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -1131,7 +1131,7 @@
   engine friendly) error messages when "exec" and "print" are used as
   statements.
 
-- Issue #21642: If the conditional if-else expression, allow an integer written
+- Issue #21642: In the conditional if-else expression, allow an integer written
   with no space between itself and the ``else`` keyword (e.g. ``True if 42else
   False``) to be valid syntax.
 
diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c
--- a/Modules/_io/stringio.c
+++ b/Modules/_io/stringio.c
@@ -711,7 +711,7 @@
     /* If newline == "", we don't translate anything.
        If newline == "\n" or newline == None, we translate to "\n", which is
        a no-op.
-       (for newline == None, TextIOWrapper translates to os.sepline, but it
+       (for newline == None, TextIOWrapper translates to os.linesep, but it
        is pointless for StringIO)
     */
     if (newline != NULL && newline[0] == '\r') {
diff --git a/Objects/object.c b/Objects/object.c
--- a/Objects/object.c
+++ b/Objects/object.c
@@ -459,7 +459,7 @@
 #ifdef Py_DEBUG
     /* PyObject_Repr() must not be called with an exception set,
        because it may clear it (directly or indirectly) and so the
-       caller looses its exception */
+       caller loses its exception */
     assert(!PyErr_Occurred());
 #endif
 
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -876,7 +876,7 @@
 #ifdef Py_DEBUG
     /* type_call() must not be called with an exception set,
        because it may clear it (directly or indirectly) and so the
-       caller looses its exception */
+       caller loses its exception */
     assert(!PyErr_Occurred());
 #endif
 
diff --git a/PC/VS9.0/kill_python.c b/PC/VS9.0/kill_python.c
--- a/PC/VS9.0/kill_python.c
+++ b/PC/VS9.0/kill_python.c
@@ -108,7 +108,7 @@
          * modules for all processes (not just the python[_d].exe ones)
          * and see if any of our DLLs are loaded (i.e. python34[_d].dll),
          * as that would also inhibit our ability to rebuild the solution.
-         * Not worth loosing sleep over though; for now, a simple check 
+         * Not worth losing sleep over though; for now, a simple check
          * for just the python executable should be sufficient.
          */
 
diff --git a/PCbuild/kill_python.c b/PCbuild/kill_python.c
--- a/PCbuild/kill_python.c
+++ b/PCbuild/kill_python.c
@@ -108,7 +108,7 @@
          * modules for all processes (not just the python[_d].exe ones)
          * and see if any of our DLLs are loaded (i.e. python34[_d].dll),
          * as that would also inhibit our ability to rebuild the solution.
-         * Not worth loosing sleep over though; for now, a simple check 
+         * Not worth losing sleep over though; for now, a simple check
          * for just the python executable should be sufficient.
          */
 
diff --git a/Python/ceval.c b/Python/ceval.c
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -1212,7 +1212,7 @@
 #ifdef Py_DEBUG
     /* PyEval_EvalFrameEx() must not be called with an exception set,
        because it may clear it (directly or indirectly) and so the
-       caller looses its exception */
+       caller loses its exception */
     assert(!PyErr_Occurred());
 #endif
 
@@ -4087,7 +4087,7 @@
 #ifdef Py_DEBUG
     /* PyEval_CallObjectWithKeywords() must not be called with an exception
        set, because it may clear it (directly or indirectly)
-       and so the caller looses its exception */
+       and so the caller loses its exception */
     assert(!PyErr_Occurred());
 #endif
 

-- 
Repository URL: https://hg.python.org/cpython

From lp_benchmark_robot at intel.com  Wed Oct  7 16:00:14 2015
From: lp_benchmark_robot at intel.com (lp_benchmark_robot at intel.com)
Date: Wed, 7 Oct 2015 15:00:14 +0100
Subject: [Python-checkins] Benchmark Results for Python Default 2015-10-07
Message-ID: <43070b8d-1f6e-41e7-95d9-afa1126479ba@irsmsx153.ger.corp.intel.com>

Results for project python_default-nightly, build date 2015-10-07 06:11:19 
commit:		b6bfc2dad87ff7e2dffc60e0d9c1dab07fc3c35a
revision date:	2015-10-07 04:54:23 +0000
environment:	Haswell-EP
	cpu:	Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz 2x18 cores, stepping 2, LLC 45 MB
	mem:	128 GB
	os:	CentOS 7.1
	kernel:	Linux 3.10.0-229.4.2.el7.x86_64

Baseline results were generated using release v3.4.3, with hash
b4cbecbc0781e89a309d03b60a1f75f8499250e6 from 2015-02-25 12:15:33+00:00

------------------------------------------------------------------------------------------
               benchmark    relative      change since      change since  current rev with
                            std_dev*          last run            v3.4.3      regrtest PGO
------------------------------------------------------------------------------------------
:-)            django_v2    0.39336%         -1.60495%          7.49150%         16.66576%
:-(              pybench    0.15681%         -0.04105%         -2.05193%          8.57239%
:-(             regex_v8    2.66830%          0.72266%         -4.47608%          0.20973%
:-|                nbody    0.12791%          1.30144%          0.14520%          9.16120%
:-|         json_dump_v2    0.33505%         -0.49813%         -0.70106%          9.53451%
:-|       normal_startup    1.06599%         -0.47143%          0.12012%          5.23171%
------------------------------------------------------------------------------------------
Note: Benchmark results are measured in seconds.
* Relative Standard Deviation (Standard Deviation/Average)

Our lab does a nightly source pull and build of the Python project and measures
performance changes against the previous stable version and the previous nightly
measurement. This is provided as a service to the community so that quality
issues with current hardware can be identified quickly.

Intel technologies' features and benefits depend on system configuration and may
require enabled hardware, software or service activation. Performance varies
depending on system configuration.


From lp_benchmark_robot at intel.com  Wed Oct  7 16:00:32 2015
From: lp_benchmark_robot at intel.com (lp_benchmark_robot at intel.com)
Date: Wed, 7 Oct 2015 15:00:32 +0100
Subject: [Python-checkins] Benchmark Results for Python 2.7 2015-10-07
Message-ID: <f70b774d-1bb7-43a4-8136-39180368f48d@irsmsx153.ger.corp.intel.com>

Results for project python_2.7-nightly, build date 2015-10-07 06:54:17 
commit:		37aee118e1a395b58aea0a746e1e3008b04350bd
revision date:	2015-10-07 03:12:02 +0000
environment:	Haswell-EP
	cpu:	Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz 2x18 cores, stepping 2, LLC 45 MB
	mem:	128 GB
	os:	CentOS 7.1
	kernel:	Linux 3.10.0-229.4.2.el7.x86_64

Baseline results were generated using release v2.7.10, with hash
15c95b7d81dcf821daade360741e00714667653f from 2015-05-23 16:02:14+00:00

------------------------------------------------------------------------------------------
               benchmark    relative      change since      change since  current rev with
                            std_dev*          last run           v2.7.10      regrtest PGO
------------------------------------------------------------------------------------------
:-)            django_v2    0.44333%         -1.37467%          3.92133%         10.00153%
:-)              pybench    0.17090%          0.04833%          6.88154%          6.28198%
:-(             regex_v8    1.18170%         -0.12320%         -2.56983%          9.26005%
:-)                nbody    0.16317%         -0.00817%          6.70789%          4.14536%
:-)         json_dump_v2    0.25562%         -0.42525%          2.92309%         13.77286%
:-(       normal_startup    1.77929%          0.07002%         -2.62910%          2.82728%
:-|              ssbench    0.63170%         -0.04144%          0.57537%          2.35317%
------------------------------------------------------------------------------------------
Note: Benchmark results for ssbench are measured in requests/second while all
other are measured in seconds.
* Relative Standard Deviation (Standard Deviation/Average)

Our lab does a nightly source pull and build of the Python project and measures
performance changes against the previous stable version and the previous nightly
measurement. This is provided as a service to the community so that quality
issues with current hardware can be identified quickly.

Intel technologies' features and benefits depend on system configuration and may
require enabled hardware, software or service activation. Performance varies
depending on system configuration.


From python-checkins at python.org  Thu Oct  8 05:35:05 2015
From: python-checkins at python.org (berker.peksag)
Date: Thu, 08 Oct 2015 03:35:05 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzE2ODAy?=
 =?utf-8?q?=3A_Document_fileno_parameter_of_socket=2Esocket=28=29?=
Message-ID: <20151008033504.55456.49787@psf.io>

https://hg.python.org/cpython/rev/f4606117d571
changeset:   98592:f4606117d571
branch:      3.4
parent:      98587:a9cbf3becfb7
user:        Berker Peksag <berker.peksag at gmail.com>
date:        Thu Oct 08 06:34:01 2015 +0300
summary:
  Issue #16802: Document fileno parameter of socket.socket()

Patch by Henrik Heimbuerger and Bar Harel.

files:
  Doc/library/socket.rst |  6 +++++-
  1 files changed, 5 insertions(+), 1 deletions(-)


diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst
--- a/Doc/library/socket.rst
+++ b/Doc/library/socket.rst
@@ -369,7 +369,11 @@
    :const:`SOCK_DGRAM`, :const:`SOCK_RAW` or perhaps one of the other ``SOCK_``
    constants. The protocol number is usually zero and may be omitted or in the
    case where the address family is :const:`AF_CAN` the protocol should be one
-   of :const:`CAN_RAW` or :const:`CAN_BCM`.
+   of :const:`CAN_RAW` or :const:`CAN_BCM`.  If *fileno* is specified, the other
+   arguments are ignored, causing the socket with the specified file descriptor
+   to return.  Unlike :func:`socket.fromfd`, *fileno* will return the same
+   socket and not a duplicate. This may help close a detached socket using
+   :meth:`socket.close()`.
 
    The newly created socket is :ref:`non-inheritable <fd_inheritance>`.
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  8 05:35:04 2015
From: python-checkins at python.org (berker.peksag)
Date: Thu, 08 Oct 2015 03:35:04 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAobWVyZ2UgMy40IC0+IDMuNSk6?=
 =?utf-8?q?_Issue_=2316802=3A_Document_fileno_parameter_of_socket=2Esocket?=
 =?utf-8?b?KCk=?=
Message-ID: <20151008033504.3293.89070@psf.io>

https://hg.python.org/cpython/rev/1d14675c6050
changeset:   98593:1d14675c6050
branch:      3.5
parent:      98589:e75e8aeb8ac7
parent:      98592:f4606117d571
user:        Berker Peksag <berker.peksag at gmail.com>
date:        Thu Oct 08 06:34:31 2015 +0300
summary:
  Issue #16802: Document fileno parameter of socket.socket()

Patch by Henrik Heimbuerger and Bar Harel.

files:
  Doc/library/socket.rst |  6 +++++-
  1 files changed, 5 insertions(+), 1 deletions(-)


diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst
--- a/Doc/library/socket.rst
+++ b/Doc/library/socket.rst
@@ -384,7 +384,11 @@
    :const:`SOCK_DGRAM`, :const:`SOCK_RAW` or perhaps one of the other ``SOCK_``
    constants. The protocol number is usually zero and may be omitted or in the
    case where the address family is :const:`AF_CAN` the protocol should be one
-   of :const:`CAN_RAW` or :const:`CAN_BCM`.
+   of :const:`CAN_RAW` or :const:`CAN_BCM`.  If *fileno* is specified, the other
+   arguments are ignored, causing the socket with the specified file descriptor
+   to return.  Unlike :func:`socket.fromfd`, *fileno* will return the same
+   socket and not a duplicate. This may help close a detached socket using
+   :meth:`socket.close()`.
 
    The newly created socket is :ref:`non-inheritable <fd_inheritance>`.
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  8 05:35:05 2015
From: python-checkins at python.org (berker.peksag)
Date: Thu, 08 Oct 2015 03:35:05 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
 =?utf-8?q?=29=3A_Issue_=2316802=3A_Document_fileno_parameter_of_socket=2E?=
 =?utf-8?q?socket=28=29?=
Message-ID: <20151008033504.7242.23885@psf.io>

https://hg.python.org/cpython/rev/9115c63cf3d2
changeset:   98594:9115c63cf3d2
parent:      98591:3291e6132a67
parent:      98593:1d14675c6050
user:        Berker Peksag <berker.peksag at gmail.com>
date:        Thu Oct 08 06:34:57 2015 +0300
summary:
  Issue #16802: Document fileno parameter of socket.socket()

Patch by Henrik Heimbuerger and Bar Harel.

files:
  Doc/library/socket.rst |  6 +++++-
  1 files changed, 5 insertions(+), 1 deletions(-)


diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst
--- a/Doc/library/socket.rst
+++ b/Doc/library/socket.rst
@@ -384,7 +384,11 @@
    :const:`SOCK_DGRAM`, :const:`SOCK_RAW` or perhaps one of the other ``SOCK_``
    constants. The protocol number is usually zero and may be omitted or in the
    case where the address family is :const:`AF_CAN` the protocol should be one
-   of :const:`CAN_RAW` or :const:`CAN_BCM`.
+   of :const:`CAN_RAW` or :const:`CAN_BCM`.  If *fileno* is specified, the other
+   arguments are ignored, causing the socket with the specified file descriptor
+   to return.  Unlike :func:`socket.fromfd`, *fileno* will return the same
+   socket and not a duplicate. This may help close a detached socket using
+   :meth:`socket.close()`.
 
    The newly created socket is :ref:`non-inheritable <fd_inheritance>`.
 

-- 
Repository URL: https://hg.python.org/cpython

From solipsis at pitrou.net  Thu Oct  8 10:43:41 2015
From: solipsis at pitrou.net (solipsis at pitrou.net)
Date: Thu, 08 Oct 2015 08:43:41 +0000
Subject: [Python-checkins] Daily reference leaks (9115c63cf3d2): sum=61491
Message-ID: <20151008084341.3275.62438@psf.io>

results for 9115c63cf3d2 on branch "default"
--------------------------------------------

test_capi leaked [5410, 5410, 5410] references, sum=16230
test_capi leaked [1421, 1423, 1423] memory blocks, sum=4267
test_functools leaked [0, 2, 2] memory blocks, sum=4
test_threading leaked [10820, 10820, 10820] references, sum=32460
test_threading leaked [2842, 2844, 2844] memory blocks, sum=8530


Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogYAlTZW', '--timeout', '7200']

From python-checkins at python.org  Thu Oct  8 11:27:11 2015
From: python-checkins at python.org (berker.peksag)
Date: Thu, 08 Oct 2015 09:27:11 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2316099=3A_RobotFil?=
 =?utf-8?q?eParser_now_supports_Crawl-delay_and_Request-rate?=
Message-ID: <20151008092710.3285.69884@psf.io>

https://hg.python.org/cpython/rev/dbed7cacfb7e
changeset:   98595:dbed7cacfb7e
user:        Berker Peksag <berker.peksag at gmail.com>
date:        Thu Oct 08 12:27:06 2015 +0300
summary:
  Issue #16099: RobotFileParser now supports Crawl-delay and Request-rate
extensions.

Patch by Nikolay Bogoychev.

files:
  Doc/library/urllib.robotparser.rst |  30 +++++-
  Doc/whatsnew/3.6.rst               |   8 +
  Lib/test/test_robotparser.py       |  90 +++++++++++++----
  Lib/urllib/robotparser.py          |  39 +++++++-
  Misc/ACKS                          |   1 +
  Misc/NEWS                          |   5 +-
  6 files changed, 147 insertions(+), 26 deletions(-)


diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst
--- a/Doc/library/urllib.robotparser.rst
+++ b/Doc/library/urllib.robotparser.rst
@@ -53,15 +53,41 @@
       Sets the time the ``robots.txt`` file was last fetched to the current
       time.
 
+   .. method:: crawl_delay(useragent)
 
-The following example demonstrates basic use of the RobotFileParser class.
+      Returns the value of the ``Crawl-delay`` parameter from ``robots.txt``
+      for the *useragent* in question.  If there is no such parameter or it
+      doesn't apply to the *useragent* specified or the ``robots.txt`` entry
+      for this parameter has invalid syntax, return ``None``.
+
+      .. versionadded:: 3.6
+
+   .. method:: request_rate(useragent)
+
+      Returns the contents of the ``Request-rate`` parameter from
+      ``robots.txt`` in the form of a :func:`~collections.namedtuple`
+      ``(requests, seconds)``.  If there is no such parameter or it doesn't
+      apply to the *useragent* specified or the ``robots.txt`` entry for this
+      parameter has invalid syntax, return ``None``.
+
+      .. versionadded:: 3.6
+
+
+The following example demonstrates basic use of the :class:`RobotFileParser`
+class::
 
    >>> import urllib.robotparser
    >>> rp = urllib.robotparser.RobotFileParser()
    >>> rp.set_url("http://www.musi-cal.com/robots.txt")
    >>> rp.read()
+   >>> rrate = rp.request_rate("*")
+   >>> rrate.requests
+   3
+   >>> rrate.seconds
+   20
+   >>> rp.crawl_delay("*")
+   6
    >>> rp.can_fetch("*", "http://www.musi-cal.com/cgi-bin/search?city=San+Francisco")
    False
    >>> rp.can_fetch("*", "http://www.musi-cal.com/")
    True
-
diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst
--- a/Doc/whatsnew/3.6.rst
+++ b/Doc/whatsnew/3.6.rst
@@ -119,6 +119,14 @@
   (Contributed by Ashley Anderson in :issue:`12006`.)
 
 
+urllib.robotparser
+------------------
+
+:class:`~urllib.robotparser.RobotFileParser` now supports ``Crawl-delay`` and
+``Request-rate`` extensions.
+(Contributed by Nikolay Bogoychev in :issue:`16099`.)
+
+
 Optimizations
 =============
 
diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py
--- a/Lib/test/test_robotparser.py
+++ b/Lib/test/test_robotparser.py
@@ -1,6 +1,7 @@
 import io
 import unittest
 import urllib.robotparser
+from collections import namedtuple
 from urllib.error import URLError, HTTPError
 from urllib.request import urlopen
 from test import support
@@ -12,7 +13,8 @@
 
 
 class RobotTestCase(unittest.TestCase):
-    def __init__(self, index=None, parser=None, url=None, good=None, agent=None):
+    def __init__(self, index=None, parser=None, url=None, good=None,
+                 agent=None, request_rate=None, crawl_delay=None):
         # workaround to make unittest discovery work (see #17066)
         if not isinstance(index, int):
             return
@@ -25,6 +27,8 @@
         self.url = url
         self.good = good
         self.agent = agent
+        self.request_rate = request_rate
+        self.crawl_delay = crawl_delay
 
     def runTest(self):
         if isinstance(self.url, tuple):
@@ -34,6 +38,18 @@
             agent = self.agent
         if self.good:
             self.assertTrue(self.parser.can_fetch(agent, url))
+            self.assertEqual(self.parser.crawl_delay(agent), self.crawl_delay)
+            # if we have actual values for request rate
+            if self.request_rate and self.parser.request_rate(agent):
+                self.assertEqual(
+                    self.parser.request_rate(agent).requests,
+                    self.request_rate.requests
+                )
+                self.assertEqual(
+                    self.parser.request_rate(agent).seconds,
+                    self.request_rate.seconds
+                )
+            self.assertEqual(self.parser.request_rate(agent), self.request_rate)
         else:
             self.assertFalse(self.parser.can_fetch(agent, url))
 
@@ -43,15 +59,17 @@
 tests = unittest.TestSuite()
 
 def RobotTest(index, robots_txt, good_urls, bad_urls,
-              agent="test_robotparser"):
+              request_rate, crawl_delay, agent="test_robotparser"):
 
     lines = io.StringIO(robots_txt).readlines()
     parser = urllib.robotparser.RobotFileParser()
     parser.parse(lines)
     for url in good_urls:
-        tests.addTest(RobotTestCase(index, parser, url, 1, agent))
+        tests.addTest(RobotTestCase(index, parser, url, 1, agent,
+                      request_rate, crawl_delay))
     for url in bad_urls:
-        tests.addTest(RobotTestCase(index, parser, url, 0, agent))
+        tests.addTest(RobotTestCase(index, parser, url, 0, agent,
+                      request_rate, crawl_delay))
 
 # Examples from http://www.robotstxt.org/wc/norobots.html (fetched 2002)
 
@@ -65,14 +83,18 @@
 
 good = ['/','/test.html']
 bad = ['/cyberworld/map/index.html','/tmp/xxx','/foo.html']
+request_rate = None
+crawl_delay = None
 
-RobotTest(1, doc, good, bad)
+RobotTest(1, doc, good, bad, request_rate, crawl_delay)
 
 # 2.
 doc = """
 # robots.txt for http://www.example.com/
 
 User-agent: *
+Crawl-delay: 1
+Request-rate: 3/15
 Disallow: /cyberworld/map/ # This is an infinite virtual URL space
 
 # Cybermapper knows where to go.
@@ -83,8 +105,10 @@
 
 good = ['/','/test.html',('cybermapper','/cyberworld/map/index.html')]
 bad = ['/cyberworld/map/index.html']
+request_rate = None  # The parameters should be equal to None since they
+crawl_delay = None   # don't apply to the cybermapper user agent
 
-RobotTest(2, doc, good, bad)
+RobotTest(2, doc, good, bad, request_rate, crawl_delay)
 
 # 3.
 doc = """
@@ -95,14 +119,18 @@
 
 good = []
 bad = ['/cyberworld/map/index.html','/','/tmp/']
+request_rate = None
+crawl_delay = None
 
-RobotTest(3, doc, good, bad)
+RobotTest(3, doc, good, bad, request_rate, crawl_delay)
 
 # Examples from http://www.robotstxt.org/wc/norobots-rfc.html (fetched 2002)
 
 # 4.
 doc = """
 User-agent: figtree
+Crawl-delay: 3
+Request-rate: 9/30
 Disallow: /tmp
 Disallow: /a%3cd.html
 Disallow: /a%2fb.html
@@ -115,8 +143,17 @@
        '/~joe/index.html'
        ]
 
-RobotTest(4, doc, good, bad, 'figtree')
-RobotTest(5, doc, good, bad, 'FigTree Robot libwww-perl/5.04')
+request_rate = namedtuple('req_rate', 'requests seconds')
+request_rate.requests = 9
+request_rate.seconds = 30
+crawl_delay = 3
+request_rate_bad = None  # not actually tested, but we still need to parse it
+crawl_delay_bad = None  # in order to accommodate the input parameters
+
+
+RobotTest(4, doc, good, bad, request_rate, crawl_delay, 'figtree' )
+RobotTest(5, doc, good, bad, request_rate_bad, crawl_delay_bad,
+          'FigTree Robot libwww-perl/5.04')
 
 # 6.
 doc = """
@@ -125,14 +162,18 @@
 Disallow: /a%3Cd.html
 Disallow: /a/b.html
 Disallow: /%7ejoe/index.html
+Crawl-delay: 3
+Request-rate: 9/banana
 """
 
 good = ['/tmp',] # XFAIL: '/a%2fb.html'
 bad = ['/tmp/','/tmp/a.html',
        '/a%3cd.html','/a%3Cd.html',"/a/b.html",
        '/%7Ejoe/index.html']
+crawl_delay = 3
+request_rate = None  # since request rate has invalid syntax, return None
 
-RobotTest(6, doc, good, bad)
+RobotTest(6, doc, good, bad, None, None)
 
 # From bug report #523041
 
@@ -140,12 +181,16 @@
 doc = """
 User-Agent: *
 Disallow: /.
+Crawl-delay: pears
 """
 
 good = ['/foo.html']
-bad = [] # Bug report says "/" should be denied, but that is not in the RFC
+bad = []  # bug report says "/" should be denied, but that is not in the RFC
 
-RobotTest(7, doc, good, bad)
+crawl_delay = None  # since crawl delay has invalid syntax, return None
+request_rate = None
+
+RobotTest(7, doc, good, bad, crawl_delay, request_rate)
 
 # From Google: http://www.google.com/support/webmasters/bin/answer.py?hl=en&answer=40364
 
@@ -154,12 +199,15 @@
 User-agent: Googlebot
 Allow: /folder1/myfile.html
 Disallow: /folder1/
+Request-rate: whale/banana
 """
 
 good = ['/folder1/myfile.html']
 bad = ['/folder1/anotherfile.html']
+crawl_delay = None
+request_rate = None  # invalid syntax, return none
 
-RobotTest(8, doc, good, bad, agent="Googlebot")
+RobotTest(8, doc, good, bad, crawl_delay, request_rate, agent="Googlebot")
 
 # 9.  This file is incorrect because "Googlebot" is a substring of
 #     "Googlebot-Mobile", so test 10 works just like test 9.
@@ -174,12 +222,12 @@
 good = []
 bad = ['/something.jpg']
 
-RobotTest(9, doc, good, bad, agent="Googlebot")
+RobotTest(9, doc, good, bad, None, None, agent="Googlebot")
 
 good = []
 bad = ['/something.jpg']
 
-RobotTest(10, doc, good, bad, agent="Googlebot-Mobile")
+RobotTest(10, doc, good, bad, None, None, agent="Googlebot-Mobile")
 
 # 11.  Get the order correct.
 doc = """
@@ -193,12 +241,12 @@
 good = []
 bad = ['/something.jpg']
 
-RobotTest(11, doc, good, bad, agent="Googlebot")
+RobotTest(11, doc, good, bad, None, None, agent="Googlebot")
 
 good = ['/something.jpg']
 bad = []
 
-RobotTest(12, doc, good, bad, agent="Googlebot-Mobile")
+RobotTest(12, doc, good, bad, None, None, agent="Googlebot-Mobile")
 
 
 # 13.  Google also got the order wrong in #8.  You need to specify the
@@ -212,7 +260,7 @@
 good = ['/folder1/myfile.html']
 bad = ['/folder1/anotherfile.html']
 
-RobotTest(13, doc, good, bad, agent="googlebot")
+RobotTest(13, doc, good, bad, None, None, agent="googlebot")
 
 
 # 14. For issue #6325 (query string support)
@@ -224,7 +272,7 @@
 good = ['/some/path']
 bad = ['/some/path?name=value']
 
-RobotTest(14, doc, good, bad)
+RobotTest(14, doc, good, bad, None, None)
 
 # 15. For issue #4108 (obey first * entry)
 doc = """
@@ -238,7 +286,7 @@
 good = ['/another/path']
 bad = ['/some/path']
 
-RobotTest(15, doc, good, bad)
+RobotTest(15, doc, good, bad, None, None)
 
 # 16. Empty query (issue #17403). Normalizing the url first.
 doc = """
@@ -250,7 +298,7 @@
 good = ['/some/path?']
 bad = ['/another/path?']
 
-RobotTest(16, doc, good, bad)
+RobotTest(16, doc, good, bad, None, None)
 
 
 class RobotHandler(BaseHTTPRequestHandler):
diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py
--- a/Lib/urllib/robotparser.py
+++ b/Lib/urllib/robotparser.py
@@ -10,7 +10,9 @@
     http://www.robotstxt.org/norobots-rfc.txt
 """
 
-import urllib.parse, urllib.request
+import collections
+import urllib.parse
+import urllib.request
 
 __all__ = ["RobotFileParser"]
 
@@ -120,10 +122,29 @@
                     if state != 0:
                         entry.rulelines.append(RuleLine(line[1], True))
                         state = 2
+                elif line[0] == "crawl-delay":
+                    if state != 0:
+                        # before trying to convert to int we need to make
+                        # sure that robots.txt has valid syntax otherwise
+                        # it will crash
+                        if line[1].strip().isdigit():
+                            entry.delay = int(line[1])
+                        state = 2
+                elif line[0] == "request-rate":
+                    if state != 0:
+                        numbers = line[1].split('/')
+                        # check if all values are sane
+                        if (len(numbers) == 2 and numbers[0].strip().isdigit()
+                            and numbers[1].strip().isdigit()):
+                            req_rate = collections.namedtuple('req_rate',
+                                                              'requests seconds')
+                            entry.req_rate = req_rate
+                            entry.req_rate.requests = int(numbers[0])
+                            entry.req_rate.seconds = int(numbers[1])
+                        state = 2
         if state == 2:
             self._add_entry(entry)
 
-
     def can_fetch(self, useragent, url):
         """using the parsed robots.txt decide if useragent can fetch url"""
         if self.disallow_all:
@@ -153,6 +174,18 @@
         # agent not found ==> access granted
         return True
 
+    def crawl_delay(self, useragent):
+        for entry in self.entries:
+            if entry.applies_to(useragent):
+                return entry.delay
+        return None
+
+    def request_rate(self, useragent):
+        for entry in self.entries:
+            if entry.applies_to(useragent):
+                return entry.req_rate
+        return None
+
     def __str__(self):
         return ''.join([str(entry) + "\n" for entry in self.entries])
 
@@ -180,6 +213,8 @@
     def __init__(self):
         self.useragents = []
         self.rulelines = []
+        self.delay = None
+        self.req_rate = None
 
     def __str__(self):
         ret = []
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -151,6 +151,7 @@
 Paul Boddie
 Matthew Boedicker
 Robin Boerdijk
+Nikolay Bogoychev
 David Bolen
 Wouter Bolsterlee
 Gawain Bolton
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -1,4 +1,4 @@
-+++++++++++
+?+++++++++++
 Python News
 +++++++++++
 
@@ -46,6 +46,9 @@
 Library
 -------
 
+- Issue #16099: RobotFileParser now supports Crawl-delay and Request-rate
+  extensions.  Patch by Nikolay Bogoychev.
+
 - Issue #25316: distutils raises OSError instead of DistutilsPlatformError
   when MSVC is not installed.
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  8 12:58:53 2015
From: python-checkins at python.org (berker.peksag)
Date: Thu, 08 Oct 2015 10:58:53 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Sort_module_names_in_whats?=
	=?utf-8?q?new/3=2E6=2Erst?=
Message-ID: <20151008105853.20775.7876@psf.io>

https://hg.python.org/cpython/rev/ed47b841f02e
changeset:   98596:ed47b841f02e
user:        Berker Peksag <berker.peksag at gmail.com>
date:        Thu Oct 08 13:58:49 2015 +0300
summary:
  Sort module names in whatsnew/3.6.rst

files:
  Doc/whatsnew/3.6.rst |  23 ++++++++++++-----------
  1 files changed, 12 insertions(+), 11 deletions(-)


diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst
--- a/Doc/whatsnew/3.6.rst
+++ b/Doc/whatsnew/3.6.rst
@@ -95,12 +95,21 @@
 Improved Modules
 ================
 
+datetime
+--------
+
+:meth:`datetime.stftime <datetime.datetime.stftime>` and
+:meth:`date.stftime <datetime.date.stftime>` methods now support ISO 8601 date
+directives ``%G``, ``%u`` and ``%V``.
+(Contributed by Ashley Anderson in :issue:`12006`.)
+
+
 operator
 --------
 
-* New object :data:`operator.subscript` makes it easier to create complex
-  indexers. For example: ``subscript[0:10:2] == slice(0, 10, 2)``
-  (Contributed by Joe Jevnik in :issue:`24379`.)
+New object :data:`operator.subscript` makes it easier to create complex
+indexers. For example: ``subscript[0:10:2] == slice(0, 10, 2)``
+(Contributed by Joe Jevnik in :issue:`24379`.)
 
 
 rlcomplete
@@ -110,14 +119,6 @@
 with underscores.  A space or a colon can be added after completed keyword.
 (Contributed by Serhiy Storchaka in :issue:`25011` and :issue:`25209`.)
 
-datetime
---------
-
-* :meth:`datetime.stftime <datetime.datetime.stftime>`  and
-  :meth:`date.stftime <datetime.date.stftime>` methods now support ISO 8601
-  date directives ``%G``, ``%u`` and ``%V``.
-  (Contributed by Ashley Anderson in :issue:`12006`.)
-
 
 urllib.robotparser
 ------------------

-- 
Repository URL: https://hg.python.org/cpython

From lp_benchmark_robot at intel.com  Thu Oct  8 16:12:03 2015
From: lp_benchmark_robot at intel.com (lp_benchmark_robot at intel.com)
Date: Thu, 8 Oct 2015 15:12:03 +0100
Subject: [Python-checkins] Benchmark Results for Python Default 2015-10-08
Message-ID: <4af6cb8c-0688-4ba6-adb7-3bf6f67bab31@irsmsx103.ger.corp.intel.com>

Results for project python_default-nightly, build date 2015-10-08 03:02:03 
commit:		3291e6132a674606af028be2d500701e5ff8285a
revision date:	2015-10-07 11:15:15 +0000
environment:	Haswell-EP
	cpu:	Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz 2x18 cores, stepping 2, LLC 45 MB
	mem:	128 GB
	os:	CentOS 7.1
	kernel:	Linux 3.10.0-229.4.2.el7.x86_64

Baseline results were generated using release v3.4.3, with hash
b4cbecbc0781e89a309d03b60a1f75f8499250e6 from 2015-02-25 12:15:33+00:00

------------------------------------------------------------------------------------------
               benchmark    relative      change since      change since  current rev with
                            std_dev*          last run            v3.4.3      regrtest PGO
------------------------------------------------------------------------------------------
:-)            django_v2    0.46296%          1.14672%          8.55231%         14.86189%
:-(              pybench    0.17124%         -0.04103%         -2.09380%          8.78514%
:-(             regex_v8    2.65849%          0.00322%         -4.47271%          4.96087%
:-|                nbody    0.06992%         -0.35455%         -0.20884%          9.39964%
:-|         json_dump_v2    0.34646%          0.42105%         -0.27706%         10.07059%
:-|       normal_startup    0.66576%          0.53474%          0.25984%          5.40658%
------------------------------------------------------------------------------------------
Note: Benchmark results are measured in seconds.
* Relative Standard Deviation (Standard Deviation/Average)

Our lab does a nightly source pull and build of the Python project and measures
performance changes against the previous stable version and the previous nightly
measurement. This is provided as a service to the community so that quality
issues with current hardware can be identified quickly.

Intel technologies' features and benefits depend on system configuration and may
require enabled hardware, software or service activation. Performance varies
depending on system configuration.


From lp_benchmark_robot at intel.com  Thu Oct  8 16:12:50 2015
From: lp_benchmark_robot at intel.com (lp_benchmark_robot at intel.com)
Date: Thu, 8 Oct 2015 15:12:50 +0100
Subject: [Python-checkins] Benchmark Results for Python 2.7 2015-10-08
Message-ID: <25570a22-09b1-4a5f-8d47-9889dcedb426@irsmsx103.ger.corp.intel.com>

Results for project python_2.7-nightly, build date 2015-10-08 03:45:04 
commit:		04815b55227f8855eb67c41361310aaa56d90626
revision date:	2015-10-07 10:39:13 +0000
environment:	Haswell-EP
	cpu:	Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz 2x18 cores, stepping 2, LLC 45 MB
	mem:	128 GB
	os:	CentOS 7.1
	kernel:	Linux 3.10.0-229.4.2.el7.x86_64

Baseline results were generated using release v2.7.10, with hash
15c95b7d81dcf821daade360741e00714667653f from 2015-05-23 16:02:14+00:00

------------------------------------------------------------------------------------------
               benchmark    relative      change since      change since  current rev with
                            std_dev*          last run           v2.7.10      regrtest PGO
------------------------------------------------------------------------------------------
:-)            django_v2    0.18215%          2.21212%          6.04671%          9.57055%
:-)              pybench    0.17620%         -0.07253%          6.81400%          6.64788%
:-(             regex_v8    1.08666%          0.34510%         -2.21586%          7.62728%
:-)                nbody    0.14237%          0.02137%          6.72783%          5.82344%
:-)         json_dump_v2    0.32811%          0.42392%          3.33462%         14.85349%
:-|       normal_startup    1.83767%          0.94491%         -1.65935%          2.70445%
:-|              ssbench    0.42751%          0.67199%          1.25123%          1.26939%
------------------------------------------------------------------------------------------
Note: Benchmark results for ssbench are measured in requests/second while all
other are measured in seconds.
* Relative Standard Deviation (Standard Deviation/Average)

Our lab does a nightly source pull and build of the Python project and measures
performance changes against the previous stable version and the previous nightly
measurement. This is provided as a service to the community so that quality
issues with current hardware can be identified quickly.

Intel technologies' features and benefits depend on system configuration and may
require enabled hardware, software or service activation. Performance varies
depending on system configuration.


From python-checkins at python.org  Thu Oct  8 18:08:04 2015
From: python-checkins at python.org (steve.dower)
Date: Thu, 08 Oct 2015 16:08:04 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E5=29=3A_Removes_deprec?=
 =?utf-8?q?ated_-n_option_from_buildbot_script=2E?=
Message-ID: <20151008160804.7236.9559@psf.io>

https://hg.python.org/cpython/rev/0b7df139a5f7
changeset:   98599:0b7df139a5f7
branch:      3.5
parent:      98597:69c4fa62b608
user:        Steve Dower <steve.dower at microsoft.com>
date:        Thu Oct 08 09:06:17 2015 -0700
summary:
  Removes deprecated -n option from buildbot script.

files:
  Tools/buildbot/test.bat |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat
--- a/Tools/buildbot/test.bat
+++ b/Tools/buildbot/test.bat
@@ -16,4 +16,4 @@
 if NOT "%1"=="" (set regrtest_args=%regrtest_args% %1) & shift & goto CheckOpts
 
 echo on
-call "%here%..\..\PCbuild\rt.bat" %rt_opts% -uall -rwW -n --timeout=3600 %regrtest_args%
+call "%here%..\..\PCbuild\rt.bat" %rt_opts% -uall -rwW --timeout=3600 %regrtest_args%

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  8 18:08:04 2015
From: python-checkins at python.org (steve.dower)
Date: Thu, 08 Oct 2015 16:08:04 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2323919=3A_Prevents?=
 =?utf-8?q?_assert_dialogs_appearing_in_the_test_suite=2E?=
Message-ID: <20151008160804.471.25310@psf.io>

https://hg.python.org/cpython/rev/62897db9ae51
changeset:   98598:62897db9ae51
parent:      98596:ed47b841f02e
user:        Steve Dower <steve.dower at microsoft.com>
date:        Thu Oct 08 09:05:36 2015 -0700
summary:
  Issue #23919: Prevents assert dialogs appearing in the test suite.

files:
  Lib/test/libregrtest/cmdline.py |   4 ++++
  Lib/test/libregrtest/setup.py   |  12 +++++++++---
  2 files changed, 13 insertions(+), 3 deletions(-)


diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py
--- a/Lib/test/libregrtest/cmdline.py
+++ b/Lib/test/libregrtest/cmdline.py
@@ -304,6 +304,10 @@
     if ns.pgo and (ns.verbose or ns.verbose2 or ns.verbose3):
         parser.error("--pgo/-v don't go together!")
 
+    if ns.nowindows:
+        print("Warning: the --nowindows (-n) option is deprecated. "
+              "Use -vv to display assertions in stderr.", file=sys.stderr)
+
     if ns.quiet:
         ns.verbose = 0
     if ns.timeout is not None:
diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py
--- a/Lib/test/libregrtest/setup.py
+++ b/Lib/test/libregrtest/setup.py
@@ -75,8 +75,11 @@
     if ns.threshold is not None:
         gc.set_threshold(ns.threshold)
 
-    if ns.nowindows:
+    try:
         import msvcrt
+    except ImportError:
+        pass
+    else:
         msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS|
                             msvcrt.SEM_NOALIGNMENTFAULTEXCEPT|
                             msvcrt.SEM_NOGPFAULTERRORBOX|
@@ -88,8 +91,11 @@
             pass
         else:
             for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]:
-                msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE)
-                msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR)
+                if ns.verbose and ns.verbose >= 2:
+                    msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE)
+                    msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR)
+                else:
+                    msvcrt.CrtSetReportMode(m, 0)
 
     support.use_resources = ns.use_resources
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  8 18:08:35 2015
From: python-checkins at python.org (steve.dower)
Date: Thu, 08 Oct 2015 16:08:35 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
	=?utf-8?q?=29=3A_Merge_from_3=2E5?=
Message-ID: <20151008160805.70970.292@psf.io>

https://hg.python.org/cpython/rev/0b7cccd9c7d5
changeset:   98600:0b7cccd9c7d5
parent:      98598:62897db9ae51
parent:      98599:0b7df139a5f7
user:        Steve Dower <steve.dower at microsoft.com>
date:        Thu Oct 08 09:06:39 2015 -0700
summary:
  Merge from 3.5

files:
  Misc/NEWS               |  2 ++
  Tools/buildbot/test.bat |  2 +-
  2 files changed, 3 insertions(+), 1 deletions(-)


diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -417,6 +417,8 @@
 Tests
 -----
 
+- Issue #23919: Prevents assert dialogs appearing in the test suite.
+
 - PCbuild\rt.bat now accepts an unlimited number of arguments to pass along
   to regrtest.py.  Previously there was a limit of 9.
 
diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat
--- a/Tools/buildbot/test.bat
+++ b/Tools/buildbot/test.bat
@@ -16,4 +16,4 @@
 if NOT "%1"=="" (set regrtest_args=%regrtest_args% %1) & shift & goto CheckOpts
 
 echo on
-call "%here%..\..\PCbuild\rt.bat" %rt_opts% -uall -rwW -n --timeout=3600 %regrtest_args%
+call "%here%..\..\PCbuild\rt.bat" %rt_opts% -uall -rwW --timeout=3600 %regrtest_args%

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  8 18:08:35 2015
From: python-checkins at python.org (steve.dower)
Date: Thu, 08 Oct 2015 16:08:35 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy41KTogSXNzdWUgIzIzOTE5?=
 =?utf-8?q?=3A_Prevents_assert_dialogs_appearing_in_the_test_suite=2E?=
Message-ID: <20151008160804.70990.52712@psf.io>

https://hg.python.org/cpython/rev/69c4fa62b608
changeset:   98597:69c4fa62b608
branch:      3.5
parent:      98593:1d14675c6050
user:        Steve Dower <steve.dower at microsoft.com>
date:        Thu Oct 08 08:56:06 2015 -0700
summary:
  Issue #23919: Prevents assert dialogs appearing in the test suite.

files:
  Lib/test/regrtest.py |  13 +++++++++++--
  Misc/NEWS            |   2 ++
  2 files changed, 13 insertions(+), 2 deletions(-)


diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py
--- a/Lib/test/regrtest.py
+++ b/Lib/test/regrtest.py
@@ -511,7 +511,13 @@
         import gc
         gc.set_threshold(ns.threshold)
     if ns.nowindows:
+        print('The --nowindows (-n) option is deprecated. '
+              'Use -vv to display assertions in stderr.')
+    try:
         import msvcrt
+    except ImportError:
+        pass
+    else:
         msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS|
                             msvcrt.SEM_NOALIGNMENTFAULTEXCEPT|
                             msvcrt.SEM_NOGPFAULTERRORBOX|
@@ -523,8 +529,11 @@
             pass
         else:
             for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]:
-                msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE)
-                msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR)
+                if ns.verbose and ns.verbose >= 2:
+                    msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE)
+                    msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR)
+                else:
+                    msvcrt.CrtSetReportMode(m, 0)
     if ns.wait:
         input("Press any key to continue...")
 
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -278,6 +278,8 @@
 Tests
 -----
 
+- Issue #23919: Prevents assert dialogs appearing in the test suite.
+
 - PCbuild\rt.bat now accepts an unlimited number of arguments to pass along
   to regrtest.py.  Previously there was a limit of 9.
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  8 19:01:27 2015
From: python-checkins at python.org (steve.dower)
Date: Thu, 08 Oct 2015 17:01:27 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy41KTogSXNzdWUgIzI1MDg5?=
 =?utf-8?q?=3A_Adds_logging_to_installer_for_case_where_launcher_is_not_se?=
 =?utf-8?q?lected?=
Message-ID: <20151008170113.70963.73432@psf.io>

https://hg.python.org/cpython/rev/1e99ba6b7c98
changeset:   98601:1e99ba6b7c98
branch:      3.5
parent:      98599:0b7df139a5f7
user:        Steve Dower <steve.dower at microsoft.com>
date:        Thu Oct 08 09:55:49 2015 -0700
summary:
  Issue #25089: Adds logging to installer for case where launcher is not selected on upgrade.

files:
  Misc/NEWS                                                    |  3 +++
  Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp |  7 +++++++
  2 files changed, 10 insertions(+), 0 deletions(-)


diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -297,6 +297,9 @@
 Windows
 -------
 
+- Issue #25089: Adds logging to installer for case where launcher is not
+  selected on upgrade.
+
 - Issue #25165: Windows uninstallation should not remove launcher if other
   versions remain
 
diff --git a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp
--- a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp
+++ b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp
@@ -687,6 +687,13 @@
             if (hr == S_FALSE) {
                 hr = LoadLauncherStateFromKey(_engine, HKEY_LOCAL_MACHINE);
             }
+            if (FAILED(hr)) {
+                BalLog(
+                    BOOTSTRAPPER_LOG_LEVEL_ERROR,
+                    "Failed to load launcher state: error code 0x%08X",
+                    hr
+                );
+            }
         } else if (BOOTSTRAPPER_RELATED_OPERATION_NONE == operation) {
             if (_command.action == BOOTSTRAPPER_ACTION_INSTALL) {
                 LOC_STRING *pLocString = nullptr;

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  8 19:01:31 2015
From: python-checkins at python.org (steve.dower)
Date: Thu, 08 Oct 2015 17:01:31 +0000
Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E5_-=3E_default?=
	=?utf-8?q?=29=3A_Merge_from_3=2E5?=
Message-ID: <20151008170114.128826.76489@psf.io>

https://hg.python.org/cpython/rev/898500fa5de3
changeset:   98602:898500fa5de3
parent:      98600:0b7cccd9c7d5
parent:      98601:1e99ba6b7c98
user:        Steve Dower <steve.dower at microsoft.com>
date:        Thu Oct 08 10:00:55 2015 -0700
summary:
  Merge from 3.5

files:
  Misc/NEWS                                                    |  3 +++
  Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp |  7 +++++++
  2 files changed, 10 insertions(+), 0 deletions(-)


diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -437,6 +437,9 @@
 Windows
 -------
 
+- Issue #25089: Adds logging to installer for case where launcher is not
+  selected on upgrade.
+
 - Issue #25165: Windows uninstallation should not remove launcher if other
   versions remain
 
diff --git a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp
--- a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp
+++ b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp
@@ -687,6 +687,13 @@
             if (hr == S_FALSE) {
                 hr = LoadLauncherStateFromKey(_engine, HKEY_LOCAL_MACHINE);
             }
+            if (FAILED(hr)) {
+                BalLog(
+                    BOOTSTRAPPER_LOG_LEVEL_ERROR,
+                    "Failed to load launcher state: error code 0x%08X",
+                    hr
+                );
+            }
         } else if (BOOTSTRAPPER_RELATED_OPERATION_NONE == operation) {
             if (_command.action == BOOTSTRAPPER_ACTION_INSTALL) {
                 LOC_STRING *pLocString = nullptr;

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Thu Oct  8 20:34:35 2015
From: python-checkins at python.org (steve.dower)
Date: Thu, 08 Oct 2015 18:34:35 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Fix_missing_import_in_libr?=
	=?utf-8?q?egrtest=2E?=
Message-ID: <20151008183434.451.16012@psf.io>

https://hg.python.org/cpython/rev/ee1ef5a97e8f
changeset:   98603:ee1ef5a97e8f
user:        Steve Dower <steve.dower at microsoft.com>
date:        Thu Oct 08 11:34:07 2015 -0700
summary:
  Fix missing import in libregrtest.

files:
  Lib/test/libregrtest/cmdline.py |  1 +
  1 files changed, 1 insertions(+), 0 deletions(-)


diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py
--- a/Lib/test/libregrtest/cmdline.py
+++ b/Lib/test/libregrtest/cmdline.py
@@ -1,5 +1,6 @@
 import argparse
 import os
+import sys
 from test import support
 
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  9 00:59:52 2015
From: python-checkins at python.org (victor.stinner)
Date: Thu, 08 Oct 2015 22:59:52 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2325318=3A_Add_=5FP?=
	=?utf-8?q?yBytesWriter_API?=
Message-ID: <20151008225951.18380.95196@psf.io>

https://hg.python.org/cpython/rev/1a2175149c5e
changeset:   98604:1a2175149c5e
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Fri Oct 09 00:33:49 2015 +0200
summary:
  Issue #25318: Add _PyBytesWriter API

Add a new private API to optimize Unicode encoders. It uses a small buffer
allocated on the stack and supports overallocation.

Use _PyBytesWriter API for UCS1 (ASCII and Latin1) and UTF-8 encoders. Enable
overallocation for the UTF-8 encoder with error handlers.

unicode_encode_ucs1(): initialize collend to collstart+1 to not check the
current character twice, we already know that it is not ASCII.

files:
  Include/unicodeobject.h    |    2 +-
  Objects/stringlib/codecs.h |   84 +----
  Objects/unicodeobject.c    |  320 +++++++++++++++++++-----
  3 files changed, 271 insertions(+), 135 deletions(-)


diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h
--- a/Include/unicodeobject.h
+++ b/Include/unicodeobject.h
@@ -908,7 +908,7 @@
     /* minimum character (default: 127, ASCII) */
     Py_UCS4 min_char;
 
-    /* If non-zero, overallocate the buffer by 25% (default: 0). */
+    /* If non-zero, overallocate the buffer (default: 0). */
     unsigned char overallocate;
 
     /* If readonly is 1, buffer is a shared string (cannot be modified)
diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h
--- a/Objects/stringlib/codecs.h
+++ b/Objects/stringlib/codecs.h
@@ -263,10 +263,7 @@
 #define MAX_SHORT_UNICHARS 300  /* largest size we'll do on the stack */
 
     Py_ssize_t i;                /* index into s of next input byte */
-    PyObject *result;            /* result string object */
     char *p;                     /* next free byte in output buffer */
-    Py_ssize_t nallocated;      /* number of result bytes allocated */
-    Py_ssize_t nneeded;            /* number of result bytes needed */
 #if STRINGLIB_SIZEOF_CHAR > 1
     PyObject *error_handler_obj = NULL;
     PyObject *exc = NULL;
@@ -275,38 +272,24 @@
 #endif
 #if STRINGLIB_SIZEOF_CHAR == 1
     const Py_ssize_t max_char_size = 2;
-    char stackbuf[MAX_SHORT_UNICHARS * 2];
 #elif STRINGLIB_SIZEOF_CHAR == 2
     const Py_ssize_t max_char_size = 3;
-    char stackbuf[MAX_SHORT_UNICHARS * 3];
 #else /*  STRINGLIB_SIZEOF_CHAR == 4 */
     const Py_ssize_t max_char_size = 4;
-    char stackbuf[MAX_SHORT_UNICHARS * 4];
 #endif
+    _PyBytesWriter writer;
 
     assert(size >= 0);
+    _PyBytesWriter_Init(&writer);
 
-    if (size <= MAX_SHORT_UNICHARS) {
-        /* Write into the stack buffer; nallocated can't overflow.
-         * At the end, we'll allocate exactly as much heap space as it
-         * turns out we need.
-         */
-        nallocated = Py_SAFE_DOWNCAST(sizeof(stackbuf), size_t, int);
-        result = NULL;   /* will allocate after we're done */
-        p = stackbuf;
+    if (size > PY_SSIZE_T_MAX / max_char_size) {
+        /* integer overflow */
+        return PyErr_NoMemory();
     }
-    else {
-        if (size > PY_SSIZE_T_MAX / max_char_size) {
-            /* integer overflow */
-            return PyErr_NoMemory();
-        }
-        /* Overallocate on the heap, and give the excess back at the end. */
-        nallocated = size * max_char_size;
-        result = PyBytes_FromStringAndSize(NULL, nallocated);
-        if (result == NULL)
-            return NULL;
-        p = PyBytes_AS_STRING(result);
-    }
+
+    p = _PyBytesWriter_Alloc(&writer, size * max_char_size);
+    if (p == NULL)
+        return NULL;
 
     for (i = 0; i < size;) {
         Py_UCS4 ch = data[i++];
@@ -338,6 +321,9 @@
             while ((endpos < size) && Py_UNICODE_IS_SURROGATE(data[endpos]))
                 endpos++;
 
+            /* Only overallocate the buffer if it's not the last write */
+            writer.overallocate = (endpos < size);
+
             switch (error_handler)
             {
             case _Py_ERROR_REPLACE:
@@ -387,29 +373,10 @@
                     repsize = PyUnicode_GET_LENGTH(rep);
 
                 if (repsize > max_char_size) {
-                    Py_ssize_t offset;
-
-                    if (result == NULL)
-                        offset = p - stackbuf;
-                    else
-                        offset = p - PyBytes_AS_STRING(result);
-
-                    if (nallocated > PY_SSIZE_T_MAX - repsize + max_char_size) {
-                        /* integer overflow */
-                        PyErr_NoMemory();
+                    p = _PyBytesWriter_Prepare(&writer, p,
+                                               repsize - max_char_size);
+                    if (p == NULL)
                         goto error;
-                    }
-                    nallocated += repsize - max_char_size;
-                    if (result != NULL) {
-                        if (_PyBytes_Resize(&result, nallocated) < 0)
-                            goto error;
-                    } else {
-                        result = PyBytes_FromStringAndSize(NULL, nallocated);
-                        if (result == NULL)
-                            goto error;
-                        Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset);
-                    }
-                    p = PyBytes_AS_STRING(result) + offset;
                 }
 
                 if (PyBytes_Check(rep)) {
@@ -437,6 +404,10 @@
 
                 i = newpos;
             }
+
+            /* If overallocation was disabled, ensure that it was the last
+               write. Otherwise, we missed an optimization */
+            assert(writer.overallocate || i == size);
         }
         else
 #if STRINGLIB_SIZEOF_CHAR > 2
@@ -461,31 +432,18 @@
 #endif /* STRINGLIB_SIZEOF_CHAR > 1 */
     }
 
-    if (result == NULL) {
-        /* This was stack allocated. */
-        nneeded = p - stackbuf;
-        assert(nneeded <= nallocated);
-        result = PyBytes_FromStringAndSize(stackbuf, nneeded);
-    }
-    else {
-        /* Cut back to size actually needed. */
-        nneeded = p - PyBytes_AS_STRING(result);
-        assert(nneeded <= nallocated);
-        _PyBytes_Resize(&result, nneeded);
-    }
-
 #if STRINGLIB_SIZEOF_CHAR > 1
     Py_XDECREF(error_handler_obj);
     Py_XDECREF(exc);
 #endif
-    return result;
+    return _PyBytesWriter_Finish(&writer, p);
 
 #if STRINGLIB_SIZEOF_CHAR > 1
  error:
     Py_XDECREF(rep);
     Py_XDECREF(error_handler_obj);
     Py_XDECREF(exc);
-    Py_XDECREF(result);
+    _PyBytesWriter_Dealloc(&writer);
     return NULL;
 #endif
 
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -163,6 +163,14 @@
             *_to++ = (to_type) *_iter++;                \
     } while (0)
 
+#ifdef MS_WINDOWS
+   /* On Windows, overallocate by 50% is the best factor */
+#  define OVERALLOCATE_FACTOR 2
+#else
+   /* On Linux, overallocate by 25% is the best factor */
+#  define OVERALLOCATE_FACTOR 4
+#endif
+
 /* This dictionary holds all interned unicode strings.  Note that references
    to strings in this dictionary are *not* counted in the string's ob_refcnt.
    When the interned string reaches a refcnt of 0 the string deallocation
@@ -338,6 +346,216 @@
 #endif
 }
 
+/* The _PyBytesWriter structure is big: it contains an embeded "stack buffer".
+   A _PyBytesWriter variable must be declared at the end of variables in a
+   function to optimize the memory allocation on the stack. */
+typedef struct {
+    /* bytes object */
+    PyObject *buffer;
+
+    /* Number of allocated size */
+    Py_ssize_t allocated;
+
+    /* Current size of the buffer (can be smaller than the allocated size) */
+    Py_ssize_t size;
+
+    /* If non-zero, overallocate the buffer (default: 0). */
+    int overallocate;
+
+    /* Stack buffer */
+    int use_stack_buffer;
+    char stack_buffer[512];
+} _PyBytesWriter;
+
+static void
+_PyBytesWriter_Init(_PyBytesWriter *writer)
+{
+    writer->buffer = NULL;
+    writer->allocated = 0;
+    writer->size = 0;
+    writer->overallocate = 0;
+    writer->use_stack_buffer = 0;
+#ifdef Py_DEBUG
+    memset(writer->stack_buffer, 0xCB, sizeof(writer->stack_buffer));
+#endif
+}
+
+static void
+_PyBytesWriter_Dealloc(_PyBytesWriter *writer)
+{
+    Py_CLEAR(writer->buffer);
+}
+
+static char*
+_PyBytesWriter_AsString(_PyBytesWriter *writer)
+{
+    if (!writer->use_stack_buffer) {
+        assert(writer->buffer != NULL);
+        return PyBytes_AS_STRING(writer->buffer);
+    }
+    else {
+        assert(writer->buffer == NULL);
+        return writer->stack_buffer;
+    }
+}
+
+Py_LOCAL_INLINE(Py_ssize_t)
+_PyBytesWriter_GetPos(_PyBytesWriter *writer, char *str)
+{
+    char *start = _PyBytesWriter_AsString(writer);
+    assert(str != NULL);
+    assert(str >= start);
+    return str - start;
+}
+
+Py_LOCAL_INLINE(void)
+_PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str)
+{
+#ifdef Py_DEBUG
+    char *start, *end;
+
+    if (!writer->use_stack_buffer) {
+        assert(writer->buffer != NULL);
+        assert(PyBytes_CheckExact(writer->buffer));
+        assert(Py_REFCNT(writer->buffer) == 1);
+    }
+    else {
+        assert(writer->buffer == NULL);
+    }
+
+    start = _PyBytesWriter_AsString(writer);
+    assert(0 <= writer->size && writer->size <= writer->allocated);
+    /* the last byte must always be null */
+    assert(start[writer->allocated] == 0);
+
+    end = start + writer->allocated;
+    assert(str != NULL);
+    assert(start <= str && str <= end);
+#endif
+}
+
+/* Add *size* bytes to the buffer.
+   str is the current pointer inside the buffer.
+   Return the updated current pointer inside the buffer.
+   Raise an exception and return NULL on error. */
+static char*
+_PyBytesWriter_Prepare(_PyBytesWriter *writer, char *str, Py_ssize_t size)
+{
+    Py_ssize_t allocated, pos;
+
+    _PyBytesWriter_CheckConsistency(writer, str);
+    assert(size >= 0);
+
+    if (size == 0) {
+        /* nothing to do */
+        return str;
+    }
+
+    if (writer->size > PY_SSIZE_T_MAX - size) {
+        PyErr_NoMemory();
+        _PyBytesWriter_Dealloc(writer);
+        return NULL;
+    }
+    writer->size += size;
+
+    allocated = writer->allocated;
+    if (writer->size <= allocated)
+        return str;
+
+    allocated = writer->size;
+    if (writer->overallocate
+        && allocated <= (PY_SSIZE_T_MAX - allocated / OVERALLOCATE_FACTOR)) {
+        /* overallocate to limit the number of realloc() */
+        allocated += allocated / OVERALLOCATE_FACTOR;
+    }
+
+    pos = _PyBytesWriter_GetPos(writer, str);
+    if (!writer->use_stack_buffer) {
+        /* Note: Don't use a bytearray object because the conversion from
+           byterray to bytes requires to copy all bytes. */
+        if (_PyBytes_Resize(&writer->buffer, allocated)) {
+            assert(writer->buffer == NULL);
+            return NULL;
+        }
+    }
+    else {
+        /* convert from stack buffer to bytes object buffer */
+        assert(writer->buffer == NULL);
+
+        writer->buffer = PyBytes_FromStringAndSize(NULL, allocated);
+        if (writer->buffer == NULL)
+            return NULL;
+
+        if (pos != 0) {
+            Py_MEMCPY(PyBytes_AS_STRING(writer->buffer),
+                      writer->stack_buffer,
+                      pos);
+        }
+
+#ifdef Py_DEBUG
+        memset(writer->stack_buffer, 0xDB, sizeof(writer->stack_buffer));
+#endif
+
+        writer->use_stack_buffer = 0;
+    }
+    writer->allocated = allocated;
+
+    str = _PyBytesWriter_AsString(writer) + pos;
+    _PyBytesWriter_CheckConsistency(writer, str);
+    return str;
+}
+
+/* Allocate the buffer to write size bytes.
+   Return the pointer to the beginning of buffer data.
+   Raise an exception and return NULL on error. */
+static char*
+_PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size)
+{
+    /* ensure that _PyBytesWriter_Alloc() is only called once */
+    assert(writer->size == 0 && writer->buffer == NULL);
+    assert(size >= 0);
+
+    writer->use_stack_buffer = 1;
+#if Py_DEBUG
+    /* the last byte is reserved, it must be '\0' */
+    writer->stack_buffer[sizeof(writer->stack_buffer) - 1] = 0;
+    writer->allocated = sizeof(writer->stack_buffer) - 1;
+#else
+    writer->allocated = sizeof(writer->stack_buffer);
+#endif
+    return _PyBytesWriter_Prepare(writer, writer->stack_buffer, size);
+}
+
+/* Get the buffer content and reset the writer.
+   Return a bytes object.
+   Raise an exception and return NULL on error. */
+static PyObject *
+_PyBytesWriter_Finish(_PyBytesWriter *writer, char *str)
+{
+    Py_ssize_t pos;
+    PyObject *result;
+
+    _PyBytesWriter_CheckConsistency(writer, str);
+
+    pos = _PyBytesWriter_GetPos(writer, str);
+    if (!writer->use_stack_buffer) {
+        if (pos != writer->allocated) {
+            if (_PyBytes_Resize(&writer->buffer, pos)) {
+                assert(writer->buffer == NULL);
+                return NULL;
+            }
+        }
+
+        result = writer->buffer;
+        writer->buffer = NULL;
+    }
+    else {
+        result = PyBytes_FromStringAndSize(writer->stack_buffer, pos);
+    }
+
+    return result;
+}
+
 #ifdef Py_DEBUG
 int
 _PyUnicode_CheckConsistency(PyObject *op, int check_content)
@@ -6460,17 +6678,15 @@
     Py_ssize_t pos=0, size;
     int kind;
     void *data;
-    /* output object */
-    PyObject *res;
     /* pointer into the output */
     char *str;
-    /* current output position */
-    Py_ssize_t ressize;
     const char *encoding = (limit == 256) ? "latin-1" : "ascii";
     const char *reason = (limit == 256) ? "ordinal not in range(256)" : "ordinal not in range(128)";
     PyObject *error_handler_obj = NULL;
     PyObject *exc = NULL;
     _Py_error_handler error_handler = _Py_ERROR_UNKNOWN;
+    /* output object */
+    _PyBytesWriter writer;
 
     if (PyUnicode_READY(unicode) == -1)
         return NULL;
@@ -6481,11 +6697,11 @@
        replacements, if we need more, we'll resize */
     if (size == 0)
         return PyBytes_FromStringAndSize(NULL, 0);
-    res = PyBytes_FromStringAndSize(NULL, size);
-    if (res == NULL)
-        return NULL;
-    str = PyBytes_AS_STRING(res);
-    ressize = size;
+
+    _PyBytesWriter_Init(&writer);
+    str = _PyBytesWriter_Alloc(&writer, size);
+    if (str == NULL)
+        return NULL;
 
     while (pos < size) {
         Py_UCS4 ch = PyUnicode_READ(kind, data, pos);
@@ -6499,15 +6715,18 @@
         else {
             Py_ssize_t requiredsize;
             PyObject *repunicode;
-            Py_ssize_t repsize, newpos, respos, i;
+            Py_ssize_t repsize, newpos, i;
             /* startpos for collecting unencodable chars */
             Py_ssize_t collstart = pos;
-            Py_ssize_t collend = pos;
+            Py_ssize_t collend = collstart + 1;
             /* find all unecodable characters */
 
             while ((collend < size) && (PyUnicode_READ(kind, data, collend) >= limit))
                 ++collend;
 
+            /* Only overallocate the buffer if it's not the last write */
+            writer.overallocate = (collend < size);
+
             /* cache callback name lookup (if not done yet, i.e. it's the first error) */
             if (error_handler == _Py_ERROR_UNKNOWN)
                 error_handler = get_error_handler(errors);
@@ -6526,8 +6745,7 @@
                 break;
 
             case _Py_ERROR_XMLCHARREFREPLACE:
-                respos = str - PyBytes_AS_STRING(res);
-                requiredsize = respos;
+                requiredsize = 0;
                 /* determine replacement size */
                 for (i = collstart; i < collend; ++i) {
                     Py_ssize_t incr;
@@ -6553,17 +6771,11 @@
                         goto overflow;
                     requiredsize += incr;
                 }
-                if (requiredsize > PY_SSIZE_T_MAX - (size - collend))
-                    goto overflow;
-                requiredsize += size - collend;
-                if (requiredsize > ressize) {
-                    if (ressize <= PY_SSIZE_T_MAX/2 && requiredsize < 2*ressize)
-                        requiredsize = 2*ressize;
-                    if (_PyBytes_Resize(&res, requiredsize))
-                        goto onError;
-                    str = PyBytes_AS_STRING(res) + respos;
-                    ressize = requiredsize;
-                }
+
+                str = _PyBytesWriter_Prepare(&writer, str, requiredsize-1);
+                if (str == NULL)
+                    goto onError;
+
                 /* generate replacement */
                 for (i = collstart; i < collend; ++i) {
                     str += sprintf(str, "&#%d;", PyUnicode_READ(kind, data, i));
@@ -6598,20 +6810,9 @@
                 if (PyBytes_Check(repunicode)) {
                     /* Directly copy bytes result to output. */
                     repsize = PyBytes_Size(repunicode);
-                    if (repsize > 1) {
-                        /* Make room for all additional bytes. */
-                        respos = str - PyBytes_AS_STRING(res);
-                        if (ressize > PY_SSIZE_T_MAX - repsize - 1) {
-                            Py_DECREF(repunicode);
-                            goto overflow;
-                        }
-                        if (_PyBytes_Resize(&res, ressize+repsize-1)) {
-                            Py_DECREF(repunicode);
-                            goto onError;
-                        }
-                        str = PyBytes_AS_STRING(res) + respos;
-                        ressize += repsize-1;
-                    }
+                    str = _PyBytesWriter_Prepare(&writer, str, repsize-1);
+                    if (str == NULL)
+                        goto onError;
                     memcpy(str, PyBytes_AsString(repunicode), repsize);
                     str += repsize;
                     pos = newpos;
@@ -6622,24 +6823,11 @@
                 /* need more space? (at least enough for what we
                    have+the replacement+the rest of the string, so
                    we won't have to check space for encodable characters) */
-                respos = str - PyBytes_AS_STRING(res);
                 repsize = PyUnicode_GET_LENGTH(repunicode);
-                requiredsize = respos;
-                if (requiredsize > PY_SSIZE_T_MAX - repsize)
-                    goto overflow;
-                requiredsize += repsize;
-                if (requiredsize > PY_SSIZE_T_MAX - (size - collend))
-                    goto overflow;
-                requiredsize += size - collend;
-                if (requiredsize > ressize) {
-                    if (ressize <= PY_SSIZE_T_MAX/2 && requiredsize < 2*ressize)
-                        requiredsize = 2*ressize;
-                    if (_PyBytes_Resize(&res, requiredsize)) {
-                        Py_DECREF(repunicode);
+                if (repsize > 1) {
+                    str = _PyBytesWriter_Prepare(&writer, str, repsize-1);
+                    if (str == NULL)
                         goto onError;
-                    }
-                    str = PyBytes_AS_STRING(res) + respos;
-                    ressize = requiredsize;
                 }
 
                 /* check if there is anything unencodable in the replacement
@@ -6657,26 +6845,23 @@
                 pos = newpos;
                 Py_DECREF(repunicode);
             }
-        }
-    }
-    /* Resize if we allocated to much */
-    size = str - PyBytes_AS_STRING(res);
-    if (size < ressize) { /* If this falls res will be NULL */
-        assert(size >= 0);
-        if (_PyBytes_Resize(&res, size) < 0)
-            goto onError;
+
+            /* If overallocation was disabled, ensure that it was the last
+               write. Otherwise, we missed an optimization */
+            assert(writer.overallocate || pos == size);
+        }
     }
 
     Py_XDECREF(error_handler_obj);
     Py_XDECREF(exc);
-    return res;
+    return _PyBytesWriter_Finish(&writer, str);
 
   overflow:
     PyErr_SetString(PyExc_OverflowError,
                     "encoded result is too long for a Python string");
 
   onError:
-    Py_XDECREF(res);
+    _PyBytesWriter_Dealloc(&writer);
     Py_XDECREF(error_handler_obj);
     Py_XDECREF(exc);
     return NULL;
@@ -13366,13 +13551,6 @@
 _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer,
                                  Py_ssize_t length, Py_UCS4 maxchar)
 {
-#ifdef MS_WINDOWS
-   /* On Windows, overallocate by 50% is the best factor */
-#  define OVERALLOCATE_FACTOR 2
-#else
-   /* On Linux, overallocate by 25% is the best factor */
-#  define OVERALLOCATE_FACTOR 4
-#endif
     Py_ssize_t newlen;
     PyObject *newbuffer;
 

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  9 01:46:58 2015
From: python-checkins at python.org (victor.stinner)
Date: Thu, 08 Oct 2015 23:46:58 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Optimize_backslashreplace_?=
	=?utf-8?q?error_handler?=
Message-ID: <20151008234645.97706.9292@psf.io>

https://hg.python.org/cpython/rev/59f4806a5add
changeset:   98605:59f4806a5add
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Fri Oct 09 01:39:28 2015 +0200
summary:
  Optimize backslashreplace error handler

Issue #25318: Optimize backslashreplace and xmlcharrefreplace error handlers in
UTF-8 encoder. Optimize also backslashreplace error handler for ASCII and
Latin1 encoders.

Use the new _PyBytesWriter API to optimize these error handlers for the
encoders. It avoids to create an exception and call the slow implementation of
the error handler.

files:
  Objects/stringlib/codecs.h |   18 ++-
  Objects/unicodeobject.c    |  195 ++++++++++++++++++------
  2 files changed, 161 insertions(+), 52 deletions(-)


diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h
--- a/Objects/stringlib/codecs.h
+++ b/Objects/stringlib/codecs.h
@@ -334,7 +334,6 @@
                 i += (endpos - startpos - 1);
                 break;
 
-
             case _Py_ERROR_SURROGATEPASS:
                 for (k=startpos; k<endpos; k++) {
                     ch = data[k];
@@ -345,6 +344,22 @@
                 i += (endpos - startpos - 1);
                 break;
 
+            case _Py_ERROR_BACKSLASHREPLACE:
+                p = backslashreplace(&writer, max_char_size, p,
+                                     unicode, startpos, endpos);
+                if (p == NULL)
+                    goto error;
+                i += (endpos - startpos - 1);
+                break;
+
+            case _Py_ERROR_XMLCHARREFREPLACE:
+                p = xmlcharrefreplace(&writer, max_char_size, p,
+                                      unicode, startpos, endpos);
+                if (p == NULL)
+                    goto error;
+                i += (endpos - startpos - 1);
+                break;
+
             case _Py_ERROR_SURROGATEESCAPE:
                 for (k=startpos; k<endpos; k++) {
                     ch = data[k];
@@ -359,7 +374,6 @@
                 startpos = k;
                 assert(startpos < endpos);
                 /* fall through the default handler */
-
             default:
                 rep = unicode_encode_call_errorhandler(
                       errors, &error_handler_obj, "utf-8", "surrogates not allowed",
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -305,9 +305,10 @@
     _Py_ERROR_UNKNOWN=0,
     _Py_ERROR_STRICT,
     _Py_ERROR_SURROGATEESCAPE,
-    _Py_ERROR_SURROGATEPASS,
     _Py_ERROR_REPLACE,
     _Py_ERROR_IGNORE,
+    _Py_ERROR_BACKSLASHREPLACE,
+    _Py_ERROR_SURROGATEPASS,
     _Py_ERROR_XMLCHARREFREPLACE,
     _Py_ERROR_OTHER
 } _Py_error_handler;
@@ -315,18 +316,18 @@
 static _Py_error_handler
 get_error_handler(const char *errors)
 {
-    if (errors == NULL)
-        return _Py_ERROR_STRICT;
-    if (strcmp(errors, "strict") == 0)
+    if (errors == NULL || strcmp(errors, "strict") == 0)
         return _Py_ERROR_STRICT;
     if (strcmp(errors, "surrogateescape") == 0)
         return _Py_ERROR_SURROGATEESCAPE;
+    if (strcmp(errors, "replace") == 0)
+        return _Py_ERROR_REPLACE;
+    if (strcmp(errors, "ignore") == 0)
+        return _Py_ERROR_IGNORE;
+    if (strcmp(errors, "backslashreplace") == 0)
+        return _Py_ERROR_BACKSLASHREPLACE;
     if (strcmp(errors, "surrogatepass") == 0)
         return _Py_ERROR_SURROGATEPASS;
-    if (strcmp(errors, "ignore") == 0)
-        return _Py_ERROR_IGNORE;
-    if (strcmp(errors, "replace") == 0)
-        return _Py_ERROR_REPLACE;
     if (strcmp(errors, "xmlcharrefreplace") == 0)
         return _Py_ERROR_XMLCHARREFREPLACE;
     return _Py_ERROR_OTHER;
@@ -771,6 +772,126 @@
         return _PyUnicode_Copy(unicode);
 }
 
+/* Implementation of the "backslashreplace" error handler for 8-bit encodings:
+   ASCII, Latin1, UTF-8, etc. */
+static char*
+backslashreplace(_PyBytesWriter *writer, Py_ssize_t prealloc_per_char,
+                  char *str,
+                 PyObject *unicode, Py_ssize_t collstart, Py_ssize_t collend)
+{
+    Py_ssize_t size, i, prealloc;
+    Py_UCS4 ch;
+    enum PyUnicode_Kind kind;
+    void *data;
+
+    assert(PyUnicode_IS_READY(unicode));
+    kind = PyUnicode_KIND(unicode);
+    data = PyUnicode_DATA(unicode);
+
+    size = 0;
+    /* determine replacement size */
+    for (i = collstart; i < collend; ++i) {
+        Py_ssize_t incr;
+
+        ch = PyUnicode_READ(kind, data, i);
+        if (ch < 0x100)
+            incr = 2+2;
+        else if (ch < 0x10000)
+            incr = 2+4;
+        else {
+            assert(ch <= MAX_UNICODE);
+            incr = 2+6;
+        }
+        if (size > PY_SSIZE_T_MAX - incr) {
+            PyErr_SetString(PyExc_OverflowError,
+                            "encoded result is too long for a Python string");
+            return NULL;
+        }
+        size += incr;
+    }
+
+    prealloc = prealloc_per_char * (collend - collstart);
+    if (size > prealloc) {
+        str = _PyBytesWriter_Prepare(writer, str, size - prealloc);
+        if (str == NULL)
+            return NULL;
+    }
+
+    /* generate replacement */
+    for (i = collstart; i < collend; ++i) {
+        ch = PyUnicode_READ(kind, data, i);
+        if (ch < 0x100)
+            str += sprintf(str, "\\x%02x", ch);
+        else if (ch < 0x10000)
+            str += sprintf(str, "\\u%04x", ch);
+        else {
+            assert(ch <= MAX_UNICODE);
+            str += sprintf(str, "\\U%08x", ch);
+        }
+    }
+    return str;
+}
+
+/* Implementation of the "xmlcharrefreplace" error handler for 8-bit encodings:
+   ASCII, Latin1, UTF-8, etc. */
+static char*
+xmlcharrefreplace(_PyBytesWriter *writer, Py_ssize_t prealloc_per_char,
+                  char *str,
+                  PyObject *unicode, Py_ssize_t collstart, Py_ssize_t collend)
+{
+    Py_ssize_t size, i, prealloc;
+    Py_UCS4 ch;
+    enum PyUnicode_Kind kind;
+    void *data;
+
+    assert(PyUnicode_IS_READY(unicode));
+    kind = PyUnicode_KIND(unicode);
+    data = PyUnicode_DATA(unicode);
+
+    size = 0;
+    /* determine replacement size */
+    for (i = collstart; i < collend; ++i) {
+        Py_ssize_t incr;
+
+        ch = PyUnicode_READ(kind, data, i);
+        if (ch < 10)
+            incr = 2+1+1;
+        else if (ch < 100)
+            incr = 2+2+1;
+        else if (ch < 1000)
+            incr = 2+3+1;
+        else if (ch < 10000)
+            incr = 2+4+1;
+        else if (ch < 100000)
+            incr = 2+5+1;
+        else if (ch < 1000000)
+            incr = 2+6+1;
+        else {
+            assert(ch <= MAX_UNICODE);
+            incr = 2+7+1;
+        }
+        if (size > PY_SSIZE_T_MAX - incr) {
+            PyErr_SetString(PyExc_OverflowError,
+                            "encoded result is too long for a Python string");
+            return NULL;
+        }
+        size += incr;
+    }
+
+    prealloc = prealloc_per_char * (collend - collstart);
+    if (size > prealloc) {
+        str = _PyBytesWriter_Prepare(writer, str, size - prealloc);
+        if (str == NULL)
+            return NULL;
+    }
+
+    /* generate replacement */
+    for (i = collstart; i < collend; ++i) {
+        str += sprintf(str, "&#%d;", PyUnicode_READ(kind, data, i));
+    }
+    return str;
+}
+
 /* --- Bloom Filters ----------------------------------------------------- */
 
 /* stuff to implement simple "bloom filters" for Unicode characters.
@@ -6713,7 +6834,6 @@
             ++pos;
         }
         else {
-            Py_ssize_t requiredsize;
             PyObject *repunicode;
             Py_ssize_t repsize, newpos, i;
             /* startpos for collecting unencodable chars */
@@ -6744,42 +6864,19 @@
                 pos = collend;
                 break;
 
-            case _Py_ERROR_XMLCHARREFREPLACE:
-                requiredsize = 0;
-                /* determine replacement size */
-                for (i = collstart; i < collend; ++i) {
-                    Py_ssize_t incr;
-
-                    ch = PyUnicode_READ(kind, data, i);
-                    if (ch < 10)
-                        incr = 2+1+1;
-                    else if (ch < 100)
-                        incr = 2+2+1;
-                    else if (ch < 1000)
-                        incr = 2+3+1;
-                    else if (ch < 10000)
-                        incr = 2+4+1;
-                    else if (ch < 100000)
-                        incr = 2+5+1;
-                    else if (ch < 1000000)
-                        incr = 2+6+1;
-                    else {
-                        assert(ch <= MAX_UNICODE);
-                        incr = 2+7+1;
-                    }
-                    if (requiredsize > PY_SSIZE_T_MAX - incr)
-                        goto overflow;
-                    requiredsize += incr;
-                }
-
-                str = _PyBytesWriter_Prepare(&writer, str, requiredsize-1);
+            case _Py_ERROR_BACKSLASHREPLACE:
+                str = backslashreplace(&writer, 1, str,
+                                       unicode, collstart, collend);
                 if (str == NULL)
                     goto onError;
-
-                /* generate replacement */
-                for (i = collstart; i < collend; ++i) {
-                    str += sprintf(str, "&#%d;", PyUnicode_READ(kind, data, i));
-                }
+                pos = collend;
+                break;
+
+            case _Py_ERROR_XMLCHARREFREPLACE:
+                str = xmlcharrefreplace(&writer, 1, str,
+                                        unicode, collstart, collend);
+                if (str == NULL)
+                    goto onError;
                 pos = collend;
                 break;
 
@@ -6810,9 +6907,11 @@
                 if (PyBytes_Check(repunicode)) {
                     /* Directly copy bytes result to output. */
                     repsize = PyBytes_Size(repunicode);
-                    str = _PyBytesWriter_Prepare(&writer, str, repsize-1);
-                    if (str == NULL)
-                        goto onError;
+                    if (repsize > 1) {
+                        str = _PyBytesWriter_Prepare(&writer, str, repsize-1);
+                        if (str == NULL)
+                            goto onError;
+                    }
                     memcpy(str, PyBytes_AsString(repunicode), repsize);
                     str += repsize;
                     pos = newpos;
@@ -6856,10 +6955,6 @@
     Py_XDECREF(exc);
     return _PyBytesWriter_Finish(&writer, str);
 
-  overflow:
-    PyErr_SetString(PyExc_OverflowError,
-                    "encoded result is too long for a Python string");
-
   onError:
     _PyBytesWriter_Dealloc(&writer);
     Py_XDECREF(error_handler_obj);

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  9 02:32:50 2015
From: python-checkins at python.org (victor.stinner)
Date: Fri, 09 Oct 2015 00:32:50 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2325318=3A_Move_=5F?=
 =?utf-8?q?PyBytesWriter_to_bytesobject=2Ec?=
Message-ID: <20151009003249.20759.13167@psf.io>

https://hg.python.org/cpython/rev/c134eddcb347
changeset:   98606:c134eddcb347
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Fri Oct 09 01:53:21 2015 +0200
summary:
  Issue #25318: Move _PyBytesWriter to bytesobject.c

Declare also the private API in bytesobject.h.

files:
  Include/bytesobject.h   |   52 ++++++
  Objects/bytesobject.c   |  193 +++++++++++++++++++++++++
  Objects/unicodeobject.c |  210 ----------------------------
  3 files changed, 245 insertions(+), 210 deletions(-)


diff --git a/Include/bytesobject.h b/Include/bytesobject.h
--- a/Include/bytesobject.h
+++ b/Include/bytesobject.h
@@ -123,6 +123,58 @@
 #define F_ALT	(1<<3)
 #define F_ZERO	(1<<4)
 
+#ifndef Py_LIMITED_API
+/* The _PyBytesWriter structure is big: it contains an embeded "stack buffer".
+   A _PyBytesWriter variable must be declared at the end of variables in a
+   function to optimize the memory allocation on the stack. */
+typedef struct {
+    /* bytes object */
+    PyObject *buffer;
+
+    /* Number of allocated size */
+    Py_ssize_t allocated;
+
+    /* Current size of the buffer (can be smaller than the allocated size) */
+    Py_ssize_t size;
+
+    /* If non-zero, overallocate the buffer (default: 0). */
+    int overallocate;
+
+    /* Stack buffer */
+    int use_stack_buffer;
+    char stack_buffer[512];
+} _PyBytesWriter;
+
+/* Initialize a bytes writer
+
+   By default, the overallocation is disabled. Set the overallocate attribute
+   to control the allocation of the buffer. */
+PyAPI_FUNC(void) _PyBytesWriter_Init(_PyBytesWriter *writer);
+
+/* Get the buffer content and reset the writer.
+   Return a bytes object.
+   Raise an exception and return NULL on error. */
+PyAPI_FUNC(PyObject *) _PyBytesWriter_Finish(_PyBytesWriter *writer,
+    char *str);
+
+/* Deallocate memory of a writer (clear its internal buffer). */
+PyAPI_FUNC(void) _PyBytesWriter_Dealloc(_PyBytesWriter *writer);
+
+/* Allocate the buffer to write size bytes.
+   Return the pointer to the beginning of buffer data.
+   Raise an exception and return NULL on error. */
+PyAPI_FUNC(char*) _PyBytesWriter_Alloc(_PyBytesWriter *writer,
+    Py_ssize_t size);
+
+/* Add *size* bytes to the buffer.
+   str is the current pointer inside the buffer.
+   Return the updated current pointer inside the buffer.
+   Raise an exception and return NULL on error. */
+PyAPI_FUNC(char*) _PyBytesWriter_Prepare(_PyBytesWriter *writer,
+    char *str,
+    Py_ssize_t size);
+#endif   /* Py_LIMITED_API */
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c
--- a/Objects/bytesobject.c
+++ b/Objects/bytesobject.c
@@ -3735,3 +3735,196 @@
     _PyObject_GC_TRACK(it);
     return (PyObject *)it;
 }
+
+
+/* _PyBytesWriter API */
+
+#ifdef MS_WINDOWS
+   /* On Windows, overallocate by 50% is the best factor */
+#  define OVERALLOCATE_FACTOR 2
+#else
+   /* On Linux, overallocate by 25% is the best factor */
+#  define OVERALLOCATE_FACTOR 4
+#endif
+
+void
+_PyBytesWriter_Init(_PyBytesWriter *writer)
+{
+    writer->buffer = NULL;
+    writer->allocated = 0;
+    writer->size = 0;
+    writer->overallocate = 0;
+    writer->use_stack_buffer = 0;
+#ifdef Py_DEBUG
+    memset(writer->stack_buffer, 0xCB, sizeof(writer->stack_buffer));
+#endif
+}
+
+void
+_PyBytesWriter_Dealloc(_PyBytesWriter *writer)
+{
+    Py_CLEAR(writer->buffer);
+}
+
+Py_LOCAL_INLINE(char*)
+_PyBytesWriter_AsString(_PyBytesWriter *writer)
+{
+    if (!writer->use_stack_buffer) {
+        assert(writer->buffer != NULL);
+        return PyBytes_AS_STRING(writer->buffer);
+    }
+    else {
+        assert(writer->buffer == NULL);
+        return writer->stack_buffer;
+    }
+}
+
+Py_LOCAL_INLINE(Py_ssize_t)
+_PyBytesWriter_GetPos(_PyBytesWriter *writer, char *str)
+{
+    char *start = _PyBytesWriter_AsString(writer);
+    assert(str != NULL);
+    assert(str >= start);
+    return str - start;
+}
+
+Py_LOCAL_INLINE(void)
+_PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str)
+{
+#ifdef Py_DEBUG
+    char *start, *end;
+
+    if (!writer->use_stack_buffer) {
+        assert(writer->buffer != NULL);
+        assert(PyBytes_CheckExact(writer->buffer));
+        assert(Py_REFCNT(writer->buffer) == 1);
+    }
+    else {
+        assert(writer->buffer == NULL);
+    }
+
+    start = _PyBytesWriter_AsString(writer);
+    assert(0 <= writer->size && writer->size <= writer->allocated);
+    /* the last byte must always be null */
+    assert(start[writer->allocated] == 0);
+
+    end = start + writer->allocated;
+    assert(str != NULL);
+    assert(start <= str && str <= end);
+#endif
+}
+
+char*
+_PyBytesWriter_Prepare(_PyBytesWriter *writer, char *str, Py_ssize_t size)
+{
+    Py_ssize_t allocated, pos;
+
+    _PyBytesWriter_CheckConsistency(writer, str);
+    assert(size >= 0);
+
+    if (size == 0) {
+        /* nothing to do */
+        return str;
+    }
+
+    if (writer->size > PY_SSIZE_T_MAX - size) {
+        PyErr_NoMemory();
+        _PyBytesWriter_Dealloc(writer);
+        return NULL;
+    }
+    writer->size += size;
+
+    allocated = writer->allocated;
+    if (writer->size <= allocated)
+        return str;
+
+    allocated = writer->size;
+    if (writer->overallocate
+        && allocated <= (PY_SSIZE_T_MAX - allocated / OVERALLOCATE_FACTOR)) {
+        /* overallocate to limit the number of realloc() */
+        allocated += allocated / OVERALLOCATE_FACTOR;
+    }
+
+    pos = _PyBytesWriter_GetPos(writer, str);
+    if (!writer->use_stack_buffer) {
+        /* Note: Don't use a bytearray object because the conversion from
+           byterray to bytes requires to copy all bytes. */
+        if (_PyBytes_Resize(&writer->buffer, allocated)) {
+            assert(writer->buffer == NULL);
+            return NULL;
+        }
+    }
+    else {
+        /* convert from stack buffer to bytes object buffer */
+        assert(writer->buffer == NULL);
+
+        writer->buffer = PyBytes_FromStringAndSize(NULL, allocated);
+        if (writer->buffer == NULL)
+            return NULL;
+
+        if (pos != 0) {
+            Py_MEMCPY(PyBytes_AS_STRING(writer->buffer),
+                      writer->stack_buffer,
+                      pos);
+        }
+
+#ifdef Py_DEBUG
+        memset(writer->stack_buffer, 0xDB, sizeof(writer->stack_buffer));
+#endif
+
+        writer->use_stack_buffer = 0;
+    }
+    writer->allocated = allocated;
+
+    str = _PyBytesWriter_AsString(writer) + pos;
+    _PyBytesWriter_CheckConsistency(writer, str);
+    return str;
+}
+
+/* Allocate the buffer to write size bytes.
+   Return the pointer to the beginning of buffer data.
+   Raise an exception and return NULL on error. */
+char*
+_PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size)
+{
+    /* ensure that _PyBytesWriter_Alloc() is only called once */
+    assert(writer->size == 0 && writer->buffer == NULL);
+    assert(size >= 0);
+
+    writer->use_stack_buffer = 1;
+#if Py_DEBUG
+    /* the last byte is reserved, it must be '\0' */
+    writer->stack_buffer[sizeof(writer->stack_buffer) - 1] = 0;
+    writer->allocated = sizeof(writer->stack_buffer) - 1;
+#else
+    writer->allocated = sizeof(writer->stack_buffer);
+#endif
+    return _PyBytesWriter_Prepare(writer, writer->stack_buffer, size);
+}
+
+PyObject *
+_PyBytesWriter_Finish(_PyBytesWriter *writer, char *str)
+{
+    Py_ssize_t pos;
+    PyObject *result;
+
+    _PyBytesWriter_CheckConsistency(writer, str);
+
+    pos = _PyBytesWriter_GetPos(writer, str);
+    if (!writer->use_stack_buffer) {
+        if (pos != writer->allocated) {
+            if (_PyBytes_Resize(&writer->buffer, pos)) {
+                assert(writer->buffer == NULL);
+                return NULL;
+            }
+        }
+
+        result = writer->buffer;
+        writer->buffer = NULL;
+    }
+    else {
+        result = PyBytes_FromStringAndSize(writer->stack_buffer, pos);
+    }
+
+    return result;
+}
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -347,216 +347,6 @@
 #endif
 }
 
-/* The _PyBytesWriter structure is big: it contains an embeded "stack buffer".
-   A _PyBytesWriter variable must be declared at the end of variables in a
-   function to optimize the memory allocation on the stack. */
-typedef struct {
-    /* bytes object */
-    PyObject *buffer;
-
-    /* Number of allocated size */
-    Py_ssize_t allocated;
-
-    /* Current size of the buffer (can be smaller than the allocated size) */
-    Py_ssize_t size;
-
-    /* If non-zero, overallocate the buffer (default: 0). */
-    int overallocate;
-
-    /* Stack buffer */
-    int use_stack_buffer;
-    char stack_buffer[512];
-} _PyBytesWriter;
-
-static void
-_PyBytesWriter_Init(_PyBytesWriter *writer)
-{
-    writer->buffer = NULL;
-    writer->allocated = 0;
-    writer->size = 0;
-    writer->overallocate = 0;
-    writer->use_stack_buffer = 0;
-#ifdef Py_DEBUG
-    memset(writer->stack_buffer, 0xCB, sizeof(writer->stack_buffer));
-#endif
-}
-
-static void
-_PyBytesWriter_Dealloc(_PyBytesWriter *writer)
-{
-    Py_CLEAR(writer->buffer);
-}
-
-static char*
-_PyBytesWriter_AsString(_PyBytesWriter *writer)
-{
-    if (!writer->use_stack_buffer) {
-        assert(writer->buffer != NULL);
-        return PyBytes_AS_STRING(writer->buffer);
-    }
-    else {
-        assert(writer->buffer == NULL);
-        return writer->stack_buffer;
-    }
-}
-
-Py_LOCAL_INLINE(Py_ssize_t)
-_PyBytesWriter_GetPos(_PyBytesWriter *writer, char *str)
-{
-    char *start = _PyBytesWriter_AsString(writer);
-    assert(str != NULL);
-    assert(str >= start);
-    return str - start;
-}
-
-Py_LOCAL_INLINE(void)
-_PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str)
-{
-#ifdef Py_DEBUG
-    char *start, *end;
-
-    if (!writer->use_stack_buffer) {
-        assert(writer->buffer != NULL);
-        assert(PyBytes_CheckExact(writer->buffer));
-        assert(Py_REFCNT(writer->buffer) == 1);
-    }
-    else {
-        assert(writer->buffer == NULL);
-    }
-
-    start = _PyBytesWriter_AsString(writer);
-    assert(0 <= writer->size && writer->size <= writer->allocated);
-    /* the last byte must always be null */
-    assert(start[writer->allocated] == 0);
-
-    end = start + writer->allocated;
-    assert(str != NULL);
-    assert(start <= str && str <= end);
-#endif
-}
-
-/* Add *size* bytes to the buffer.
-   str is the current pointer inside the buffer.
-   Return the updated current pointer inside the buffer.
-   Raise an exception and return NULL on error. */
-static char*
-_PyBytesWriter_Prepare(_PyBytesWriter *writer, char *str, Py_ssize_t size)
-{
-    Py_ssize_t allocated, pos;
-
-    _PyBytesWriter_CheckConsistency(writer, str);
-    assert(size >= 0);
-
-    if (size == 0) {
-        /* nothing to do */
-        return str;
-    }
-
-    if (writer->size > PY_SSIZE_T_MAX - size) {
-        PyErr_NoMemory();
-        _PyBytesWriter_Dealloc(writer);
-        return NULL;
-    }
-    writer->size += size;
-
-    allocated = writer->allocated;
-    if (writer->size <= allocated)
-        return str;
-
-    allocated = writer->size;
-    if (writer->overallocate
-        && allocated <= (PY_SSIZE_T_MAX - allocated / OVERALLOCATE_FACTOR)) {
-        /* overallocate to limit the number of realloc() */
-        allocated += allocated / OVERALLOCATE_FACTOR;
-    }
-
-    pos = _PyBytesWriter_GetPos(writer, str);
-    if (!writer->use_stack_buffer) {
-        /* Note: Don't use a bytearray object because the conversion from
-           byterray to bytes requires to copy all bytes. */
-        if (_PyBytes_Resize(&writer->buffer, allocated)) {
-            assert(writer->buffer == NULL);
-            return NULL;
-        }
-    }
-    else {
-        /* convert from stack buffer to bytes object buffer */
-        assert(writer->buffer == NULL);
-
-        writer->buffer = PyBytes_FromStringAndSize(NULL, allocated);
-        if (writer->buffer == NULL)
-            return NULL;
-
-        if (pos != 0) {
-            Py_MEMCPY(PyBytes_AS_STRING(writer->buffer),
-                      writer->stack_buffer,
-                      pos);
-        }
-
-#ifdef Py_DEBUG
-        memset(writer->stack_buffer, 0xDB, sizeof(writer->stack_buffer));
-#endif
-
-        writer->use_stack_buffer = 0;
-    }
-    writer->allocated = allocated;
-
-    str = _PyBytesWriter_AsString(writer) + pos;
-    _PyBytesWriter_CheckConsistency(writer, str);
-    return str;
-}
-
-/* Allocate the buffer to write size bytes.
-   Return the pointer to the beginning of buffer data.
-   Raise an exception and return NULL on error. */
-static char*
-_PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size)
-{
-    /* ensure that _PyBytesWriter_Alloc() is only called once */
-    assert(writer->size == 0 && writer->buffer == NULL);
-    assert(size >= 0);
-
-    writer->use_stack_buffer = 1;
-#if Py_DEBUG
-    /* the last byte is reserved, it must be '\0' */
-    writer->stack_buffer[sizeof(writer->stack_buffer) - 1] = 0;
-    writer->allocated = sizeof(writer->stack_buffer) - 1;
-#else
-    writer->allocated = sizeof(writer->stack_buffer);
-#endif
-    return _PyBytesWriter_Prepare(writer, writer->stack_buffer, size);
-}
-
-/* Get the buffer content and reset the writer.
-   Return a bytes object.
-   Raise an exception and return NULL on error. */
-static PyObject *
-_PyBytesWriter_Finish(_PyBytesWriter *writer, char *str)
-{
-    Py_ssize_t pos;
-    PyObject *result;
-
-    _PyBytesWriter_CheckConsistency(writer, str);
-
-    pos = _PyBytesWriter_GetPos(writer, str);
-    if (!writer->use_stack_buffer) {
-        if (pos != writer->allocated) {
-            if (_PyBytes_Resize(&writer->buffer, pos)) {
-                assert(writer->buffer == NULL);
-                return NULL;
-            }
-        }
-
-        result = writer->buffer;
-        writer->buffer = NULL;
-    }
-    else {
-        result = PyBytes_FromStringAndSize(writer->stack_buffer, pos);
-    }
-
-    return result;
-}
-
 #ifdef Py_DEBUG
 int
 _PyUnicode_CheckConsistency(PyObject *op, int check_content)

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  9 02:52:38 2015
From: python-checkins at python.org (victor.stinner)
Date: Fri, 09 Oct 2015 00:52:38 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2325318=3A_Fix_comp?=
	=?utf-8?q?ilation_error?=
Message-ID: <20151009005238.20779.49084@psf.io>

https://hg.python.org/cpython/rev/e9c1404d6bd9
changeset:   98607:e9c1404d6bd9
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Fri Oct 09 02:52:16 2015 +0200
summary:
  Issue #25318: Fix compilation error

Replace "#if Py_DEBUG" with "#ifdef Py_DEBUG".

files:
  Objects/bytesobject.c |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c
--- a/Objects/bytesobject.c
+++ b/Objects/bytesobject.c
@@ -3892,7 +3892,7 @@
     assert(size >= 0);
 
     writer->use_stack_buffer = 1;
-#if Py_DEBUG
+#ifdef Py_DEBUG
     /* the last byte is reserved, it must be '\0' */
     writer->stack_buffer[sizeof(writer->stack_buffer) - 1] = 0;
     writer->allocated = sizeof(writer->stack_buffer) - 1;

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  9 02:55:28 2015
From: python-checkins at python.org (raymond.hettinger)
Date: Fri, 09 Oct 2015 00:55:28 +0000
Subject: [Python-checkins] =?utf-8?q?test=3A_test?=
Message-ID: <20151009005528.7262.70312@psf.io>

https://hg.python.org/test/rev/199cb49e866b
changeset:   226:199cb49e866b
user:        Raymond Hettinger <python at rcn.com>
date:        Thu Oct 08 20:55:24 2015 -0400
summary:
  test

files:
  ab |  0 
  1 files changed, 0 insertions(+), 0 deletions(-)


diff --git a/ab b/ab
new file mode 100644

-- 
Repository URL: https://hg.python.org/test

From python-checkins at python.org  Fri Oct  9 03:14:26 2015
From: python-checkins at python.org (raymond.hettinger)
Date: Fri, 09 Oct 2015 01:14:26 +0000
Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzI1MzI2?=
 =?utf-8?q?=3A__Improve_an_obscure_error_message=2E?=
Message-ID: <20151009011420.97720.71338@psf.io>

https://hg.python.org/cpython/rev/4d1bd86d3bbd
changeset:   98608:4d1bd86d3bbd
branch:      2.7
parent:      98586:04815b55227f
user:        Raymond Hettinger <python at rcn.com>
date:        Thu Oct 08 21:14:15 2015 -0400
summary:
  Issue #25326:  Improve an obscure error message.

files:
  Objects/abstract.c |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Objects/abstract.c b/Objects/abstract.c
--- a/Objects/abstract.c
+++ b/Objects/abstract.c
@@ -259,7 +259,7 @@
          pb->bf_getcharbuffer == NULL ||
          pb->bf_getsegcount == NULL) {
         PyErr_SetString(PyExc_TypeError,
-                        "expected a character buffer object");
+                        "expected a string or other character buffer object");
         return -1;
     }
     if ((*pb->bf_getsegcount)(obj,NULL) != 1) {

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  9 03:38:50 2015
From: python-checkins at python.org (victor.stinner)
Date: Fri, 09 Oct 2015 01:38:50 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2325318=3A_Avoid_sp?=
 =?utf-8?q?rintf=28=29_in_backslashreplace=28=29?=
Message-ID: <20151009013850.18382.52485@psf.io>

https://hg.python.org/cpython/rev/9cf89366bbcb
changeset:   98609:9cf89366bbcb
parent:      98607:e9c1404d6bd9
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Fri Oct 09 03:17:30 2015 +0200
summary:
  Issue #25318: Avoid sprintf() in backslashreplace()

Rewrite backslashreplace() to be closer to PyCodec_BackslashReplaceErrors().

Add also unit tests for non-BMP characters.

files:
  Lib/test/test_codecs.py |   6 ++++--
  Objects/unicodeobject.c |  27 +++++++++++++++++++--------
  2 files changed, 23 insertions(+), 10 deletions(-)


diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -3155,7 +3155,8 @@
             ('[\x80\xff\u20ac]', 'ignore', b'[]'),
             ('[\x80\xff\u20ac]', 'replace', b'[???]'),
             ('[\x80\xff\u20ac]', 'xmlcharrefreplace', b'[&#128;&#255;&#8364;]'),
-            ('[\x80\xff\u20ac]', 'backslashreplace', b'[\\x80\\xff\\u20ac]'),
+            ('[\x80\xff\u20ac\U000abcde]', 'backslashreplace',
+             b'[\\x80\\xff\\u20ac\\U000abcde]'),
             ('[\udc80\udcff]', 'surrogateescape', b'[\x80\xff]'),
         ):
             with self.subTest(data=data, error_handler=error_handler,
@@ -3197,7 +3198,8 @@
         for data, error_handler, expected in (
             ('[\u20ac\udc80]', 'ignore', b'[]'),
             ('[\u20ac\udc80]', 'replace', b'[??]'),
-            ('[\u20ac\udc80]', 'backslashreplace', b'[\\u20ac\\udc80]'),
+            ('[\u20ac\U000abcde]', 'backslashreplace',
+             b'[\\u20ac\\U000abcde]'),
             ('[\u20ac\udc80]', 'xmlcharrefreplace', b'[&#8364;&#56448;]'),
             ('[\udc80\udcff]', 'surrogateescape', b'[\x80\xff]'),
         ):
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -610,14 +610,25 @@
     /* generate replacement */
     for (i = collstart; i < collend; ++i) {
         ch = PyUnicode_READ(kind, data, i);
-        if (ch < 0x100)
-            str += sprintf(str, "\\x%02x", ch);
-        else if (ch < 0x10000)
-            str += sprintf(str, "\\u%04x", ch);
-        else {
-            assert(ch <= MAX_UNICODE);
-            str += sprintf(str, "\\U%08x", ch);
-        }
+        *str++ = '\\';
+        if (ch >= 0x00010000) {
+            *str++ = 'U';
+            *str++ = Py_hexdigits[(ch>>28)&0xf];
+            *str++ = Py_hexdigits[(ch>>24)&0xf];
+            *str++ = Py_hexdigits[(ch>>20)&0xf];
+            *str++ = Py_hexdigits[(ch>>16)&0xf];
+            *str++ = Py_hexdigits[(ch>>12)&0xf];
+            *str++ = Py_hexdigits[(ch>>8)&0xf];
+        }
+        else if (ch >= 0x100) {
+            *str++ = 'u';
+            *str++ = Py_hexdigits[(ch>>12)&0xf];
+            *str++ = Py_hexdigits[(ch>>8)&0xf];
+        }
+        else
+            *str++ = 'x';
+        *str++ = Py_hexdigits[(ch>>4)&0xf];
+        *str++ = Py_hexdigits[ch&0xf];
     }
     return str;
 }

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  9 03:38:52 2015
From: python-checkins at python.org (victor.stinner)
Date: Fri, 09 Oct 2015 01:38:52 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2325318=3A_cleanup_?=
	=?utf-8?q?code_=5FPyBytesWriter?=
Message-ID: <20151009013850.475.612@psf.io>

https://hg.python.org/cpython/rev/c53dcf1d6967
changeset:   98611:c53dcf1d6967
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Fri Oct 09 03:38:24 2015 +0200
summary:
  Issue #25318: cleanup code _PyBytesWriter

Rename "stack buffer" to "small buffer".

Add also an assertion in _PyBytesWriter_GetPos().

files:
  Include/bytesobject.h |   4 +-
  Objects/bytesobject.c |  34 +++++++++++++++---------------
  2 files changed, 19 insertions(+), 19 deletions(-)


diff --git a/Include/bytesobject.h b/Include/bytesobject.h
--- a/Include/bytesobject.h
+++ b/Include/bytesobject.h
@@ -141,8 +141,8 @@
     int overallocate;
 
     /* Stack buffer */
-    int use_stack_buffer;
-    char stack_buffer[512];
+    int use_small_buffer;
+    char small_buffer[512];
 } _PyBytesWriter;
 
 /* Initialize a bytes writer
diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c
--- a/Objects/bytesobject.c
+++ b/Objects/bytesobject.c
@@ -3754,9 +3754,9 @@
     writer->allocated = 0;
     writer->size = 0;
     writer->overallocate = 0;
-    writer->use_stack_buffer = 0;
+    writer->use_small_buffer = 0;
 #ifdef Py_DEBUG
-    memset(writer->stack_buffer, 0xCB, sizeof(writer->stack_buffer));
+    memset(writer->small_buffer, 0xCB, sizeof(writer->small_buffer));
 #endif
 }
 
@@ -3769,13 +3769,13 @@
 Py_LOCAL_INLINE(char*)
 _PyBytesWriter_AsString(_PyBytesWriter *writer)
 {
-    if (!writer->use_stack_buffer) {
+    if (!writer->use_small_buffer) {
         assert(writer->buffer != NULL);
         return PyBytes_AS_STRING(writer->buffer);
     }
     else {
         assert(writer->buffer == NULL);
-        return writer->stack_buffer;
+        return writer->small_buffer;
     }
 }
 
@@ -3785,6 +3785,7 @@
     char *start = _PyBytesWriter_AsString(writer);
     assert(str != NULL);
     assert(str >= start);
+    assert(str - start <= writer->allocated);
     return str - start;
 }
 
@@ -3794,7 +3795,7 @@
 #ifdef Py_DEBUG
     char *start, *end;
 
-    if (!writer->use_stack_buffer) {
+    if (!writer->use_small_buffer) {
         assert(writer->buffer != NULL);
         assert(PyBytes_CheckExact(writer->buffer));
         assert(Py_REFCNT(writer->buffer) == 1);
@@ -3846,7 +3847,7 @@
     }
 
     pos = _PyBytesWriter_GetPos(writer, str);
-    if (!writer->use_stack_buffer) {
+    if (!writer->use_small_buffer) {
         /* Note: Don't use a bytearray object because the conversion from
            byterray to bytes requires to copy all bytes. */
         if (_PyBytes_Resize(&writer->buffer, allocated)) {
@@ -3864,15 +3865,14 @@
 
         if (pos != 0) {
             Py_MEMCPY(PyBytes_AS_STRING(writer->buffer),
-                      writer->stack_buffer,
+                      writer->small_buffer,
                       pos);
         }
 
+        writer->use_small_buffer = 0;
 #ifdef Py_DEBUG
-        memset(writer->stack_buffer, 0xDB, sizeof(writer->stack_buffer));
+        memset(writer->small_buffer, 0xDB, sizeof(writer->small_buffer));
 #endif
-
-        writer->use_stack_buffer = 0;
     }
     writer->allocated = allocated;
 
@@ -3891,15 +3891,15 @@
     assert(writer->size == 0 && writer->buffer == NULL);
     assert(size >= 0);
 
-    writer->use_stack_buffer = 1;
+    writer->use_small_buffer = 1;
 #ifdef Py_DEBUG
     /* the last byte is reserved, it must be '\0' */
-    writer->stack_buffer[sizeof(writer->stack_buffer) - 1] = 0;
-    writer->allocated = sizeof(writer->stack_buffer) - 1;
+    writer->allocated = sizeof(writer->small_buffer) - 1;
+    writer->small_buffer[writer->allocated] = 0;
 #else
-    writer->allocated = sizeof(writer->stack_buffer);
+    writer->allocated = sizeof(writer->small_buffer);
 #endif
-    return _PyBytesWriter_Prepare(writer, writer->stack_buffer, size);
+    return _PyBytesWriter_Prepare(writer, writer->small_buffer, size);
 }
 
 PyObject *
@@ -3911,7 +3911,7 @@
     _PyBytesWriter_CheckConsistency(writer, str);
 
     pos = _PyBytesWriter_GetPos(writer, str);
-    if (!writer->use_stack_buffer) {
+    if (!writer->use_small_buffer) {
         if (pos != writer->allocated) {
             if (_PyBytes_Resize(&writer->buffer, pos)) {
                 assert(writer->buffer == NULL);
@@ -3923,7 +3923,7 @@
         writer->buffer = NULL;
     }
     else {
-        result = PyBytes_FromStringAndSize(writer->stack_buffer, pos);
+        result = PyBytes_FromStringAndSize(writer->small_buffer, pos);
     }
 
     return result;

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  9 03:38:56 2015
From: python-checkins at python.org (victor.stinner)
Date: Fri, 09 Oct 2015 01:38:56 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2325318=3A_Fix_back?=
	=?utf-8?q?slashreplace=28=29?=
Message-ID: <20151009013850.2683.39292@psf.io>

https://hg.python.org/cpython/rev/0a522f68d275
changeset:   98610:0a522f68d275
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Fri Oct 09 03:37:11 2015 +0200
summary:
  Issue #25318: Fix backslashreplace()

Fix code to estimate the needed space.

files:
  Objects/unicodeobject.c |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -590,7 +590,7 @@
             incr = 2+4;
         else {
             assert(ch <= MAX_UNICODE);
-            incr = 2+6;
+            incr = 2+8;
         }
         if (size > PY_SSIZE_T_MAX - incr) {
             PyErr_SetString(PyExc_OverflowError,

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  9 06:03:57 2015
From: python-checkins at python.org (raymond.hettinger)
Date: Fri, 09 Oct 2015 04:03:57 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2325298=3A__Add_loc?=
 =?utf-8?q?k_and_rlock_weakref_tests_=28Contributed_by_Nir_Soffer=29=2E?=
Message-ID: <20151009040357.18388.16368@psf.io>

https://hg.python.org/cpython/rev/45903695e86c
changeset:   98612:45903695e86c
user:        Raymond Hettinger <python at rcn.com>
date:        Fri Oct 09 00:03:51 2015 -0400
summary:
  Issue #25298:  Add lock and rlock weakref tests (Contributed by Nir Soffer).

files:
  Lib/test/lock_tests.py |  12 ++++++++++++
  1 files changed, 12 insertions(+), 0 deletions(-)


diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py
--- a/Lib/test/lock_tests.py
+++ b/Lib/test/lock_tests.py
@@ -7,6 +7,7 @@
 from _thread import start_new_thread, TIMEOUT_MAX
 import threading
 import unittest
+import weakref
 
 from test import support
 
@@ -198,6 +199,17 @@
         self.assertFalse(results[0])
         self.assertTimeout(results[1], 0.5)
 
+    def test_weakref_exists(self):
+        lock = self.locktype()
+        ref = weakref.ref(lock)
+        self.assertIsNotNone(ref())
+
+    def test_weakref_deleted(self):
+        lock = self.locktype()
+        ref = weakref.ref(lock)
+        del lock
+        self.assertIsNone(ref())
+
 
 class LockTests(BaseLockTests):
     """

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  9 06:43:00 2015
From: python-checkins at python.org (raymond.hettinger)
Date: Fri, 09 Oct 2015 04:43:00 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Make_comparison_more_consi?=
	=?utf-8?q?stent?=
Message-ID: <20151009044300.97706.75364@psf.io>

https://hg.python.org/cpython/rev/7978f187b10a
changeset:   98613:7978f187b10a
user:        Raymond Hettinger <python at rcn.com>
date:        Fri Oct 09 00:42:47 2015 -0400
summary:
  Make comparison more consistent

files:
  Python/bltinmodule.c |  2 +-
  1 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -331,7 +331,7 @@
             Py_DECREF(it);
             return NULL;
         }
-        if (cmp == 1) {
+        if (cmp > 0) {
             Py_DECREF(it);
             Py_RETURN_TRUE;
         }

-- 
Repository URL: https://hg.python.org/cpython

From python-checkins at python.org  Fri Oct  9 07:35:30 2015
From: python-checkins at python.org (raymond.hettinger)
Date: Fri, 09 Oct 2015 05:35:30 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Hoist_constant_expression_?=
	=?utf-8?q?out_of_the_inner_loop=2E?=
Message-ID: <20151009053529.3295.49294@psf.io>

https://hg.python.org/cpython/rev/1aae9b6a6929
changeset:   98614:1aae9b6a6929
user:        Raymond Hettinger <python at rcn.com>
date:        Fri Oct 09 01:34:08 2015 -0400
summary:
  Hoist constant expression out of the inner loop.

files:
  Python/bltinmodule.c |  6 +++---
  1 files changed, 3 insertions(+), 3 deletions(-)


diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -469,6 +469,7 @@
     PyObject *it = lz->it;
     long ok;
     PyObject *(*iternext)(PyObject *);
+    int checktrue = lz->func == Py_None || lz->func == (PyObject *)&PyBool_Type;
 
     iternext = *Py_TYPE(it)->tp_iternext;
     for (;;) {
@@ -476,12 +477,11 @@
         if (item == NULL)
             return NULL;
 
-        if (lz->func == Py_None || lz->func == (PyObject *)&PyBool_Type) {
+        if (checktrue) {
             ok = PyObject_IsTrue(item);
         } else {
             PyObject *good;
-            good = PyObject_CallFunctionObjArgs(lz->func,
-                                                item, NULL);
+            good = PyObject_CallFunctionObjArgs(lz->func, item, NULL);
             if (good == NULL) {
                 Py_DECREF(item);
                 return NULL;

-- 
Repository URL: https://hg.python.org/cpython

From solipsis at pitrou.net  Fri Oct  9 10:44:42 2015
From: solipsis at pitrou.net (solipsis at pitrou.net)
Date: Fri, 09 Oct 2015 08:44:42 +0000
Subject: [Python-checkins] Daily reference leaks (1aae9b6a6929): sum=17880
Message-ID: <20151009084442.18368.25524@psf.io>

results for 1aae9b6a6929 on branch "default"
--------------------------------------------

test_asyncio leaked [3, 0, 0] memory blocks, sum=3
test_capi leaked [1598, 1598, 1598] references, sum=4794
test_capi leaked [387, 389, 389] memory blocks, sum=1165
test_functools leaked [0, 2, 2] memory blocks, sum=4
test_threading leaked [3196, 3196, 3196] references, sum=9588
test_threading leaked [774, 776, 776] memory blocks, sum=2326


Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogikNhry', '--timeout', '7200']

From python-checkins at python.org  Fri Oct  9 12:21:56 2015
From: python-checkins at python.org (victor.stinner)
Date: Fri, 09 Oct 2015 10:21:56 +0000
Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2325349=3A_Optimize?=
 =?utf-8?q?_bytes_=25_args_using_the_new_private_=5FPyBytesWriter_API?=
Message-ID: <20151009102155.3273.45766@psf.io>

https://hg.python.org/cpython/rev/b2f3cbdc0f2d
changeset:   98615:b2f3cbdc0f2d
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Fri Oct 09 11:48:06 2015 +0200
summary:
  Issue #25349: Optimize bytes % args using the new private _PyBytesWriter API

* Thanks to the _PyBytesWriter API, output smaller than 512 bytes are allocated
  on the stack and so avoid calling _PyBytes_Resize(). Because of that, change
  the default buffer size to fmtcnt instead of fmtcnt+100.
* Rely on _PyBytesWriter algorithm to overallocate the buffer instead of using
  a custom code. For example, _PyBytesWriter uses a different overallocation
  factor (25% or 50%) depending on the platform to get best performances.
* Disable overallocation for the last write.
* Replace C loops to fill characters with memset()
* Add also many comments to _PyBytes_Format()
* Remove unused FORMATBUFLEN constant
* Avoid the creation of a temporary bytes object when formatting a floating
  point number (when no custom formatting option is used)
* Fix also reference leaks on error handling
* Use Py_MEMCPY() to copy bytes between two formatters (%)

files:
  Misc/NEWS             |    2 +
  Objects/bytesobject.c |  187 ++++++++++++++++++++---------
  2 files changed, 130 insertions(+), 59 deletions(-)


diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,8 @@
 Core and Builtins
 -----------------
 
+- Issue #25349: Optimize bytes % args using the new private _PyBytesWriter API.
+
 - Issue #24806: Prevent builtin types that are not allowed to be subclassed from
   being subclassed through multiple inheritance.
 
diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c
--- a/Objects/bytesobject.c
+++ b/Objects/bytesobject.c
@@ -409,12 +409,15 @@
 
 /* Returns a new reference to a PyBytes object, or NULL on failure. */
 
-static PyObject *
-formatfloat(PyObject *v, int flags, int prec, int type)
+static char*
+formatfloat(PyObject *v, int flags, int prec, int type,
+            PyObject **p_result, _PyBytesWriter *writer, char *str,
+            Py_ssize_t prealloc)
 {
     char *p;
     PyObject *result;
     double x;
+    size_t len;
 
     x = PyFloat_AsDouble(v);
     if (x == -1.0 && PyErr_Occurred()) {
@@ -431,9 +434,23 @@
 
     if (p == NULL)
         return NULL;
-    result = PyBytes_FromStringAndSize(p, strlen(p));
+
+    len = strlen(p);
+    if (writer != NULL) {
+        if ((Py_ssize_t)len > prealloc) {
+            str = _PyBytesWriter_Prepare(writer, str, len - prealloc);
+            if (str == NULL)
+                return NULL;
+        }
+        Py_MEMCPY(str, p, len);
+        str += len;
+        return str;
+    }
+
+    result = PyBytes_FromStringAndSize(p, len);
     PyMem_Free(p);
-    return result;
+    *p_result = result;
+    return str;
 }
 
 static PyObject *
@@ -557,36 +574,32 @@
     return NULL;
 }
 
-/* fmt%(v1,v2,...) is roughly equivalent to sprintf(fmt, v1, v2, ...)
-
-   FORMATBUFLEN is the length of the buffer in which the ints &
-   chars are formatted. XXX This is a magic number. Each formatting
-   routine does bounds checking to ensure no overflow, but a better
-   solution may be to malloc a buffer of appropriate size for each
-   format. For now, the current solution is sufficient.
-*/
-#define FORMATBUFLEN (size_t)120
+/* fmt%(v1,v2,...) is roughly equivalent to sprintf(fmt, v1, v2, ...) */
 
 PyObject *
 _PyBytes_Format(PyObject *format, PyObject *args)
 {
     char *fmt, *res;
     Py_ssize_t arglen, argidx;
-    Py_ssize_t reslen, rescnt, fmtcnt;
+    Py_ssize_t fmtcnt;
     int args_owned = 0;
-    PyObject *result;
     PyObject *dict = NULL;
+    _PyBytesWriter writer;
+
     if (format == NULL || !PyBytes_Check(format) || args == NULL) {
         PyErr_BadInternalCall();
         return NULL;
     }
     fmt = PyBytes_AS_STRING(format);
     fmtcnt = PyBytes_GET_SIZE(format);
-    reslen = rescnt = fmtcnt + 100;
-    result = PyBytes_FromStringAndSize((char *)NULL, reslen);
-    if (result == NULL)
+
+    _PyBytesWriter_Init(&writer);
+
+    res = _PyBytesWriter_Alloc(&writer, fmtcnt);
+    if (res == NULL)
         return NULL;
-    res = PyBytes_AsString(result);
+    writer.overallocate = 1;
+
     if (PyTuple_Check(args)) {
         arglen = PyTuple_GET_SIZE(args);
         argidx = 0;
@@ -600,18 +613,25 @@
         !PyByteArray_Check(args)) {
             dict = args;
     }
+
     while (--fmtcnt >= 0) {
         if (*fmt != '%') {
-            if (--rescnt < 0) {
-                rescnt = fmtcnt + 100;
-                reslen += rescnt;
-                if (_PyBytes_Resize(&result, reslen))
-                    return NULL;
-                res = PyBytes_AS_STRING(result)
-                    + reslen - rescnt;
-                --rescnt;
+            Py_ssize_t len;
+            char *pos;
+
+            pos = strchr(fmt + 1, '%');
+            if (pos != NULL)
+                len = pos - fmt;
+            else {
+                len = PyBytes_GET_SIZE(format);
+                len -= (fmt - PyBytes_AS_STRING(format));
             }
-            *res++ = *fmt++;
+            assert(len != 0);
+
+            Py_MEMCPY(res, fmt, len);
+            res += len;
+            fmt += len;
+            fmtcnt -= (len - 1);
         }
         else {
             /* Got a format specifier */
@@ -626,6 +646,10 @@
             int sign;
             Py_ssize_t len = 0;
             char onechar; /* For byte_converter() */
+            Py_ssize_t alloc;
+#ifdef Py_DEBUG
+            char *before;
+#endif
 
             fmt++;
             if (*fmt == '(') {
@@ -673,6 +697,8 @@
                 arglen = -1;
                 argidx = -2;
             }
+
+            /* Parse flags. Example: "%+i" => flags=F_SIGN. */
             while (--fmtcnt >= 0) {
                 switch (c = *fmt++) {
                 case '-': flags |= F_LJUST; continue;
@@ -683,6 +709,8 @@
                 }
                 break;
             }
+
+            /* Parse width. Example: "%10s" => width=10 */
             if (c == '*') {
                 v = getnextarg(args, arglen, &argidx);
                 if (v == NULL)
@@ -717,6 +745,8 @@
                     width = width*10 + (c - '0');
                 }
             }
+
+            /* Parse precision. Example: "%.3f" => prec=3 */
             if (c == '.') {
                 prec = 0;
                 if (--fmtcnt >= 0)
@@ -771,6 +801,12 @@
                 if (v == NULL)
                     goto error;
             }
+
+            if (fmtcnt < 0) {
+                /* last writer: disable writer overallocation */
+                writer.overallocate = 0;
+            }
+
             sign = 0;
             fill = ' ';
             switch (c) {
@@ -778,6 +814,7 @@
                 pbuf = "%";
                 len = 1;
                 break;
+
             case 'r':
                 // %r is only for 2/3 code; 3 only code should use %a
             case 'a':
@@ -790,6 +827,7 @@
                 if (prec >= 0 && len > prec)
                     len = prec;
                 break;
+
             case 's':
                 // %s is only for 2/3 code; 3 only code should use %b
             case 'b':
@@ -799,6 +837,7 @@
                 if (prec >= 0 && len > prec)
                     len = prec;
                 break;
+
             case 'i':
             case 'd':
             case 'u':
@@ -815,14 +854,24 @@
                 if (flags & F_ZERO)
                     fill = '0';
                 break;
+
             case 'e':
             case 'E':
             case 'f':
             case 'F':
             case 'g':
             case 'G':
-                temp = formatfloat(v, flags, prec, c);
-                if (temp == NULL)
+                if (width == -1 && prec == -1
+                    && !(flags & (F_SIGN | F_BLANK)))
+                {
+                    /* Fast path */
+                    res = formatfloat(v, flags, prec, c, NULL, &writer, res, 1);
+                    if (res == NULL)
+                        goto error;
+                    continue;
+                }
+
+                if (!formatfloat(v, flags, prec, c, &temp, NULL, res, 1))
                     goto error;
                 pbuf = PyBytes_AS_STRING(temp);
                 len = PyBytes_GET_SIZE(temp);
@@ -830,12 +879,14 @@
                 if (flags & F_ZERO)
                     fill = '0';
                 break;
+
             case 'c':
                 pbu