From python-checkins at python.org Mon Sep 1 00:02:55 2014 From: python-checkins at python.org (jason.coombs) Date: Mon, 1 Sep 2014 00:02:55 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogIzIyMzE1OiBBZGQg?= =?utf-8?q?test_to_capture_the_failure=2E?= Message-ID: <3hmT7M2Yxvz7Lrs@mail.python.org> http://hg.python.org/cpython/rev/7304b9b95438 changeset: 92285:7304b9b95438 branch: 3.4 parent: 92283:c3cecf8e7497 user: Jason R. Coombs date: Sun Aug 31 15:02:42 2014 -0400 summary: #22315: Add test to capture the failure. files: Lib/distutils/tests/test_dir_util.py | 29 ++++++++++++++++ 1 files changed, 29 insertions(+), 0 deletions(-) diff --git a/Lib/distutils/tests/test_dir_util.py b/Lib/distutils/tests/test_dir_util.py --- a/Lib/distutils/tests/test_dir_util.py +++ b/Lib/distutils/tests/test_dir_util.py @@ -3,7 +3,9 @@ import os import stat import sys +import contextlib +from distutils import dir_util, errors from distutils.dir_util import (mkpath, remove_tree, create_tree, copy_tree, ensure_relative) @@ -11,6 +13,20 @@ from distutils.tests import support from test.support import run_unittest + + at contextlib.context_manager +def patch_obj(obj, attr, replacement): + """ + A poor man's mock.patch.object + """ + orig = getattr(obj, attr) + try: + setattr(obj, attr, replacement) + yield + finally: + setattr(obj, attr, orig) + + class DirUtilTestCase(support.TempdirManager, unittest.TestCase): def _log(self, msg, *args): @@ -119,6 +135,19 @@ self.assertEqual(ensure_relative('c:\\home\\foo'), 'c:home\\foo') self.assertEqual(ensure_relative('home\\foo'), 'home\\foo') + def test_copy_tree_exception_in_listdir(self): + """ + An exception in listdir should raise a DistutilsFileError + """ + def new_listdir(path): + raise OSError() + # simulate a transient network error or other failure invoking listdir + with patch_obj(os, 'listdir', new_listdir): + args = 'src', None + exc = errors.DistutilsFileError + self.assertRaises(exc, dir_util.copy_tree, *args) + + def test_suite(): return unittest.makeSuite(DirUtilTestCase) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 1 00:02:56 2014 From: python-checkins at python.org (jason.coombs) Date: Mon, 1 Sep 2014 00:02:56 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogIzIyMzE1OiBVc2Ug?= =?utf-8?q?technique_outlined_in_test=5Ffile=5Futil?= Message-ID: <3hmT7N4KbGz7Lrw@mail.python.org> http://hg.python.org/cpython/rev/300cd36eb25c changeset: 92286:300cd36eb25c branch: 3.4 user: Jason R. Coombs date: Sun Aug 31 17:31:32 2014 -0400 summary: #22315: Use technique outlined in test_file_util files: Lib/distutils/tests/test_dir_util.py | 27 ++------------- 1 files changed, 5 insertions(+), 22 deletions(-) diff --git a/Lib/distutils/tests/test_dir_util.py b/Lib/distutils/tests/test_dir_util.py --- a/Lib/distutils/tests/test_dir_util.py +++ b/Lib/distutils/tests/test_dir_util.py @@ -3,7 +3,7 @@ import os import stat import sys -import contextlib +from unittest.mock import patch from distutils import dir_util, errors from distutils.dir_util import (mkpath, remove_tree, create_tree, copy_tree, @@ -14,19 +14,6 @@ from test.support import run_unittest - at contextlib.context_manager -def patch_obj(obj, attr, replacement): - """ - A poor man's mock.patch.object - """ - orig = getattr(obj, attr) - try: - setattr(obj, attr, replacement) - yield - finally: - setattr(obj, attr, orig) - - class DirUtilTestCase(support.TempdirManager, unittest.TestCase): def _log(self, msg, *args): @@ -135,17 +122,13 @@ self.assertEqual(ensure_relative('c:\\home\\foo'), 'c:home\\foo') self.assertEqual(ensure_relative('home\\foo'), 'home\\foo') - def test_copy_tree_exception_in_listdir(self): + @patch('os.listdir', side_effect=OSError()) + def test_copy_tree_exception_in_listdir(self, listdir): """ An exception in listdir should raise a DistutilsFileError """ - def new_listdir(path): - raise OSError() - # simulate a transient network error or other failure invoking listdir - with patch_obj(os, 'listdir', new_listdir): - args = 'src', None - exc = errors.DistutilsFileError - self.assertRaises(exc, dir_util.copy_tree, *args) + with self.assertRaises(errors.DistutilsFileError): + dir_util.copy_tree('src', None) def test_suite(): -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 1 00:02:57 2014 From: python-checkins at python.org (jason.coombs) Date: Mon, 1 Sep 2014 00:02:57 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogIzIyMzE1OiBQcm92?= =?utf-8?q?ide_an_actual_directory_during_test_invocation=2E?= Message-ID: <3hmT7P5g1tz7Lrx@mail.python.org> http://hg.python.org/cpython/rev/3402813338db changeset: 92287:3402813338db branch: 3.4 user: Jason R. Coombs date: Sun Aug 31 17:37:35 2014 -0400 summary: #22315: Provide an actual directory during test invocation. files: Lib/distutils/tests/test_dir_util.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Lib/distutils/tests/test_dir_util.py b/Lib/distutils/tests/test_dir_util.py --- a/Lib/distutils/tests/test_dir_util.py +++ b/Lib/distutils/tests/test_dir_util.py @@ -128,7 +128,7 @@ An exception in listdir should raise a DistutilsFileError """ with self.assertRaises(errors.DistutilsFileError): - dir_util.copy_tree('src', None) + dir_util.copy_tree(self.target, None) def test_suite(): -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 1 00:02:58 2014 From: python-checkins at python.org (jason.coombs) Date: Mon, 1 Sep 2014 00:02:58 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogIzIyMzE1OiBVc2Ug?= =?utf-8?q?an_existent_directory_for_=27src=27_to_trigger_appropriate_beha?= =?utf-8?q?vior=2E?= Message-ID: <3hmT7Q6xrWz7LsF@mail.python.org> http://hg.python.org/cpython/rev/ddb0f90620b7 changeset: 92288:ddb0f90620b7 branch: 3.4 user: Jason R. Coombs date: Sun Aug 31 17:51:22 2014 -0400 summary: #22315: Use an existent directory for 'src' to trigger appropriate behavior. files: Lib/distutils/tests/test_dir_util.py | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diff --git a/Lib/distutils/tests/test_dir_util.py b/Lib/distutils/tests/test_dir_util.py --- a/Lib/distutils/tests/test_dir_util.py +++ b/Lib/distutils/tests/test_dir_util.py @@ -128,7 +128,8 @@ An exception in listdir should raise a DistutilsFileError """ with self.assertRaises(errors.DistutilsFileError): - dir_util.copy_tree(self.target, None) + src = self.tempdirs[-1] + dir_util.copy_tree(src, None) def test_suite(): -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 1 00:03:00 2014 From: python-checkins at python.org (jason.coombs) Date: Mon, 1 Sep 2014 00:03:00 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogIzIyMzE1OiBVc2Ug?= =?utf-8?q?advertised_API_for_OSError?= Message-ID: <3hmT7S1BrHz7Ls3@mail.python.org> http://hg.python.org/cpython/rev/75a5cc4ef31c changeset: 92289:75a5cc4ef31c branch: 3.4 user: Jason R. Coombs date: Sun Aug 31 17:42:20 2014 -0400 summary: #22315: Use advertised API for OSError files: Lib/distutils/dir_util.py | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diff --git a/Lib/distutils/dir_util.py b/Lib/distutils/dir_util.py --- a/Lib/distutils/dir_util.py +++ b/Lib/distutils/dir_util.py @@ -125,12 +125,11 @@ try: names = os.listdir(src) except OSError as e: - (errno, errstr) = e if dry_run: names = [] else: raise DistutilsFileError( - "error listing files in '%s': %s" % (src, errstr)) + "error listing files in '%s': %s" % (src, e.strerror)) if not dry_run: mkpath(dst, verbose=verbose) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 1 00:03:01 2014 From: python-checkins at python.org (jason.coombs) Date: Mon, 1 Sep 2014 00:03:01 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?q?=29=3A_Merge_with_3=2E4=3B_Closes_=2322315?= Message-ID: <3hmT7T2wDjz7LsG@mail.python.org> http://hg.python.org/cpython/rev/640c575ab3e1 changeset: 92290:640c575ab3e1 parent: 92284:2b2da4ae86b4 parent: 92289:75a5cc4ef31c user: Jason R. Coombs date: Sun Aug 31 18:02:18 2014 -0400 summary: Merge with 3.4; Closes #22315 files: Lib/distutils/dir_util.py | 3 +-- Lib/distutils/tests/test_dir_util.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/Lib/distutils/dir_util.py b/Lib/distutils/dir_util.py --- a/Lib/distutils/dir_util.py +++ b/Lib/distutils/dir_util.py @@ -125,12 +125,11 @@ try: names = os.listdir(src) except OSError as e: - (errno, errstr) = e if dry_run: names = [] else: raise DistutilsFileError( - "error listing files in '%s': %s" % (src, errstr)) + "error listing files in '%s': %s" % (src, e.strerror)) if not dry_run: mkpath(dst, verbose=verbose) diff --git a/Lib/distutils/tests/test_dir_util.py b/Lib/distutils/tests/test_dir_util.py --- a/Lib/distutils/tests/test_dir_util.py +++ b/Lib/distutils/tests/test_dir_util.py @@ -3,7 +3,9 @@ import os import stat import sys +from unittest.mock import patch +from distutils import dir_util, errors from distutils.dir_util import (mkpath, remove_tree, create_tree, copy_tree, ensure_relative) @@ -11,6 +13,7 @@ from distutils.tests import support from test.support import run_unittest + class DirUtilTestCase(support.TempdirManager, unittest.TestCase): def _log(self, msg, *args): @@ -119,6 +122,16 @@ self.assertEqual(ensure_relative('c:\\home\\foo'), 'c:home\\foo') self.assertEqual(ensure_relative('home\\foo'), 'home\\foo') + @patch('os.listdir', side_effect=OSError()) + def test_copy_tree_exception_in_listdir(self, listdir): + """ + An exception in listdir should raise a DistutilsFileError + """ + with self.assertRaises(errors.DistutilsFileError): + src = self.tempdirs[-1] + dir_util.copy_tree(src, None) + + def test_suite(): return unittest.makeSuite(DirUtilTestCase) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 1 01:57:27 2014 From: python-checkins at python.org (alex.gaynor) Date: Mon, 1 Sep 2014 01:57:27 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_476=3A_Better_document_th?= =?utf-8?q?e_warning_mechanism_discussed_on_python-dev?= Message-ID: <3hmWgW1PTCz7LrX@mail.python.org> http://hg.python.org/peps/rev/d0de5805cbab changeset: 5537:d0de5805cbab user: Alex Gaynor date: Sun Aug 31 16:57:21 2014 -0700 summary: PEP 476: Better document the warning mechanism discussed on python-dev files: pep-0476.txt | 9 +++++++++ 1 files changed, 9 insertions(+), 0 deletions(-) diff --git a/pep-0476.txt b/pep-0476.txt --- a/pep-0476.txt +++ b/pep-0476.txt @@ -115,6 +115,15 @@ following this PEP will be modified to emit a warning in cases that would raise an Exception in Python 3.5. +Warnings +-------- + +To support this warning, in 3.4.next a new ``verify_mode`` is introduced +``CERT_WARN``, which is equivilant to ``CERT_NONE``, except in cases that would +fail as ``CERT_REQUIRED`` or fail the hostname check emits a warning. In +3.4.next the ``httplib`` module will set this as the ``verify_mode`` if the +default context is used. + Other protocols =============== -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Mon Sep 1 05:19:09 2014 From: python-checkins at python.org (guido.van.rossum) Date: Mon, 1 Sep 2014 05:19:09 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_Clarify_space_around_colon_in?= =?utf-8?q?_slice_=28severa_cases=29=2E?= Message-ID: <3hmc8F6RCDz7LrX@mail.python.org> http://hg.python.org/peps/rev/e98737176f1d changeset: 5538:e98737176f1d user: Guido van Rossum date: Sun Aug 31 20:18:33 2014 -0700 summary: Clarify space around colon in slice (severa cases). files: pep-0008.txt | 25 +++++++++++++++++++++++-- 1 files changed, 23 insertions(+), 2 deletions(-) diff --git a/pep-0008.txt b/pep-0008.txt --- a/pep-0008.txt +++ b/pep-0008.txt @@ -408,6 +408,27 @@ Yes: if x == 4: print x, y; x, y = y, x No: if x == 4 : print x , y ; x , y = y , x +- However, in a slice the colon acts like a binary operator, and + should have equal amounts on either side (treating it as the + operator with the lowest priority). In an extended slice, both + colons must have the same amount of spacing applied. Exception: + when a slice parameter is omitted, the space is omitted. + + Yes:: + + ham[1:9], ham[1:9:3], ham[:9:3], ham[1::3], ham[1:9:] + ham[lower:upper], ham[lower:upper:], ham[lower::step] + ham[lower+offset : upper+offset] + ham[: upper_fn(x) : step_fn(x)], ham[:: step_fn(x)] + ham[lower + offset : upper + offset] + + No:: + + ham[lower + offset:upper + offset] + ham[1: 9], ham[1 :9], ham[1:9 :3] + ham[lower : : upper] + ham[ : upper] + - Immediately before the open parenthesis that starts the argument list of a function call:: @@ -417,8 +438,8 @@ - Immediately before the open parenthesis that starts an indexing or slicing:: - Yes: dict['key'] = list[index] - No: dict ['key'] = list [index] + Yes: dct['key'] = lst[index] + No: dct ['key'] = lst [index] - More than one space around an assignment (or other) operator to align it with another. -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Mon Sep 1 09:08:43 2014 From: python-checkins at python.org (ned.deily) Date: Mon, 1 Sep 2014 09:08:43 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzIyMzIw?= =?utf-8?q?=3A_Fix_broken_link_in_the_General_Python_FAQ=2E?= Message-ID: <3hmjF74H5Bz7Lmn@mail.python.org> http://hg.python.org/cpython/rev/241f9aa9fb89 changeset: 92291:241f9aa9fb89 branch: 2.7 parent: 92281:446d4dfcc220 user: Ned Deily date: Sun Aug 31 23:57:13 2014 -0700 summary: Issue #22320: Fix broken link in the General Python FAQ. Original patch by Josh Lynn. files: Doc/faq/general.rst | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -167,7 +167,8 @@ several useful pieces of freely distributable software. The source will compile and run out of the box on most UNIX platforms. -Consult the `Developer FAQ `__ for more +Consult the `Getting Started section of the Python Developer's Guide +`__ for more information on getting the source code and compiling it. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 1 09:08:44 2014 From: python-checkins at python.org (ned.deily) Date: Mon, 1 Sep 2014 09:08:44 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzIyMzIw?= =?utf-8?q?=3A_Fix_broken_link_in_the_General_Python_FAQ=2E?= Message-ID: <3hmjF85fmFz7LrC@mail.python.org> http://hg.python.org/cpython/rev/3eaba8a0cb3a changeset: 92292:3eaba8a0cb3a branch: 3.4 parent: 92289:75a5cc4ef31c user: Ned Deily date: Mon Sep 01 00:06:18 2014 -0700 summary: Issue #22320: Fix broken link in the General Python FAQ. Original patch by Josh Lynn. files: Doc/faq/general.rst | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -167,7 +167,8 @@ several useful pieces of freely distributable software. The source will compile and run out of the box on most UNIX platforms. -Consult the `Developer FAQ `__ for more +Consult the `Getting Started section of the Python Developer's Guide +`__ for more information on getting the source code and compiling it. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 1 09:08:46 2014 From: python-checkins at python.org (ned.deily) Date: Mon, 1 Sep 2014 09:08:46 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?q?=29=3A_Issue_=2322320=3A_merge_from_3=2E4?= Message-ID: <3hmjFB0SZ2z7Lr3@mail.python.org> http://hg.python.org/cpython/rev/c81e2b6eccd8 changeset: 92293:c81e2b6eccd8 parent: 92290:640c575ab3e1 parent: 92292:3eaba8a0cb3a user: Ned Deily date: Mon Sep 01 00:08:00 2014 -0700 summary: Issue #22320: merge from 3.4 files: Doc/faq/general.rst | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -167,7 +167,8 @@ several useful pieces of freely distributable software. The source will compile and run out of the box on most UNIX platforms. -Consult the `Developer FAQ `__ for more +Consult the `Getting Started section of the Python Developer's Guide +`__ for more information on getting the source code and compiling it. -- Repository URL: http://hg.python.org/cpython From solipsis at pitrou.net Mon Sep 1 10:41:45 2014 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Mon, 01 Sep 2014 10:41:45 +0200 Subject: [Python-checkins] Daily reference leaks (640c575ab3e1): sum=151940 Message-ID: results for 640c575ab3e1 on branch "default" -------------------------------------------- test_codecs leaked [5825, 5825, 5825] references, sum=17475 test_codecs leaked [1172, 1174, 1174] memory blocks, sum=3520 test_collections leaked [0, 2, 0] references, sum=2 test_distutils leaked [37735, 37735, 37735] references, sum=113205 test_distutils leaked [5909, 5911, 5911] memory blocks, sum=17731 test_functools leaked [0, 0, 3] memory blocks, sum=3 test_site leaked [0, 0, 2] references, sum=2 test_site leaked [0, 0, 2] memory blocks, sum=2 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogHPiXJp', '-x'] From python-checkins at python.org Mon Sep 1 11:29:33 2014 From: python-checkins at python.org (berker.peksag) Date: Mon, 1 Sep 2014 11:29:33 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzE5NDQ3?= =?utf-8?q?=3A_Suppress_output_of_py=5Fcompile=2Ecompile=28=29=2E?= Message-ID: <3hmmMd48fGz7Llx@mail.python.org> http://hg.python.org/cpython/rev/2d0bcb653085 changeset: 92294:2d0bcb653085 branch: 3.4 parent: 92292:3eaba8a0cb3a user: Berker Peksag date: Mon Sep 01 12:29:53 2014 +0300 summary: Issue #19447: Suppress output of py_compile.compile(). files: Lib/test/test_py_compile.py | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py --- a/Lib/test/test_py_compile.py +++ b/Lib/test/test_py_compile.py @@ -94,7 +94,8 @@ def test_bad_coding(self): bad_coding = os.path.join(os.path.dirname(__file__), 'bad_coding2.py') - self.assertIsNone(py_compile.compile(bad_coding, doraise=False)) + with support.captured_stderr(): + self.assertIsNone(py_compile.compile(bad_coding, doraise=False)) self.assertFalse(os.path.exists( importlib.util.cache_from_source(bad_coding))) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 1 11:29:34 2014 From: python-checkins at python.org (berker.peksag) Date: Mon, 1 Sep 2014 11:29:34 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?q?=29=3A_Issue_=2319447=3A_Suppress_output_of_py=5Fcompile=2Ecomp?= =?utf-8?b?aWxlKCku?= Message-ID: <3hmmMf5rzCz7Ln0@mail.python.org> http://hg.python.org/cpython/rev/a8ef9d7c4d20 changeset: 92295:a8ef9d7c4d20 parent: 92293:c81e2b6eccd8 parent: 92294:2d0bcb653085 user: Berker Peksag date: Mon Sep 01 12:30:17 2014 +0300 summary: Issue #19447: Suppress output of py_compile.compile(). files: Lib/test/test_py_compile.py | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py --- a/Lib/test/test_py_compile.py +++ b/Lib/test/test_py_compile.py @@ -94,7 +94,8 @@ def test_bad_coding(self): bad_coding = os.path.join(os.path.dirname(__file__), 'bad_coding2.py') - self.assertIsNone(py_compile.compile(bad_coding, doraise=False)) + with support.captured_stderr(): + self.assertIsNone(py_compile.compile(bad_coding, doraise=False)) self.assertFalse(os.path.exists( importlib.util.cache_from_source(bad_coding))) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 1 11:32:59 2014 From: python-checkins at python.org (berker.peksag) Date: Mon, 1 Sep 2014 11:32:59 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E4=29=3A_Remove_unused_?= =?utf-8?q?imports=2E?= Message-ID: <3hmmRb3jBvz7Llx@mail.python.org> http://hg.python.org/cpython/rev/45d9f4172451 changeset: 92296:45d9f4172451 branch: 3.4 parent: 92294:2d0bcb653085 user: Berker Peksag date: Mon Sep 01 12:33:12 2014 +0300 summary: Remove unused imports. files: Lib/test/test_py_compile.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py --- a/Lib/test/test_py_compile.py +++ b/Lib/test/test_py_compile.py @@ -3,11 +3,11 @@ import py_compile import shutil import stat -import sys import tempfile import unittest -from test import support, script_helper +from test import support + class PyCompileTests(unittest.TestCase): -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 1 11:33:00 2014 From: python-checkins at python.org (berker.peksag) Date: Mon, 1 Sep 2014 11:33:00 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?q?=29=3A_Remove_unused_imports=2E?= Message-ID: <3hmmRc5K9Tz7Llx@mail.python.org> http://hg.python.org/cpython/rev/5f1d5e1f6c1a changeset: 92297:5f1d5e1f6c1a parent: 92295:a8ef9d7c4d20 parent: 92296:45d9f4172451 user: Berker Peksag date: Mon Sep 01 12:33:41 2014 +0300 summary: Remove unused imports. files: Lib/test/test_py_compile.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py --- a/Lib/test/test_py_compile.py +++ b/Lib/test/test_py_compile.py @@ -3,11 +3,11 @@ import py_compile import shutil import stat -import sys import tempfile import unittest -from test import support, script_helper +from test import support + class PyCompileTests(unittest.TestCase): -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 1 23:59:23 2014 From: python-checkins at python.org (benjamin.peterson) Date: Mon, 1 Sep 2014 23:59:23 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_a_more_universal_unicode_c?= =?utf-8?q?har_example?= Message-ID: <3hn50q73p4z7LkK@mail.python.org> http://hg.python.org/cpython/rev/68d08df36620 changeset: 92298:68d08df36620 user: Benjamin Peterson date: Mon Sep 01 17:59:18 2014 -0400 summary: a more universal unicode char example files: Doc/library/functions.rst | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -158,8 +158,7 @@ Return the string representing a character whose Unicode codepoint is the integer *i*. For example, ``chr(97)`` returns the string ``'a'``, while - ``chr(12491)`` returns the string ``'?'``. This is the inverse of - :func:`ord`. + ``chr(9731)`` returns the string ``'?'``. This is the inverse of :func:`ord`. The valid range for the argument is from 0 through 1,114,111 (0x10FFFF in base 16). :exc:`ValueError` will be raised if *i* is outside that range. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Tue Sep 2 00:04:08 2014 From: python-checkins at python.org (benjamin.peterson) Date: Tue, 2 Sep 2014 00:04:08 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_try_to_pick_a_unicode_char?= =?utf-8?q?_that_latex_understands?= Message-ID: <3hn56J3MDlz7LjM@mail.python.org> http://hg.python.org/cpython/rev/6f65c9d11cc7 changeset: 92299:6f65c9d11cc7 user: Benjamin Peterson date: Mon Sep 01 18:04:02 2014 -0400 summary: try to pick a unicode char that latex understands files: Doc/library/functions.rst | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -158,7 +158,7 @@ Return the string representing a character whose Unicode codepoint is the integer *i*. For example, ``chr(97)`` returns the string ``'a'``, while - ``chr(9731)`` returns the string ``'?'``. This is the inverse of :func:`ord`. + ``chr(9835)`` returns the string ``'?'``. This is the inverse of :func:`ord`. The valid range for the argument is from 0 through 1,114,111 (0x10FFFF in base 16). :exc:`ValueError` will be raised if *i* is outside that range. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Tue Sep 2 00:26:40 2014 From: python-checkins at python.org (benjamin.peterson) Date: Tue, 2 Sep 2014 00:26:40 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_sigma_actually_works_in_la?= =?utf-8?b?dGV4Li4u?= Message-ID: <3hn5cJ1Qjgz7LmQ@mail.python.org> http://hg.python.org/cpython/rev/3dafa5f1dc04 changeset: 92300:3dafa5f1dc04 user: Benjamin Peterson date: Mon Sep 01 18:26:22 2014 -0400 summary: sigma actually works in latex... files: Doc/library/functions.rst | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -158,7 +158,7 @@ Return the string representing a character whose Unicode codepoint is the integer *i*. For example, ``chr(97)`` returns the string ``'a'``, while - ``chr(9835)`` returns the string ``'?'``. This is the inverse of :func:`ord`. + ``chr(931)`` returns the string ``'?'``. This is the inverse of :func:`ord`. The valid range for the argument is from 0 through 1,114,111 (0x10FFFF in base 16). :exc:`ValueError` will be raised if *i* is outside that range. @@ -1061,8 +1061,8 @@ Given a string representing one Unicode character, return an integer representing the Unicode code point of that character. For example, - ``ord('a')`` returns the integer ``97`` and ``ord('?')`` returns - ``12491``. This is the inverse of :func:`chr`. + ``ord('a')`` returns the integer ``97`` and ``ord('?')`` returns ``931``. + This is the inverse of :func:`chr`. .. function:: pow(x, y[, z]) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Tue Sep 2 04:51:06 2014 From: python-checkins at python.org (benjamin.peterson) Date: Tue, 2 Sep 2014 04:51:06 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_docs_have_moved?= Message-ID: <3hnCTQ5Rjdz7Ljf@mail.python.org> http://hg.python.org/peps/rev/30a9bd37c3c9 changeset: 5539:30a9bd37c3c9 user: Benjamin Peterson date: Mon Sep 01 22:46:19 2014 -0400 summary: docs have moved files: pep-0101.txt | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pep-0101.txt b/pep-0101.txt --- a/pep-0101.txt +++ b/pep-0101.txt @@ -425,10 +425,10 @@ release for an older version, don't change the current link. ___ If this is a final release (even a maintenance release), also unpack - the HTML docs to - /data/ftp.python.org/pub/docs.python.org/release/X.Y.Z on dinsdale. - If it is a release of a security-fix-only version, tell the DE to - build a version with the "version switcher" and put it there. + the HTML docs to /srv/docs.python.org/release/X.Y.Z on + docs-backend.psf.io. Make sure the files are in group "docs". If it + is a release of a security-fix-only version, tell the DE to build a + version with the "version switcher" and put it there. ___ Let the DE check if the docs are built and work all right. -- Repository URL: http://hg.python.org/peps From solipsis at pitrou.net Tue Sep 2 09:34:02 2014 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Tue, 02 Sep 2014 09:34:02 +0200 Subject: [Python-checkins] Daily reference leaks (3dafa5f1dc04): sum=151932 Message-ID: results for 3dafa5f1dc04 on branch "default" -------------------------------------------- test_codecs leaked [5825, 5825, 5825] references, sum=17475 test_codecs leaked [1172, 1174, 1174] memory blocks, sum=3520 test_collections leaked [-2, 0, 0] references, sum=-2 test_distutils leaked [37735, 37735, 37735] references, sum=113205 test_distutils leaked [5909, 5911, 5911] memory blocks, sum=17731 test_functools leaked [0, 0, 3] memory blocks, sum=3 test_site leaked [0, 2, -2] references, sum=0 test_site leaked [0, 2, -2] memory blocks, sum=0 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogQX61t6', '-x'] From python-checkins at python.org Tue Sep 2 11:50:08 2014 From: python-checkins at python.org (victor.stinner) Date: Tue, 2 Sep 2014 11:50:08 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogQ2xvc2VzICMyMjI1?= =?utf-8?q?8=3A_Fix_the_the_internal_function_set=5Finheritable=28=29_on_I?= =?utf-8?q?llumos=2E?= Message-ID: <3hnNmw5wWdz7LjM@mail.python.org> http://hg.python.org/cpython/rev/27cef7476f2b changeset: 92301:27cef7476f2b branch: 3.4 parent: 92296:45d9f4172451 user: Victor Stinner date: Tue Sep 02 11:41:04 2014 +0200 summary: Closes #22258: Fix the the internal function set_inheritable() on Illumos. This platform exposes the function ioctl(FIOCLEX), but calling it fails with errno is ENOTTY: "Inappropriate ioctl for device". set_inheritable() now falls back to the slower fcntl() (F_GETFD and then F_SETFD). files: Misc/NEWS | 5 +++ Python/fileutils.c | 48 ++++++++++++++++++++++++--------- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,11 @@ Core and Builtins ----------------- +- Issue #22258: Fix the the internal function set_inheritable() on Illumos. + This platform exposes the function ``ioctl(FIOCLEX)``, but calling it fails + with errno is ENOTTY: "Inappropriate ioctl for device". set_inheritable() + now falls back to the slower ``fcntl()`` (``F_GETFD`` and then ``F_SETFD``). + - Issue #21669: With the aid of heuristics in SyntaxError.__init__, the parser now attempts to generate more meaningful (or at least more search engine friendly) error messages when "exec" and "print" are used as diff --git a/Python/fileutils.c b/Python/fileutils.c --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -622,10 +622,12 @@ #ifdef MS_WINDOWS HANDLE handle; DWORD flags; -#elif defined(HAVE_SYS_IOCTL_H) && defined(FIOCLEX) && defined(FIONCLEX) +#else +#if defined(HAVE_SYS_IOCTL_H) && defined(FIOCLEX) && defined(FIONCLEX) + static int ioctl_works = -1; int request; int err; -#elif defined(HAVE_FCNTL_H) +#endif int flags; int res; #endif @@ -671,20 +673,38 @@ } return 0; -#elif defined(HAVE_SYS_IOCTL_H) && defined(FIOCLEX) && defined(FIONCLEX) - if (inheritable) - request = FIONCLEX; - else - request = FIOCLEX; - err = ioctl(fd, request, NULL); - if (err) { - if (raise) - PyErr_SetFromErrno(PyExc_OSError); - return -1; +#else + +#if defined(HAVE_SYS_IOCTL_H) && defined(FIOCLEX) && defined(FIONCLEX) + if (ioctl_works != 0) { + /* fast-path: ioctl() only requires one syscall */ + if (inheritable) + request = FIONCLEX; + else + request = FIOCLEX; + err = ioctl(fd, request, NULL); + if (!err) { + ioctl_works = 1; + return 0; + } + + if (errno != ENOTTY) { + if (raise) + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + else { + /* Issue #22258: Here, ENOTTY means "Inappropriate ioctl for + device". The ioctl is declared but not supported by the kernel. + Remember that ioctl() doesn't work. It is the case on + Illumos-based OS for example. */ + ioctl_works = 0; + } + /* fallback to fcntl() if ioctl() does not work */ } - return 0; +#endif -#else + /* slow-path: fcntl() requires two syscalls */ flags = fcntl(fd, F_GETFD); if (flags < 0) { if (raise) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Tue Sep 2 11:50:10 2014 From: python-checkins at python.org (victor.stinner) Date: Tue, 2 Sep 2014 11:50:10 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?q?=29=3A_=28Merge_3=2E4=29_Closes_=2322258=3A_Fix_the_the_interna?= =?utf-8?q?l_function_set=5Finheritable=28=29_on?= Message-ID: <3hnNmy0Zzmz7LkG@mail.python.org> http://hg.python.org/cpython/rev/4a51c45f405b changeset: 92302:4a51c45f405b parent: 92300:3dafa5f1dc04 parent: 92301:27cef7476f2b user: Victor Stinner date: Tue Sep 02 11:49:48 2014 +0200 summary: (Merge 3.4) Closes #22258: Fix the the internal function set_inheritable() on Illumos. This platform exposes the function ioctl(FIOCLEX), but calling it fails with errno is ENOTTY: "Inappropriate ioctl for device". set_inheritable() now falls back to the slower fcntl() (F_GETFD and then F_SETFD). files: Misc/NEWS | 5 +++ Python/fileutils.c | 48 ++++++++++++++++++++++++--------- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,11 @@ Core and Builtins ----------------- +- Issue #22258: Fix the the internal function set_inheritable() on Illumos. + This platform exposes the function ``ioctl(FIOCLEX)``, but calling it fails + with errno is ENOTTY: "Inappropriate ioctl for device". set_inheritable() + now falls back to the slower ``fcntl()`` (``F_GETFD`` and then ``F_SETFD``). + - Issue #21389: Displaying the __qualname__ of the underlying function in the repr of a bound method. diff --git a/Python/fileutils.c b/Python/fileutils.c --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -625,10 +625,12 @@ #ifdef MS_WINDOWS HANDLE handle; DWORD flags; -#elif defined(HAVE_SYS_IOCTL_H) && defined(FIOCLEX) && defined(FIONCLEX) +#else +#if defined(HAVE_SYS_IOCTL_H) && defined(FIOCLEX) && defined(FIONCLEX) + static int ioctl_works = -1; int request; int err; -#elif defined(HAVE_FCNTL_H) +#endif int flags; int res; #endif @@ -674,20 +676,38 @@ } return 0; -#elif defined(HAVE_SYS_IOCTL_H) && defined(FIOCLEX) && defined(FIONCLEX) - if (inheritable) - request = FIONCLEX; - else - request = FIOCLEX; - err = ioctl(fd, request, NULL); - if (err) { - if (raise) - PyErr_SetFromErrno(PyExc_OSError); - return -1; +#else + +#if defined(HAVE_SYS_IOCTL_H) && defined(FIOCLEX) && defined(FIONCLEX) + if (ioctl_works != 0) { + /* fast-path: ioctl() only requires one syscall */ + if (inheritable) + request = FIONCLEX; + else + request = FIOCLEX; + err = ioctl(fd, request, NULL); + if (!err) { + ioctl_works = 1; + return 0; + } + + if (errno != ENOTTY) { + if (raise) + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + else { + /* Issue #22258: Here, ENOTTY means "Inappropriate ioctl for + device". The ioctl is declared but not supported by the kernel. + Remember that ioctl() doesn't work. It is the case on + Illumos-based OS for example. */ + ioctl_works = 0; + } + /* fallback to fcntl() if ioctl() does not work */ } - return 0; +#endif -#else + /* slow-path: fcntl() requires two syscalls */ flags = fcntl(fd, F_GETFD); if (flags < 0) { if (raise) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Tue Sep 2 13:15:55 2014 From: python-checkins at python.org (nick.coghlan) Date: Tue, 2 Sep 2014 13:15:55 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_440=3A_Missed_one_update_?= =?utf-8?q?for_single_number_releases?= Message-ID: <3hnQgv6qnDz7Ljl@mail.python.org> http://hg.python.org/peps/rev/ff38b758e584 changeset: 5540:ff38b758e584 user: Nick Coghlan date: Tue Sep 02 21:15:46 2014 +1000 summary: PEP 440: Missed one update for single number releases files: pep-0440.txt | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/pep-0440.txt b/pep-0440.txt --- a/pep-0440.txt +++ b/pep-0440.txt @@ -178,7 +178,7 @@ The release segment consists of one or more non-negative integer values, separated by dots:: - N[.N]+ + N(.N)* Final releases within a project MUST be numbered in a consistently increasing fashion, otherwise automated tools will not be able to upgrade -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Tue Sep 2 19:39:28 2014 From: python-checkins at python.org (guido.van.rossum) Date: Tue, 2 Sep 2014 19:39:28 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Closes_=2321527=3A_Add_def?= =?utf-8?q?ault_number_of_workers_to_ThreadPoolExecutor=2E_=28Claudiu?= Message-ID: <3hnbBS1dz0z7LjM@mail.python.org> http://hg.python.org/cpython/rev/2805b0dca798 changeset: 92303:2805b0dca798 user: Guido van Rossum date: Tue Sep 02 10:39:18 2014 -0700 summary: Closes #21527: Add default number of workers to ThreadPoolExecutor. (Claudiu Popa.) files: Doc/library/concurrent.futures.rst | 10 +++++++++- Lib/concurrent/futures/thread.py | 7 ++++++- Lib/test/test_concurrent_futures.py | 6 ++++++ Misc/NEWS | 3 +++ 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -115,11 +115,19 @@ executor.submit(wait_on_future) -.. class:: ThreadPoolExecutor(max_workers) +.. class:: ThreadPoolExecutor(max_workers=None) An :class:`Executor` subclass that uses a pool of at most *max_workers* threads to execute calls asynchronously. + .. versionchanged:: 3.5 + If *max_workers* is ``None`` or + not given, it will default to the number of processors on the machine, + multiplied by ``5``, assuming that :class:`ThreadPoolExecutor` is often + used to overlap I/O instead of CPU work and the number of workers + should be higher than the number of workers + for :class:`ProcessPoolExecutor`. + .. _threadpoolexecutor-example: diff --git a/Lib/concurrent/futures/thread.py b/Lib/concurrent/futures/thread.py --- a/Lib/concurrent/futures/thread.py +++ b/Lib/concurrent/futures/thread.py @@ -10,6 +10,7 @@ import queue import threading import weakref +import os # Workers are created as daemon threads. This is done to allow the interpreter # to exit when there are still idle threads in a ThreadPoolExecutor's thread @@ -80,13 +81,17 @@ _base.LOGGER.critical('Exception in worker', exc_info=True) class ThreadPoolExecutor(_base.Executor): - def __init__(self, max_workers): + def __init__(self, max_workers=None): """Initializes a new ThreadPoolExecutor instance. Args: max_workers: The maximum number of threads that can be used to execute the given calls. """ + if max_workers is None: + # Use this number because ThreadPoolExecutor is often + # used to overlap I/O instead of CPU work. + max_workers = (os.cpu_count() or 1) * 5 if max_workers <= 0: raise ValueError("max_workers must be greater than 0") diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py --- a/Lib/test/test_concurrent_futures.py +++ b/Lib/test/test_concurrent_futures.py @@ -11,6 +11,7 @@ from test.script_helper import assert_python_ok +import os import sys import threading import time @@ -444,6 +445,11 @@ self.executor.shutdown(wait=True) self.assertCountEqual(finished, range(10)) + def test_default_workers(self): + executor = self.executor_type() + self.assertEqual(executor._max_workers, + (os.cpu_count() or 1) * 5) + class ProcessPoolExecutorTest(ProcessPoolMixin, ExecutorTest, unittest.TestCase): def test_killed_child(self): diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -129,6 +129,9 @@ Library ------- +- Issue #21527: Add a default number of workers to ThreadPoolExecutor equal + to 5 times the number of CPUs. Patch by Claudiu Popa. + - Issue #22216: smtplib now resets its state more completely after a quit. The most obvious consequence of the previous behavior was a STARTTLS failure during a connect/starttls/quit/connect/starttls sequence. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Tue Sep 2 21:12:25 2014 From: python-checkins at python.org (victor.stinner) Date: Tue, 2 Sep 2014 21:12:25 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_471=3A_Minor_change=2C_re?= =?utf-8?q?name_scandir=28=29_parameter_from_=22directory=22_to_=22path=22?= Message-ID: <3hndFj64Bxz7LjM@mail.python.org> http://hg.python.org/peps/rev/c375c79c2d5c changeset: 5541:c375c79c2d5c user: Victor Stinner date: Tue Sep 02 21:12:14 2014 +0200 summary: PEP 471: Minor change, rename scandir() parameter from "directory" to "path" files: pep-0471.txt | 33 +++++++++++++++++---------------- 1 files changed, 17 insertions(+), 16 deletions(-) diff --git a/pep-0471.txt b/pep-0471.txt --- a/pep-0471.txt +++ b/pep-0471.txt @@ -91,11 +91,11 @@ module in the standard library, ``scandir``, that takes a single, optional string as its argument:: - scandir(directory='.') -> generator of DirEntry objects + scandir(path='.') -> generator of DirEntry objects Like ``listdir``, ``scandir`` calls the operating system's directory iteration system calls to get the names of the files in the given -``directory``, but it's different from ``listdir`` in two ways: +``path``, but it's different from ``listdir`` in two ways: * Instead of returning bare filename strings, it returns lightweight ``DirEntry`` objects that hold the filename string and provide @@ -106,16 +106,17 @@ as a true iterator instead of returning the full list immediately. ``scandir()`` yields a ``DirEntry`` object for each file and -sub-directory in ``directory``. Just like ``listdir``, the ``'.'`` +sub-directory in ``path``. Just like ``listdir``, the ``'.'`` and ``'..'`` pseudo-directories are skipped, and the entries are yielded in system-dependent order. Each ``DirEntry`` object has the following attributes and methods: -* ``name``: the entry's filename, relative to the ``directory`` +* ``name``: the entry's filename, relative to the scandir ``path`` argument (corresponds to the return values of ``os.listdir``) * ``path``: the entry's full path name (not necessarily an absolute - path) -- the equivalent of ``os.path.join(directory, entry.name)`` + path) -- the equivalent of ``os.path.join(scandir_path, + entry.name)`` * ``is_dir(*, follow_symlinks=True)``: similar to ``pathlib.Path.is_dir()``, but the return value is cached on the @@ -124,7 +125,7 @@ * ``is_file(*, follow_symlinks=True)``: similar to ``pathlib.Path.is_file()``, but the return value is cached on the - ``DirEntry`` object; doesn't require a system call in most cases; + ``DirEntry`` object; doesn't require a system call in most cases; don't follow symbolic links if ``follow_symlinks`` is False * ``is_symlink()``: similar to ``pathlib.Path.is_symlink()``, but the @@ -147,9 +148,9 @@ first call. Like the other functions in the ``os`` module, ``scandir()`` accepts -either a bytes or str object for the ``directory`` parameter, and +either a bytes or str object for the ``path`` parameter, and returns the ``DirEntry.name`` and ``DirEntry.path`` attributes with -the same type as ``directory``. However, it is *strongly recommended* +the same type as ``path``. However, it is *strongly recommended* to use the str type, as this ensures cross-platform support for Unicode filenames. (On Windows, bytes filenames have been deprecated since Python 3.3). @@ -183,10 +184,10 @@ use of the ``DirEntry.stat()`` method and ``DirEntry.path`` attribute:: - def get_tree_size(directory): - """Return total size of files in directory and subdirs.""" + def get_tree_size(path): + """Return total size of files in given path and subdirs.""" total = 0 - for entry in os.scandir(directory): + for entry in os.scandir(path): if entry.is_dir(follow_symlinks=False): total += get_tree_size(entry.path) else: @@ -256,13 +257,13 @@ For example, below is a version of the ``get_tree_size()`` example shown above, but with fine-grained error handling added:: - def get_tree_size(directory): - """Return total size of files in directory and subdirs. If + def get_tree_size(path): + """Return total size of files in path and subdirs. If is_dir() or stat() fails, print an error message to stderr and assume zero size (for example, file has been deleted). """ total = 0 - for entry in os.scandir(directory): + for entry in os.scandir(path): try: is_dir = entry.is_dir(follow_symlinks=False) except OSError as error: @@ -534,12 +535,12 @@ ``OSError``) during iteration, leading to a rather ugly, hand-made iteration loop:: - it = os.scandir(directory) + it = os.scandir(path) while True: try: entry = next(it) except OSError as error: - handle_error(directory, error) + handle_error(path, error) except StopIteration: break -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Tue Sep 2 23:01:57 2014 From: python-checkins at python.org (victor.stinner) Date: Tue, 2 Sep 2014 23:01:57 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_pytime=2Eh=3A_remove_dupli?= =?utf-8?q?cated_=22=23ifndef_Py=5FLIMITED=5FAPI=22?= Message-ID: <3hngh54MHBz7LjY@mail.python.org> http://hg.python.org/cpython/rev/41853d9d60ff changeset: 92304:41853d9d60ff user: Victor Stinner date: Sun Aug 31 15:48:55 2014 +0200 summary: pytime.h: remove duplicated "#ifndef Py_LIMITED_API" files: Include/pytime.h | 3 --- 1 files changed, 0 insertions(+), 3 deletions(-) diff --git a/Include/pytime.h b/Include/pytime.h --- a/Include/pytime.h +++ b/Include/pytime.h @@ -13,8 +13,6 @@ extern "C" { #endif -#ifndef Py_LIMITED_API - #ifdef HAVE_GETTIMEOFDAY typedef struct timeval _PyTime_timeval; #else @@ -96,7 +94,6 @@ /* Initialize time. Return 0 on success, raise an exception and return -1 on error. */ PyAPI_FUNC(int) _PyTime_Init(void); -#endif /* Py_LIMITED_API */ #ifdef __cplusplus } -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Tue Sep 2 23:01:58 2014 From: python-checkins at python.org (victor.stinner) Date: Tue, 2 Sep 2014 23:01:58 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2322043=3A_Fix_=5FP?= =?utf-8?q?yTime=5Fgettimeofday=28=29_if_HAVE=5FGETTIMEOFDAY?= Message-ID: <3hngh65j6dz7LjY@mail.python.org> http://hg.python.org/cpython/rev/330bd57685fc changeset: 92305:330bd57685fc user: Victor Stinner date: Tue Sep 02 23:01:40 2014 +0200 summary: Issue #22043: Fix _PyTime_gettimeofday() if HAVE_GETTIMEOFDAY Ensure also that the tv_usec field is consistent: in range [0; 999999]. files: Python/pytime.c | 7 ++----- 1 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Python/pytime.c b/Python/pytime.c --- a/Python/pytime.c +++ b/Python/pytime.c @@ -37,7 +37,6 @@ info->resolution = timeIncrement * 1e-7; info->adjustable = 1; } - return 0; #else /* MS_WINDOWS */ int err; @@ -67,11 +66,9 @@ else info->resolution = 1e-9; } - return 0; #else /* HAVE_CLOCK_GETTIME */ /* test gettimeofday() */ -#ifdef HAVE_GETTIMEOFDAY #ifdef GETTIMEOFDAY_NO_TZ err = gettimeofday(tp); #else @@ -89,10 +86,10 @@ info->monotonic = 0; info->adjustable = 1; } - return 0; -#endif /* HAVE_GETTIMEOFDAY */ #endif /* !HAVE_CLOCK_GETTIME */ #endif /* !MS_WINDOWS */ + assert(0 <= tp->tv_usec && tp->tv_usec < 1000 * 1000); + return 0; } void -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Tue Sep 2 23:26:17 2014 From: python-checkins at python.org (victor.stinner) Date: Tue, 2 Sep 2014 23:26:17 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2322043=3A_time=2Em?= =?utf-8?q?onotonic=28=29_is_now_always_available?= Message-ID: <3hnhD92xYvz7LjT@mail.python.org> http://hg.python.org/cpython/rev/b12857782041 changeset: 92306:b12857782041 user: Victor Stinner date: Tue Sep 02 23:18:25 2014 +0200 summary: Issue #22043: time.monotonic() is now always available threading.Lock.acquire(), threading.RLock.acquire() and socket operations now use a monotonic clock, instead of the system clock, when a timeout is used. files: Doc/library/time.rst | 4 +- Doc/whatsnew/3.5.rst | 5 + Include/pytime.h | 18 ++ Lib/queue.py | 5 +- Lib/sched.py | 5 +- Lib/socketserver.py | 5 +- Lib/subprocess.py | 5 +- Lib/telnetlib.py | 5 +- Lib/test/test_selectors.py | 5 +- Lib/threading.py | 5 +- Lib/trace.py | 5 +- Misc/NEWS | 5 + Modules/_threadmodule.c | 4 +- Modules/gcmodule.c | 6 +- Modules/socketmodule.c | 4 +- Modules/timemodule.c | 141 +------------------- Python/pytime.c | 175 +++++++++++++++++++++++++ 17 files changed, 226 insertions(+), 176 deletions(-) diff --git a/Doc/library/time.rst b/Doc/library/time.rst --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -315,9 +315,9 @@ processes running for more than 49 days. On more recent versions of Windows and on other operating systems, :func:`monotonic` is system-wide. - Availability: Windows, Mac OS X, Linux, FreeBSD, OpenBSD, Solaris. - .. versionadded:: 3.3 + .. versionchanged:: 3.5 + The function is now always available. .. function:: perf_counter() diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -238,6 +238,11 @@ :meth:`socket.socket.send`. (contributed by Giampaolo Rodola' in :issue:`17552`) +time +---- + +The :func:`time.monotonic` function is now always available (:issue`22043`). + wsgiref ------- diff --git a/Include/pytime.h b/Include/pytime.h --- a/Include/pytime.h +++ b/Include/pytime.h @@ -91,6 +91,24 @@ long *nsec, _PyTime_round_t); +/* Get the time of a monotonic clock, i.e. a clock that cannot go backwards. + The clock is not affected by system clock updates. The reference point of + the returned value is undefined, so that only the difference between the + results of consecutive calls is valid. + + The function never fails. _PyTime_Init() ensures that a monotonic clock + is available and works. */ +PyAPI_FUNC(void) _PyTime_monotonic( + _PyTime_timeval *tp); + +/* Similar to _PyTime_monotonic(), fill also info (if set) with information of + the function used to get the time. + + Return 0 on success, raise an exception and return -1 on error. */ +PyAPI_FUNC(int) _PyTime_monotonic_info( + _PyTime_timeval *tp, + _Py_clock_info_t *info); + /* Initialize time. Return 0 on success, raise an exception and return -1 on error. */ PyAPI_FUNC(int) _PyTime_Init(void); diff --git a/Lib/queue.py b/Lib/queue.py --- a/Lib/queue.py +++ b/Lib/queue.py @@ -6,10 +6,7 @@ import dummy_threading as threading from collections import deque from heapq import heappush, heappop -try: - from time import monotonic as time -except ImportError: - from time import time +from time import monotonic as time __all__ = ['Empty', 'Full', 'Queue', 'PriorityQueue', 'LifoQueue'] diff --git a/Lib/sched.py b/Lib/sched.py --- a/Lib/sched.py +++ b/Lib/sched.py @@ -35,10 +35,7 @@ import threading except ImportError: import dummy_threading as threading -try: - from time import monotonic as _time -except ImportError: - from time import time as _time +from time import monotonic as _time __all__ = ["scheduler"] diff --git a/Lib/socketserver.py b/Lib/socketserver.py --- a/Lib/socketserver.py +++ b/Lib/socketserver.py @@ -136,10 +136,7 @@ import threading except ImportError: import dummy_threading as threading -try: - from time import monotonic as time -except ImportError: - from time import time as time +from time import monotonic as time __all__ = ["TCPServer","UDPServer","ForkingUDPServer","ForkingTCPServer", "ThreadingUDPServer","ThreadingTCPServer","BaseRequestHandler", diff --git a/Lib/subprocess.py b/Lib/subprocess.py --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -365,10 +365,7 @@ import builtins import warnings import errno -try: - from time import monotonic as _time -except ImportError: - from time import time as _time +from time import monotonic as _time # Exception classes used by this module. class SubprocessError(Exception): pass diff --git a/Lib/telnetlib.py b/Lib/telnetlib.py --- a/Lib/telnetlib.py +++ b/Lib/telnetlib.py @@ -36,10 +36,7 @@ import sys import socket import selectors -try: - from time import monotonic as _time -except ImportError: - from time import time as _time +from time import monotonic as _time __all__ = ["Telnet"] diff --git a/Lib/test/test_selectors.py b/Lib/test/test_selectors.py --- a/Lib/test/test_selectors.py +++ b/Lib/test/test_selectors.py @@ -8,10 +8,7 @@ from time import sleep import unittest import unittest.mock -try: - from time import monotonic as time -except ImportError: - from time import time as time +from time import monotonic as time try: import resource except ImportError: diff --git a/Lib/threading.py b/Lib/threading.py --- a/Lib/threading.py +++ b/Lib/threading.py @@ -3,10 +3,7 @@ import sys as _sys import _thread -try: - from time import monotonic as _time -except ImportError: - from time import time as _time +from time import monotonic as _time from traceback import format_exc as _format_exc from _weakrefset import WeakSet from itertools import islice as _islice diff --git a/Lib/trace.py b/Lib/trace.py --- a/Lib/trace.py +++ b/Lib/trace.py @@ -59,10 +59,7 @@ import dis import pickle from warnings import warn as _warn -try: - from time import monotonic as _time -except ImportError: - from time import time as _time +from time import monotonic as _time try: import threading diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -129,6 +129,11 @@ Library ------- +- Issue #22043: time.monotonic() is now always available. + ``threading.Lock.acquire()``, ``threading.RLock.acquire()`` and socket + operations now use a monotonic clock, instead of the system clock, when a + timeout is used. + - Issue #21527: Add a default number of workers to ThreadPoolExecutor equal to 5 times the number of CPUs. Patch by Claudiu Popa. diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -57,7 +57,7 @@ if (microseconds > 0) { - _PyTime_gettimeofday(&endtime); + _PyTime_monotonic(&endtime); endtime.tv_sec += microseconds / (1000 * 1000); endtime.tv_usec += microseconds % (1000 * 1000); } @@ -83,7 +83,7 @@ /* If we're using a timeout, recompute the timeout after processing * signals, since those can take time. */ if (microseconds > 0) { - _PyTime_gettimeofday(&curtime); + _PyTime_monotonic(&curtime); microseconds = ((endtime.tv_sec - curtime.tv_sec) * 1000000 + (endtime.tv_usec - curtime.tv_usec)); diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -25,7 +25,7 @@ #include "Python.h" #include "frameobject.h" /* for PyFrame_ClearFreeList */ -#include "pytime.h" /* for _PyTime_gettimeofday, _PyTime_INTERVAL */ +#include "pytime.h" /* for _PyTime_monotonic, _PyTime_INTERVAL */ /* Get an object's GC head */ #define AS_GC(o) ((PyGC_Head *)(o)-1) @@ -919,7 +919,7 @@ for (i = 0; i < NUM_GENERATIONS; i++) PySys_FormatStderr(" %zd", gc_list_size(GEN_HEAD(i))); - _PyTime_gettimeofday(&t1); + _PyTime_monotonic(&t1); PySys_WriteStderr("\n"); } @@ -1025,7 +1025,7 @@ } if (debug & DEBUG_STATS) { _PyTime_timeval t2; - _PyTime_gettimeofday(&t2); + _PyTime_monotonic(&t2); if (m == 0 && n == 0) PySys_WriteStderr("gc: done"); diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -680,7 +680,7 @@ double interval = s->sock_timeout; \ int has_timeout = s->sock_timeout > 0.0; \ if (has_timeout) { \ - _PyTime_gettimeofday(&now); \ + _PyTime_monotonic(&now); \ deadline = now; \ _PyTime_ADD_SECONDS(deadline, s->sock_timeout); \ } \ @@ -691,7 +691,7 @@ if (!has_timeout || \ (!CHECK_ERRNO(EWOULDBLOCK) && !CHECK_ERRNO(EAGAIN))) \ break; \ - _PyTime_gettimeofday(&now); \ + _PyTime_monotonic(&now); \ interval = _PyTime_INTERVAL(now, deadline); \ } \ } \ diff --git a/Modules/timemodule.c b/Modules/timemodule.c --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -37,10 +37,6 @@ #endif /* MS_WINDOWS */ #endif /* !__WATCOMC__ || __QNX__ */ -#if defined(__APPLE__) -#include -#endif - /* Forward declarations */ static int floatsleep(double); static PyObject* floattime(_Py_clock_info_t *info); @@ -899,122 +895,15 @@ should not be relied on."); #endif /* HAVE_WORKING_TZSET */ -#if defined(MS_WINDOWS) || defined(__APPLE__) \ - || (defined(HAVE_CLOCK_GETTIME) \ - && (defined(CLOCK_HIGHRES) || defined(CLOCK_MONOTONIC))) -#define PYMONOTONIC -#endif - -#ifdef PYMONOTONIC -static PyObject* +static PyObject * pymonotonic(_Py_clock_info_t *info) { -#if defined(MS_WINDOWS) - static ULONGLONG (*GetTickCount64) (void) = NULL; - static ULONGLONG (CALLBACK *Py_GetTickCount64)(void); - static int has_getickcount64 = -1; - double result; - - if (has_getickcount64 == -1) { - /* GetTickCount64() was added to Windows Vista */ - if (winver.dwMajorVersion >= 6) { - HINSTANCE hKernel32; - hKernel32 = GetModuleHandleW(L"KERNEL32"); - *(FARPROC*)&Py_GetTickCount64 = GetProcAddress(hKernel32, - "GetTickCount64"); - has_getickcount64 = (Py_GetTickCount64 != NULL); - } - else - has_getickcount64 = 0; - } - - if (has_getickcount64) { - ULONGLONG ticks; - ticks = Py_GetTickCount64(); - result = (double)ticks * 1e-3; - } - else { - static DWORD last_ticks = 0; - static DWORD n_overflow = 0; - DWORD ticks; - - ticks = GetTickCount(); - if (ticks < last_ticks) - n_overflow++; - last_ticks = ticks; - - result = ldexp(n_overflow, 32); - result += ticks; - result *= 1e-3; - } - - if (info) { - DWORD timeAdjustment, timeIncrement; - BOOL isTimeAdjustmentDisabled, ok; - if (has_getickcount64) - info->implementation = "GetTickCount64()"; - else - info->implementation = "GetTickCount()"; - info->monotonic = 1; - ok = GetSystemTimeAdjustment(&timeAdjustment, &timeIncrement, - &isTimeAdjustmentDisabled); - if (!ok) { - PyErr_SetFromWindowsErr(0); - return NULL; - } - info->resolution = timeIncrement * 1e-7; - info->adjustable = 0; - } - return PyFloat_FromDouble(result); - -#elif defined(__APPLE__) - static mach_timebase_info_data_t timebase; - uint64_t time; - double secs; - - if (timebase.denom == 0) { - /* According to the Technical Q&A QA1398, mach_timebase_info() cannot - fail: https://developer.apple.com/library/mac/#qa/qa1398/ */ - (void)mach_timebase_info(&timebase); - } - - time = mach_absolute_time(); - secs = (double)time * timebase.numer / timebase.denom * 1e-9; - if (info) { - info->implementation = "mach_absolute_time()"; - info->resolution = (double)timebase.numer / timebase.denom * 1e-9; - info->monotonic = 1; - info->adjustable = 0; - } - return PyFloat_FromDouble(secs); - -#elif defined(HAVE_CLOCK_GETTIME) && (defined(CLOCK_HIGHRES) || defined(CLOCK_MONOTONIC)) - struct timespec tp; -#ifdef CLOCK_HIGHRES - const clockid_t clk_id = CLOCK_HIGHRES; - const char *function = "clock_gettime(CLOCK_HIGHRES)"; -#else - const clockid_t clk_id = CLOCK_MONOTONIC; - const char *function = "clock_gettime(CLOCK_MONOTONIC)"; -#endif - - if (clock_gettime(clk_id, &tp) != 0) { - PyErr_SetFromErrno(PyExc_OSError); + _PyTime_timeval tv; + if (_PyTime_monotonic_info(&tv, info) < 0) { + assert(info != NULL); return NULL; } - - if (info) { - struct timespec res; - info->monotonic = 1; - info->implementation = function; - info->adjustable = 0; - if (clock_getres(clk_id, &res) == 0) - info->resolution = res.tv_sec + res.tv_nsec * 1e-9; - else - info->resolution = 1e-9; - } - return PyFloat_FromDouble(tp.tv_sec + tp.tv_nsec * 1e-9); -#endif + return PyFloat_FromDouble((double)tv.tv_sec + tv.tv_usec * 1e-6); } static PyObject * @@ -1027,7 +916,6 @@ "monotonic() -> float\n\ \n\ Monotonic clock, cannot go backward."); -#endif /* PYMONOTONIC */ static PyObject* perf_counter(_Py_clock_info_t *info) @@ -1035,20 +923,7 @@ #ifdef WIN32_PERF_COUNTER return win_perf_counter(info); #else - -#ifdef PYMONOTONIC - static int use_monotonic = 1; - - if (use_monotonic) { - PyObject *res = pymonotonic(info); - if (res != NULL) - return res; - use_monotonic = 0; - PyErr_Clear(); - } -#endif - return floattime(info); - + return pymonotonic(info); #endif } @@ -1216,10 +1091,8 @@ else if (strcmp(name, "clock") == 0) obj = pyclock(&info); #endif -#ifdef PYMONOTONIC else if (strcmp(name, "monotonic") == 0) obj = pymonotonic(&info); -#endif else if (strcmp(name, "perf_counter") == 0) obj = perf_counter(&info); else if (strcmp(name, "process_time") == 0) @@ -1411,9 +1284,7 @@ #ifdef HAVE_WORKING_TZSET {"tzset", time_tzset, METH_NOARGS, tzset_doc}, #endif -#ifdef PYMONOTONIC {"monotonic", time_monotonic, METH_NOARGS, monotonic_doc}, -#endif {"process_time", time_process_time, METH_NOARGS, process_time_doc}, {"perf_counter", time_perf_counter, METH_NOARGS, perf_counter_doc}, {"get_clock_info", time_get_clock_info, METH_VARARGS, get_clock_info_doc}, diff --git a/Python/pytime.c b/Python/pytime.c --- a/Python/pytime.c +++ b/Python/pytime.c @@ -3,6 +3,14 @@ #include #endif +#if defined(__APPLE__) +#include /* mach_absolute_time(), mach_timebase_info() */ +#endif + +#ifdef MS_WINDOWS +static OSVERSIONINFOEX winver; +#endif + static int pygettimeofday(_PyTime_timeval *tp, _Py_clock_info_t *info, int raise) { @@ -109,6 +117,160 @@ return pygettimeofday(tp, info, 1); } +static int +pymonotonic(_PyTime_timeval *tp, _Py_clock_info_t *info, int raise) +{ +#ifdef Py_DEBUG + static _PyTime_timeval last = {-1, -1}; +#endif +#if defined(MS_WINDOWS) + static ULONGLONG (*GetTickCount64) (void) = NULL; + static ULONGLONG (CALLBACK *Py_GetTickCount64)(void); + static int has_gettickcount64 = -1; + ULONGLONG result; + + assert(info == NULL || raise); + + if (has_gettickcount64 == -1) { + /* GetTickCount64() was added to Windows Vista */ + has_gettickcount64 = (winver.dwMajorVersion >= 6); + if (has_gettickcount64) { + HINSTANCE hKernel32; + hKernel32 = GetModuleHandleW(L"KERNEL32"); + *(FARPROC*)&Py_GetTickCount64 = GetProcAddress(hKernel32, + "GetTickCount64"); + assert(Py_GetTickCount64 != NULL); + } + } + + if (has_gettickcount64) { + result = Py_GetTickCount64(); + } + else { + static DWORD last_ticks = 0; + static DWORD n_overflow = 0; + DWORD ticks; + + ticks = GetTickCount(); + if (ticks < last_ticks) + n_overflow++; + last_ticks = ticks; + + result = (ULONGLONG)n_overflow << 32; + result += ticks; + } + + tp->tv_sec = result / 1000; + tp->tv_usec = (result % 1000) * 1000; + + if (info) { + DWORD timeAdjustment, timeIncrement; + BOOL isTimeAdjustmentDisabled, ok; + if (has_gettickcount64) + info->implementation = "GetTickCount64()"; + else + info->implementation = "GetTickCount()"; + info->monotonic = 1; + ok = GetSystemTimeAdjustment(&timeAdjustment, &timeIncrement, + &isTimeAdjustmentDisabled); + if (!ok) { + PyErr_SetFromWindowsErr(0); + return -1; + } + info->resolution = timeIncrement * 1e-7; + info->adjustable = 0; + } + +#elif defined(__APPLE__) + static mach_timebase_info_data_t timebase; + uint64_t time; + + if (timebase.denom == 0) { + /* According to the Technical Q&A QA1398, mach_timebase_info() cannot + fail: https://developer.apple.com/library/mac/#qa/qa1398/ */ + (void)mach_timebase_info(&timebase); + } + + time = mach_absolute_time(); + + /* nanoseconds => microseconds */ + time /= 1000; + /* apply timebase factor */ + time *= timebase.numer; + time /= timebase.denom; + tp->tv_sec = time / (1000 * 1000); + tp->tv_usec = time % (1000 * 1000); + + if (info) { + info->implementation = "mach_absolute_time()"; + info->resolution = (double)timebase.numer / timebase.denom * 1e-9; + info->monotonic = 1; + info->adjustable = 0; + } + +#else + struct timespec ts; +#ifdef CLOCK_HIGHRES + const clockid_t clk_id = CLOCK_HIGHRES; + const char *implementation = "clock_gettime(CLOCK_HIGHRES)"; +#else + const clockid_t clk_id = CLOCK_MONOTONIC; + const char *implementation = "clock_gettime(CLOCK_MONOTONIC)"; +#endif + + assert(info == NULL || raise); + + if (clock_gettime(clk_id, &ts) != 0) { + if (raise) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + tp->tv_sec = 0; + tp->tv_usec = 0; + return -1; + } + + if (info) { + struct timespec res; + info->monotonic = 1; + info->implementation = implementation; + info->adjustable = 0; + if (clock_getres(clk_id, &res) != 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + info->resolution = res.tv_sec + res.tv_nsec * 1e-9; + } + tp->tv_sec = ts.tv_sec; + tp->tv_usec = ts.tv_nsec / 1000; +#endif + assert(0 <= tp->tv_usec && tp->tv_usec < 1000 * 1000); +#ifdef Py_DEBUG + /* monotonic clock cannot go backward */ + assert(tp->tv_sec > last.tv_sec + || (tp->tv_sec == last.tv_sec && tp->tv_usec >= last.tv_usec)); + last = *tp; +#endif + return 0; +} + +void +_PyTime_monotonic(_PyTime_timeval *tp) +{ + if (pymonotonic(tp, NULL, 0) < 0) { + /* cannot happen, _PyTime_Init() checks that pymonotonic() works */ + assert(0); + tp->tv_sec = 0; + tp->tv_usec = 0; + } +} + +int +_PyTime_monotonic_info(_PyTime_timeval *tp, _Py_clock_info_t *info) +{ + return pymonotonic(tp, info, 1); +} + static void error_time_t_overflow(void) { @@ -245,8 +407,21 @@ _PyTime_Init(void) { _PyTime_timeval tv; + +#ifdef MS_WINDOWS + winver.dwOSVersionInfoSize = sizeof(winver); + if (!GetVersionEx((OSVERSIONINFO*)&winver)) { + PyErr_SetFromWindowsErr(0); + return -1; + } +#endif + /* ensure that the system clock works */ if (_PyTime_gettimeofday_info(&tv, NULL) < 0) return -1; + + /* ensure that the operating system provides a monotonic clock */ + if (_PyTime_monotonic_info(&tv, NULL) < 0) + return -1; return 0; } -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 3 09:44:06 2014 From: python-checkins at python.org (victor.stinner) Date: Wed, 3 Sep 2014 09:44:06 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2322043=3A_Fix_pymo?= =?utf-8?q?notonic=28=29=2C_use_tv=5Fusec=3D-1_as_a_marker_to_skip?= Message-ID: <3hnxx25gtyz7LjX@mail.python.org> http://hg.python.org/cpython/rev/9deef14393d5 changeset: 92307:9deef14393d5 user: Victor Stinner date: Wed Sep 03 09:43:48 2014 +0200 summary: Issue #22043: Fix pymonotonic(), use tv_usec=-1 as a marker to skip the monotonic test files: Python/pytime.c | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Python/pytime.c b/Python/pytime.c --- a/Python/pytime.c +++ b/Python/pytime.c @@ -121,7 +121,7 @@ pymonotonic(_PyTime_timeval *tp, _Py_clock_info_t *info, int raise) { #ifdef Py_DEBUG - static _PyTime_timeval last = {-1, -1}; + static _PyTime_timeval last = {0, -1}; #endif #if defined(MS_WINDOWS) static ULONGLONG (*GetTickCount64) (void) = NULL; @@ -247,7 +247,8 @@ assert(0 <= tp->tv_usec && tp->tv_usec < 1000 * 1000); #ifdef Py_DEBUG /* monotonic clock cannot go backward */ - assert(tp->tv_sec > last.tv_sec + assert(last.tv_usec == -1 + || tp->tv_sec > last.tv_sec || (tp->tv_sec == last.tv_sec && tp->tv_usec >= last.tv_usec)); last = *tp; #endif -- Repository URL: http://hg.python.org/cpython From solipsis at pitrou.net Wed Sep 3 10:36:00 2014 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Wed, 03 Sep 2014 10:36:00 +0200 Subject: [Python-checkins] Daily reference leaks (b12857782041): sum=151934 Message-ID: results for b12857782041 on branch "default" -------------------------------------------- test_codecs leaked [5825, 5825, 5825] references, sum=17475 test_codecs leaked [1172, 1174, 1174] memory blocks, sum=3520 test_collections leaked [2, 0, -2] references, sum=0 test_distutils leaked [37735, 37735, 37735] references, sum=113205 test_distutils leaked [5909, 5911, 5911] memory blocks, sum=17731 test_functools leaked [0, 0, 3] memory blocks, sum=3 test_site leaked [-2, 0, 2] references, sum=0 test_site leaked [-2, 0, 2] memory blocks, sum=0 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/refloggfQtJ8', '-x'] From python-checkins at python.org Wed Sep 3 22:19:01 2014 From: python-checkins at python.org (terry.reedy) Date: Wed, 3 Sep 2014 22:19:01 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzIxOTMz?= =?utf-8?q?=3A_Users_can_now_change_the_font_size_for_example_code=2E?= Message-ID: <3hpGh52w3jz7LjQ@mail.python.org> http://hg.python.org/cpython/rev/ce14092430b6 changeset: 92308:ce14092430b6 branch: 3.4 parent: 92301:27cef7476f2b user: Terry Jan Reedy date: Wed Sep 03 16:17:41 2014 -0400 summary: Issue #21933: Users can now change the font size for example code. Original patch by Lita Cho. files: Lib/turtledemo/__main__.py | 93 +++++++++++++++++++------ 1 files changed, 71 insertions(+), 22 deletions(-) diff --git a/Lib/turtledemo/__main__.py b/Lib/turtledemo/__main__.py --- a/Lib/turtledemo/__main__.py +++ b/Lib/turtledemo/__main__.py @@ -17,14 +17,17 @@ The (syntax coloured) source code appears in the left source code window. IT CANNOT BE EDITED, but ONLY VIEWED! - - Press START button to start the demo. - - Stop execution by pressing the STOP button. - - Clear screen by pressing the CLEAR button. - - Restart by pressing the START button again. + The demo viewer windows can be resized. The divider between text + and canvas can be moved by grabbing it with the mouse. The text font + size can be changed from the menu and with Control/Command- '-'/'+'. + It can also be changed on most systems with Control-mousewheel. - SPECIAL demos are those which run EVENTDRIVEN. - (For example clock.py - or oldTurtleDemo.py which - in the end expects a mouse click.): + Press START button to start the demo. + Stop execution by pressing the STOP button. + Clear screen by pressing the CLEAR button. + Restart by pressing the START button again. + + SPECIAL demos, such as clock.py are those which run EVENTDRIVEN. Press START button to start the demo. @@ -87,7 +90,7 @@ from tkinter import * from idlelib.Percolator import Percolator from idlelib.ColorDelegator import ColorDelegator -from idlelib.textView import view_text # TextViewer +from idlelib.textView import view_text from importlib import reload from turtledemo import __doc__ as about_turtledemo @@ -95,6 +98,7 @@ import time demo_dir = os.path.dirname(os.path.abspath(__file__)) +darwin = sys.platform == 'darwin' STARTUP = 1 READY = 2 @@ -104,7 +108,11 @@ menufont = ("Arial", 12, NORMAL) btnfont = ("Arial", 12, 'bold') -txtfont = ('Lucida Console', 8, 'normal') +txtfont = ['Lucida Console', 10, 'normal'] + +MINIMUM_FONT_SIZE = 6 +MAXIMUM_FONT_SIZE = 100 +font_sizes = [8, 9, 10, 11, 12, 14, 18, 20, 22, 24, 30] def getExampleEntries(): return [entry[:-3] for entry in os.listdir(demo_dir) if @@ -123,7 +131,7 @@ root.title('Python turtle-graphics examples') root.wm_protocol("WM_DELETE_WINDOW", self._destroy) - if sys.platform == 'darwin': + if darwin: import subprocess # Make sure we are the currently activated OS X application # so that our menu bar appears. @@ -136,8 +144,7 @@ '-e', 'end tell', ], stderr=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - ) + stdout=subprocess.DEVNULL,) root.grid_rowconfigure(0, weight=1) root.grid_columnconfigure(0, weight=1) @@ -147,9 +154,11 @@ self.mBar = Menu(root, relief=RAISED, borderwidth=2) self.mBar.add_cascade(menu=self.makeLoadDemoMenu(self.mBar), - label='Examples', underline=0, font=menufont) + label='Examples', underline=0) + self.mBar.add_cascade(menu=self.makeFontMenu(self.mBar), + label='Fontsize', underline=0) self.mBar.add_cascade(menu=self.makeHelpMenu(self.mBar), - label='Help', underline=0, font=menufont) + label='Help', underline=0) root['menu'] = self.mBar pane = PanedWindow(orient=HORIZONTAL, sashwidth=5, @@ -203,7 +212,7 @@ hbar['command'] = text.xview hbar.pack(side=BOTTOM, fill=X) - text['font'] = txtfont + text['font'] = tuple(txtfont) text['yscrollcommand'] = vbar.set text['xscrollcommand'] = hbar.set text.pack(side=LEFT, fill=BOTH, expand=1) @@ -216,7 +225,7 @@ turtle._Screen._canvas = self._canvas = canvas = turtle.ScrolledCanvas( root, 800, 600, self.canvwidth, self.canvheight) canvas.adjustScrolls() - canvas._rootwindow.bind('', self.onResize) + self.makeBindings(canvas._rootwindow) canvas._canvas['borderwidth'] = 0 self.screen = _s_ = turtle.Screen() @@ -225,6 +234,35 @@ turtle.RawTurtle.screens = [_s_] return canvas + def makeBindings(self, widget): + widget.bind('', self.onResize) + + shortcut = 'Command' if darwin else 'Control' + widget.bind_all('<%s-minus>' % shortcut, self.decrease_size) + widget.bind_all('<%s-underscore>' % shortcut, self.decrease_size) + widget.bind_all('<%s-equal>' % shortcut, self.increase_size) + widget.bind_all('<%s-plus>' % shortcut, self.increase_size) + widget.bind_all('', self.update_mousewheel) + widget.bind('', self.increase_size) + widget.bind('', self.decrease_size) + + def set_txtsize(self, size): + txtfont[1] = size + self.text['font'] = tuple(txtfont) + self.output_lbl['text'] = 'Font size %d' % size + + def decrease_size(self, dummy=None): + self.set_txtsize(max(txtfont[1] - 1, MINIMUM_FONT_SIZE)) + + def increase_size(self, dummy=None): + self.set_txtsize(min(txtfont[1] + 1, MAXIMUM_FONT_SIZE)) + + def update_mousewheel(self, event): + # For wheel up, event.delte = 120 on Windows, -1 on darwin. + # X-11 sends Control-Button-4 event instead. + (self.decrease_size() if (event.delta < 0 and not darwin) + else self.increase_size()) + def configGUI(self, start, stop, clear, txt="", color="blue"): self.start_btn.config(state=start, bg="#d00" if start == NORMAL else "#fca") @@ -238,13 +276,25 @@ menu = Menu(master) for entry in getExampleEntries(): - def loadexample(x): - def emit(): - self.loadfile(x) - return emit + def load(entry=entry): + self.loadfile(entry) menu.add_command(label=entry, underline=0, - font=menufont, command=loadexample(entry)) + font=menufont, command=load) + return menu + def makeFontMenu(self, master): + menu = Menu(master) + menu.add_command(label="Decrease (C-'-')", command=self.decrease_size, + font=menufont) + menu.add_command(label="Increase (C-'+')", command=self.increase_size, + font=menufont) + menu.add_separator() + + for size in font_sizes: + def resize(size=size): + self.set_txtsize(size) + menu.add_command(label=str(size), underline=0, + font=menufont, command=resize) return menu def makeHelpMenu(self, master): @@ -254,7 +304,6 @@ def show(help_label=help_label, help_file=help_file): view_text(self.root, help_label, help_file) menu.add_command(label=help_label, font=menufont, command=show) - return menu def refreshCanvas(self): -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 3 22:19:02 2014 From: python-checkins at python.org (terry.reedy) Date: Wed, 3 Sep 2014 22:19:02 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?q?=29=3A_Issue_=2321933=3A_Merge_with_3=2E4?= Message-ID: <3hpGh65dHfz7Ljn@mail.python.org> http://hg.python.org/cpython/rev/e2e0c9f90a81 changeset: 92309:e2e0c9f90a81 parent: 92307:9deef14393d5 parent: 92308:ce14092430b6 user: Terry Jan Reedy date: Wed Sep 03 16:18:34 2014 -0400 summary: Issue #21933: Merge with 3.4 files: Lib/turtledemo/__main__.py | 93 +++++++++++++++++++------ 1 files changed, 71 insertions(+), 22 deletions(-) diff --git a/Lib/turtledemo/__main__.py b/Lib/turtledemo/__main__.py --- a/Lib/turtledemo/__main__.py +++ b/Lib/turtledemo/__main__.py @@ -17,14 +17,17 @@ The (syntax coloured) source code appears in the left source code window. IT CANNOT BE EDITED, but ONLY VIEWED! - - Press START button to start the demo. - - Stop execution by pressing the STOP button. - - Clear screen by pressing the CLEAR button. - - Restart by pressing the START button again. + The demo viewer windows can be resized. The divider between text + and canvas can be moved by grabbing it with the mouse. The text font + size can be changed from the menu and with Control/Command- '-'/'+'. + It can also be changed on most systems with Control-mousewheel. - SPECIAL demos are those which run EVENTDRIVEN. - (For example clock.py - or oldTurtleDemo.py which - in the end expects a mouse click.): + Press START button to start the demo. + Stop execution by pressing the STOP button. + Clear screen by pressing the CLEAR button. + Restart by pressing the START button again. + + SPECIAL demos, such as clock.py are those which run EVENTDRIVEN. Press START button to start the demo. @@ -87,7 +90,7 @@ from tkinter import * from idlelib.Percolator import Percolator from idlelib.ColorDelegator import ColorDelegator -from idlelib.textView import view_text # TextViewer +from idlelib.textView import view_text from importlib import reload from turtledemo import __doc__ as about_turtledemo @@ -95,6 +98,7 @@ import time demo_dir = os.path.dirname(os.path.abspath(__file__)) +darwin = sys.platform == 'darwin' STARTUP = 1 READY = 2 @@ -104,7 +108,11 @@ menufont = ("Arial", 12, NORMAL) btnfont = ("Arial", 12, 'bold') -txtfont = ('Lucida Console', 8, 'normal') +txtfont = ['Lucida Console', 10, 'normal'] + +MINIMUM_FONT_SIZE = 6 +MAXIMUM_FONT_SIZE = 100 +font_sizes = [8, 9, 10, 11, 12, 14, 18, 20, 22, 24, 30] def getExampleEntries(): return [entry[:-3] for entry in os.listdir(demo_dir) if @@ -123,7 +131,7 @@ root.title('Python turtle-graphics examples') root.wm_protocol("WM_DELETE_WINDOW", self._destroy) - if sys.platform == 'darwin': + if darwin: import subprocess # Make sure we are the currently activated OS X application # so that our menu bar appears. @@ -136,8 +144,7 @@ '-e', 'end tell', ], stderr=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - ) + stdout=subprocess.DEVNULL,) root.grid_rowconfigure(0, weight=1) root.grid_columnconfigure(0, weight=1) @@ -147,9 +154,11 @@ self.mBar = Menu(root, relief=RAISED, borderwidth=2) self.mBar.add_cascade(menu=self.makeLoadDemoMenu(self.mBar), - label='Examples', underline=0, font=menufont) + label='Examples', underline=0) + self.mBar.add_cascade(menu=self.makeFontMenu(self.mBar), + label='Fontsize', underline=0) self.mBar.add_cascade(menu=self.makeHelpMenu(self.mBar), - label='Help', underline=0, font=menufont) + label='Help', underline=0) root['menu'] = self.mBar pane = PanedWindow(orient=HORIZONTAL, sashwidth=5, @@ -203,7 +212,7 @@ hbar['command'] = text.xview hbar.pack(side=BOTTOM, fill=X) - text['font'] = txtfont + text['font'] = tuple(txtfont) text['yscrollcommand'] = vbar.set text['xscrollcommand'] = hbar.set text.pack(side=LEFT, fill=BOTH, expand=1) @@ -216,7 +225,7 @@ turtle._Screen._canvas = self._canvas = canvas = turtle.ScrolledCanvas( root, 800, 600, self.canvwidth, self.canvheight) canvas.adjustScrolls() - canvas._rootwindow.bind('', self.onResize) + self.makeBindings(canvas._rootwindow) canvas._canvas['borderwidth'] = 0 self.screen = _s_ = turtle.Screen() @@ -225,6 +234,35 @@ turtle.RawTurtle.screens = [_s_] return canvas + def makeBindings(self, widget): + widget.bind('', self.onResize) + + shortcut = 'Command' if darwin else 'Control' + widget.bind_all('<%s-minus>' % shortcut, self.decrease_size) + widget.bind_all('<%s-underscore>' % shortcut, self.decrease_size) + widget.bind_all('<%s-equal>' % shortcut, self.increase_size) + widget.bind_all('<%s-plus>' % shortcut, self.increase_size) + widget.bind_all('', self.update_mousewheel) + widget.bind('', self.increase_size) + widget.bind('', self.decrease_size) + + def set_txtsize(self, size): + txtfont[1] = size + self.text['font'] = tuple(txtfont) + self.output_lbl['text'] = 'Font size %d' % size + + def decrease_size(self, dummy=None): + self.set_txtsize(max(txtfont[1] - 1, MINIMUM_FONT_SIZE)) + + def increase_size(self, dummy=None): + self.set_txtsize(min(txtfont[1] + 1, MAXIMUM_FONT_SIZE)) + + def update_mousewheel(self, event): + # For wheel up, event.delte = 120 on Windows, -1 on darwin. + # X-11 sends Control-Button-4 event instead. + (self.decrease_size() if (event.delta < 0 and not darwin) + else self.increase_size()) + def configGUI(self, start, stop, clear, txt="", color="blue"): self.start_btn.config(state=start, bg="#d00" if start == NORMAL else "#fca") @@ -238,13 +276,25 @@ menu = Menu(master) for entry in getExampleEntries(): - def loadexample(x): - def emit(): - self.loadfile(x) - return emit + def load(entry=entry): + self.loadfile(entry) menu.add_command(label=entry, underline=0, - font=menufont, command=loadexample(entry)) + font=menufont, command=load) + return menu + def makeFontMenu(self, master): + menu = Menu(master) + menu.add_command(label="Decrease (C-'-')", command=self.decrease_size, + font=menufont) + menu.add_command(label="Increase (C-'+')", command=self.increase_size, + font=menufont) + menu.add_separator() + + for size in font_sizes: + def resize(size=size): + self.set_txtsize(size) + menu.add_command(label=str(size), underline=0, + font=menufont, command=resize) return menu def makeHelpMenu(self, master): @@ -254,7 +304,6 @@ def show(help_label=help_label, help_file=help_file): view_text(self.root, help_label, help_file) menu.add_command(label=help_label, font=menufont, command=show) - return menu def refreshCanvas(self): -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 3 23:33:54 2014 From: python-checkins at python.org (victor.stinner) Date: Wed, 3 Sep 2014 23:33:54 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzIyMzMx?= =?utf-8?q?=3A_Skip_test=5Finterrupted=5Fwrite=5Ftext=28=29_on_FreeBSD_old?= =?utf-8?q?er_than_8=2E0?= Message-ID: <3hpJLV4f3Kz7LjP@mail.python.org> http://hg.python.org/cpython/rev/baa372eb731c changeset: 92310:baa372eb731c branch: 3.4 parent: 92308:ce14092430b6 user: Victor Stinner date: Wed Sep 03 23:32:28 2014 +0200 summary: Issue #22331: Skip test_interrupted_write_text() on FreeBSD older than 8.0 files: Lib/test/test_io.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -3298,6 +3298,8 @@ def test_interrupted_write_buffered(self): self.check_interrupted_write(b"xy", b"xy", mode="wb") + # Issue #22331: The test hangs on FreeBSD 7.2 + @support.requires_freebsd_version(8) def test_interrupted_write_text(self): self.check_interrupted_write("xy", b"xy", mode="w", encoding="ascii") -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 3 23:33:55 2014 From: python-checkins at python.org (victor.stinner) Date: Wed, 3 Sep 2014 23:33:55 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?q?=29=3A_=28Merge_3=2E4=29_Issue_=2322331=3A_Skip_test=5Finterrup?= =?utf-8?q?ted=5Fwrite=5Ftext=28=29_on_FreeBSD_older?= Message-ID: <3hpJLW6Md9z7LkG@mail.python.org> http://hg.python.org/cpython/rev/340d48347295 changeset: 92311:340d48347295 parent: 92309:e2e0c9f90a81 parent: 92310:baa372eb731c user: Victor Stinner date: Wed Sep 03 23:33:43 2014 +0200 summary: (Merge 3.4) Issue #22331: Skip test_interrupted_write_text() on FreeBSD older than 8.0 files: Lib/test/test_io.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -3383,6 +3383,8 @@ def test_interrupted_write_buffered(self): self.check_interrupted_write(b"xy", b"xy", mode="wb") + # Issue #22331: The test hangs on FreeBSD 7.2 + @support.requires_freebsd_version(8) def test_interrupted_write_text(self): self.check_interrupted_write("xy", b"xy", mode="w", encoding="ascii") -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 3 23:48:55 2014 From: python-checkins at python.org (victor.stinner) Date: Wed, 3 Sep 2014 23:48:55 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzIyMzMy?= =?utf-8?q?=3A_test=5Fmultiprocessing=5Fmain=5Fhandling_is_now_skipped_if_?= =?utf-8?q?sem=5Fopen?= Message-ID: <3hpJgq2qVBz7LjZ@mail.python.org> http://hg.python.org/cpython/rev/db9eb9cba1ec changeset: 92312:db9eb9cba1ec branch: 3.4 parent: 92310:baa372eb731c user: Victor Stinner date: Wed Sep 03 23:48:08 2014 +0200 summary: Issue #22332: test_multiprocessing_main_handling is now skipped if sem_open implementation is broken (ex: skipped on FreeBSD 6.4). files: Lib/test/test_multiprocessing_main_handling.py | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/Lib/test/test_multiprocessing_main_handling.py b/Lib/test/test_multiprocessing_main_handling.py --- a/Lib/test/test_multiprocessing_main_handling.py +++ b/Lib/test/test_multiprocessing_main_handling.py @@ -22,6 +22,9 @@ import multiprocessing AVAILABLE_START_METHODS = set(multiprocessing.get_all_start_methods()) +# Issue #22332: Skip tests if sem_open implementation is broken. +support.import_module('multiprocessing.synchronize') + verbose = support.verbose test_source = """\ -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 3 23:48:56 2014 From: python-checkins at python.org (victor.stinner) Date: Wed, 3 Sep 2014 23:48:56 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?q?=29=3A_=28Merge_3=2E4=29_Issue_=2322332=3A_test=5Fmultiprocessi?= =?utf-8?q?ng=5Fmain=5Fhandling_is_now_skipped_if?= Message-ID: <3hpJgr4WKcz7Ljf@mail.python.org> http://hg.python.org/cpython/rev/2f21d920d00d changeset: 92313:2f21d920d00d parent: 92311:340d48347295 parent: 92312:db9eb9cba1ec user: Victor Stinner date: Wed Sep 03 23:48:37 2014 +0200 summary: (Merge 3.4) Issue #22332: test_multiprocessing_main_handling is now skipped if sem_open implementation is broken (ex: skipped on FreeBSD 6.4). files: Lib/test/test_multiprocessing_main_handling.py | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/Lib/test/test_multiprocessing_main_handling.py b/Lib/test/test_multiprocessing_main_handling.py --- a/Lib/test/test_multiprocessing_main_handling.py +++ b/Lib/test/test_multiprocessing_main_handling.py @@ -22,6 +22,9 @@ import multiprocessing AVAILABLE_START_METHODS = set(multiprocessing.get_all_start_methods()) +# Issue #22332: Skip tests if sem_open implementation is broken. +support.import_module('multiprocessing.synchronize') + verbose = support.verbose test_source = """\ -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 4 00:36:27 2014 From: python-checkins at python.org (victor.stinner) Date: Thu, 4 Sep 2014 00:36:27 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzIwOTU3?= =?utf-8?q?=3A_test=5Fsmtpnet_now_uses_support=2Etransient=5Finternet=28?= =?utf-8?q?=29_to_call?= Message-ID: <3hpKkg1DNZz7Ljb@mail.python.org> http://hg.python.org/cpython/rev/b7200cde1b68 changeset: 92314:b7200cde1b68 branch: 3.4 parent: 92312:db9eb9cba1ec user: Victor Stinner date: Thu Sep 04 00:35:43 2014 +0200 summary: Issue #20957: test_smtpnet now uses support.transient_internet() to call check_ssl_verifiy(), so only test_connect_using_sslcontext_verified() is skipped if smtp.gmail.com cannot be joined, not the whole file. files: Lib/test/test_smtpnet.py | 7 +++++-- 1 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_smtpnet.py b/Lib/test/test_smtpnet.py --- a/Lib/test/test_smtpnet.py +++ b/Lib/test/test_smtpnet.py @@ -42,7 +42,6 @@ class SmtpSSLTest(unittest.TestCase): testServer = 'smtp.gmail.com' remotePort = 465 - can_verify = check_ssl_verifiy(testServer, remotePort) def test_connect(self): support.get_attribute(smtplib, 'SMTP_SSL') @@ -66,8 +65,12 @@ server.ehlo() server.quit() - @unittest.skipUnless(can_verify, "SSL certificate can't be verified") def test_connect_using_sslcontext_verified(self): + with support.transient_internet(self.testServer): + can_verify = check_ssl_verifiy(self.testServer, self.remotePort) + if not can_verify: + self.skipTest("SSL certificate can't be verified") + support.get_attribute(smtplib, 'SMTP_SSL') context = ssl.create_default_context() with support.transient_internet(self.testServer): -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 4 00:36:28 2014 From: python-checkins at python.org (victor.stinner) Date: Thu, 4 Sep 2014 00:36:28 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?q?=29=3A_=28Merge_3=2E4=29_Issue_=2320957=3A_test=5Fsmtpnet_now_u?= =?utf-8?q?ses_support=2Etransient=5Finternet=28=29_to?= Message-ID: <3hpKkh2txsz7LkT@mail.python.org> http://hg.python.org/cpython/rev/85511d4a846e changeset: 92315:85511d4a846e parent: 92313:2f21d920d00d parent: 92314:b7200cde1b68 user: Victor Stinner date: Thu Sep 04 00:36:09 2014 +0200 summary: (Merge 3.4) Issue #20957: test_smtpnet now uses support.transient_internet() to call check_ssl_verifiy(), so only test_connect_using_sslcontext_verified() is skipped if smtp.gmail.com cannot be joined, not the whole file. files: Lib/test/test_smtpnet.py | 7 +++++-- 1 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_smtpnet.py b/Lib/test/test_smtpnet.py --- a/Lib/test/test_smtpnet.py +++ b/Lib/test/test_smtpnet.py @@ -42,7 +42,6 @@ class SmtpSSLTest(unittest.TestCase): testServer = 'smtp.gmail.com' remotePort = 465 - can_verify = check_ssl_verifiy(testServer, remotePort) def test_connect(self): support.get_attribute(smtplib, 'SMTP_SSL') @@ -66,8 +65,12 @@ server.ehlo() server.quit() - @unittest.skipUnless(can_verify, "SSL certificate can't be verified") def test_connect_using_sslcontext_verified(self): + with support.transient_internet(self.testServer): + can_verify = check_ssl_verifiy(self.testServer, self.remotePort) + if not can_verify: + self.skipTest("SSL certificate can't be verified") + support.get_attribute(smtplib, 'SMTP_SSL') context = ssl.create_default_context() with support.transient_internet(self.testServer): -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 4 00:54:02 2014 From: python-checkins at python.org (victor.stinner) Date: Thu, 4 Sep 2014 00:54:02 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzIxNDQw?= =?utf-8?q?=3A_Backport_changeset_4ebf97299b18_to_branch_3=2E4=2C_use?= Message-ID: <3hpL6y4b5Tz7LjR@mail.python.org> http://hg.python.org/cpython/rev/706ec07100d6 changeset: 92316:706ec07100d6 branch: 3.4 parent: 92314:b7200cde1b68 user: Victor Stinner date: Thu Sep 04 00:49:01 2014 +0200 summary: Issue #21440: Backport changeset 4ebf97299b18 to branch 3.4, use support.rmtree() and support.unlink() in test_zipfile & test_tarfile files: Lib/test/test_tarfile.py | 39 +++++++++++++-------------- Lib/test/test_zipfile.py | 23 +++++++-------- 2 files changed, 30 insertions(+), 32 deletions(-) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1,7 +1,6 @@ import sys import os import io -import shutil from hashlib import md5 import unittest @@ -480,16 +479,16 @@ # Test hardlink extraction (e.g. bug #857297). with tarfile.open(tarname, errorlevel=1, encoding="iso8859-1") as tar: tar.extract("ustar/regtype", TEMPDIR) - self.addCleanup(os.remove, os.path.join(TEMPDIR, "ustar/regtype")) + self.addCleanup(support.unlink, os.path.join(TEMPDIR, "ustar/regtype")) tar.extract("ustar/lnktype", TEMPDIR) - self.addCleanup(os.remove, os.path.join(TEMPDIR, "ustar/lnktype")) + self.addCleanup(support.unlink, os.path.join(TEMPDIR, "ustar/lnktype")) with open(os.path.join(TEMPDIR, "ustar/lnktype"), "rb") as f: data = f.read() self.assertEqual(md5sum(data), md5_regtype) tar.extract("ustar/symtype", TEMPDIR) - self.addCleanup(os.remove, os.path.join(TEMPDIR, "ustar/symtype")) + self.addCleanup(support.unlink, os.path.join(TEMPDIR, "ustar/symtype")) with open(os.path.join(TEMPDIR, "ustar/symtype"), "rb") as f: data = f.read() self.assertEqual(md5sum(data), md5_regtype) @@ -522,7 +521,7 @@ self.assertEqual(tarinfo.mtime, file_mtime, errmsg) finally: tar.close() - shutil.rmtree(DIR) + support.rmtree(DIR) def test_extract_directory(self): dirtype = "ustar/dirtype" @@ -537,7 +536,7 @@ if sys.platform != "win32": self.assertEqual(os.stat(extracted).st_mode & 0o777, 0o755) finally: - shutil.rmtree(DIR) + support.rmtree(DIR) def test_init_close_fobj(self): # Issue #7341: Close the internal file object in the TarFile @@ -901,7 +900,7 @@ fobj.seek(4096) fobj.truncate() s = os.stat(name) - os.remove(name) + support.unlink(name) return s.st_blocks == 0 else: return False @@ -1034,7 +1033,7 @@ finally: tar.close() finally: - os.rmdir(path) + support.rmdir(path) @unittest.skipUnless(hasattr(os, "link"), "Missing hardlink implementation") @@ -1054,8 +1053,8 @@ finally: tar.close() finally: - os.remove(target) - os.remove(link) + support.unlink(target) + support.unlink(link) @support.skip_unless_symlink def test_symlink_size(self): @@ -1069,7 +1068,7 @@ finally: tar.close() finally: - os.remove(path) + support.unlink(path) def test_add_self(self): # Test for #1257255. @@ -1116,7 +1115,7 @@ finally: tar.close() finally: - shutil.rmtree(tempdir) + support.rmtree(tempdir) def test_filter(self): tempdir = os.path.join(TEMPDIR, "filter") @@ -1152,7 +1151,7 @@ finally: tar.close() finally: - shutil.rmtree(tempdir) + support.rmtree(tempdir) # Guarantee that stored pathnames are not modified. Don't # remove ./ or ../ or double slashes. Still make absolute @@ -1180,9 +1179,9 @@ tar.close() if not dir: - os.remove(foo) + support.unlink(foo) else: - os.rmdir(foo) + support.rmdir(foo) self.assertEqual(t.name, cmp_path or path.replace(os.sep, "/")) @@ -1213,8 +1212,8 @@ finally: tar.close() finally: - os.unlink(temparchive) - shutil.rmtree(tempdir) + support.unlink(temparchive) + support.rmtree(tempdir) def test_pathnames(self): self._test_pathname("foo") @@ -1314,7 +1313,7 @@ # Test for issue #8464: Create files with correct # permissions. if os.path.exists(tmpname): - os.remove(tmpname) + support.unlink(tmpname) original_umask = os.umask(0o022) try: @@ -1668,7 +1667,7 @@ def setUp(self): self.tarname = tmpname if os.path.exists(self.tarname): - os.remove(self.tarname) + support.unlink(self.tarname) def _create_testtar(self, mode="w:"): with tarfile.open(tarname, encoding="iso8859-1") as src: @@ -2175,7 +2174,7 @@ def tearDownModule(): if os.path.exists(TEMPDIR): - shutil.rmtree(TEMPDIR) + support.rmtree(TEMPDIR) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -3,7 +3,6 @@ import sys import importlib.util import time -import shutil import struct import zipfile import unittest @@ -12,7 +11,7 @@ from tempfile import TemporaryFile from random import randint, random, getrandbits -from test.support import (TESTFN, findfile, unlink, +from test.support import (TESTFN, findfile, unlink, rmtree, requires_zlib, requires_bz2, requires_lzma, captured_stdout, check_warnings) @@ -691,7 +690,7 @@ self.assertNotIn('mod2.txt', names) finally: - shutil.rmtree(TESTFN2) + rmtree(TESTFN2) def test_write_python_directory_filtered(self): os.mkdir(TESTFN2) @@ -711,7 +710,7 @@ self.assertNotIn('mod2.py', names) finally: - shutil.rmtree(TESTFN2) + rmtree(TESTFN2) def test_write_non_pyfile(self): with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: @@ -741,7 +740,7 @@ self.assertNotIn('mod1.pyo', names) finally: - shutil.rmtree(TESTFN2) + rmtree(TESTFN2) class ExtractTests(unittest.TestCase): @@ -767,7 +766,7 @@ os.remove(writtenfile) # remove the test file subdirectories - shutil.rmtree(os.path.join(os.getcwd(), 'ziptest2dir')) + rmtree(os.path.join(os.getcwd(), 'ziptest2dir')) def test_extract_all(self): with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: @@ -785,7 +784,7 @@ os.remove(outfile) # remove the test file subdirectories - shutil.rmtree(os.path.join(os.getcwd(), 'ziptest2dir')) + rmtree(os.path.join(os.getcwd(), 'ziptest2dir')) def check_file(self, filename, content): self.assertTrue(os.path.isfile(filename)) @@ -867,12 +866,12 @@ msg='extract %r: %r != %r' % (arcname, writtenfile, correctfile)) self.check_file(correctfile, content) - shutil.rmtree('target') + rmtree('target') with zipfile.ZipFile(TESTFN2, 'r') as zipfp: zipfp.extractall(targetpath) self.check_file(correctfile, content) - shutil.rmtree('target') + rmtree('target') correctfile = os.path.join(os.getcwd(), *fixedname.split('/')) @@ -881,12 +880,12 @@ self.assertEqual(writtenfile, correctfile, msg="extract %r" % arcname) self.check_file(correctfile, content) - shutil.rmtree(fixedname.split('/')[0]) + rmtree(fixedname.split('/')[0]) with zipfile.ZipFile(TESTFN2, 'r') as zipfp: zipfp.extractall() self.check_file(correctfile, content) - shutil.rmtree(fixedname.split('/')[0]) + rmtree(fixedname.split('/')[0]) os.remove(TESTFN2) @@ -1643,7 +1642,7 @@ self.assertTrue(zipf.filelist[0].filename.endswith("x/")) def tearDown(self): - shutil.rmtree(TESTFN2) + rmtree(TESTFN2) if os.path.exists(TESTFN): unlink(TESTFN) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 4 00:54:03 2014 From: python-checkins at python.org (victor.stinner) Date: Thu, 4 Sep 2014 00:54:03 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzIxNDQw?= =?utf-8?q?=3A_test=5Fzipfile=3A_replace_last_direct_calls_to_os=2Eremove?= =?utf-8?q?=28=29_with?= Message-ID: <3hpL6z6q6Mz7Ljp@mail.python.org> http://hg.python.org/cpython/rev/6b4d31641109 changeset: 92317:6b4d31641109 branch: 3.4 user: Victor Stinner date: Thu Sep 04 00:51:09 2014 +0200 summary: Issue #21440: test_zipfile: replace last direct calls to os.remove() with support.unlink() files: Lib/test/test_zipfile.py | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -717,7 +717,7 @@ with open(TESTFN, 'w') as f: f.write('most definitely not a python file') self.assertRaises(RuntimeError, zipfp.writepy, TESTFN) - os.remove(TESTFN) + unlink(TESTFN) def test_write_pyfile_bad_syntax(self): os.mkdir(TESTFN2) @@ -763,7 +763,7 @@ with open(writtenfile, "rb") as f: self.assertEqual(fdata.encode(), f.read()) - os.remove(writtenfile) + unlink(writtenfile) # remove the test file subdirectories rmtree(os.path.join(os.getcwd(), 'ziptest2dir')) @@ -781,7 +781,7 @@ with open(outfile, "rb") as f: self.assertEqual(fdata.encode(), f.read()) - os.remove(outfile) + unlink(outfile) # remove the test file subdirectories rmtree(os.path.join(os.getcwd(), 'ziptest2dir')) @@ -887,7 +887,7 @@ self.check_file(correctfile, content) rmtree(fixedname.split('/')[0]) - os.remove(TESTFN2) + unlink(TESTFN2) class OtherTests(unittest.TestCase): @@ -1755,7 +1755,7 @@ def tearDown(self): for sep, fn in self.arcfiles.items(): - os.remove(fn) + unlink(fn) unlink(TESTFN) unlink(TESTFN2) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 4 00:54:05 2014 From: python-checkins at python.org (victor.stinner) Date: Thu, 4 Sep 2014 00:54:05 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?b?KTogKE1lcmdlIDMuNCkgSXNzdWUgIzIxNDQwOiB0ZXN0X3ppcGZpbGU6IHJl?= =?utf-8?q?place_last_direct_calls_to?= Message-ID: <3hpL711cPyz7LkJ@mail.python.org> http://hg.python.org/cpython/rev/73ce1afc6ee2 changeset: 92318:73ce1afc6ee2 parent: 92315:85511d4a846e parent: 92317:6b4d31641109 user: Victor Stinner date: Thu Sep 04 00:51:25 2014 +0200 summary: (Merge 3.4) Issue #21440: test_zipfile: replace last direct calls to os.remove() with support.unlink() files: Lib/test/test_zipfile.py | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -717,7 +717,7 @@ with open(TESTFN, 'w') as f: f.write('most definitely not a python file') self.assertRaises(RuntimeError, zipfp.writepy, TESTFN) - os.remove(TESTFN) + unlink(TESTFN) def test_write_pyfile_bad_syntax(self): os.mkdir(TESTFN2) @@ -763,7 +763,7 @@ with open(writtenfile, "rb") as f: self.assertEqual(fdata.encode(), f.read()) - os.remove(writtenfile) + unlink(writtenfile) # remove the test file subdirectories rmtree(os.path.join(os.getcwd(), 'ziptest2dir')) @@ -781,7 +781,7 @@ with open(outfile, "rb") as f: self.assertEqual(fdata.encode(), f.read()) - os.remove(outfile) + unlink(outfile) # remove the test file subdirectories rmtree(os.path.join(os.getcwd(), 'ziptest2dir')) @@ -887,7 +887,7 @@ self.check_file(correctfile, content) rmtree(fixedname.split('/')[0]) - os.remove(TESTFN2) + unlink(TESTFN2) class OtherTests(unittest.TestCase): @@ -1755,7 +1755,7 @@ def tearDown(self): for sep, fn in self.arcfiles.items(): - os.remove(fn) + unlink(fn) unlink(TESTFN) unlink(TESTFN2) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 4 01:03:03 2014 From: python-checkins at python.org (victor.stinner) Date: Thu, 4 Sep 2014 01:03:03 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Fix_timeout_in_test=2Efork?= =?utf-8?q?=5Fwait?= Message-ID: <3hpLKM6pM4z7LjN@mail.python.org> http://hg.python.org/cpython/rev/797c68e67188 changeset: 92319:797c68e67188 user: Victor Stinner date: Thu Sep 04 01:02:17 2014 +0200 summary: Fix timeout in test.fork_wait files: Lib/test/fork_wait.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Lib/test/fork_wait.py b/Lib/test/fork_wait.py --- a/Lib/test/fork_wait.py +++ b/Lib/test/fork_wait.py @@ -52,7 +52,7 @@ deadline = time.monotonic() + 10.0 while len(self.alive) < NUM_THREADS: time.sleep(0.1) - if time.monotonic() <= deadline: + if deadline < time.monotonic(): break a = sorted(self.alive.keys()) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 4 01:25:03 2014 From: python-checkins at python.org (benjamin.peterson) Date: Thu, 4 Sep 2014 01:25:03 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_this_is_a_standards_track_pep?= Message-ID: <3hpLpl3chQz7Ljk@mail.python.org> http://hg.python.org/peps/rev/6d6de2e0749f changeset: 5542:6d6de2e0749f user: Benjamin Peterson date: Wed Sep 03 19:24:58 2014 -0400 summary: this is a standards track pep files: pep-0477.txt | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/pep-0477.txt b/pep-0477.txt --- a/pep-0477.txt +++ b/pep-0477.txt @@ -5,7 +5,7 @@ Author: Donald Stufft Nick Coghlan Status: Active -Type: Process +Type: Standards Track Content-Type: text/x-rst Created: 26-Aug-2014 Post-History: 1-Sep-2014 -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Thu Sep 4 01:29:58 2014 From: python-checkins at python.org (terry.reedy) Date: Thu, 4 Sep 2014 01:29:58 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzIyMDUx?= =?utf-8?q?=3A_remove_unneeded_reload_that_allowed_bad_code=2E?= Message-ID: <3hpLwQ5J42z7LkS@mail.python.org> http://hg.python.org/cpython/rev/b76d854f580e changeset: 92320:b76d854f580e branch: 2.7 parent: 92291:241f9aa9fb89 user: Terry Jan Reedy date: Wed Sep 03 19:29:11 2014 -0400 summary: Issue #22051: remove unneeded reload that allowed bad code. files: Demo/turtle/turtleDemo.py | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diff --git a/Demo/turtle/turtleDemo.py b/Demo/turtle/turtleDemo.py --- a/Demo/turtle/turtleDemo.py +++ b/Demo/turtle/turtleDemo.py @@ -210,7 +210,6 @@ direc, fname = os.path.split(filename) self.root.title(fname[6:-3]+" - a Python turtle graphics example") self.module = __import__(fname[:-3]) - reload(self.module) self.configGUI(NORMAL, NORMAL, DISABLED, DISABLED, "Press start button", "red") self.state = READY -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 4 01:29:59 2014 From: python-checkins at python.org (terry.reedy) Date: Thu, 4 Sep 2014 01:29:59 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzIyMDUx?= =?utf-8?q?=3A_remove_unneeded_reload_that_allowed_bad_code=2E?= Message-ID: <3hpLwR6yXsz7LlV@mail.python.org> http://hg.python.org/cpython/rev/55d4f6c2be2d changeset: 92321:55d4f6c2be2d branch: 3.4 parent: 92317:6b4d31641109 user: Terry Jan Reedy date: Wed Sep 03 19:29:17 2014 -0400 summary: Issue #22051: remove unneeded reload that allowed bad code. files: Lib/turtledemo/__main__.py | 2 -- 1 files changed, 0 insertions(+), 2 deletions(-) diff --git a/Lib/turtledemo/__main__.py b/Lib/turtledemo/__main__.py --- a/Lib/turtledemo/__main__.py +++ b/Lib/turtledemo/__main__.py @@ -91,7 +91,6 @@ from idlelib.Percolator import Percolator from idlelib.ColorDelegator import ColorDelegator from idlelib.textView import view_text -from importlib import reload from turtledemo import __doc__ as about_turtledemo import turtle @@ -322,7 +321,6 @@ self.text.delete("1.0", "end") self.text.insert("1.0", chars) self.root.title(filename + " - a Python turtle graphics example") - reload(self.module) self.configGUI(NORMAL, DISABLED, DISABLED, "Press start button", "red") self.state = READY -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 4 01:30:01 2014 From: python-checkins at python.org (terry.reedy) Date: Thu, 4 Sep 2014 01:30:01 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?q?=29=3A_Merge_with_3=2E4?= Message-ID: <3hpLwT1Z7rz7LlT@mail.python.org> http://hg.python.org/cpython/rev/9569b52e6e21 changeset: 92322:9569b52e6e21 parent: 92319:797c68e67188 parent: 92321:55d4f6c2be2d user: Terry Jan Reedy date: Wed Sep 03 19:29:31 2014 -0400 summary: Merge with 3.4 files: Lib/turtledemo/__main__.py | 2 -- 1 files changed, 0 insertions(+), 2 deletions(-) diff --git a/Lib/turtledemo/__main__.py b/Lib/turtledemo/__main__.py --- a/Lib/turtledemo/__main__.py +++ b/Lib/turtledemo/__main__.py @@ -91,7 +91,6 @@ from idlelib.Percolator import Percolator from idlelib.ColorDelegator import ColorDelegator from idlelib.textView import view_text -from importlib import reload from turtledemo import __doc__ as about_turtledemo import turtle @@ -322,7 +321,6 @@ self.text.delete("1.0", "end") self.text.insert("1.0", chars) self.root.title(filename + " - a Python turtle graphics example") - reload(self.module) self.configGUI(NORMAL, DISABLED, DISABLED, "Press start button", "red") self.state = READY -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 4 01:43:37 2014 From: python-checkins at python.org (terry.reedy) Date: Thu, 4 Sep 2014 01:43:37 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_NEWS_items_for?= =?utf-8?q?_turtledemo=2E?= Message-ID: <3hpMD973VYz7LjR@mail.python.org> http://hg.python.org/cpython/rev/78a61a78969a changeset: 92323:78a61a78969a branch: 2.7 parent: 92320:b76d854f580e user: Terry Jan Reedy date: Wed Sep 03 19:39:18 2014 -0400 summary: NEWS items for turtledemo. files: Misc/NEWS | 12 ++++++++++++ 1 files changed, 12 insertions(+), 0 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -19,6 +19,18 @@ Library ------- +- Issue #22051: turtledemo no longer reloads examples to re-run them. + Initialization of variables and gui setup should be done in main(), + which is called each time a demo is run, but not on import. + +_ Issue #21597: The separator between the turtledemo text pane and the drawing + canvas can now be grabbed and dragged with a mouse. The code text pane can + be widened to easily view or copy the full width of the text. The canvas + can be widened on small screens. Original patches by Jan Kanis and Lita Cho. + +- Issue #18132: Turtledemo buttons no longer disappear when the window is + shrunk. Original patches by Jan Kanis and Lita Cho. + - Issue #22312: Fix ntpath.splitdrive IndexError. - Issue #22216: smtplib now resets its state more completely after a quit. The -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 4 01:43:39 2014 From: python-checkins at python.org (terry.reedy) Date: Thu, 4 Sep 2014 01:43:39 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E4=29=3A_NEWS_items_for?= =?utf-8?q?_turtledemo=2E?= Message-ID: <3hpMDC1qjYz7LjR@mail.python.org> http://hg.python.org/cpython/rev/948be7abfc45 changeset: 92324:948be7abfc45 branch: 3.4 parent: 92321:55d4f6c2be2d user: Terry Jan Reedy date: Wed Sep 03 19:39:25 2014 -0400 summary: NEWS items for turtledemo. files: Misc/NEWS | 16 ++++++++++++++++ 1 files changed, 16 insertions(+), 0 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,22 @@ Library ------- +- Issue #22051: turtledemo no longer reloads examples to re-run them. + Initialization of variables and gui setup should be done in main(), + which is called each time a demo is run, but not on import. + +- Issue #21933: Turtledemo users can change the code font size with a menu + selection or control(command) '-' or '+' or control-mousewheel. + Original patch by Lita Cho. + +_ Issue #21597: The separator between the turtledemo text pane and the drawing + canvas can now be grabbed and dragged with a mouse. The code text pane can + be widened to easily view or copy the full width of the text. The canvas + can be widened on small screens. Original patches by Jan Kanis and Lita Cho. + +- Issue #18132: Turtledemo buttons no longer disappear when the window is + shrunk. Original patches by Jan Kanis and Lita Cho. + - Issue #22216: smtplib now resets its state more completely after a quit. The most obvious consequence of the previous behavior was a STARTTLS failure during a connect/starttls/quit/connect/starttls sequence. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 4 01:43:40 2014 From: python-checkins at python.org (terry.reedy) Date: Thu, 4 Sep 2014 01:43:40 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?q?=29=3A_merge_with_3=2E4?= Message-ID: <3hpMDD3lTrz7Lk6@mail.python.org> http://hg.python.org/cpython/rev/f7bc5ecb0a95 changeset: 92325:f7bc5ecb0a95 parent: 92322:9569b52e6e21 parent: 92324:948be7abfc45 user: Terry Jan Reedy date: Wed Sep 03 19:43:12 2014 -0400 summary: merge with 3.4 files: Misc/NEWS | 16 ++++++++++++++++ 1 files changed, 16 insertions(+), 0 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -129,6 +129,22 @@ Library ------- +- Issue #22051: turtledemo no longer reloads examples to re-run them. + Initialization of variables and gui setup should be done in main(), + which is called each time a demo is run, but not on import. + +- Issue #21933: Turtledemo users can change the code font size with a menu + selection or control(command) '-' or '+' or control-mousewheel. + Original patch by Lita Cho. + +_ Issue #21597: The separator between the turtledemo text pane and the drawing + canvas can now be grabbed and dragged with a mouse. The code text pane can + be widened to easily view or copy the full width of the text. The canvas + can be widened on small screens. Original patches by Jan Kanis and Lita Cho. + +- Issue #18132: Turtledemo buttons no longer disappear when the window is + shrunk. Original patches by Jan Kanis and Lita Cho. + - Issue #22043: time.monotonic() is now always available. ``threading.Lock.acquire()``, ``threading.RLock.acquire()`` and socket operations now use a monotonic clock, instead of the system clock, when a -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 4 01:54:32 2014 From: python-checkins at python.org (terry.reedy) Date: Thu, 4 Sep 2014 01:54:32 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_NEWS_items_for?= =?utf-8?q?_Idle=2E?= Message-ID: <3hpMSm58f9z7Ljk@mail.python.org> http://hg.python.org/cpython/rev/db95df7f309e changeset: 92326:db95df7f309e branch: 2.7 parent: 92323:78a61a78969a user: Terry Jan Reedy date: Wed Sep 03 19:52:39 2014 -0400 summary: NEWS items for Idle. files: Misc/NEWS | 9 +++++++++ 1 files changed, 9 insertions(+), 0 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -121,6 +121,15 @@ - Issue #22199: Make get_makefile_filename() available in Lib/sysconfig.py for 2.7 to match other versions of sysconfig. +IDLE +---- + +- Issue #17390: Adjust Editor window title; remove 'Python', + move version to end. + +- Issue #14105: Idle debugger breakpoints no longer disappear + when inseting or deleting lines. + Extension Modules ----------------- -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 4 01:54:33 2014 From: python-checkins at python.org (terry.reedy) Date: Thu, 4 Sep 2014 01:54:33 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E4=29=3A_NEWS_items_for?= =?utf-8?q?_Idle=2E?= Message-ID: <3hpMSn725Gz7Ljk@mail.python.org> http://hg.python.org/cpython/rev/b2e9d845bc6c changeset: 92327:b2e9d845bc6c branch: 3.4 parent: 92324:948be7abfc45 user: Terry Jan Reedy date: Wed Sep 03 19:52:46 2014 -0400 summary: NEWS items for Idle. files: Misc/NEWS | 10 ++++++++-- 1 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -46,7 +46,7 @@ can be widened on small screens. Original patches by Jan Kanis and Lita Cho. - Issue #18132: Turtledemo buttons no longer disappear when the window is - shrunk. Original patches by Jan Kanis and Lita Cho. + shrunk. Original patches by Jan Kanis and Lita Cho. - Issue #22216: smtplib now resets its state more completely after a quit. The most obvious consequence of the previous behavior was a STARTTLS failure @@ -285,7 +285,13 @@ IDLE ---- -- Issue #17172: Add the ability to run turtledemo from Idle. +- Issue #17390: Adjust Editor window title; remove 'Python', + move version to end. + +- Issue #14105: Idle debugger breakpoints no longer disappear + when inseting or deleting lines. + +- Issue #17172: Turtledemo can now be run from Idle. Currently, the entry is on the Help menu, but it may move to Run. Patch by Ramchandra Apt and Lita Cho. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 4 01:54:35 2014 From: python-checkins at python.org (terry.reedy) Date: Thu, 4 Sep 2014 01:54:35 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?q?=29=3A_NEWS_items_for_Idle=2E?= Message-ID: <3hpMSq1mBYz7LkY@mail.python.org> http://hg.python.org/cpython/rev/c69be794a9c6 changeset: 92328:c69be794a9c6 parent: 92325:f7bc5ecb0a95 parent: 92327:b2e9d845bc6c user: Terry Jan Reedy date: Wed Sep 03 19:54:06 2014 -0400 summary: NEWS items for Idle. files: Misc/NEWS | 8 +++++++- 1 files changed, 7 insertions(+), 1 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -802,7 +802,13 @@ IDLE ---- -- Issue #17172: Add the ability to run turtledemo from Idle. +- Issue #17390: Adjust Editor window title; remove 'Python', + move version to end. + +- Issue #14105: Idle debugger breakpoints no longer disappear + when inseting or deleting lines. + +- Issue #17172: Turtledemo can now be run from Idle. Currently, the entry is on the Help menu, but it may move to Run. Patch by Ramchandra Apt and Lita Cho. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 4 01:55:09 2014 From: python-checkins at python.org (nick.coghlan) Date: Thu, 4 Sep 2014 01:55:09 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_477=3A_With_GvR=27s_appro?= =?utf-8?q?val=2C_Benjamin_is_the_BDFL-Delegate?= Message-ID: <3hpMTT6YZ1z7Ljk@mail.python.org> http://hg.python.org/peps/rev/909954e4ba43 changeset: 5543:909954e4ba43 user: Nick Coghlan date: Thu Sep 04 09:54:58 2014 +1000 summary: PEP 477: With GvR's approval, Benjamin is the BDFL-Delegate files: pep-0477.txt | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diff --git a/pep-0477.txt b/pep-0477.txt --- a/pep-0477.txt +++ b/pep-0477.txt @@ -4,7 +4,8 @@ Last-Modified: $Date$ Author: Donald Stufft Nick Coghlan -Status: Active +BDFL-Delegate: Benjamin Peterson +Status: Draft Type: Standards Track Content-Type: text/x-rst Created: 26-Aug-2014 -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Thu Sep 4 04:22:24 2014 From: python-checkins at python.org (benjamin.peterson) Date: Thu, 4 Sep 2014 04:22:24 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_accept_pep_477?= Message-ID: <3hpQlN3jc5z7Lkc@mail.python.org> http://hg.python.org/peps/rev/6d1b12bd4a3a changeset: 5544:6d1b12bd4a3a user: Benjamin Peterson date: Wed Sep 03 22:22:19 2014 -0400 summary: accept pep 477 files: pep-0477.txt | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/pep-0477.txt b/pep-0477.txt --- a/pep-0477.txt +++ b/pep-0477.txt @@ -5,7 +5,7 @@ Author: Donald Stufft Nick Coghlan BDFL-Delegate: Benjamin Peterson -Status: Draft +Status: Accepted Type: Standards Track Content-Type: text/x-rst Created: 26-Aug-2014 -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Thu Sep 4 06:20:08 2014 From: python-checkins at python.org (nick.coghlan) Date: Thu, 4 Sep 2014 06:20:08 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_Resolution_link_for_PEP_477_a?= =?utf-8?q?cceptance?= Message-ID: <3hpTMD0Sn8z7LlB@mail.python.org> http://hg.python.org/peps/rev/da85a173e556 changeset: 5545:da85a173e556 user: Nick Coghlan date: Thu Sep 04 14:19:57 2014 +1000 summary: Resolution link for PEP 477 acceptance files: pep-0477.txt | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/pep-0477.txt b/pep-0477.txt --- a/pep-0477.txt +++ b/pep-0477.txt @@ -10,6 +10,7 @@ Content-Type: text/x-rst Created: 26-Aug-2014 Post-History: 1-Sep-2014 +Resolution: https://mail.python.org/pipermail/python-dev/2014-September/136238.html Abstract -- Repository URL: http://hg.python.org/peps From solipsis at pitrou.net Thu Sep 4 09:27:41 2014 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Thu, 04 Sep 2014 09:27:41 +0200 Subject: [Python-checkins] Daily reference leaks (c69be794a9c6): sum=151928 Message-ID: results for c69be794a9c6 on branch "default" -------------------------------------------- test_codecs leaked [5825, 5825, 5825] references, sum=17475 test_codecs leaked [1172, 1174, 1174] memory blocks, sum=3520 test_collections leaked [-4, 0, 0] references, sum=-4 test_collections leaked [-2, 0, 0] memory blocks, sum=-2 test_distutils leaked [37735, 37735, 37735] references, sum=113205 test_distutils leaked [5909, 5911, 5911] memory blocks, sum=17731 test_functools leaked [0, 0, 3] memory blocks, sum=3 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogB8sGiT', '-x'] From python-checkins at python.org Thu Sep 4 09:29:49 2014 From: python-checkins at python.org (victor.stinner) Date: Thu, 4 Sep 2014 09:29:49 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2322258=3A_Fix_typo?= =?utf-8?q?_in_Misc/NEWS?= Message-ID: <3hpYZ519FQz7Lk2@mail.python.org> http://hg.python.org/cpython/rev/9d5386a22e68 changeset: 92329:9d5386a22e68 user: Victor Stinner date: Thu Sep 04 09:29:39 2014 +0200 summary: Issue #22258: Fix typo in Misc/NEWS files: Misc/NEWS | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,7 +10,7 @@ Core and Builtins ----------------- -- Issue #22258: Fix the the internal function set_inheritable() on Illumos. +- Issue #22258: Fix the internal function set_inheritable() on Illumos. This platform exposes the function ``ioctl(FIOCLEX)``, but calling it fails with errno is ENOTTY: "Inappropriate ioctl for device". set_inheritable() now falls back to the slower ``fcntl()`` (``F_GETFD`` and then ``F_SETFD``). -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 4 09:38:55 2014 From: python-checkins at python.org (victor.stinner) Date: Thu, 4 Sep 2014 09:38:55 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2322333=3A_Add_debu?= =?utf-8?q?g_traces_to_test=5Fthreaded=5Fimport?= Message-ID: <3hpYmb52Vmz7Lk2@mail.python.org> http://hg.python.org/cpython/rev/f8d2834e1269 changeset: 92330:f8d2834e1269 user: Victor Stinner date: Thu Sep 04 09:38:38 2014 +0200 summary: Issue #22333: Add debug traces to test_threaded_import files: Lib/test/test_threaded_import.py | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_threaded_import.py b/Lib/test/test_threaded_import.py --- a/Lib/test/test_threaded_import.py +++ b/Lib/test/test_threaded_import.py @@ -118,8 +118,10 @@ t = threading.Thread(target=task, args=(N, done, done_tasks, errors,)) t.start() - self.assertTrue(done.wait(60)) - self.assertFalse(errors) + completed = done.wait(60) + dbg_info = 'done: %s/%s' % (len(done_tasks), N) + self.assertFalse(errors, dbg_info) + self.assertTrue(completed, dbg_info) if verbose: print("OK.") -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 4 09:53:32 2014 From: python-checkins at python.org (victor.stinner) Date: Thu, 4 Sep 2014 09:53:32 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2322334=3A_Add_debu?= =?utf-8?q?g_traces_to_test=5Ftcl?= Message-ID: <3hpZ5S1zS8z7Lk9@mail.python.org> http://hg.python.org/cpython/rev/e6cfbc3e2ed9 changeset: 92331:e6cfbc3e2ed9 user: Victor Stinner date: Thu Sep 04 09:53:16 2014 +0200 summary: Issue #22334: Add debug traces to test_tcl files: Lib/test/test_tcl.py | 8 ++++++-- 1 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -499,8 +499,9 @@ (1, '2', (3.4,)) if self.wantobjects else ('1', '2', '3.4')), ] + tk_patchlevel = get_tk_patchlevel() if tcl_version >= (8, 5): - if not self.wantobjects or get_tk_patchlevel() < (8, 5, 5): + if not self.wantobjects or tk_patchlevel < (8, 5, 5): # Before 8.5.5 dicts were converted to lists through string expected = ('12', '\u20ac', '\xe2\x82\xac', '3.4') else: @@ -509,8 +510,11 @@ (call('dict', 'create', 12, '\u20ac', b'\xe2\x82\xac', (3.4,)), expected), ] + dbg_info = ('want objects? %s, Tcl version: %s, Tk patchlevel: %s' + % (self.wantobjects, tcl_version, tk_patchlevel)) for arg, res in testcases: - self.assertEqual(splitlist(arg), res, msg=arg) + self.assertEqual(splitlist(arg), res, + 'arg=%a, %s' % (arg, dbg_info)) self.assertRaises(TclError, splitlist, '{') def test_split(self): -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 4 10:42:49 2014 From: python-checkins at python.org (lukasz.langa) Date: Thu, 4 Sep 2014 10:42:49 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Fix_=2319546=3A_onfigparse?= =?utf-8?q?r_exceptions_expose_implementation_details=2E__Patch_by?= Message-ID: <3hpbBK23jbz7Ljn@mail.python.org> http://hg.python.org/cpython/rev/2b14665b7bce changeset: 92332:2b14665b7bce parent: 92328:c69be794a9c6 user: ?ukasz Langa date: Thu Sep 04 01:36:33 2014 -0700 summary: Fix #19546: onfigparser exceptions expose implementation details. Patch by Claudiu Popa. files: Lib/configparser.py | 12 ++-- Lib/test/test_configparser.py | 52 +++++++++++++++++++++++ Misc/NEWS | 4 + 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/Lib/configparser.py b/Lib/configparser.py --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -410,7 +410,7 @@ v = map[var] except KeyError: raise InterpolationMissingOptionError( - option, section, rest, var) + option, section, rest, var) from None if "%" in v: self._interpolate_some(parser, option, accum, v, section, map, depth + 1) @@ -482,7 +482,7 @@ "More than one ':' found: %r" % (rest,)) except (KeyError, NoSectionError, NoOptionError): raise InterpolationMissingOptionError( - option, section, rest, ":".join(path)) + option, section, rest, ":".join(path)) from None if "$" in v: self._interpolate_some(parser, opt, accum, v, sect, dict(parser.items(sect, raw=True)), @@ -515,7 +515,7 @@ value = value % vars except KeyError as e: raise InterpolationMissingOptionError( - option, section, rawval, e.args[0]) + option, section, rawval, e.args[0]) from None else: break if value and "%(" in value: @@ -647,7 +647,7 @@ try: opts = self._sections[section].copy() except KeyError: - raise NoSectionError(section) + raise NoSectionError(section) from None opts.update(self._defaults) return list(opts.keys()) @@ -876,7 +876,7 @@ try: sectdict = self._sections[section] except KeyError: - raise NoSectionError(section) + raise NoSectionError(section) from None sectdict[self.optionxform(option)] = value def write(self, fp, space_around_delimiters=True): @@ -917,7 +917,7 @@ try: sectdict = self._sections[section] except KeyError: - raise NoSectionError(section) + raise NoSectionError(section) from None option = self.optionxform(option) existed = option in sectdict if existed: diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -1763,6 +1763,58 @@ self.assertEqual(s['k2'], 'v2') self.assertEqual(s['k3'], 'v3;#//still v3# and still v3') +class ExceptionContextTestCase(unittest.TestCase): + """ Test that implementation details doesn't leak + through raising exceptions. """ + + def test_get_basic_interpolation(self): + parser = configparser.ConfigParser() + parser.read_string(""" + [Paths] + home_dir: /Users + my_dir: %(home_dir1)s/lumberjack + my_pictures: %(my_dir)s/Pictures + """) + cm = self.assertRaises(configparser.InterpolationMissingOptionError) + with cm: + parser.get('Paths', 'my_dir') + self.assertIs(cm.exception.__suppress_context__, True) + + def test_get_extended_interpolation(self): + parser = configparser.ConfigParser( + interpolation=configparser.ExtendedInterpolation()) + parser.read_string(""" + [Paths] + home_dir: /Users + my_dir: ${home_dir1}/lumberjack + my_pictures: ${my_dir}/Pictures + """) + cm = self.assertRaises(configparser.InterpolationMissingOptionError) + with cm: + parser.get('Paths', 'my_dir') + self.assertIs(cm.exception.__suppress_context__, True) + + def test_missing_options(self): + parser = configparser.ConfigParser() + parser.read_string(""" + [Paths] + home_dir: /Users + """) + with self.assertRaises(configparser.NoSectionError) as cm: + parser.options('test') + self.assertIs(cm.exception.__suppress_context__, True) + + def test_missing_section(self): + config = configparser.ConfigParser() + with self.assertRaises(configparser.NoSectionError) as cm: + config.set('Section1', 'an_int', '15') + self.assertIs(cm.exception.__suppress_context__, True) + + def test_remove_option(self): + config = configparser.ConfigParser() + with self.assertRaises(configparser.NoSectionError) as cm: + config.remove_option('Section1', 'an_int') + self.assertIs(cm.exception.__suppress_context__, True) if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -129,6 +129,10 @@ Library ------- +- Issue #19546: configparser exceptions no longer expose implementation details. + Chained KeyErrors are removed, which leads to cleaner tracebacks. Patch by + Claudiu Popa. + - Issue #22051: turtledemo no longer reloads examples to re-run them. Initialization of variables and gui setup should be done in main(), which is called each time a demo is run, but not on import. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 4 10:42:50 2014 From: python-checkins at python.org (lukasz.langa) Date: Thu, 4 Sep 2014 10:42:50 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_default_-=3E_default?= =?utf-8?q?=29=3A_Merge_fix_for_=2319546=3A_configparser_exceptions_leak_i?= =?utf-8?q?mplementation_details?= Message-ID: <3hpbBL4sJgz7LkM@mail.python.org> http://hg.python.org/cpython/rev/554ead559f24 changeset: 92333:554ead559f24 parent: 92331:e6cfbc3e2ed9 parent: 92332:2b14665b7bce user: ?ukasz Langa date: Thu Sep 04 01:42:29 2014 -0700 summary: Merge fix for #19546: configparser exceptions leak implementation details files: Lib/configparser.py | 12 ++-- Lib/test/test_configparser.py | 52 +++++++++++++++++++++++ Misc/NEWS | 4 + 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/Lib/configparser.py b/Lib/configparser.py --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -410,7 +410,7 @@ v = map[var] except KeyError: raise InterpolationMissingOptionError( - option, section, rest, var) + option, section, rest, var) from None if "%" in v: self._interpolate_some(parser, option, accum, v, section, map, depth + 1) @@ -482,7 +482,7 @@ "More than one ':' found: %r" % (rest,)) except (KeyError, NoSectionError, NoOptionError): raise InterpolationMissingOptionError( - option, section, rest, ":".join(path)) + option, section, rest, ":".join(path)) from None if "$" in v: self._interpolate_some(parser, opt, accum, v, sect, dict(parser.items(sect, raw=True)), @@ -515,7 +515,7 @@ value = value % vars except KeyError as e: raise InterpolationMissingOptionError( - option, section, rawval, e.args[0]) + option, section, rawval, e.args[0]) from None else: break if value and "%(" in value: @@ -647,7 +647,7 @@ try: opts = self._sections[section].copy() except KeyError: - raise NoSectionError(section) + raise NoSectionError(section) from None opts.update(self._defaults) return list(opts.keys()) @@ -876,7 +876,7 @@ try: sectdict = self._sections[section] except KeyError: - raise NoSectionError(section) + raise NoSectionError(section) from None sectdict[self.optionxform(option)] = value def write(self, fp, space_around_delimiters=True): @@ -917,7 +917,7 @@ try: sectdict = self._sections[section] except KeyError: - raise NoSectionError(section) + raise NoSectionError(section) from None option = self.optionxform(option) existed = option in sectdict if existed: diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -1763,6 +1763,58 @@ self.assertEqual(s['k2'], 'v2') self.assertEqual(s['k3'], 'v3;#//still v3# and still v3') +class ExceptionContextTestCase(unittest.TestCase): + """ Test that implementation details doesn't leak + through raising exceptions. """ + + def test_get_basic_interpolation(self): + parser = configparser.ConfigParser() + parser.read_string(""" + [Paths] + home_dir: /Users + my_dir: %(home_dir1)s/lumberjack + my_pictures: %(my_dir)s/Pictures + """) + cm = self.assertRaises(configparser.InterpolationMissingOptionError) + with cm: + parser.get('Paths', 'my_dir') + self.assertIs(cm.exception.__suppress_context__, True) + + def test_get_extended_interpolation(self): + parser = configparser.ConfigParser( + interpolation=configparser.ExtendedInterpolation()) + parser.read_string(""" + [Paths] + home_dir: /Users + my_dir: ${home_dir1}/lumberjack + my_pictures: ${my_dir}/Pictures + """) + cm = self.assertRaises(configparser.InterpolationMissingOptionError) + with cm: + parser.get('Paths', 'my_dir') + self.assertIs(cm.exception.__suppress_context__, True) + + def test_missing_options(self): + parser = configparser.ConfigParser() + parser.read_string(""" + [Paths] + home_dir: /Users + """) + with self.assertRaises(configparser.NoSectionError) as cm: + parser.options('test') + self.assertIs(cm.exception.__suppress_context__, True) + + def test_missing_section(self): + config = configparser.ConfigParser() + with self.assertRaises(configparser.NoSectionError) as cm: + config.set('Section1', 'an_int', '15') + self.assertIs(cm.exception.__suppress_context__, True) + + def test_remove_option(self): + config = configparser.ConfigParser() + with self.assertRaises(configparser.NoSectionError) as cm: + config.remove_option('Section1', 'an_int') + self.assertIs(cm.exception.__suppress_context__, True) if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -129,6 +129,10 @@ Library ------- +- Issue #19546: configparser exceptions no longer expose implementation details. + Chained KeyErrors are removed, which leads to cleaner tracebacks. Patch by + Claudiu Popa. + - Issue #22051: turtledemo no longer reloads examples to re-run them. Initialization of variables and gui setup should be done in main(), which is called each time a demo is run, but not on import. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 4 17:30:34 2014 From: python-checkins at python.org (victor.stinner) Date: Thu, 4 Sep 2014 17:30:34 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2321951=3A_Fix_AsOb?= =?utf-8?q?j=28=29_of_the_=5Ftkinter_module=3A_raise_MemoryError_on_memory?= Message-ID: <3hpmDp1Mcsz7LjS@mail.python.org> http://hg.python.org/cpython/rev/9ab404cdcaa1 changeset: 92334:9ab404cdcaa1 user: Victor Stinner date: Thu Sep 04 17:29:52 2014 +0200 summary: Issue #21951: Fix AsObj() of the _tkinter module: raise MemoryError on memory allocation failure files: Modules/_tkinter.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -913,8 +913,10 @@ return NULL; } argv = (Tcl_Obj **) ckalloc(((size_t)size) * sizeof(Tcl_Obj *)); - if(!argv) - return 0; + if(!argv) { + PyErr_NoMemory(); + return NULL; + } for (i = 0; i < size; i++) argv[i] = AsObj(PySequence_Fast_GET_ITEM(value,i)); result = Tcl_NewListObj(size, argv); -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 4 17:43:12 2014 From: python-checkins at python.org (benjamin.peterson) Date: Thu, 4 Sep 2014 17:43:12 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_remove_script_?= =?utf-8?q?that_now_lives_at_https=3A//github=2Ecom/python/docsbuild-scrip?= =?utf-8?q?ts?= Message-ID: <3hpmWN2C1Nz7LjR@mail.python.org> http://hg.python.org/cpython/rev/77687e7e6d9d changeset: 92335:77687e7e6d9d branch: 2.7 parent: 92326:db95df7f309e user: Benjamin Peterson date: Mon Sep 01 23:00:11 2014 -0400 summary: remove script that now lives at https://github.com/python/docsbuild-scripts files: Doc/tools/dailybuild.py | 135 ---------------------------- 1 files changed, 0 insertions(+), 135 deletions(-) diff --git a/Doc/tools/dailybuild.py b/Doc/tools/dailybuild.py deleted file mode 100755 --- a/Doc/tools/dailybuild.py +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# Runs the daily build of the Python docs on dinsdale.python.org. -# -# Usages: -# -# dailybuild.py [-q] -# -# without any arguments builds docs for all branches configured in the global -# BRANCHES value. -q selects "quick build", which means to build only HTML. -# -# dailybuild.py [-q] [-d] -# -# builds one version, where is an SVN checkout directory of the -# Python branch to build docs for, and is the directory where the -# result should be placed. If -d is given, the docs are built even if the -# branch is in development mode (i.e. version contains a, b or c). -# -# This script is not run from the checkout, so if you want to change how the -# daily build is run, you must replace it on dinsdale. This is necessary, for -# example, after the release of a new minor version. -# -# 03/2010, Georg Brandl - -import os -import sys -import getopt - - -BUILDROOT = '/home/gbrandl/docbuild' -SPHINXBUILD = os.path.join(BUILDROOT, 'sphinx-env/bin/sphinx-build') -WWWROOT = '/data/ftp.python.org/pub/docs.python.org' - -BRANCHES = [ - # checkout, target, isdev - (BUILDROOT + '/python34', WWWROOT + '/3.4', False), - (BUILDROOT + '/python35', WWWROOT + '/3.5', True), - (BUILDROOT + '/python27', WWWROOT + '/2.7', False), -] - - -def _files_changed(old, new): - with open(old, 'rb') as fp1, open(new, 'rb') as fp2: - st1 = os.fstat(fp1.fileno()) - st2 = os.fstat(fp2.fileno()) - if st1.st_size != st2.st_size: - return False - if st1.st_mtime >= st2.st_mtime: - return True - while True: - one = fp1.read(4096) - two = fp2.read(4096) - if one != two: - return False - if one == '': - break - return True - -def build_one(checkout, target, isdev, quick): - print 'Doc autobuild started in %s' % checkout - os.chdir(checkout) - print 'Running hg pull --update' - os.system('hg pull --update') - print 'Running make autobuild' - maketarget = 'autobuild-' + ('html' if quick else - ('dev' if isdev else 'stable')) - if os.WEXITSTATUS(os.system('cd Doc; make SPHINXBUILD=%s %s' % (SPHINXBUILD, maketarget))) == 2: - print '*' * 80 - return - print('Computing changed files') - changed = [] - for dirpath, dirnames, filenames in os.walk('Doc/build/html/'): - dir_rel = dirpath[len('Doc/build/html/'):] - for fn in filenames: - local_path = os.path.join(dirpath, fn) - rel_path = os.path.join(dir_rel, fn) - target_path = os.path.join(target, rel_path) - if (os.path.exists(target_path) and - not _files_changed(target_path, local_path)): - changed.append(rel_path) - print 'Copying HTML files to %s' % target - os.system('cp -a Doc/build/html/* %s' % target) - if not quick: - print 'Copying dist files' - os.system('mkdir -p %s/archives' % target) - os.system('cp -a Doc/dist/* %s/archives' % target) - changed.append('archives/') - for fn in os.listdir(os.path.join(target, 'archives')): - changed.append('archives/' + fn) - print '%s files changed' % len(changed) - if changed: - target_ino = os.stat(target).st_ino - targets_dir = os.path.dirname(target) - prefixes = [] - for fn in os.listdir(targets_dir): - if os.stat(os.path.join(targets_dir, fn)).st_ino == target_ino: - prefixes.append(fn) - to_purge = [] - for prefix in prefixes: - to_purge.extend(prefix + "/" + p for p in changed) - purge_cmd = 'curl -X PURGE "https://docs.python.org/{%s}"' % ','.join(to_purge) - print("Running CDN purge") - os.system(purge_cmd) - print 'Finished' - print '=' * 80 - -def usage(): - print 'Usage:' - print ' %s' % sys.argv[0] - print 'or' - print ' %s [-d] ' % sys.argv[0] - sys.exit(1) - - -if __name__ == '__main__': - try: - opts, args = getopt.getopt(sys.argv[1:], 'dq') - except getopt.error: - usage() - quick = devel = False - for opt, _ in opts: - if opt == '-q': - quick = True - if opt == '-d': - devel = True - if devel and not args: - usage() - if args: - if len(args) != 2: - usage() - build_one(os.path.abspath(args[0]), os.path.abspath(args[1]), devel, quick) - else: - for checkout, dest, devel in BRANCHES: - build_one(checkout, dest, devel, quick) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 4 17:50:20 2014 From: python-checkins at python.org (benjamin.peterson) Date: Thu, 4 Sep 2014 17:50:20 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_the_correct_ro?= =?utf-8?q?le_is_=3Ameth=3A_not_=3Amethod=3A?= Message-ID: <3hpmgc0WSRz7Ljf@mail.python.org> http://hg.python.org/cpython/rev/f17ab9fed3b0 changeset: 92336:f17ab9fed3b0 branch: 2.7 user: Benjamin Peterson date: Thu Sep 04 11:50:14 2014 -0400 summary: the correct role is :meth: not :method: files: Doc/library/argparse.rst | 14 ++++++-------- 1 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -1213,19 +1213,17 @@ ^^^^^^^^^^^^^^ Action classes implement the Action API, a callable which returns a callable -which processes arguments from the command-line. Any object which follows -this API may be passed as the ``action`` parameter to -:method:`add_argument`. +which processes arguments from the command-line. Any object which follows this +API may be passed as the ``action`` parameter to :meth:`add_argument`. .. class:: Action(option_strings, dest, nargs=None, const=None, default=None, type=None, choices=None, required=False, help=None, metavar=None) -Action objects are used by an ArgumentParser to represent the information -needed to parse a single argument from one or more strings from the -command line. The Action class must accept the two positional arguments -plus any keyword arguments passed to :method:`ArgumentParser.add_argument` -except for the ``action`` itself. +Action objects are used by an ArgumentParser to represent the information needed +to parse a single argument from one or more strings from the command line. The +Action class must accept the two positional arguments plus any keyword arguments +passed to :meth:`ArgumentParser.add_argument` except for the ``action`` itself. Instances of Action (or return value of any callable to the ``action`` parameter) should have attributes "dest", "option_strings", "default", "type", -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 4 21:00:19 2014 From: python-checkins at python.org (antoine.pitrou) Date: Thu, 4 Sep 2014 21:00:19 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2320421=3A_Add_a_?= =?utf-8?q?=2Eversion=28=29_method_to_SSL_sockets_exposing_the_actual?= Message-ID: <3hprtq1hdhz7LlY@mail.python.org> http://hg.python.org/cpython/rev/648685f8d5e9 changeset: 92337:648685f8d5e9 parent: 92334:9ab404cdcaa1 user: Antoine Pitrou date: Thu Sep 04 21:00:10 2014 +0200 summary: Issue #20421: Add a .version() method to SSL sockets exposing the actual protocol version in use. files: Doc/library/ssl.rst | 18 ++++++- Lib/ssl.py | 9 ++++ Lib/test/test_ssl.py | 67 ++++++++++++++++++++++--------- Misc/NEWS | 3 + Modules/_ssl.c | 13 ++++++ 5 files changed, 86 insertions(+), 24 deletions(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -910,10 +910,10 @@ .. method:: SSLSocket.selected_npn_protocol() - Returns the protocol that was selected during the TLS/SSL handshake. If - :meth:`SSLContext.set_npn_protocols` was not called, or if the other party - does not support NPN, or if the handshake has not yet happened, this will - return ``None``. + Returns the higher-level protocol that was selected during the TLS/SSL + handshake. If :meth:`SSLContext.set_npn_protocols` was not called, or + if the other party does not support NPN, or if the handshake has not yet + happened, this will return ``None``. .. versionadded:: 3.3 @@ -925,6 +925,16 @@ returned socket should always be used for further communication with the other side of the connection, rather than the original socket. +.. method:: SSLSocket.version() + + Return the actual SSL protocol version negotiated by the connection + as a string, or ``None`` is no secure connection is established. + As of this writing, possible return values include ``"SSLv2"``, + ``"SSLv3"``, ``"TLSv1"``, ``"TLSv1.1"`` and ``"TLSv1.2"``. + Recent OpenSSL versions may define more return values. + + .. versionadded:: 3.5 + .. attribute:: SSLSocket.context The :class:`SSLContext` object this SSL socket is tied to. If the SSL diff --git a/Lib/ssl.py b/Lib/ssl.py --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -861,6 +861,15 @@ return None return self._sslobj.tls_unique_cb() + def version(self): + """ + Return a string identifying the protocol version used by the + current SSL channel, or None if there is no established channel. + """ + if self._sslobj is None: + return None + return self._sslobj.version() + def wrap_socket(sock, keyfile=None, certfile=None, server_side=False, cert_reqs=CERT_NONE, diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1942,7 +1942,8 @@ 'compression': s.compression(), 'cipher': s.cipher(), 'peercert': s.getpeercert(), - 'client_npn_protocol': s.selected_npn_protocol() + 'client_npn_protocol': s.selected_npn_protocol(), + 'version': s.version(), }) s.close() stats['server_npn_protocols'] = server.selected_protocols @@ -1950,6 +1951,13 @@ def try_protocol_combo(server_protocol, client_protocol, expect_success, certsreqs=None, server_options=0, client_options=0): + """ + Try to SSL-connect using *client_protocol* to *server_protocol*. + If *expect_success* is true, assert that the connection succeeds, + if it's false, assert that the connection fails. + Also, if *expect_success* is a string, assert that it is the protocol + version actually used by the connection. + """ if certsreqs is None: certsreqs = ssl.CERT_NONE certtype = { @@ -1979,8 +1987,8 @@ ctx.load_cert_chain(CERTFILE) ctx.load_verify_locations(CERTFILE) try: - server_params_test(client_context, server_context, - chatty=False, connectionchatty=False) + stats = server_params_test(client_context, server_context, + chatty=False, connectionchatty=False) # Protocol mismatch can result in either an SSLError, or a # "Connection reset by peer" error. except ssl.SSLError: @@ -1995,6 +2003,10 @@ "Client protocol %s succeeded with server protocol %s!" % (ssl.get_protocol_name(client_protocol), ssl.get_protocol_name(server_protocol))) + elif (expect_success is not True + and expect_success != stats['version']): + raise AssertionError("version mismatch: expected %r, got %r" + % (expect_success, stats['version'])) class ThreadedTests(unittest.TestCase): @@ -2225,17 +2237,17 @@ sys.stdout.write( " SSL2 client to SSL23 server test unexpectedly failed:\n %s\n" % str(x)) - try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, True) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, 'SSLv3') try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True) - try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, True) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1') - try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, True, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_OPTIONAL) try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, ssl.CERT_OPTIONAL) - try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, True, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) - try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, True, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_REQUIRED) try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, ssl.CERT_REQUIRED) - try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, True, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) # Server with specific SSL options try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, @@ -2252,9 +2264,9 @@ """Connecting to an SSLv3 server with various client options""" if support.verbose: sys.stdout.write("\n") - try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True) - try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_OPTIONAL) - try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3') + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_REQUIRED) if hasattr(ssl, 'PROTOCOL_SSLv2'): try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False) try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, False, @@ -2262,7 +2274,7 @@ try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLSv1, False) if no_sslv2_implies_sslv3_hello(): # No SSLv2 => client will use an SSLv3 hello on recent OpenSSLs - try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, True, + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, 'SSLv3', client_options=ssl.OP_NO_SSLv2) @skip_if_broken_ubuntu_ssl @@ -2270,9 +2282,9 @@ """Connecting to a TLSv1 server with various client options""" if support.verbose: sys.stdout.write("\n") - try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True) - try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_OPTIONAL) - try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1') + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) if hasattr(ssl, 'PROTOCOL_SSLv2'): try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False) try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False) @@ -2287,14 +2299,14 @@ Testing against older TLS versions.""" if support.verbose: sys.stdout.write("\n") - try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_1, True) + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') if hasattr(ssl, 'PROTOCOL_SSLv2'): try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv2, False) try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv3, False) try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv23, False, client_options=ssl.OP_NO_TLSv1_1) - try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1_1, True) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1, False) try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_1, False) @@ -2307,7 +2319,7 @@ Testing against older TLS versions.""" if support.verbose: sys.stdout.write("\n") - try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_2, True, + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_2, 'TLSv1.2', server_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2, client_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2,) if hasattr(ssl, 'PROTOCOL_SSLv2'): @@ -2316,7 +2328,7 @@ try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv23, False, client_options=ssl.OP_NO_TLSv1_2) - try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1_2, True) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1_2, 'TLSv1.2') try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1, False) try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_2, False) try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_1, False) @@ -2697,6 +2709,21 @@ s.connect((HOST, server.port)) self.assertIn("no shared cipher", str(server.conn_errors[0])) + def test_version_basic(self): + """ + Basic tests for SSLSocket.version(). + More tests are done in the test_protocol_*() methods. + """ + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with ThreadedEchoServer(CERTFILE, + ssl_version=ssl.PROTOCOL_TLSv1, + chatty=False) as server: + with context.wrap_socket(socket.socket()) as s: + self.assertIs(s.version(), None) + s.connect((HOST, server.port)) + self.assertEqual(s.version(), "TLSv1") + self.assertIs(s.version(), None) + @unittest.skipUnless(ssl.HAS_ECDH, "test requires ECDH-enabled OpenSSL") def test_default_ecdh_curve(self): # Issue #21015: elliptic curve-based Diffie Hellman key exchange diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -129,6 +129,9 @@ Library ------- +- Issue #20421: Add a .version() method to SSL sockets exposing the actual + protocol version in use. + - Issue #19546: configparser exceptions no longer expose implementation details. Chained KeyErrors are removed, which leads to cleaner tracebacks. Patch by Claudiu Popa. diff --git a/Modules/_ssl.c b/Modules/_ssl.c --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -1402,6 +1402,18 @@ return NULL; } +static PyObject *PySSL_version(PySSLSocket *self) +{ + const char *version; + + if (self->ssl == NULL) + Py_RETURN_NONE; + version = SSL_get_version(self->ssl); + if (!strcmp(version, "unknown")) + Py_RETURN_NONE; + return PyUnicode_FromString(version); +} + #ifdef OPENSSL_NPN_NEGOTIATED static PyObject *PySSL_selected_npn_protocol(PySSLSocket *self) { const unsigned char *out; @@ -1939,6 +1951,7 @@ {"peer_certificate", (PyCFunction)PySSL_peercert, METH_VARARGS, PySSL_peercert_doc}, {"cipher", (PyCFunction)PySSL_cipher, METH_NOARGS}, + {"version", (PyCFunction)PySSL_version, METH_NOARGS}, #ifdef OPENSSL_NPN_NEGOTIATED {"selected_npn_protocol", (PyCFunction)PySSL_selected_npn_protocol, METH_NOARGS}, #endif -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 4 22:33:37 2014 From: python-checkins at python.org (alex.gaynor) Date: Thu, 4 Sep 2014 22:33:37 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzIwNDIx?= =?utf-8?q?=3A_Add_a_=2Eversion=28=29_method_to_SSL_sockets_exposing_the_a?= =?utf-8?q?ctual?= Message-ID: <3hptyT0fJ5z7Ljp@mail.python.org> http://hg.python.org/cpython/rev/16c86a6bdbe2 changeset: 92338:16c86a6bdbe2 branch: 2.7 parent: 92336:f17ab9fed3b0 user: Alex Gaynor date: Thu Sep 04 13:33:22 2014 -0700 summary: Issue #20421: Add a .version() method to SSL sockets exposing the actual protocol version in use. Backport from default. files: Doc/library/ssl.rst | 18 ++++++- Lib/ssl.py | 9 ++++ Lib/test/test_ssl.py | 67 ++++++++++++++++++++++--------- Misc/NEWS | 1 + Modules/_ssl.c | 13 ++++++ 5 files changed, 84 insertions(+), 24 deletions(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -867,10 +867,10 @@ .. method:: SSLSocket.selected_npn_protocol() - Returns the protocol that was selected during the TLS/SSL handshake. If - :meth:`SSLContext.set_npn_protocols` was not called, or if the other party - does not support NPN, or if the handshake has not yet happened, this will - return ``None``. + Returns the higher-level protocol that was selected during the TLS/SSL + handshake. If :meth:`SSLContext.set_npn_protocols` was not called, or + if the other party does not support NPN, or if the handshake has not yet + happened, this will return ``None``. .. versionadded:: 2.7.9 @@ -882,6 +882,16 @@ returned socket should always be used for further communication with the other side of the connection, rather than the original socket. +.. method:: SSLSocket.version() + + Return the actual SSL protocol version negotiated by the connection + as a string, or ``None`` is no secure connection is established. + As of this writing, possible return values include ``"SSLv2"``, + ``"SSLv3"``, ``"TLSv1"``, ``"TLSv1.1"`` and ``"TLSv1.2"``. + Recent OpenSSL versions may define more return values. + + .. versionadded:: 3.5 + .. attribute:: SSLSocket.context The :class:`SSLContext` object this SSL socket is tied to. If the SSL diff --git a/Lib/ssl.py b/Lib/ssl.py --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -862,6 +862,15 @@ return None return self._sslobj.tls_unique_cb() + def version(self): + """ + Return a string identifying the protocol version used by the + current SSL channel, or None if there is no established channel. + """ + if self._sslobj is None: + return None + return self._sslobj.version() + def wrap_socket(sock, keyfile=None, certfile=None, server_side=False, cert_reqs=CERT_NONE, diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1904,7 +1904,8 @@ 'compression': s.compression(), 'cipher': s.cipher(), 'peercert': s.getpeercert(), - 'client_npn_protocol': s.selected_npn_protocol() + 'client_npn_protocol': s.selected_npn_protocol(), + 'version': s.version(), }) s.close() stats['server_npn_protocols'] = server.selected_protocols @@ -1912,6 +1913,13 @@ def try_protocol_combo(server_protocol, client_protocol, expect_success, certsreqs=None, server_options=0, client_options=0): + """ + Try to SSL-connect using *client_protocol* to *server_protocol*. + If *expect_success* is true, assert that the connection succeeds, + if it's false, assert that the connection fails. + Also, if *expect_success* is a string, assert that it is the protocol + version actually used by the connection. + """ if certsreqs is None: certsreqs = ssl.CERT_NONE certtype = { @@ -1941,8 +1949,8 @@ ctx.load_cert_chain(CERTFILE) ctx.load_verify_locations(CERTFILE) try: - server_params_test(client_context, server_context, - chatty=False, connectionchatty=False) + stats = server_params_test(client_context, server_context, + chatty=False, connectionchatty=False) # Protocol mismatch can result in either an SSLError, or a # "Connection reset by peer" error. except ssl.SSLError: @@ -1957,6 +1965,10 @@ "Client protocol %s succeeded with server protocol %s!" % (ssl.get_protocol_name(client_protocol), ssl.get_protocol_name(server_protocol))) + elif (expect_success is not True + and expect_success != stats['version']): + raise AssertionError("version mismatch: expected %r, got %r" + % (expect_success, stats['version'])) class ThreadedTests(unittest.TestCase): @@ -2186,17 +2198,17 @@ sys.stdout.write( " SSL2 client to SSL23 server test unexpectedly failed:\n %s\n" % str(x)) - try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, True) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, 'SSLv3') try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True) - try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, True) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1') - try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, True, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_OPTIONAL) try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, ssl.CERT_OPTIONAL) - try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, True, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) - try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, True, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_REQUIRED) try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, ssl.CERT_REQUIRED) - try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, True, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) # Server with specific SSL options try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, @@ -2213,9 +2225,9 @@ """Connecting to an SSLv3 server with various client options""" if support.verbose: sys.stdout.write("\n") - try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True) - try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_OPTIONAL) - try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3') + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_REQUIRED) if hasattr(ssl, 'PROTOCOL_SSLv2'): try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False) try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, False, @@ -2223,7 +2235,7 @@ try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLSv1, False) if no_sslv2_implies_sslv3_hello(): # No SSLv2 => client will use an SSLv3 hello on recent OpenSSLs - try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, True, + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, 'SSLv3', client_options=ssl.OP_NO_SSLv2) @skip_if_broken_ubuntu_ssl @@ -2231,9 +2243,9 @@ """Connecting to a TLSv1 server with various client options""" if support.verbose: sys.stdout.write("\n") - try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True) - try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_OPTIONAL) - try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1') + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) if hasattr(ssl, 'PROTOCOL_SSLv2'): try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False) try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False) @@ -2248,14 +2260,14 @@ Testing against older TLS versions.""" if support.verbose: sys.stdout.write("\n") - try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_1, True) + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') if hasattr(ssl, 'PROTOCOL_SSLv2'): try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv2, False) try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv3, False) try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv23, False, client_options=ssl.OP_NO_TLSv1_1) - try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1_1, True) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1, False) try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_1, False) @@ -2268,7 +2280,7 @@ Testing against older TLS versions.""" if support.verbose: sys.stdout.write("\n") - try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_2, True, + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_2, 'TLSv1.2', server_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2, client_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2,) if hasattr(ssl, 'PROTOCOL_SSLv2'): @@ -2277,7 +2289,7 @@ try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv23, False, client_options=ssl.OP_NO_TLSv1_2) - try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1_2, True) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1_2, 'TLSv1.2') try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1, False) try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_2, False) try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_1, False) @@ -2619,6 +2631,21 @@ s.connect((HOST, server.port)) self.assertIn("no shared cipher", str(server.conn_errors[0])) + def test_version_basic(self): + """ + Basic tests for SSLSocket.version(). + More tests are done in the test_protocol_*() methods. + """ + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with ThreadedEchoServer(CERTFILE, + ssl_version=ssl.PROTOCOL_TLSv1, + chatty=False) as server: + with closing(context.wrap_socket(socket.socket())) as s: + self.assertIs(s.version(), None) + s.connect((HOST, server.port)) + self.assertEqual(s.version(), "TLSv1") + self.assertIs(s.version(), None) + @unittest.skipUnless(ssl.HAS_ECDH, "test requires ECDH-enabled OpenSSL") def test_default_ecdh_curve(self): # Issue #21015: elliptic curve-based Diffie Hellman key exchange diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -171,6 +171,7 @@ ------- - Issue #22160: The bundled version of OpenSSL has been updated to 1.0.1i. +version in use. What's New in Python 2.7.8? diff --git a/Modules/_ssl.c b/Modules/_ssl.c --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -1384,6 +1384,18 @@ return NULL; } +static PyObject *PySSL_version(PySSLSocket *self) +{ + const char *version; + + if (self->ssl == NULL) + Py_RETURN_NONE; + version = SSL_get_version(self->ssl); + if (!strcmp(version, "unknown")) + Py_RETURN_NONE; + return PyUnicode_FromString(version); +} + #ifdef OPENSSL_NPN_NEGOTIATED static PyObject *PySSL_selected_npn_protocol(PySSLSocket *self) { const unsigned char *out; @@ -1907,6 +1919,7 @@ {"peer_certificate", (PyCFunction)PySSL_peercert, METH_VARARGS, PySSL_peercert_doc}, {"cipher", (PyCFunction)PySSL_cipher, METH_NOARGS}, + {"version", (PyCFunction)PySSL_version, METH_NOARGS}, #ifdef OPENSSL_NPN_NEGOTIATED {"selected_npn_protocol", (PyCFunction)PySSL_selected_npn_protocol, METH_NOARGS}, #endif -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 4 22:37:18 2014 From: python-checkins at python.org (alex.gaynor) Date: Thu, 4 Sep 2014 22:37:18 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_Fixed_the_vers?= =?utf-8?q?ionadded_in_the_docs_for_the_backport_in_16c86a6bdbe2?= Message-ID: <3hpv2k2D7Pz7Ljf@mail.python.org> http://hg.python.org/cpython/rev/390e910b2f96 changeset: 92339:390e910b2f96 branch: 2.7 user: Alex Gaynor date: Thu Sep 04 13:37:07 2014 -0700 summary: Fixed the versionadded in the docs for the backport in 16c86a6bdbe2 files: Doc/library/ssl.rst | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -890,7 +890,7 @@ ``"SSLv3"``, ``"TLSv1"``, ``"TLSv1.1"`` and ``"TLSv1.2"``. Recent OpenSSL versions may define more return values. - .. versionadded:: 3.5 + .. versionadded:: 2.7.9 .. attribute:: SSLSocket.context -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 5 01:16:44 2014 From: python-checkins at python.org (victor.stinner) Date: Fri, 5 Sep 2014 01:16:44 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2322290=3A_PyObject?= =?utf-8?q?=5FCall=28=29_now_fails_with_an_assertion_error_when_called?= Message-ID: <3hpyZh1tkjz7LjT@mail.python.org> http://hg.python.org/cpython/rev/16e3d240456f changeset: 92340:16e3d240456f parent: 92337:648685f8d5e9 user: Victor Stinner date: Fri Sep 05 01:10:29 2014 +0200 summary: Issue #22290: PyObject_Call() now fails with an assertion error when called with an exception set. This new assertion helps to understand if the exception was already set before calling the function or raised by the function. files: Objects/abstract.c | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diff --git a/Objects/abstract.c b/Objects/abstract.c --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2074,6 +2074,11 @@ { ternaryfunc call; + /* PyObject_Call() must not be called with an exception set, + because it may clear it (directly or indirectly) and so the + caller looses its exception */ + assert(!PyErr_Occurred()); + if ((call = func->ob_type->tp_call) != NULL) { PyObject *result; if (Py_EnterRecursiveCall(" while calling a Python object")) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 5 03:01:15 2014 From: python-checkins at python.org (terry.reedy) Date: Fri, 5 Sep 2014 03:01:15 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzIxOTMz?= =?utf-8?q?=3A_Make_Control-Mousewhell_only_change_font_size_and_not_also_?= =?utf-8?q?scroll=2E?= Message-ID: <3hq0vH5YKwz7LjT@mail.python.org> http://hg.python.org/cpython/rev/ecc98ea50bc3 changeset: 92341:ecc98ea50bc3 branch: 3.4 parent: 92327:b2e9d845bc6c user: Terry Jan Reedy date: Thu Sep 04 20:59:41 2014 -0400 summary: Issue #21933: Make Control-Mousewhell only change font size and not also scroll. Original patch by Serhiy Storchaka. files: Lib/turtledemo/__main__.py | 40 +++++++++++++------------ 1 files changed, 21 insertions(+), 19 deletions(-) diff --git a/Lib/turtledemo/__main__.py b/Lib/turtledemo/__main__.py --- a/Lib/turtledemo/__main__.py +++ b/Lib/turtledemo/__main__.py @@ -19,8 +19,9 @@ The demo viewer windows can be resized. The divider between text and canvas can be moved by grabbing it with the mouse. The text font - size can be changed from the menu and with Control/Command- '-'/'+'. - It can also be changed on most systems with Control-mousewheel. + size can be changed from the menu and with Control/Command '-'/'+'. + It can also be changed on most systems with Control-mousewheel + when the mouse is over the text. Press START button to start the demo. Stop execution by pressing the STOP button. @@ -210,10 +211,19 @@ self.hbar = hbar = Scrollbar(text_frame, name='hbar', orient=HORIZONTAL) hbar['command'] = text.xview hbar.pack(side=BOTTOM, fill=X) + text['yscrollcommand'] = vbar.set + text['xscrollcommand'] = hbar.set text['font'] = tuple(txtfont) - text['yscrollcommand'] = vbar.set - text['xscrollcommand'] = hbar.set + shortcut = 'Command' if darwin else 'Control' + text.bind_all('<%s-minus>' % shortcut, self.decrease_size) + text.bind_all('<%s-underscore>' % shortcut, self.decrease_size) + text.bind_all('<%s-equal>' % shortcut, self.increase_size) + text.bind_all('<%s-plus>' % shortcut, self.increase_size) + text.bind('', self.update_mousewheel) + text.bind('', self.increase_size) + text.bind('', self.decrease_size) + text.pack(side=LEFT, fill=BOTH, expand=1) return text_frame @@ -224,7 +234,7 @@ turtle._Screen._canvas = self._canvas = canvas = turtle.ScrolledCanvas( root, 800, 600, self.canvwidth, self.canvheight) canvas.adjustScrolls() - self.makeBindings(canvas._rootwindow) + canvas._rootwindow.bind('', self.onResize) canvas._canvas['borderwidth'] = 0 self.screen = _s_ = turtle.Screen() @@ -233,18 +243,6 @@ turtle.RawTurtle.screens = [_s_] return canvas - def makeBindings(self, widget): - widget.bind('', self.onResize) - - shortcut = 'Command' if darwin else 'Control' - widget.bind_all('<%s-minus>' % shortcut, self.decrease_size) - widget.bind_all('<%s-underscore>' % shortcut, self.decrease_size) - widget.bind_all('<%s-equal>' % shortcut, self.increase_size) - widget.bind_all('<%s-plus>' % shortcut, self.increase_size) - widget.bind_all('', self.update_mousewheel) - widget.bind('', self.increase_size) - widget.bind('', self.decrease_size) - def set_txtsize(self, size): txtfont[1] = size self.text['font'] = tuple(txtfont) @@ -252,15 +250,19 @@ def decrease_size(self, dummy=None): self.set_txtsize(max(txtfont[1] - 1, MINIMUM_FONT_SIZE)) + return 'break' def increase_size(self, dummy=None): self.set_txtsize(min(txtfont[1] + 1, MAXIMUM_FONT_SIZE)) + return 'break' def update_mousewheel(self, event): # For wheel up, event.delte = 120 on Windows, -1 on darwin. # X-11 sends Control-Button-4 event instead. - (self.decrease_size() if (event.delta < 0 and not darwin) - else self.increase_size()) + if (event.delta < 0) == (not darwin): + return self.decrease_size() + else: + return self.increase_size() def configGUI(self, start, stop, clear, txt="", color="blue"): self.start_btn.config(state=start, -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 5 03:01:17 2014 From: python-checkins at python.org (terry.reedy) Date: Fri, 5 Sep 2014 03:01:17 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?q?=29=3A_Merge_with_3=2E4?= Message-ID: <3hq0vK1BzYz7Ljr@mail.python.org> http://hg.python.org/cpython/rev/034ebeb8be2d changeset: 92342:034ebeb8be2d parent: 92340:16e3d240456f parent: 92341:ecc98ea50bc3 user: Terry Jan Reedy date: Thu Sep 04 21:00:52 2014 -0400 summary: Merge with 3.4 files: Lib/turtledemo/__main__.py | 40 +++++++++++++------------ 1 files changed, 21 insertions(+), 19 deletions(-) diff --git a/Lib/turtledemo/__main__.py b/Lib/turtledemo/__main__.py --- a/Lib/turtledemo/__main__.py +++ b/Lib/turtledemo/__main__.py @@ -19,8 +19,9 @@ The demo viewer windows can be resized. The divider between text and canvas can be moved by grabbing it with the mouse. The text font - size can be changed from the menu and with Control/Command- '-'/'+'. - It can also be changed on most systems with Control-mousewheel. + size can be changed from the menu and with Control/Command '-'/'+'. + It can also be changed on most systems with Control-mousewheel + when the mouse is over the text. Press START button to start the demo. Stop execution by pressing the STOP button. @@ -210,10 +211,19 @@ self.hbar = hbar = Scrollbar(text_frame, name='hbar', orient=HORIZONTAL) hbar['command'] = text.xview hbar.pack(side=BOTTOM, fill=X) + text['yscrollcommand'] = vbar.set + text['xscrollcommand'] = hbar.set text['font'] = tuple(txtfont) - text['yscrollcommand'] = vbar.set - text['xscrollcommand'] = hbar.set + shortcut = 'Command' if darwin else 'Control' + text.bind_all('<%s-minus>' % shortcut, self.decrease_size) + text.bind_all('<%s-underscore>' % shortcut, self.decrease_size) + text.bind_all('<%s-equal>' % shortcut, self.increase_size) + text.bind_all('<%s-plus>' % shortcut, self.increase_size) + text.bind('', self.update_mousewheel) + text.bind('', self.increase_size) + text.bind('', self.decrease_size) + text.pack(side=LEFT, fill=BOTH, expand=1) return text_frame @@ -224,7 +234,7 @@ turtle._Screen._canvas = self._canvas = canvas = turtle.ScrolledCanvas( root, 800, 600, self.canvwidth, self.canvheight) canvas.adjustScrolls() - self.makeBindings(canvas._rootwindow) + canvas._rootwindow.bind('', self.onResize) canvas._canvas['borderwidth'] = 0 self.screen = _s_ = turtle.Screen() @@ -233,18 +243,6 @@ turtle.RawTurtle.screens = [_s_] return canvas - def makeBindings(self, widget): - widget.bind('', self.onResize) - - shortcut = 'Command' if darwin else 'Control' - widget.bind_all('<%s-minus>' % shortcut, self.decrease_size) - widget.bind_all('<%s-underscore>' % shortcut, self.decrease_size) - widget.bind_all('<%s-equal>' % shortcut, self.increase_size) - widget.bind_all('<%s-plus>' % shortcut, self.increase_size) - widget.bind_all('', self.update_mousewheel) - widget.bind('', self.increase_size) - widget.bind('', self.decrease_size) - def set_txtsize(self, size): txtfont[1] = size self.text['font'] = tuple(txtfont) @@ -252,15 +250,19 @@ def decrease_size(self, dummy=None): self.set_txtsize(max(txtfont[1] - 1, MINIMUM_FONT_SIZE)) + return 'break' def increase_size(self, dummy=None): self.set_txtsize(min(txtfont[1] + 1, MAXIMUM_FONT_SIZE)) + return 'break' def update_mousewheel(self, event): # For wheel up, event.delte = 120 on Windows, -1 on darwin. # X-11 sends Control-Button-4 event instead. - (self.decrease_size() if (event.delta < 0 and not darwin) - else self.increase_size()) + if (event.delta < 0) == (not darwin): + return self.decrease_size() + else: + return self.increase_size() def configGUI(self, start, stop, clear, txt="", color="blue"): self.start_btn.config(state=start, -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 5 04:41:04 2014 From: python-checkins at python.org (benjamin.peterson) Date: Fri, 5 Sep 2014 04:41:04 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_fix_malformed_?= =?utf-8?q?use_cmdoption_directive?= Message-ID: <3hq36S4nbjz7LjV@mail.python.org> http://hg.python.org/cpython/rev/41617012fa1e changeset: 92343:41617012fa1e branch: 2.7 parent: 92339:390e910b2f96 user: Benjamin Peterson date: Thu Sep 04 22:40:34 2014 -0400 summary: fix malformed use cmdoption directive files: Doc/library/compileall.rst | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -20,7 +20,8 @@ .. program:: compileall -.. cmdoption:: [directory|file]... +.. cmdoption:: directory ... + file ... Positional arguments are files to compile or directories that contain source files, traversed recursively. If no argument is given, behave as if -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 5 05:07:14 2014 From: python-checkins at python.org (benjamin.peterson) Date: Fri, 5 Sep 2014 05:07:14 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_remove_automat?= =?utf-8?q?ic_svn_checkout_of_sphinx_and_its_deps_from_Doc/Makefile?= Message-ID: <3hq3hf3GYDz7LjM@mail.python.org> http://hg.python.org/cpython/rev/48033d90c61d changeset: 92344:48033d90c61d branch: 2.7 user: Benjamin Peterson date: Thu Sep 04 23:07:03 2014 -0400 summary: remove automatic svn checkout of sphinx and its deps from Doc/Makefile Users are now required to install Sphinx themselves as is already the case with the Python 3 branch. files: .hgignore | 5 - Doc/Makefile | 54 ++++-------- Doc/README.txt | 54 +++++++++---- Doc/make.bat | 152 ++++++++++++++++++++++++++---------- Misc/NEWS | 3 + 5 files changed, 167 insertions(+), 101 deletions(-) diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -21,11 +21,6 @@ reflog.txt$ tags$ Lib/plat-mac/errors.rsrc.df.rsrc -Doc/tools/sphinx/ -Doc/tools/docutils/ -Doc/tools/jinja/ -Doc/tools/jinja2/ -Doc/tools/pygments/ Misc/python.pc Modules/Setup$ Modules/Setup.config diff --git a/Doc/Makefile b/Doc/Makefile --- a/Doc/Makefile +++ b/Doc/Makefile @@ -5,7 +5,7 @@ # You can set these variables from the command line. PYTHON = python -SVNROOT = http://svn.python.org/projects +SPHINXBUILD = sphinx-build SPHINXOPTS = PAPER = SOURCES = @@ -14,18 +14,19 @@ ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_paper_size=$(PAPER) \ $(SPHINXOPTS) . build/$(BUILDER) $(SOURCES) -.PHONY: help checkout update build html htmlhelp latex text changes linkcheck \ +.PHONY: help build html htmlhelp latex text changes linkcheck \ suspicious coverage doctest pydoc-topics htmlview clean dist check serve \ autobuild-dev autobuild-stable help: @echo "Please use \`make ' where is one of" @echo " clean to remove build files" - @echo " update to update build tools" @echo " html to make standalone HTML files" + @echo " htmlview to open the index page built by the html target in your browser" @echo " htmlhelp to make HTML files and a HTML help project" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " text to make plain text files" + @echo " epub to make EPUB files" @echo " changes to make an overview over all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @echo " coverage to check documentation coverage for library and C API" @@ -36,30 +37,8 @@ @echo " check to run a check for frequent markup errors" @echo " serve to serve the documentation on the localhost (8000)" -# Note: if you update versions here, do the same in make.bat and README.txt -checkout: - @if [ ! -d tools/sphinx ]; then \ - echo "Checking out Sphinx..."; \ - svn checkout $(SVNROOT)/external/Sphinx-1.0.7/sphinx tools/sphinx; \ - fi - @if [ ! -d tools/docutils ]; then \ - echo "Checking out Docutils..."; \ - svn checkout $(SVNROOT)/external/docutils-0.6/docutils tools/docutils; \ - fi - @if [ ! -d tools/jinja2 ]; then \ - echo "Checking out Jinja..."; \ - svn checkout $(SVNROOT)/external/Jinja-2.3.1/jinja2 tools/jinja2; \ - fi - @if [ ! -d tools/pygments ]; then \ - echo "Checking out Pygments..."; \ - svn checkout $(SVNROOT)/external/Pygments-1.3.1/pygments tools/pygments; \ - fi - -update: clean checkout - -build: checkout - mkdir -p build/$(BUILDER) build/doctrees - $(PYTHON) tools/sphinx-build.py $(ALLSPHINXOPTS) +build: + $(SPHINXBUILD) $(ALLSPHINXOPTS) @echo html: BUILDER = html @@ -81,6 +60,10 @@ text: build @echo "Build finished; the text files are in build/text." +epub: BUILDER = epub +epub: build + @echo "Build finished; the epub files are in build/epub." + changes: BUILDER = changes changes: build @echo "The overview file is in build/changes." @@ -114,17 +97,13 @@ pydoc-topics: BUILDER = pydoc-topics pydoc-topics: build @echo "Building finished; now copy build/pydoc-topics/topics.py" \ - "to Lib/pydoc_data/topics.py" + "to ../Lib/pydoc_data/topics.py" htmlview: html $(PYTHON) -c "import webbrowser; webbrowser.open('build/html/index.html')" clean: -rm -rf build/* - -rm -rf tools/sphinx - -rm -rf tools/pygments - -rm -rf tools/jinja2 - -rm -rf tools/docutils dist: rm -rf dist @@ -164,6 +143,11 @@ cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-letter.zip cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-letter.tar.bz2 + # copy the epub build + rm -rf build/epub + make epub + cp -pPR build/epub/Python.epub dist/python-$(DISTVERSION)-docs.epub + check: $(PYTHON) tools/rstlint.py -i tools @@ -174,7 +158,6 @@ # for development releases: always build autobuild-dev: - make update make dist SPHINXOPTS='-A daily=1 -A versionswitcher=1' -make suspicious @@ -182,10 +165,9 @@ autobuild-html: make html SPHINXOPTS='-A daily=1 -A versionswitcher=1' -# for stable releases: only build if not in pre-release stage (alpha, beta) -# release candidate downloads are okay, since the stable tree can be in that stage +# for stable releases: only build if not in pre-release stage (alpha, beta, rc) autobuild-stable: - @case $(DISTVERSION) in *[ab]*) \ + @case $(DISTVERSION) in *[abc]*) \ echo "Not building; $(DISTVERSION) is not a release version."; \ exit 1;; \ esac diff --git a/Doc/README.txt b/Doc/README.txt --- a/Doc/README.txt +++ b/Doc/README.txt @@ -3,54 +3,69 @@ This directory contains the reStructuredText (reST) sources to the Python documentation. You don't need to build them yourself, prebuilt versions are -available at https://docs.python.org/2/download.html +available at . -Documentation on the authoring Python documentation, including information about +Documentation on authoring Python documentation, including information about both style and markup, is available in the "Documenting Python" chapter of the -documentation. +developers guide . Building the docs ================= -You need to have Python 2 installed; the toolset used to build the -docs is written in Python. It is called *Sphinx*, it is not included in this -tree, but maintained separately. Also needed are the docutils, supplying the -base markup that Sphinx uses, Jinja, a templating engine, and optionally -Pygments, a code highlighter. +You need to have Sphinx installed; it is the toolset +used to build the docs. It is not included in this tree, but maintained +separately and available from PyPI . Using make ---------- -Luckily, a Makefile has been prepared so that on Unix, provided you have -installed Python and Subversion, you can just run :: +A Makefile has been prepared so that on Unix, provided you have installed +Sphinx, you can just run :: make html -to check out the necessary toolset in the `tools/` subdirectory and build the -HTML output files. To view the generated HTML, point your favorite browser at -the top-level index `build/html/index.html` after running "make". +to build the HTML output files. On Windows, we try to emulate the Makefile as closely as possible with a ``make.bat`` file. +To use a Python interpreter that's not called ``python``, use the standard +way to set Makefile variables, using e.g. :: + + make html PYTHON=python3 + +On Windows, set the PYTHON environment variable instead. + +To use a specific sphinx-build (something other than ``sphinx-build``), set +the SPHINXBUILD variable. + Available make targets are: + * "clean", which removes all build files. + * "html", which builds standalone HTML files for offline viewing. + * "htmlview", which re-uses the "html" builder, but then opens the main page + in your default web browser. + * "htmlhelp", which builds HTML files and a HTML Help project file usable to convert them into a single Compiled HTML (.chm) file -- these are popular under Microsoft Windows, but very handy on every platform. - To create the CHM file, you need to run the Microsoft HTML Help Workshop over - the generated project (.hhp) file. + To create the CHM file, you need to run the Microsoft HTML Help Workshop + over the generated project (.hhp) file. The make.bat script does this for + you on Windows. * "latex", which builds LaTeX source files as input to "pdflatex" to produce PDF documents. * "text", which builds a plain text file for each source file. + * "epub", which builds an EPUB document, suitable to be viewed on e-book + readers. + * "linkcheck", which checks all external references to see whether they are broken, redirected or malformed, and outputs this information to stdout as well as a plain-text (.txt) file. @@ -70,7 +85,12 @@ * "suspicious", which checks the parsed markup for text that looks like malformed and thus unconverted reST. -A "make update" updates the Subversion checkouts in `tools/`. + * "check", which checks for frequent markup errors. + + * "serve", which serves the build/html directory on port 8000. + + * "dist", (Unix only) which creates distributable archives of HTML, text, + PDF, and EPUB builds. Without make @@ -78,7 +98,7 @@ Install the Sphinx package and its dependencies from PyPI. -Then, from the ``Docs`` directory, run :: +Then, from the ``Doc`` directory, run :: sphinx-build -b . build/ diff --git a/Doc/make.bat b/Doc/make.bat --- a/Doc/make.bat +++ b/Doc/make.bat @@ -1,58 +1,124 @@ -@@echo off + at echo off setlocal -set SVNROOT=http://svn.python.org/projects -if "%PYTHON%" EQU "" set PYTHON=..\pcbuild\python -if "%HTMLHELP%" EQU "" set HTMLHELP=%ProgramFiles%\HTML Help Workshop\hhc.exe +pushd %~dp0 + +set this=%~n0 + +if "%SPHINXBUILD%" EQU "" set SPHINXBUILD=sphinx-build +if "%PYTHON%" EQU "" set PYTHON=py + +if DEFINED ProgramFiles(x86) set _PRGMFLS=%ProgramFiles(x86)% +if NOT DEFINED ProgramFiles(x86) set _PRGMFLS=%ProgramFiles% +if "%HTMLHELP%" EQU "" set HTMLHELP=%_PRGMFLS%\HTML Help Workshop\hhc.exe + if "%DISTVERSION%" EQU "" for /f "usebackq" %%v in (`%PYTHON% tools/sphinxext/patchlevel.py`) do set DISTVERSION=%%v +if "%BUILDDIR%" EQU "" set BUILDDIR=build + +rem Targets that don't require sphinx-build if "%1" EQU "" goto help -if "%1" EQU "html" goto build -if "%1" EQU "htmlhelp" goto build -if "%1" EQU "latex" goto build -if "%1" EQU "text" goto build -if "%1" EQU "suspicious" goto build -if "%1" EQU "linkcheck" goto build -if "%1" EQU "changes" goto build -if "%1" EQU "checkout" goto checkout -if "%1" EQU "update" goto update +if "%1" EQU "help" goto help +if "%1" EQU "check" goto check +if "%1" EQU "serve" goto serve +if "%1" == "clean" ( + rmdir /q /s %BUILDDIR% + goto end +) + +%SPHINXBUILD% 2> nul +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + goto end +) + +rem Targets that do require sphinx-build and have their own label +if "%1" EQU "htmlview" goto htmlview + +rem Everything else +goto build :help -set this=%~n0 -echo HELP +echo.usage: %this% BUILDER [filename ...] echo. -echo %this% checkout -echo %this% update -echo %this% html -echo %this% htmlhelp -echo %this% latex -echo %this% text -echo %this% suspicious -echo %this% linkcheck -echo %this% changes +echo.Call %this% with the desired Sphinx builder as the first argument, e.g. +echo.``%this% html`` or ``%this% doctest``. Interesting targets that are +echo.always available include: echo. -goto end - -:checkout -svn co %SVNROOT%/external/Sphinx-1.0.7/sphinx tools/sphinx -svn co %SVNROOT%/external/docutils-0.6/docutils tools/docutils -svn co %SVNROOT%/external/Jinja-2.3.1/jinja2 tools/jinja2 -svn co %SVNROOT%/external/Pygments-1.3.1/pygments tools/pygments -goto end - -:update -svn update tools/sphinx -svn update tools/docutils -svn update tools/jinja2 -svn update tools/pygments +echo. Provided by Sphinx: +echo. html, htmlhelp, latex, text +echo. suspicious, linkcheck, changes, doctest +echo. Provided by this script: +echo. clean, check, serve, htmlview +echo. +echo.All arguments past the first one are passed through to sphinx-build as +echo.filenames to build or are ignored. See README.txt in this directory or +echo.the documentation for your version of Sphinx for more exhaustive lists +echo.of available targets and descriptions of each. +echo. +echo.This script assumes that the SPHINXBUILD environment variable contains +echo.a legitimate command for calling sphinx-build, or that sphinx-build is +echo.on your PATH if SPHINXBUILD is not set. Options for sphinx-build can +echo.be passed by setting the SPHINXOPTS environment variable. goto end :build -if not exist build mkdir build -if not exist build\%1 mkdir build\%1 -if not exist build\doctrees mkdir build\doctrees -cmd /C %PYTHON% tools\sphinx-build.py -b%1 -dbuild\doctrees . build\%* -if "%1" EQU "htmlhelp" "%HTMLHELP%" build\htmlhelp\python%DISTVERSION:.=%.hhp +if NOT "%PAPER%" == "" ( + set SPHINXOPTS=-D latex_paper_size=%PAPER% %SPHINXOPTS% +) +cmd /C %SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . %BUILDDIR%\%* + +if "%1" EQU "htmlhelp" ( + if not exist "%HTMLHELP%" ( + echo. + echo.The HTML Help Workshop was not found. Set the HTMLHELP variable + echo.to the path to hhc.exe or download and install it from + echo.http://msdn.microsoft.com/en-us/library/ms669985 + rem Set errorlevel to 1 and exit + cmd /C exit /b 1 + goto end + ) + cmd /C "%HTMLHELP%" build\htmlhelp\python%DISTVERSION:.=%.hhp + rem hhc.exe seems to always exit with code 1, reset to 0 for less than 2 + if not errorlevel 2 cmd /C exit /b 0 +) + +echo. +if errorlevel 1 ( + echo.Build failed (exit code %ERRORLEVEL%^), check for error messages + echo.above. Any output will be found in %BUILDDIR%\%1 +) else ( + echo.Build succeeded. All output should be in %BUILDDIR%\%1 +) +goto end + +:htmlview +if NOT "%2" EQU "" ( + echo.Can't specify filenames to build with htmlview target, ignoring. +) +cmd /C %this% html + +if EXIST %BUILDDIR%\html\index.html ( + echo.Opening %BUILDDIR%\html\index.html in the default web browser... + start %BUILDDIR%\html\index.html +) + +goto end + +:check +cmd /C %PYTHON% tools\rstlint.py -i tools +goto end + +:serve +cmd /C %PYTHON% ..\Tools\scripts\serve.py %BUILDDIR%\html goto end :end +popd diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -158,6 +158,9 @@ Build ----- +- The documentation Makefile no longer automatically downloads Sphinx. Users are + now required to have Sphinx already installed to build the documentation. + - Issue #21958: Define HAVE_ROUND when building with Visual Studio 2013 and above. Patch by Zachary Turner. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 5 05:26:01 2014 From: python-checkins at python.org (benjamin.peterson) Date: Fri, 5 Sep 2014 05:26:01 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?devguide=3A_update_for_recent_changes?= =?utf-8?q?_in_doc_building?= Message-ID: <3hq46K1p8Zz7LjY@mail.python.org> http://hg.python.org/devguide/rev/2ed6c4b04334 changeset: 711:2ed6c4b04334 user: Benjamin Peterson date: Thu Sep 04 23:25:50 2014 -0400 summary: update for recent changes in doc building files: documenting.rst | 17 +++++++++-------- 1 files changed, 9 insertions(+), 8 deletions(-) diff --git a/documenting.rst b/documenting.rst --- a/documenting.rst +++ b/documenting.rst @@ -1451,13 +1451,8 @@ below. You can view the documentation after building the HTML by pointing a browser at the file :file:`Doc/build/html/index.html`. -In the Python 2.7 and 3.3 branches, the Sphinx toolchain will be checked out -using Subversion from ``svn.python.org`` by the Makefile. This toolchain will -need an installed Python 2 to run. - -In the Python 3.4 and later branches, you are expected to have installed a -recent version of Sphinx on your system, so that the Makefile can find the -``sphinx-build`` command. +You are expected to have installed a recent version of Sphinx on your system or +in a virtualenv_, so that the Makefile can find the ``sphinx-build`` command. Using make / make.bat @@ -1469,7 +1464,9 @@ cd Doc make html -or alternatively ``make -C Doc html``. This builds the output as HTML. +or alternatively ``make -C Doc html``. This builds the output as HTML. You can +specify the location of the ``sphinx-build`` command with the ``SPHINXBUILD`` +make variable. For Windows users there is a :file:`make.bat` batchfile that tries to work like ``make`` does. @@ -1478,6 +1475,9 @@ * "html", which builds standalone HTML files for offline viewing. + * "htmlview", which builds the standalone HTML files and then opens a web + browser to display them. + * "htmlhelp", which builds HTML files and a HTML Help project file usable to convert them into a single Compiled HTML (.chm) file -- these are popular under Microsoft Windows, but very handy on every platform. @@ -1526,3 +1526,4 @@ .. _Jinja: http://jinja.pocoo.org/ .. _Pygments: http://pygments.org/ .. _Sphinx: http://sphinx-doc.org/ +.. _virtualenv: https://virtualenv.pypa.io/ -- Repository URL: http://hg.python.org/devguide From python-checkins at python.org Fri Sep 5 09:11:38 2014 From: python-checkins at python.org (serhiy.storchaka) Date: Fri, 5 Sep 2014 09:11:38 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Silenced_some_warnings_abo?= =?utf-8?q?ut_comparison_between_signed_and_unsigned_integer?= Message-ID: <3hq96f4WVFz7Lkg@mail.python.org> http://hg.python.org/cpython/rev/858747aac9e9 changeset: 92345:858747aac9e9 parent: 92342:034ebeb8be2d user: Serhiy Storchaka date: Fri Sep 05 10:10:23 2014 +0300 summary: Silenced some warnings about comparison between signed and unsigned integer expressions. files: Modules/_pickle.c | 6 +++--- Parser/node.c | 4 ++-- Parser/pgenmain.c | 7 ++++--- Parser/printgrammar.c | 5 +++-- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -1456,7 +1456,7 @@ pdata[1] = (unsigned char)(*value & 0xff); len = 2; } - else if (*value <= 0xffffffffL) { + else if ((size_t)*value <= 0xffffffffUL) { pdata[0] = LONG_BINGET; pdata[1] = (unsigned char)(*value & 0xff); pdata[2] = (unsigned char)((*value >> 8) & 0xff); @@ -1513,7 +1513,7 @@ pdata[1] = (unsigned char)idx; len = 2; } - else if (idx <= 0xffffffffL) { + else if ((size_t)idx <= 0xffffffffUL) { pdata[0] = LONG_BINPUT; pdata[1] = (unsigned char)(idx & 0xff); pdata[2] = (unsigned char)((idx >> 8) & 0xff); @@ -2013,7 +2013,7 @@ header[1] = (unsigned char)size; len = 2; } - else if (size <= 0xffffffffL) { + else if ((size_t)size <= 0xffffffffUL) { header[0] = BINBYTES; header[1] = (unsigned char)(size & 0xff); header[2] = (unsigned char)((size >> 8) & 0xff); diff --git a/Parser/node.c b/Parser/node.c --- a/Parser/node.c +++ b/Parser/node.c @@ -70,8 +70,8 @@ * Note that this would be straightforward if a node stored its current * capacity. The code is tricky to avoid that. */ -#define XXXROUNDUP(n) ((n) <= 1 ? (n) : \ - (n) <= 128 ? _Py_SIZE_ROUND_UP((n), 4) : \ +#define XXXROUNDUP(n) ((n) <= 1 ? (n) : \ + (n) <= 128 ? (int)_Py_SIZE_ROUND_UP((n), 4) : \ fancy_roundup(n)) diff --git a/Parser/pgenmain.c b/Parser/pgenmain.c --- a/Parser/pgenmain.c +++ b/Parser/pgenmain.c @@ -96,10 +96,11 @@ fprintf(stderr, "Parsing error %d, line %d.\n", err.error, err.lineno); if (err.text != NULL) { - size_t i; + size_t len; + int i; fprintf(stderr, "%s", err.text); - i = strlen(err.text); - if (i == 0 || err.text[i-1] != '\n') + len = strlen(err.text); + if (len == 0 || err.text[len-1] != '\n') fprintf(stderr, "\n"); for (i = 0; i < err.offset; i++) { if (err.text[i] == '\t') diff --git a/Parser/printgrammar.c b/Parser/printgrammar.c --- a/Parser/printgrammar.c +++ b/Parser/printgrammar.c @@ -84,7 +84,7 @@ printdfas(grammar *g, FILE *fp) { dfa *d; - int i, j; + int i, j, n; printstates(g, fp); fprintf(fp, "static dfa dfas[%d] = {\n", g->g_ndfas); @@ -93,7 +93,8 @@ fprintf(fp, " {%d, \"%s\", %d, %d, states_%d,\n", d->d_type, d->d_name, d->d_initial, d->d_nstates, i); fprintf(fp, " \""); - for (j = 0; j < NBYTES(g->g_ll.ll_nlabels); j++) + n = NBYTES(g->g_ll.ll_nlabels); + for (j = 0; j < n; j++) fprintf(fp, "\\%03o", d->d_first[j] & 0xff); fprintf(fp, "\"},\n"); } -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 5 09:26:26 2014 From: python-checkins at python.org (serhiy.storchaka) Date: Fri, 5 Sep 2014 09:26:26 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzIyMjIx?= =?utf-8?q?=3A_Backported_fixes_from_Python_3_=28issue_=2318960=29=2E?= Message-ID: <3hq9Rk0RzZz7LlH@mail.python.org> http://hg.python.org/cpython/rev/dd1e21f17b1c changeset: 92346:dd1e21f17b1c branch: 2.7 parent: 92344:48033d90c61d user: Serhiy Storchaka date: Fri Sep 05 10:22:05 2014 +0300 summary: Issue #22221: Backported fixes from Python 3 (issue #18960). * Now the source encoding declaration on the second line isn't effective if the first line contains anything except a comment. This affects compile(), eval() and exec() too. * IDLE now ignores the source encoding declaration on the second line if the first line contains anything except a comment. * 2to3 and the findnocoding.py script now ignore the source encoding declaration on the second line if the first line contains anything except a comment. files: Lib/idlelib/IOBinding.py | 3 +++ Lib/lib2to3/pgen2/tokenize.py | 3 +++ Lib/test/test_compile.py | 17 ++++++++++++++++- Misc/NEWS | 10 ++++++++++ Parser/tokenizer.c | 20 +++++++++++++++++--- Tools/scripts/findnocoding.py | 4 +++- 6 files changed, 52 insertions(+), 5 deletions(-) diff --git a/Lib/idlelib/IOBinding.py b/Lib/idlelib/IOBinding.py --- a/Lib/idlelib/IOBinding.py +++ b/Lib/idlelib/IOBinding.py @@ -72,6 +72,7 @@ encoding = encoding.lower() coding_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)') +blank_re = re.compile(r'^[ \t\f]*(?:[#\r\n]|$)') class EncodingMessage(SimpleDialog): "Inform user that an encoding declaration is needed." @@ -130,6 +131,8 @@ match = coding_re.match(line) if match is not None: break + if not blank_re.match(line): + return None else: return None name = match.group(1) diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -237,6 +237,7 @@ toks_append(tokval) cookie_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)') +blank_re = re.compile(r'^[ \t\f]*(?:[#\r\n]|$)') def _get_normal_name(orig_enc): """Imitates get_normal_name in tokenizer.c.""" @@ -309,6 +310,8 @@ encoding = find_cookie(first) if encoding: return encoding, [first] + if not blank_re.match(first): + return default, [first] second = read_or_stop() if not second: diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -412,9 +412,24 @@ l = lambda: "foo" self.assertIsNone(l.__doc__) - def test_unicode_encoding(self): + @test_support.requires_unicode + def test_encoding(self): + code = b'# -*- coding: badencoding -*-\npass\n' + self.assertRaises(SyntaxError, compile, code, 'tmp', 'exec') code = u"# -*- coding: utf-8 -*-\npass\n" self.assertRaises(SyntaxError, compile, code, "tmp", "exec") + code = 'u"\xc2\xa4"\n' + self.assertEqual(eval(code), u'\xc2\xa4') + code = u'u"\xc2\xa4"\n' + self.assertEqual(eval(code), u'\xc2\xa4') + code = '# -*- coding: latin1 -*-\nu"\xc2\xa4"\n' + self.assertEqual(eval(code), u'\xc2\xa4') + code = '# -*- coding: utf-8 -*-\nu"\xc2\xa4"\n' + self.assertEqual(eval(code), u'\xa4') + code = '# -*- coding: iso8859-15 -*-\nu"\xc2\xa4"\n' + self.assertEqual(eval(code), test_support.u(r'\xc2\u20ac')) + code = 'u"""\\\n# -*- coding: utf-8 -*-\n\xc2\xa4"""\n' + self.assertEqual(eval(code), u'# -*- coding: utf-8 -*-\n\xc2\xa4') def test_subscripts(self): # SF bug 1448804 diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Core and Builtins ----------------- +- Issue #22221: Now the source encoding declaration on the second line isn't + effective if the first line contains anything except a comment. + - Issue #22023: Fix ``%S``, ``%R`` and ``%V`` formats of :c:func:`PyUnicode_FromFormat`. @@ -124,6 +127,9 @@ IDLE ---- +- Issue #22221: IDLE now ignores the source encoding declaration on the second + line if the first line contains anything except a comment. + - Issue #17390: Adjust Editor window title; remove 'Python', move version to end. @@ -140,6 +146,10 @@ Tools/Demos ----------- +- Issue #22221: 2to3 and the findnocoding.py script now ignore the source + encoding declaration on the second line if the first line contains anything + except a comment. + - Issue #22201: Command-line interface of the zipfile module now correctly extracts ZIP files with directory entries. Patch by Ryan Wilson. diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -259,11 +259,25 @@ char * cs; int r = 1; - if (tok->cont_line) + if (tok->cont_line) { /* It's a continuation line, so it can't be a coding spec. */ + tok->read_coding_spec = 1; return 1; + } cs = get_coding_spec(line, size); - if (cs != NULL) { + if (!cs) { + Py_ssize_t i; + for (i = 0; i < size; i++) { + if (line[i] == '#' || line[i] == '\n' || line[i] == '\r') + break; + if (line[i] != ' ' && line[i] != '\t' && line[i] != '\014') { + /* Stop checking coding spec after a line containing + * anything except a comment. */ + tok->read_coding_spec = 1; + break; + } + } + } else { tok->read_coding_spec = 1; if (tok->encoding == NULL) { assert(tok->decoding_state == 1); /* raw */ @@ -688,7 +702,7 @@ if (newl[0]) { if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl)) return error_ret(tok); - if (tok->enc == NULL && newl[1]) { + if (tok->enc == NULL && !tok->read_coding_spec && newl[1]) { if (!check_coding_spec(newl[0]+1, newl[1] - newl[0], tok, buf_setreadl)) return error_ret(tok); diff --git a/Tools/scripts/findnocoding.py b/Tools/scripts/findnocoding.py --- a/Tools/scripts/findnocoding.py +++ b/Tools/scripts/findnocoding.py @@ -33,6 +33,7 @@ decl_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)') +blank_re = re.compile(r'^[ \t\f]*(?:[#\r\n]|$)') def get_declaration(line): match = decl_re.match(line) @@ -57,7 +58,8 @@ line1 = infile.readline() line2 = infile.readline() - if get_declaration(line1) or get_declaration(line2): + if (get_declaration(line1) or + blank_re.match(line1) and get_declaration(line2)): # the file does have an encoding declaration, so trust it infile.close() return False -- Repository URL: http://hg.python.org/cpython From solipsis at pitrou.net Fri Sep 5 09:54:47 2014 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Fri, 05 Sep 2014 09:54:47 +0200 Subject: [Python-checkins] Daily reference leaks (034ebeb8be2d): sum=151932 Message-ID: results for 034ebeb8be2d on branch "default" -------------------------------------------- test_codecs leaked [5825, 5825, 5825] references, sum=17475 test_codecs leaked [1172, 1174, 1174] memory blocks, sum=3520 test_collections leaked [0, 2, 0] references, sum=2 test_distutils leaked [37735, 37735, 37735] references, sum=113205 test_distutils leaked [5909, 5911, 5911] memory blocks, sum=17731 test_functools leaked [0, 0, 3] memory blocks, sum=3 test_site leaked [-2, 0, 0] references, sum=-2 test_site leaked [-2, 0, 0] memory blocks, sum=-2 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflog0EOwfh', '-x'] From python-checkins at python.org Fri Sep 5 10:11:59 2014 From: python-checkins at python.org (serhiy.storchaka) Date: Fri, 5 Sep 2014 10:11:59 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzIyMjIx?= =?utf-8?q?=3A_Add_tests_for_compile=28=29_with_source_encoding_cookie=2E?= Message-ID: <3hqBSH5nBQz7Ljf@mail.python.org> http://hg.python.org/cpython/rev/13cd8ea4cafe changeset: 92347:13cd8ea4cafe branch: 3.4 parent: 92341:ecc98ea50bc3 user: Serhiy Storchaka date: Fri Sep 05 11:00:56 2014 +0300 summary: Issue #22221: Add tests for compile() with source encoding cookie. files: Lib/test/test_compile.py | 23 ++++++++++++++++++++--- 1 files changed, 20 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -304,9 +304,26 @@ l = lambda: "foo" self.assertIsNone(l.__doc__) -## def test_unicode_encoding(self): -## code = "# -*- coding: utf-8 -*-\npass\n" -## self.assertRaises(SyntaxError, compile, code, "tmp", "exec") + def test_encoding(self): + code = b'# -*- coding: badencoding -*-\npass\n' + self.assertRaises(SyntaxError, compile, code, 'tmp', 'exec') + code = '# -*- coding: badencoding -*-\n"\xc2\xa4"\n' + compile(code, 'tmp', 'exec') + self.assertEqual(eval(code), '\xc2\xa4') + code = '"\xc2\xa4"\n' + self.assertEqual(eval(code), '\xc2\xa4') + code = b'"\xc2\xa4"\n' + self.assertEqual(eval(code), '\xa4') + code = b'# -*- coding: latin1 -*-\n"\xc2\xa4"\n' + self.assertEqual(eval(code), '\xc2\xa4') + code = b'# -*- coding: utf-8 -*-\n"\xc2\xa4"\n' + self.assertEqual(eval(code), '\xa4') + code = b'# -*- coding: iso8859-15 -*-\n"\xc2\xa4"\n' + self.assertEqual(eval(code), '\xc2\u20ac') + code = '"""\\\n# -*- coding: iso8859-15 -*-\n\xc2\xa4"""\n' + self.assertEqual(eval(code), '# -*- coding: iso8859-15 -*-\n\xc2\xa4') + code = b'"""\\\n# -*- coding: iso8859-15 -*-\n\xc2\xa4"""\n' + self.assertEqual(eval(code), '# -*- coding: iso8859-15 -*-\n\xa4') def test_subscripts(self): # SF bug 1448804 -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 5 10:12:00 2014 From: python-checkins at python.org (serhiy.storchaka) Date: Fri, 5 Sep 2014 10:12:00 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?q?=29=3A_Issue_=2322221=3A_Add_tests_for_compile=28=29_with_sourc?= =?utf-8?q?e_encoding_cookie=2E?= Message-ID: <3hqBSJ73B8z7LjY@mail.python.org> http://hg.python.org/cpython/rev/9d335a54d728 changeset: 92348:9d335a54d728 parent: 92345:858747aac9e9 parent: 92347:13cd8ea4cafe user: Serhiy Storchaka date: Fri Sep 05 11:01:43 2014 +0300 summary: Issue #22221: Add tests for compile() with source encoding cookie. files: Lib/test/test_compile.py | 23 ++++++++++++++++++++--- 1 files changed, 20 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -304,9 +304,26 @@ l = lambda: "foo" self.assertIsNone(l.__doc__) -## def test_unicode_encoding(self): -## code = "# -*- coding: utf-8 -*-\npass\n" -## self.assertRaises(SyntaxError, compile, code, "tmp", "exec") + def test_encoding(self): + code = b'# -*- coding: badencoding -*-\npass\n' + self.assertRaises(SyntaxError, compile, code, 'tmp', 'exec') + code = '# -*- coding: badencoding -*-\n"\xc2\xa4"\n' + compile(code, 'tmp', 'exec') + self.assertEqual(eval(code), '\xc2\xa4') + code = '"\xc2\xa4"\n' + self.assertEqual(eval(code), '\xc2\xa4') + code = b'"\xc2\xa4"\n' + self.assertEqual(eval(code), '\xa4') + code = b'# -*- coding: latin1 -*-\n"\xc2\xa4"\n' + self.assertEqual(eval(code), '\xc2\xa4') + code = b'# -*- coding: utf-8 -*-\n"\xc2\xa4"\n' + self.assertEqual(eval(code), '\xa4') + code = b'# -*- coding: iso8859-15 -*-\n"\xc2\xa4"\n' + self.assertEqual(eval(code), '\xc2\u20ac') + code = '"""\\\n# -*- coding: iso8859-15 -*-\n\xc2\xa4"""\n' + self.assertEqual(eval(code), '# -*- coding: iso8859-15 -*-\n\xc2\xa4') + code = b'"""\\\n# -*- coding: iso8859-15 -*-\n\xc2\xa4"""\n' + self.assertEqual(eval(code), '# -*- coding: iso8859-15 -*-\n\xa4') def test_subscripts(self): # SF bug 1448804 -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 5 12:16:34 2014 From: python-checkins at python.org (victor.stinner) Date: Fri, 5 Sep 2014 12:16:34 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogcmVncnRlc3Q6IGJh?= =?utf-8?q?ckport_=22=5B__1/399=5D=22_progress_back_from_Python_3?= Message-ID: <3hqFD227zVz7Lk2@mail.python.org> http://hg.python.org/cpython/rev/425c8bbc2ee7 changeset: 92349:425c8bbc2ee7 branch: 2.7 parent: 92346:dd1e21f17b1c user: Victor Stinner date: Fri Sep 05 12:12:11 2014 +0200 summary: regrtest: backport "[ 1/399]" progress back from Python 3 The progress bar helps a lot to analyze noisy buildbot logs, to find quickly where errors occurred. files: Lib/test/regrtest.py | 19 +++++++++++++++---- 1 files changed, 15 insertions(+), 4 deletions(-) diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -475,8 +475,12 @@ if bad: return tests = test_forever() + test_count = '' + test_count_width = 3 else: tests = iter(selected) + test_count = '/{}'.format(len(selected)) + test_count_width = len(test_count) - 1 if use_mp: try: @@ -521,8 +525,6 @@ output.put((None, None, None, None)) return result = json.loads(result) - if not quiet: - stdout = test+'\n'+stdout output.put((test, stdout.rstrip(), stderr.rstrip(), result)) except BaseException: output.put((None, None, None, None)) @@ -531,6 +533,7 @@ for worker in workers: worker.start() finished = 0 + test_index = 1 try: while finished < use_mp: test, stdout, stderr, result = output.get() @@ -547,15 +550,23 @@ assert result[1] == 'KeyboardInterrupt' raise KeyboardInterrupt # What else? accumulate_result(test, result) + if not quiet: + fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" + print(fmt.format( + test_count_width, test_index, test_count, + len(bad), test)) + test_index += 1 except KeyboardInterrupt: interrupted = True pending.close() for worker in workers: worker.join() else: - for test in tests: + for test_index, test in enumerate(tests, 1): if not quiet: - print test + fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" + print(fmt.format( + test_count_width, test_index, test_count, len(bad), test)) sys.stdout.flush() if trace: # If we're tracing code coverage, then we don't exit with status -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 5 16:03:40 2014 From: python-checkins at python.org (benjamin.peterson) Date: Fri, 5 Sep 2014 16:03:40 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_backport_our_o?= =?utf-8?q?wn_copy_of_the_ref-counting_extension?= Message-ID: <3hqLG46JcFz7LjX@mail.python.org> http://hg.python.org/cpython/rev/9d41ceea3b8b changeset: 92350:9d41ceea3b8b branch: 2.7 user: Benjamin Peterson date: Fri Sep 05 10:03:26 2014 -0400 summary: backport our own copy of the ref-counting extension files: Doc/conf.py | 4 +- Doc/tools/sphinxext/c_annotations.py | 120 +++++++++++++++ 2 files changed, 122 insertions(+), 2 deletions(-) diff --git a/Doc/conf.py b/Doc/conf.py --- a/Doc/conf.py +++ b/Doc/conf.py @@ -13,8 +13,8 @@ # General configuration # --------------------- -extensions = ['sphinx.ext.refcounting', 'sphinx.ext.coverage', - 'sphinx.ext.doctest', 'pyspecific'] +extensions = ['sphinx.ext.coverage', 'sphinx.ext.doctest', + 'pyspecific', 'c_annotations'] templates_path = ['tools/sphinxext'] # General substitutions. diff --git a/Doc/tools/sphinxext/c_annotations.py b/Doc/tools/sphinxext/c_annotations.py new file mode 100644 --- /dev/null +++ b/Doc/tools/sphinxext/c_annotations.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- +""" + c_annotations.py + ~~~~~~~~~~~~~~~~ + + Supports annotations for C API elements: + + * reference count annotations for C API functions. Based on + refcount.py and anno-api.py in the old Python documentation tools. + + * stable API annotations + + Usage: Set the `refcount_file` config value to the path to the reference + count data file. + + :copyright: Copyright 2007-2013 by Georg Brandl. + :license: Python license. +""" + +from os import path +from docutils import nodes +from docutils.parsers.rst import directives + +from sphinx import addnodes +from sphinx.domains.c import CObject + + +class RCEntry: + def __init__(self, name): + self.name = name + self.args = [] + self.result_type = '' + self.result_refs = None + + +class Annotations(dict): + @classmethod + def fromfile(cls, filename): + d = cls() + fp = open(filename, 'r') + try: + for line in fp: + line = line.strip() + if line[:1] in ("", "#"): + # blank lines and comments + continue + parts = line.split(":", 4) + if len(parts) != 5: + raise ValueError("Wrong field count in %r" % line) + function, type, arg, refcount, comment = parts + # Get the entry, creating it if needed: + try: + entry = d[function] + except KeyError: + entry = d[function] = RCEntry(function) + if not refcount or refcount == "null": + refcount = None + else: + refcount = int(refcount) + # Update the entry with the new parameter or the result + # information. + if arg: + entry.args.append((arg, type, refcount)) + else: + entry.result_type = type + entry.result_refs = refcount + finally: + fp.close() + return d + + def add_annotations(self, app, doctree): + for node in doctree.traverse(addnodes.desc_content): + par = node.parent + if par['domain'] != 'c': + continue + if par['stableabi']: + node.insert(0, nodes.emphasis(' Part of the stable ABI.', + ' Part of the stable ABI.', + classes=['stableabi'])) + if par['objtype'] != 'function': + continue + if not par[0].has_key('names') or not par[0]['names']: + continue + name = par[0]['names'][0] + if name.startswith("c."): + name = name[2:] + entry = self.get(name) + if not entry: + continue + elif entry.result_type not in ("PyObject*", "PyVarObject*"): + continue + if entry.result_refs is None: + rc = 'Return value: Always NULL.' + elif entry.result_refs: + rc = 'Return value: New reference.' + else: + rc = 'Return value: Borrowed reference.' + node.insert(0, nodes.emphasis(rc, rc, classes=['refcount'])) + + +def init_annotations(app): + refcounts = Annotations.fromfile( + path.join(app.srcdir, app.config.refcount_file)) + app.connect('doctree-read', refcounts.add_annotations) + + +def setup(app): + app.add_config_value('refcount_file', '', True) + app.connect('builder-inited', init_annotations) + + # monkey-patch C object... + CObject.option_spec = { + 'noindex': directives.flag, + 'stableabi': directives.flag, + } + old_handle_signature = CObject.handle_signature + def new_handle_signature(self, sig, signode): + signode.parent['stableabi'] = 'stableabi' in self.options + return old_handle_signature(self, sig, signode) + CObject.handle_signature = new_handle_signature -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 5 19:30:55 2014 From: python-checkins at python.org (alex.gaynor) Date: Fri, 5 Sep 2014 19:30:55 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_PEP_476=3A_Updates_based_on_G?= =?utf-8?q?uido=27s_feedback_on_python-dev?= Message-ID: <3hqQsC0K3Gz7Lm3@mail.python.org> http://hg.python.org/peps/rev/67652719e187 changeset: 5546:67652719e187 user: Alex Gaynor date: Fri Sep 05 10:23:19 2014 -0700 summary: PEP 476: Updates based on Guido's feedback on python-dev files: pep-0476.txt | 22 +++++----------------- 1 files changed, 5 insertions(+), 17 deletions(-) diff --git a/pep-0476.txt b/pep-0476.txt --- a/pep-0476.txt +++ b/pep-0476.txt @@ -111,19 +111,6 @@ Twisted's 14.0 release made this same change, and it has been met with almost no opposition. -In order to make this transition as smooth as possible, the next 3.4.x release -following this PEP will be modified to emit a warning in cases that would raise -an Exception in Python 3.5. - -Warnings --------- - -To support this warning, in 3.4.next a new ``verify_mode`` is introduced -``CERT_WARN``, which is equivilant to ``CERT_NONE``, except in cases that would -fail as ``CERT_REQUIRED`` or fail the hostname check emits a warning. In -3.4.next the ``httplib`` module will set this as the ``verify_mode`` if the -default context is used. - Other protocols =============== @@ -142,10 +129,11 @@ Python Versions =============== -This PEP proposes making these changes to ``default`` (Python 3) branch. I -strongly believe these changes also belong in Python 2, but doing them in a -patch-release isn't reasonable, and there is strong opposition to doing a 2.8 -release. +This PEP describes changes that will occur on both the 3.4.x, 3.5 and 2.7.X +branches. For 2.7.X this will require backporting the ``context`` +(``SSLContext``) argument to ``httplib``, in addition to the features already +backported in +:pep:`466`. Copyright ========= -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Fri Sep 5 21:09:01 2014 From: python-checkins at python.org (victor.stinner) Date: Fri, 5 Sep 2014 21:09:01 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzIyMzQw?= =?utf-8?q?=3A_Fix_Python_3_warnings_in_Python_2_tests?= Message-ID: <3hqT2P63jQz7Lkb@mail.python.org> http://hg.python.org/cpython/rev/0675b3a55941 changeset: 92351:0675b3a55941 branch: 2.7 user: Victor Stinner date: Fri Sep 05 21:05:05 2014 +0200 summary: Issue #22340: Fix Python 3 warnings in Python 2 tests files: Lib/idlelib/idle_test/test_calltips.py | 2 +- Lib/sqlite3/test/dbapi.py | 4 +- Lib/sqlite3/test/types.py | 10 ++- Lib/sqlite3/test/userfunctions.py | 20 ++++-- Lib/test/test_collections.py | 43 ++++++++----- Lib/test/test_hash.py | 3 +- Lib/test/test_hmac.py | 16 ++-- Lib/test/test_ssl.py | 3 +- 8 files changed, 63 insertions(+), 38 deletions(-) diff --git a/Lib/idlelib/idle_test/test_calltips.py b/Lib/idlelib/idle_test/test_calltips.py --- a/Lib/idlelib/idle_test/test_calltips.py +++ b/Lib/idlelib/idle_test/test_calltips.py @@ -163,7 +163,7 @@ # In 3.x, get_entity changed from 'instance method' to module function # since 'self' not used. Use dummy instance until change 2.7 also. def test_bad_entity(self): - self.assertIsNone(CTi.get_entity('1/0')) + self.assertIsNone(CTi.get_entity('1//0')) def test_good_entity(self): self.assertIs(CTi.get_entity('int'), int) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -24,6 +24,7 @@ import unittest import sys import sqlite3 as sqlite +from test import test_support try: import threading except ImportError: @@ -653,7 +654,8 @@ ts = sqlite.TimestampFromTicks(42) def CheckBinary(self): - b = sqlite.Binary(chr(0) + "'") + with test_support.check_py3k_warnings(): + b = sqlite.Binary(chr(0) + "'") class ExtensionTests(unittest.TestCase): def CheckScriptStringSql(self): diff --git a/Lib/sqlite3/test/types.py b/Lib/sqlite3/test/types.py --- a/Lib/sqlite3/test/types.py +++ b/Lib/sqlite3/test/types.py @@ -24,6 +24,7 @@ import datetime import unittest import sqlite3 as sqlite +from test import test_support try: import zlib except ImportError: @@ -67,7 +68,8 @@ self.assertEqual(row[0], val) def CheckBlob(self): - val = buffer("Guglhupf") + with test_support.check_py3k_warnings(): + val = buffer("Guglhupf") self.cur.execute("insert into test(b) values (?)", (val,)) self.cur.execute("select b from test") row = self.cur.fetchone() @@ -231,7 +233,8 @@ def CheckBlob(self): # default - val = buffer("Guglhupf") + with test_support.check_py3k_warnings(): + val = buffer("Guglhupf") self.cur.execute("insert into test(bin) values (?)", (val,)) self.cur.execute("select bin from test") row = self.cur.fetchone() @@ -347,7 +350,8 @@ def CheckBinaryInputForConverter(self): testdata = "abcdefg" * 10 - result = self.con.execute('select ? as "x [bin]"', (buffer(zlib.compress(testdata)),)).fetchone()[0] + with test_support.check_py3k_warnings(): + result = self.con.execute('select ? as "x [bin]"', (buffer(zlib.compress(testdata)),)).fetchone()[0] self.assertEqual(testdata, result) class DateTimeTests(unittest.TestCase): diff --git a/Lib/sqlite3/test/userfunctions.py b/Lib/sqlite3/test/userfunctions.py --- a/Lib/sqlite3/test/userfunctions.py +++ b/Lib/sqlite3/test/userfunctions.py @@ -24,6 +24,7 @@ import unittest import sqlite3 as sqlite +from test import test_support def func_returntext(): return "foo" @@ -36,7 +37,8 @@ def func_returnnull(): return None def func_returnblob(): - return buffer("blob") + with test_support.check_py3k_warnings(): + return buffer("blob") def func_returnlonglong(): return 1<<31 def func_raiseexception(): @@ -202,8 +204,9 @@ cur = self.con.cursor() cur.execute("select returnblob()") val = cur.fetchone()[0] - self.assertEqual(type(val), buffer) - self.assertEqual(val, buffer("blob")) + with test_support.check_py3k_warnings(): + self.assertEqual(type(val), buffer) + self.assertEqual(val, buffer("blob")) def CheckFuncReturnLongLong(self): cur = self.con.cursor() @@ -246,7 +249,8 @@ def CheckParamBlob(self): cur = self.con.cursor() - cur.execute("select isblob(?)", (buffer("blob"),)) + with test_support.check_py3k_warnings(): + cur.execute("select isblob(?)", (buffer("blob"),)) val = cur.fetchone()[0] self.assertEqual(val, 1) @@ -269,8 +273,9 @@ b blob ) """) - cur.execute("insert into test(t, i, f, n, b) values (?, ?, ?, ?, ?)", - ("foo", 5, 3.14, None, buffer("blob"),)) + with test_support.check_py3k_warnings(): + cur.execute("insert into test(t, i, f, n, b) values (?, ?, ?, ?, ?)", + ("foo", 5, 3.14, None, buffer("blob"),)) self.con.create_aggregate("nostep", 1, AggrNoStep) self.con.create_aggregate("nofinalize", 1, AggrNoFinalize) @@ -362,7 +367,8 @@ def CheckAggrCheckParamBlob(self): cur = self.con.cursor() - cur.execute("select checkType('blob', ?)", (buffer("blob"),)) + with test_support.check_py3k_warnings(): + cur.execute("select checkType('blob', ?)", (buffer("blob"),)) val = cur.fetchone()[0] self.assertEqual(val, 1) diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -8,13 +8,14 @@ from random import randrange, shuffle import keyword import re -import sets import sys from collections import Hashable, Iterable, Iterator from collections import Sized, Container, Callable from collections import Set, MutableSet from collections import Mapping, MutableMapping from collections import Sequence, MutableSequence +with test_support.check_warnings(('', DeprecationWarning)): + import sets TestNT = namedtuple('TestNT', 'x y z') # type used for pickle tests @@ -713,10 +714,12 @@ self.assertTrue(r1 < r3) self.assertFalse(r1 < r1) self.assertFalse(r1 < r2) - # python 2 only, cross-type compares will succeed - f1 < l3 - f1 < l1 - f1 < l2 + + with test_support.check_py3k_warnings(): + # python 2 only, cross-type compares will succeed + f1 < l3 + f1 < l1 + f1 < l2 # any subset self.assertTrue(f1 <= f3) @@ -728,10 +731,12 @@ self.assertTrue(r1 <= r3) self.assertTrue(r1 <= r1) self.assertFalse(r1 <= r2) - # python 2 only, cross-type compares will succeed - f1 <= l3 - f1 <= l1 - f1 <= l2 + + with test_support.check_py3k_warnings(): + # python 2 only, cross-type compares will succeed + f1 <= l3 + f1 <= l1 + f1 <= l2 # proper superset self.assertTrue(f3 > f1) @@ -743,10 +748,12 @@ self.assertTrue(r3 > r1) self.assertFalse(r1 > r1) self.assertFalse(r2 > r1) - # python 2 only, cross-type compares will succeed - f1 > l3 - f1 > l1 - f1 > l2 + + with test_support.check_py3k_warnings(): + # python 2 only, cross-type compares will succeed + f1 > l3 + f1 > l1 + f1 > l2 # any superset self.assertTrue(f3 >= f1) @@ -758,10 +765,12 @@ self.assertTrue(r3 >= r1) self.assertTrue(r1 >= r1) self.assertFalse(r2 >= r1) - # python 2 only, cross-type compares will succeed - f1 >= l3 - f1 >=l1 - f1 >= l2 + + with test_support.check_py3k_warnings(): + # python 2 only, cross-type compares will succeed + f1 >= l3 + f1 >=l1 + f1 >= l2 # equality self.assertTrue(f1 == f1) diff --git a/Lib/test/test_hash.py b/Lib/test/test_hash.py --- a/Lib/test/test_hash.py +++ b/Lib/test/test_hash.py @@ -215,7 +215,8 @@ repr_ = 'buffer("abc")' def test_empty_string(self): - self.assertEqual(hash(buffer("")), 0) + with test_support.check_py3k_warnings(): + self.assertEqual(hash(buffer("")), 0) class DatetimeTests(HashRandomizationTests): def get_hash_command(self, repr_): diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py --- a/Lib/test/test_hmac.py +++ b/Lib/test/test_hmac.py @@ -389,10 +389,11 @@ a, b = "foo?", "foo?" self.assertTrue(hmac.compare_digest(a, b)) - # subclasses are supported by ignore __eq__ - class mystr(str): - def __eq__(self, other): - return False + with test_support.check_py3k_warnings(): + # subclasses are supported by ignore __eq__ + class mystr(str): + def __eq__(self, other): + return False a, b = mystr("foobar"), mystr("foobar") self.assertTrue(hmac.compare_digest(a, b)) @@ -401,9 +402,10 @@ a, b = mystr("foobar"), mystr("foobaz") self.assertFalse(hmac.compare_digest(a, b)) - class mybytes(bytes): - def __eq__(self, other): - return False + with test_support.check_py3k_warnings(): + class mybytes(bytes): + def __eq__(self, other): + return False a, b = mybytes(b"foobar"), mybytes(b"foobar") self.assertTrue(hmac.compare_digest(a, b)) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -2365,7 +2365,8 @@ # now fetch the same data from the HTTPS server url = 'https://%s:%d/%s' % ( HOST, server.port, os.path.split(CERTFILE)[1]) - f = urllib.urlopen(url) + with support.check_py3k_warnings(): + f = urllib.urlopen(url) try: dlen = f.info().getheader("content-length") if dlen and (int(dlen) > 0): -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 5 21:41:37 2014 From: python-checkins at python.org (victor.stinner) Date: Fri, 5 Sep 2014 21:41:37 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzIyMzQw?= =?utf-8?q?=3A_Fix_test=5Fcollections_if_the_sets_module_was_already_impor?= =?utf-8?q?ted?= Message-ID: <3hqTm15XHVz7Ln0@mail.python.org> http://hg.python.org/cpython/rev/407653078135 changeset: 92352:407653078135 branch: 2.7 user: Victor Stinner date: Fri Sep 05 21:41:25 2014 +0200 summary: Issue #22340: Fix test_collections if the sets module was already imported files: Lib/test/test_collections.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -14,8 +14,8 @@ from collections import Set, MutableSet from collections import Mapping, MutableMapping from collections import Sequence, MutableSequence -with test_support.check_warnings(('', DeprecationWarning)): - import sets +# Silence deprecation warning +sets = test_support.import_module('sets', deprecated=True) TestNT = namedtuple('TestNT', 'x y z') # type used for pickle tests -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 5 22:28:57 2014 From: python-checkins at python.org (benjamin.peterson) Date: Fri, 5 Sep 2014 22:28:57 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_use_correct_article_=28closes?= =?utf-8?q?_=2322342=29?= Message-ID: <3hqVpd62Lrz7LkP@mail.python.org> http://hg.python.org/peps/rev/4b03ae00a76b changeset: 5547:4b03ae00a76b user: Benjamin Peterson date: Fri Sep 05 16:28:49 2014 -0400 summary: use correct article (closes #22342) files: pep-0380.txt | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/pep-0380.txt b/pep-0380.txt --- a/pep-0380.txt +++ b/pep-0380.txt @@ -207,7 +207,7 @@ The rationale behind most of the semantics presented above stems from the desire to be able to refactor generator code. It should be -possible to take an section of code containing one or more ``yield`` +possible to take a section of code containing one or more ``yield`` expressions, move it into a separate function (using the usual techniques to deal with references to variables in the surrounding scope, etc.), and call the new function using a ``yield from`` -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Fri Sep 5 22:38:34 2014 From: python-checkins at python.org (serhiy.storchaka) Date: Fri, 5 Sep 2014 22:38:34 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzIwNjQz?= =?utf-8?q?=3A_Removed_unneeded_=28and_wrong=29_class_directives=2E?= Message-ID: <3hqW1k2fJQz7Ljt@mail.python.org> http://hg.python.org/cpython/rev/bc4e26755a13 changeset: 92353:bc4e26755a13 branch: 3.4 parent: 92347:13cd8ea4cafe user: Serhiy Storchaka date: Fri Sep 05 23:27:36 2014 +0300 summary: Issue #20643: Removed unneeded (and wrong) class directives. files: Doc/reference/expressions.rst | 5 +---- 1 files changed, 1 insertions(+), 4 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -400,7 +400,6 @@ is already executing raises a :exc:`ValueError` exception. .. index:: exception: StopIteration -.. class:: generator .. method:: generator.__next__() @@ -410,7 +409,7 @@ :meth:`~generator.__next__` method, the current yield expression always evaluates to :const:`None`. The execution then continues to the next yield expression, where the generator is suspended again, and the value of the - :token:`expression_list` is returned to :meth:`next`'s caller. If the + :token:`expression_list` is returned to :meth:`__next__`'s caller. If the generator exits without yielding another value, a :exc:`StopIteration` exception is raised. @@ -450,8 +449,6 @@ other exception, it is propagated to the caller. :meth:`close` does nothing if the generator has already exited due to an exception or normal exit. -.. class:: . - .. index:: single: yield; examples Examples -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 5 22:38:35 2014 From: python-checkins at python.org (serhiy.storchaka) Date: Fri, 5 Sep 2014 22:38:35 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?q?=29=3A_Issue_=2320643=3A_Removed_unneeded_=28and_wrong=29_class?= =?utf-8?q?_directives=2E?= Message-ID: <3hqW1l4G0cz7Ljy@mail.python.org> http://hg.python.org/cpython/rev/060e347c9a23 changeset: 92354:060e347c9a23 parent: 92348:9d335a54d728 parent: 92353:bc4e26755a13 user: Serhiy Storchaka date: Fri Sep 05 23:28:19 2014 +0300 summary: Issue #20643: Removed unneeded (and wrong) class directives. files: Doc/reference/expressions.rst | 5 +---- 1 files changed, 1 insertions(+), 4 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -400,7 +400,6 @@ is already executing raises a :exc:`ValueError` exception. .. index:: exception: StopIteration -.. class:: generator .. method:: generator.__next__() @@ -410,7 +409,7 @@ :meth:`~generator.__next__` method, the current yield expression always evaluates to :const:`None`. The execution then continues to the next yield expression, where the generator is suspended again, and the value of the - :token:`expression_list` is returned to :meth:`next`'s caller. If the + :token:`expression_list` is returned to :meth:`__next__`'s caller. If the generator exits without yielding another value, a :exc:`StopIteration` exception is raised. @@ -450,8 +449,6 @@ other exception, it is propagated to the caller. :meth:`close` does nothing if the generator has already exited due to an exception or normal exit. -.. class:: . - .. index:: single: yield; examples Examples -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Fri Sep 5 22:38:37 2014 From: python-checkins at python.org (serhiy.storchaka) Date: Fri, 5 Sep 2014 22:38:37 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzIwNjQz?= =?utf-8?q?=3A_Fixed_references_to_the_next=28=29_method_=28distinguish_fr?= =?utf-8?q?om_the?= Message-ID: <3hqW1n07Tmz7Lls@mail.python.org> http://hg.python.org/cpython/rev/6dba9db360d0 changeset: 92355:6dba9db360d0 branch: 2.7 parent: 92352:407653078135 user: Serhiy Storchaka date: Fri Sep 05 23:34:12 2014 +0300 summary: Issue #20643: Fixed references to the next() method (distinguish from the next() function). files: Doc/c-api/typeobj.rst | 2 +- Doc/glossary.rst | 4 ++-- Doc/library/2to3.rst | 2 +- Doc/library/collections.rst | 4 ++-- Doc/library/stdtypes.rst | 8 ++++---- Doc/reference/expressions.rst | 9 ++++----- Doc/reference/simple_stmts.rst | 16 ++++++++-------- Doc/tutorial/classes.rst | 10 +++++----- 8 files changed, 27 insertions(+), 28 deletions(-) diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -770,7 +770,7 @@ exception may or may not be set. When another error occurs, it must return *NULL* too. Its presence normally signals that the instances of this type are iterators (although classic instances always have this function, even if - they don't define a :meth:`next` method). + they don't define a :meth:`~iterator.next` method). Iterator types should also define the :c:member:`~PyTypeObject.tp_iter` function, and that function should return the iterator instance itself (not a new iterator diff --git a/Doc/glossary.rst b/Doc/glossary.rst --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -408,10 +408,10 @@ iterator An object representing a stream of data. Repeated calls to the iterator's - :meth:`next` method return successive items in the stream. When no more + :meth:`~generator.next` method return successive items in the stream. When no more data are available a :exc:`StopIteration` exception is raised instead. At this point, the iterator object is exhausted and any further calls to its - :meth:`next` method just raise :exc:`StopIteration` again. Iterators are + :meth:`~generator.next` method just raise :exc:`StopIteration` again. Iterators are required to have an :meth:`__iter__` method that returns the iterator object itself so every iterator is also iterable and may be used in most places where other iterables are accepted. One notable exception is code diff --git a/Doc/library/2to3.rst b/Doc/library/2to3.rst --- a/Doc/library/2to3.rst +++ b/Doc/library/2to3.rst @@ -289,7 +289,7 @@ .. 2to3fixer:: next Converts the use of iterator's :meth:`~iterator.next` methods to the - :func:`next` function. It also renames :meth:`next` methods to + :func:`next` function. It also renames :meth:`~iterator.next` methods to :meth:`~iterator.__next__`. .. 2to3fixer:: nonzero diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -954,8 +954,8 @@ .. class:: Iterator - ABC for classes that provide the :meth:`__iter__` and :meth:`next` methods. - See also the definition of :term:`iterator`. + ABC for classes that provide the :meth:`~iterator.__iter__` and + :meth:`~iterator.next` methods. See also the definition of :term:`iterator`. .. class:: Sequence MutableSequence diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -652,7 +652,7 @@ specific types are not important beyond their implementation of the iterator protocol. -The intention of the protocol is that once an iterator's :meth:`next` method +The intention of the protocol is that once an iterator's :meth:`~iterator.next` method raises :exc:`StopIteration`, it will continue to do so on subsequent calls. Implementations that do not obey this property are deemed broken. (This constraint was added in Python 2.3; in Python 2.2, various iterators are broken @@ -667,9 +667,9 @@ Python's :term:`generator`\s provide a convenient way to implement the iterator protocol. If a container object's :meth:`__iter__` method is implemented as a generator, it will automatically return an iterator object (technically, a -generator object) supplying the :meth:`__iter__` and :meth:`next` methods. More -information about generators can be found in :ref:`the documentation for the -yield expression `. +generator object) supplying the :meth:`~iterator.__iter__` and +:meth:`~iterator.next` methods. More information about generators can be found +in :ref:`the documentation for the yield expression `. .. _typesseq: diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -431,22 +431,21 @@ is already executing raises a :exc:`ValueError` exception. .. index:: exception: StopIteration -.. class:: generator .. method:: generator.next() Starts the execution of a generator function or resumes it at the last executed :keyword:`yield` expression. When a generator function is resumed with a - :meth:`next` method, the current :keyword:`yield` expression always evaluates to + :meth:`~generator.next` method, the current :keyword:`yield` expression + always evaluates to :const:`None`. The execution then continues to the next :keyword:`yield` expression, where the generator is suspended again, and the value of the - :token:`expression_list` is returned to :meth:`next`'s caller. If the generator + :token:`expression_list` is returned to :meth:`~generator.next`'s caller. + If the generator exits without yielding another value, a :exc:`StopIteration` exception is raised. -.. class:: . - .. method:: generator.send(value) Resumes the execution and "sends" a value into the generator function. The diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -506,16 +506,16 @@ When a generator function is called, it returns an iterator known as a generator iterator, or more commonly, a generator. The body of the generator function is -executed by calling the generator's :meth:`next` method repeatedly until it -raises an exception. +executed by calling the generator's :meth:`~generator.next` method repeatedly +until it raises an exception. When a :keyword:`yield` statement is executed, the state of the generator is -frozen and the value of :token:`expression_list` is returned to :meth:`next`'s -caller. By "frozen" we mean that all local state is retained, including the -current bindings of local variables, the instruction pointer, and the internal -evaluation stack: enough information is saved so that the next time :meth:`next` -is invoked, the function can proceed exactly as if the :keyword:`yield` -statement were just another external call. +frozen and the value of :token:`expression_list` is returned to +:meth:`~generator.next`'s caller. By "frozen" we mean that all local state is +retained, including the current bindings of local variables, the instruction +pointer, and the internal evaluation stack: enough information is saved so that +the next time :meth:`~generator.next` is invoked, the function can proceed +exactly as if the :keyword:`yield` statement were just another external call. As of Python version 2.5, the :keyword:`yield` statement is now allowed in the :keyword:`try` clause of a :keyword:`try` ... :keyword:`finally` construct. If diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -788,8 +788,8 @@ Having seen the mechanics behind the iterator protocol, it is easy to add iterator behavior to your classes. Define an :meth:`__iter__` method which -returns an object with a :meth:`next` method. If the class defines -:meth:`next`, then :meth:`__iter__` can just return ``self``:: +returns an object with a :meth:`~iterator.next` method. If the class +defines :meth:`~iterator.next`, then :meth:`__iter__` can just return ``self``:: class Reverse: """Iterator for looping over a sequence backwards.""" @@ -825,7 +825,7 @@ :term:`Generator`\s are a simple and powerful tool for creating iterators. They are written like regular functions but use the :keyword:`yield` statement -whenever they want to return data. Each time :meth:`next` is called, the +whenever they want to return data. Each time :func:`next` is called on it, the generator resumes where it left-off (it remembers all the data values and which statement was last executed). An example shows that generators can be trivially easy to create:: @@ -846,8 +846,8 @@ Anything that can be done with generators can also be done with class based iterators as described in the previous section. What makes generators so -compact is that the :meth:`__iter__` and :meth:`next` methods are created -automatically. +compact is that the :meth:`__iter__` and :meth:`~generator.next` methods +are created automatically. Another key feature is that the local variables and execution state are automatically saved between calls. This made the function easier to write and -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 6 00:58:35 2014 From: python-checkins at python.org (ned.deily) Date: Sat, 6 Sep 2014 00:58:35 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_Update_OS_X_in?= =?utf-8?q?staller_build_script_for_changes_to_documentation_build=3A?= Message-ID: <3hqZ7H3Dswz7Ljy@mail.python.org> http://hg.python.org/cpython/rev/ec052852524d changeset: 92356:ec052852524d branch: 2.7 user: Ned Deily date: Fri Sep 05 15:51:54 2014 -0700 summary: Update OS X installer build script for changes to documentation build: as of 2.7.9, doc builds require an externally installed sphinx-build like 3.4+ builds do. files: Mac/BuildScript/build-installer.py | 16 +++++----------- 1 files changed, 5 insertions(+), 11 deletions(-) diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -615,8 +615,7 @@ # Ensure ws have access to hg and to sphinx-build. # You may have to create links in /usr/bin for them. runCommand('hg --version') - if getVersionTuple() >= (3, 4): - runCommand('sphinx-build --version') + runCommand('sphinx-build --version') def parseOptions(args=None): """ @@ -929,15 +928,10 @@ docdir = os.path.join(rootDir, 'pydocs') curDir = os.getcwd() os.chdir(buildDir) - # The Doc build changed for 3.4 (technically, for 3.4.1) - if getVersionTuple() < (3, 4): - # This step does an svn checkout of sphinx and its dependencies - runCommand('make update') - runCommand("make html PYTHON='%s'" % os.path.abspath(sys.executable)) - else: - runCommand('make clean') - # Assume sphinx-build is on our PATH, checked in checkEnvironment - runCommand('make html') + # The Doc build changed for 3.4 (technically, for 3.4.1) and for 2.7.9 + runCommand('make clean') + # Assume sphinx-build is on our PATH, checked in checkEnvironment + runCommand('make html') os.chdir(curDir) if not os.path.exists(docdir): os.mkdir(docdir) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 6 00:58:36 2014 From: python-checkins at python.org (ned.deily) Date: Sat, 6 Sep 2014 00:58:36 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogRG9jdW1lbnQgMi43?= =?utf-8?q?=2E9_changes_in_OS_X_installer_build_requirements=3A?= Message-ID: <3hqZ7J4vH7z7Ljy@mail.python.org> http://hg.python.org/cpython/rev/322f77ee6d5a changeset: 92357:322f77ee6d5a branch: 2.7 user: Ned Deily date: Fri Sep 05 15:52:45 2014 -0700 summary: Document 2.7.9 changes in OS X installer build requirements: because the Doc Makefile has been changed to no longer download sphinx and its dependencies, build-installer.py now requires that there be an externally-supplied sphinx-build available. files: Mac/BuildScript/README.txt | 21 +++++++++++++++------ 1 files changed, 15 insertions(+), 6 deletions(-) diff --git a/Mac/BuildScript/README.txt b/Mac/BuildScript/README.txt --- a/Mac/BuildScript/README.txt +++ b/Mac/BuildScript/README.txt @@ -11,9 +11,9 @@ For Python 2.7.x and 3.x, PSF practice is to build two installer variants for each release. -Beginning with Python 2.7.8, we plan to drop binary installer support for +Beginning with Python 2.7.9, we plan to drop binary installer support for Mac OS X 10.3.9 and 10.4.x systems. To ease the transition, for Python 2.7.7 -only there will be three installers provided: +and 2.7.8 there were three installers provided: 1. DEPRECATED - 32-bit-only, i386 and PPC universal, capable on running on all machines supported by Mac OS X 10.3.9 through (at least) 10.9:: @@ -41,7 +41,8 @@ * ``MacOSX10.4u`` SDK (later SDKs do not support PPC G3 processors) * ``MACOSX_DEPLOYMENT_TARGET=10.3`` * Apple ``gcc-4.0`` - * system Python 2.5 for documentation build with Sphinx + * bootstrap non-framework Python 2.7 for documentation build with + Sphinx (as of 2.7.9) - alternate build environments: @@ -76,7 +77,8 @@ * ``MacOSX10.5`` SDK * ``MACOSX_DEPLOYMENT_TARGET=10.5`` * Apple ``gcc-4.2`` - * system Python 2.5+ for documentation build with Sphinx + * bootstrap non-framework Python 2.7 for documentation build with + Sphinx (as of 2.7.9) - alternate build environments: @@ -110,7 +112,8 @@ * ``MacOSX10.6`` SDK * ``MACOSX_DEPLOYMENT_TARGET=10.6`` * Apple ``gcc-4.2`` - * system Python 2.6 for documentation build with Sphinx + * bootstrap non-framework Python 2.7 for documentation build with + Sphinx (as of 2.7.9) - alternate build environments: @@ -134,7 +137,13 @@ interfere with the build. * The documentation for the release is built using Sphinx - because it is included in the installer. + because it is included in the installer. For 2.7.x up to and including + 2.7.8, the ``Doc/Makefile`` used ``svn`` to download repos of + ``Sphinx`` and its dependencies. Beginning with 2.7.9, the ``Doc/Makefile`` + assumes there is an externally-provided ``sphinx-build`` and requires at + least Python 2.6 to run. Because of this, it is no longer possible to + build a 2.7.9 or later installer on OS X 10.5 using the Apple-supplied + Python 2.5. * It is safest to start each variant build with an empty source directory populated with a fresh copy of the untarred source. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 6 00:58:37 2014 From: python-checkins at python.org (ned.deily) Date: Sat, 6 Sep 2014 00:58:37 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E4=29=3A_Keep_Mac/build?= =?utf-8?q?-installer=2Epy_in_sync_across_branches_=28affects_2=2E7_only?= =?utf-8?b?KS4=?= Message-ID: <3hqZ7K6G0Xz7Lkw@mail.python.org> http://hg.python.org/cpython/rev/b424d9679308 changeset: 92358:b424d9679308 branch: 3.4 parent: 92353:bc4e26755a13 user: Ned Deily date: Fri Sep 05 15:57:05 2014 -0700 summary: Keep Mac/build-installer.py in sync across branches (affects 2.7 only). files: Mac/BuildScript/build-installer.py | 16 +++++----------- 1 files changed, 5 insertions(+), 11 deletions(-) diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -615,8 +615,7 @@ # Ensure ws have access to hg and to sphinx-build. # You may have to create links in /usr/bin for them. runCommand('hg --version') - if getVersionTuple() >= (3, 4): - runCommand('sphinx-build --version') + runCommand('sphinx-build --version') def parseOptions(args=None): """ @@ -929,15 +928,10 @@ docdir = os.path.join(rootDir, 'pydocs') curDir = os.getcwd() os.chdir(buildDir) - # The Doc build changed for 3.4 (technically, for 3.4.1) - if getVersionTuple() < (3, 4): - # This step does an svn checkout of sphinx and its dependencies - runCommand('make update') - runCommand("make html PYTHON='%s'" % os.path.abspath(sys.executable)) - else: - runCommand('make clean') - # Assume sphinx-build is on our PATH, checked in checkEnvironment - runCommand('make html') + # The Doc build changed for 3.4 (technically, for 3.4.1) and for 2.7.9 + runCommand('make clean') + # Assume sphinx-build is on our PATH, checked in checkEnvironment + runCommand('make html') os.chdir(curDir) if not os.path.exists(docdir): os.mkdir(docdir) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 6 00:58:39 2014 From: python-checkins at python.org (ned.deily) Date: Sat, 6 Sep 2014 00:58:39 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?q?=29=3A_Keep_Mac/build-installer=2Epy_in_sync_across_branches_?= =?utf-8?q?=28affects_2=2E7_only=29=2E?= Message-ID: <3hqZ7M0sMTz7Lml@mail.python.org> http://hg.python.org/cpython/rev/7fe96ffb2dc6 changeset: 92359:7fe96ffb2dc6 parent: 92354:060e347c9a23 parent: 92358:b424d9679308 user: Ned Deily date: Fri Sep 05 15:57:54 2014 -0700 summary: Keep Mac/build-installer.py in sync across branches (affects 2.7 only). files: Mac/BuildScript/build-installer.py | 16 +++++----------- 1 files changed, 5 insertions(+), 11 deletions(-) diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -615,8 +615,7 @@ # Ensure ws have access to hg and to sphinx-build. # You may have to create links in /usr/bin for them. runCommand('hg --version') - if getVersionTuple() >= (3, 4): - runCommand('sphinx-build --version') + runCommand('sphinx-build --version') def parseOptions(args=None): """ @@ -929,15 +928,10 @@ docdir = os.path.join(rootDir, 'pydocs') curDir = os.getcwd() os.chdir(buildDir) - # The Doc build changed for 3.4 (technically, for 3.4.1) - if getVersionTuple() < (3, 4): - # This step does an svn checkout of sphinx and its dependencies - runCommand('make update') - runCommand("make html PYTHON='%s'" % os.path.abspath(sys.executable)) - else: - runCommand('make clean') - # Assume sphinx-build is on our PATH, checked in checkEnvironment - runCommand('make html') + # The Doc build changed for 3.4 (technically, for 3.4.1) and for 2.7.9 + runCommand('make clean') + # Assume sphinx-build is on our PATH, checked in checkEnvironment + runCommand('make html') os.chdir(curDir) if not os.path.exists(docdir): os.mkdir(docdir) -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 6 04:28:51 2014 From: python-checkins at python.org (guido.van.rossum) Date: Sat, 6 Sep 2014 04:28:51 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Add_PEP-411-compliant_note?= =?utf-8?q?_about_asyncio_being_provisional=2E_Fixes_issue_=2322346=2E?= Message-ID: <3hqfnv3cz2z7Lkw@mail.python.org> http://hg.python.org/cpython/rev/f8f3e83c9528 changeset: 92360:f8f3e83c9528 user: Guido van Rossum date: Fri Sep 05 19:28:40 2014 -0700 summary: Add PEP-411-compliant note about asyncio being provisional. Fixes issue #22346. files: Doc/library/asyncio.rst | 7 +++++++ 1 files changed, 7 insertions(+), 0 deletions(-) diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -4,6 +4,13 @@ .. module:: asyncio :synopsis: Asynchronous I/O, event loop, coroutines and tasks. +.. note:: + + The asyncio package has been included in the standard library on a + :term:`provisional basis `. Backwards incompatible + changes (up to and including removal of the module) may occur if deemed + necessary by the core developers. + .. versionadded:: 3.4 **Source code:** :source:`Lib/asyncio/` -- Repository URL: http://hg.python.org/cpython From solipsis at pitrou.net Sat Sep 6 10:38:25 2014 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Sat, 06 Sep 2014 10:38:25 +0200 Subject: [Python-checkins] Daily reference leaks (7fe96ffb2dc6): sum=151930 Message-ID: results for 7fe96ffb2dc6 on branch "default" -------------------------------------------- test_codecs leaked [5825, 5825, 5825] references, sum=17475 test_codecs leaked [1172, 1174, 1174] memory blocks, sum=3520 test_collections leaked [0, -2, 2] references, sum=0 test_distutils leaked [37735, 37735, 37735] references, sum=113205 test_distutils leaked [5909, 5911, 5911] memory blocks, sum=17731 test_functools leaked [0, 0, 3] memory blocks, sum=3 test_site leaked [-2, 0, 0] references, sum=-2 test_site leaked [-2, 0, 0] memory blocks, sum=-2 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogOtKgWZ', '-x'] From python-checkins at python.org Sat Sep 6 11:47:21 2014 From: python-checkins at python.org (nick.coghlan) Date: Sat, 6 Sep 2014 11:47:21 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzIyMzQ2?= =?utf-8?q?=3A_also_include_asyncio_PEP_411_notice_in_3=2E4?= Message-ID: <3hqrWs1VRPz7MnQ@mail.python.org> http://hg.python.org/cpython/rev/fefe7822e6b8 changeset: 92361:fefe7822e6b8 branch: 3.4 parent: 92358:b424d9679308 user: Nick Coghlan date: Sat Sep 06 19:43:06 2014 +1000 summary: Issue #22346: also include asyncio PEP 411 notice in 3.4 files: Doc/library/asyncio.rst | 7 +++++++ 1 files changed, 7 insertions(+), 0 deletions(-) diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -4,6 +4,13 @@ .. module:: asyncio :synopsis: Asynchronous I/O, event loop, coroutines and tasks. +.. note:: + + The asyncio package has been included in the standard library on a + :term:`provisional basis `. Backwards incompatible + changes (up to and including removal of the module) may occur if deemed + necessary by the core developers. + .. versionadded:: 3.4 **Source code:** :source:`Lib/asyncio/` -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 6 11:47:22 2014 From: python-checkins at python.org (nick.coghlan) Date: Sat, 6 Sep 2014 11:47:22 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?q?=29=3A_Null_merge_from_3=2E4_backport?= Message-ID: <3hqrWt36Dzz7MnQ@mail.python.org> http://hg.python.org/cpython/rev/258481642dd7 changeset: 92362:258481642dd7 parent: 92360:f8f3e83c9528 parent: 92361:fefe7822e6b8 user: Nick Coghlan date: Sat Sep 06 19:44:41 2014 +1000 summary: Null merge from 3.4 backport files: -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 6 12:40:19 2014 From: python-checkins at python.org (nick.coghlan) Date: Sat, 6 Sep 2014 12:40:19 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzIyMjk1?= =?utf-8?q?=3A_Adopt_=27python_-m_pip=27_as_the_preferred_invocation?= Message-ID: <3hqshz5QhVz7MpF@mail.python.org> http://hg.python.org/cpython/rev/e8447da8791d changeset: 92363:e8447da8791d branch: 3.4 parent: 92361:fefe7822e6b8 user: Nick Coghlan date: Sat Sep 06 20:38:23 2014 +1000 summary: Issue #22295: Adopt 'python -m pip' as the preferred invocation files: Doc/distributing/index.rst | 13 +++++++++- Doc/glossary.rst | 8 ++++++ Doc/installing/index.rst | 30 +++++++++++++++++-------- 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst --- a/Doc/distributing/index.rst +++ b/Doc/distributing/index.rst @@ -93,9 +93,18 @@ versions of Python. The currently recommended build and distribution tools can be installed -using ``pip``:: +by invoking the ``pip`` module at the command line:: - pip install setuptools wheel twine + python -m pip install setuptools wheel twine + +.. note:: + + For POSIX users (including Mac OS X and Linux users), these instructions + assume the use of a :term:`virtual environment`. + + For Windows users, these instructions assume that the option to + adjust the system PATH environment variable was selected when installing + Python. The Python Packaging User Guide includes more details on the `currently recommended tools`_. diff --git a/Doc/glossary.rst b/Doc/glossary.rst --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -857,6 +857,14 @@ dictionary view to become a full list use ``list(dictview)``. See :ref:`dict-views`. + virtual environment + A cooperatively isolated runtime environment that allows Python users + and applications to install and upgrade Python distribution packages + without interfering with the behaviour of other Python applications + running on the same system. + + See also :ref:`scripts-pyvenv` + virtual machine A computer defined entirely in software. Python's virtual machine executes the :term:`bytecode` emitted by the bytecode compiler. diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -40,6 +40,10 @@ * ``pyvenv`` is the standard tool for creating virtual environments, and has been part of Python since Python 3.3. Starting with Python 3.4, it defaults to installing ``pip`` into all created virtual environments +* ``virtualenv`` is a third party alternative (and predecessor) to + ``pyvenv``. It allows virtual environments to be used on versions of + Python prior to 3.4, which either don't provide ``pyvenv`` at all, or + aren't able to automatically install ``pip`` into created environments. * the `Python Package Index `__ is a public repository of open source licensed packages made available for use by other Python users @@ -63,27 +67,33 @@ =========== The standard packaging tools are all designed to be used from the command -line. For Windows users, the examples below assume that the option to -adjust the system PATH environment variable was selected when installing -Python. For Linux users, the command to install into the system version of -Python 3 is likely to be ``pip3`` rather than ``pip``. +line. The following command will install the latest version of a module and its dependencies from the Python Package Index:: - pip install SomePackage + python -m pip install SomePackage + +.. note:: + + For POSIX users (including Mac OS X and Linux users), the examples in + this guide assume the use of a :term:`virtual environment`. + + For Windows users, the examples in this guide assume that the option to + adjust the system PATH environment variable was selected when installing + Python. It's also possible to specify an exact or minimum version directly on the command line:: - pip install SomePackage==1.0.4 # specific version - pip install 'SomePackage>=1.0.4' # minimum version + python -m pip install SomePackage==1.0.4 # specific version + python -m pip install 'SomePackage>=1.0.4' # minimum version Normally, if a suitable module is already installed, attempting to install it again will have no effect. Upgrading existing modules must be requested explicitly:: - pip install --upgrade SomePackage + python -m pip install --upgrade SomePackage More information and resources regarding ``pip`` and its capabilities can be found in the `Python Packaging User Guide `__. @@ -120,8 +130,8 @@ ... install packages just for the current user? ----------------------------------------------- -Passing the ``--user`` option to ``pip install`` will install a package -just for the current user, rather than for all users of the system. +Passing the ``--user`` option to ``python -m pip install`` will install a +package just for the current user, rather than for all users of the system. ... install scientific Python packages? -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 6 12:40:21 2014 From: python-checkins at python.org (nick.coghlan) Date: Sat, 6 Sep 2014 12:40:21 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?q?=29=3A_Merge_issue_=2322295_fix_from_3=2E4?= Message-ID: <3hqsj1144jz7NCX@mail.python.org> http://hg.python.org/cpython/rev/a969b42e6e2b changeset: 92364:a969b42e6e2b parent: 92362:258481642dd7 parent: 92363:e8447da8791d user: Nick Coghlan date: Sat Sep 06 20:40:00 2014 +1000 summary: Merge issue #22295 fix from 3.4 files: Doc/distributing/index.rst | 13 +++++++++- Doc/glossary.rst | 8 ++++++ Doc/installing/index.rst | 30 +++++++++++++++++-------- 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst --- a/Doc/distributing/index.rst +++ b/Doc/distributing/index.rst @@ -93,9 +93,18 @@ versions of Python. The currently recommended build and distribution tools can be installed -using ``pip``:: +by invoking the ``pip`` module at the command line:: - pip install setuptools wheel twine + python -m pip install setuptools wheel twine + +.. note:: + + For POSIX users (including Mac OS X and Linux users), these instructions + assume the use of a :term:`virtual environment`. + + For Windows users, these instructions assume that the option to + adjust the system PATH environment variable was selected when installing + Python. The Python Packaging User Guide includes more details on the `currently recommended tools`_. diff --git a/Doc/glossary.rst b/Doc/glossary.rst --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -858,6 +858,14 @@ dictionary view to become a full list use ``list(dictview)``. See :ref:`dict-views`. + virtual environment + A cooperatively isolated runtime environment that allows Python users + and applications to install and upgrade Python distribution packages + without interfering with the behaviour of other Python applications + running on the same system. + + See also :ref:`scripts-pyvenv` + virtual machine A computer defined entirely in software. Python's virtual machine executes the :term:`bytecode` emitted by the bytecode compiler. diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -40,6 +40,10 @@ * ``pyvenv`` is the standard tool for creating virtual environments, and has been part of Python since Python 3.3. Starting with Python 3.4, it defaults to installing ``pip`` into all created virtual environments +* ``virtualenv`` is a third party alternative (and predecessor) to + ``pyvenv``. It allows virtual environments to be used on versions of + Python prior to 3.4, which either don't provide ``pyvenv`` at all, or + aren't able to automatically install ``pip`` into created environments. * the `Python Package Index `__ is a public repository of open source licensed packages made available for use by other Python users @@ -63,27 +67,33 @@ =========== The standard packaging tools are all designed to be used from the command -line. For Windows users, the examples below assume that the option to -adjust the system PATH environment variable was selected when installing -Python. For Linux users, the command to install into the system version of -Python 3 is likely to be ``pip3`` rather than ``pip``. +line. The following command will install the latest version of a module and its dependencies from the Python Package Index:: - pip install SomePackage + python -m pip install SomePackage + +.. note:: + + For POSIX users (including Mac OS X and Linux users), the examples in + this guide assume the use of a :term:`virtual environment`. + + For Windows users, the examples in this guide assume that the option to + adjust the system PATH environment variable was selected when installing + Python. It's also possible to specify an exact or minimum version directly on the command line:: - pip install SomePackage==1.0.4 # specific version - pip install 'SomePackage>=1.0.4' # minimum version + python -m pip install SomePackage==1.0.4 # specific version + python -m pip install 'SomePackage>=1.0.4' # minimum version Normally, if a suitable module is already installed, attempting to install it again will have no effect. Upgrading existing modules must be requested explicitly:: - pip install --upgrade SomePackage + python -m pip install --upgrade SomePackage More information and resources regarding ``pip`` and its capabilities can be found in the `Python Packaging User Guide `__. @@ -120,8 +130,8 @@ ... install packages just for the current user? ----------------------------------------------- -Passing the ``--user`` option to ``pip install`` will install a package -just for the current user, rather than for all users of the system. +Passing the ``--user`` option to ``python -m pip install`` will install a +package just for the current user, rather than for all users of the system. ... install scientific Python packages? -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 6 19:08:26 2014 From: python-checkins at python.org (serhiy.storchaka) Date: Sat, 6 Sep 2014 19:08:26 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2322215=3A_Now_Valu?= =?utf-8?q?eError_is_raised_instead_of_TypeError_when_str_or_bytes?= Message-ID: <3hr2Jp0FrFz7LjV@mail.python.org> http://hg.python.org/cpython/rev/25032ec29315 changeset: 92365:25032ec29315 user: Serhiy Storchaka date: Sat Sep 06 20:07:17 2014 +0300 summary: Issue #22215: Now ValueError is raised instead of TypeError when str or bytes argument contains not permitted null character or byte. files: Lib/test/test_builtin.py | 4 ++-- Lib/test/test_fileio.py | 4 ++-- Lib/test/test_getargs2.py | 10 +++++----- Lib/test/test_io.py | 4 ++-- Lib/test/test_site.py | 2 +- Misc/NEWS | 3 +++ Modules/_io/fileio.c | 2 +- Modules/_tkinter.c | 4 ++-- Modules/posixmodule.c | 2 +- Modules/socketmodule.c | 2 +- Objects/bytesobject.c | 4 ++-- Objects/unicodeobject.c | 10 +++++----- Python/bltinmodule.c | 4 ++-- Python/getargs.c | 25 ++++++++++++------------- 14 files changed, 41 insertions(+), 39 deletions(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -312,11 +312,11 @@ self.assertRaises(TypeError, compile) self.assertRaises(ValueError, compile, 'print(42)\n', '', 'badmode') self.assertRaises(ValueError, compile, 'print(42)\n', '', 'single', 0xff) - self.assertRaises(TypeError, compile, chr(0), 'f', 'exec') + self.assertRaises(ValueError, compile, chr(0), 'f', 'exec') self.assertRaises(TypeError, compile, 'pass', '?', 'exec', mode='eval', source='0', filename='tmp') compile('print("\xe5")\n', '', 'exec') - self.assertRaises(TypeError, compile, chr(0), 'f', 'exec') + self.assertRaises(ValueError, compile, chr(0), 'f', 'exec') self.assertRaises(ValueError, compile, str('a = 1'), 'f', 'bad') # test the optimize argument diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -361,8 +361,8 @@ def testConstructorHandlesNULChars(self): fn_with_NUL = 'foo\0bar' - self.assertRaises(TypeError, _FileIO, fn_with_NUL, 'w') - self.assertRaises(TypeError, _FileIO, bytes(fn_with_NUL, 'ascii'), 'w') + self.assertRaises(ValueError, _FileIO, fn_with_NUL, 'w') + self.assertRaises(ValueError, _FileIO, bytes(fn_with_NUL, 'ascii'), 'w') def testInvalidFd(self): self.assertRaises(ValueError, _FileIO, -10) diff --git a/Lib/test/test_getargs2.py b/Lib/test/test_getargs2.py --- a/Lib/test/test_getargs2.py +++ b/Lib/test/test_getargs2.py @@ -482,7 +482,7 @@ def test_s(self): from _testcapi import getargs_s self.assertEqual(getargs_s('abc\xe9'), b'abc\xc3\xa9') - self.assertRaises(TypeError, getargs_s, 'nul:\0') + self.assertRaises(ValueError, getargs_s, 'nul:\0') self.assertRaises(TypeError, getargs_s, b'bytes') self.assertRaises(TypeError, getargs_s, bytearray(b'bytearray')) self.assertRaises(TypeError, getargs_s, memoryview(b'memoryview')) @@ -509,7 +509,7 @@ def test_z(self): from _testcapi import getargs_z self.assertEqual(getargs_z('abc\xe9'), b'abc\xc3\xa9') - self.assertRaises(TypeError, getargs_z, 'nul:\0') + self.assertRaises(ValueError, getargs_z, 'nul:\0') self.assertRaises(TypeError, getargs_z, b'bytes') self.assertRaises(TypeError, getargs_z, bytearray(b'bytearray')) self.assertRaises(TypeError, getargs_z, memoryview(b'memoryview')) @@ -537,7 +537,7 @@ from _testcapi import getargs_y self.assertRaises(TypeError, getargs_y, 'abc\xe9') self.assertEqual(getargs_y(b'bytes'), b'bytes') - self.assertRaises(TypeError, getargs_y, b'nul:\0') + self.assertRaises(ValueError, getargs_y, b'nul:\0') self.assertRaises(TypeError, getargs_y, bytearray(b'bytearray')) self.assertRaises(TypeError, getargs_y, memoryview(b'memoryview')) self.assertRaises(TypeError, getargs_y, None) @@ -577,7 +577,7 @@ def test_u(self): from _testcapi import getargs_u self.assertEqual(getargs_u('abc\xe9'), 'abc\xe9') - self.assertRaises(TypeError, getargs_u, 'nul:\0') + self.assertRaises(ValueError, getargs_u, 'nul:\0') self.assertRaises(TypeError, getargs_u, b'bytes') self.assertRaises(TypeError, getargs_u, bytearray(b'bytearray')) self.assertRaises(TypeError, getargs_u, memoryview(b'memoryview')) @@ -595,7 +595,7 @@ def test_Z(self): from _testcapi import getargs_Z self.assertEqual(getargs_Z('abc\xe9'), 'abc\xe9') - self.assertRaises(TypeError, getargs_Z, 'nul:\0') + self.assertRaises(ValueError, getargs_Z, 'nul:\0') self.assertRaises(TypeError, getargs_Z, b'bytes') self.assertRaises(TypeError, getargs_Z, bytearray(b'bytearray')) self.assertRaises(TypeError, getargs_Z, memoryview(b'memoryview')) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -363,8 +363,8 @@ def test_open_handles_NUL_chars(self): fn_with_NUL = 'foo\0bar' - self.assertRaises(TypeError, self.open, fn_with_NUL, 'w') - self.assertRaises(TypeError, self.open, bytes(fn_with_NUL, 'ascii'), 'w') + self.assertRaises(ValueError, self.open, fn_with_NUL, 'w') + self.assertRaises(ValueError, self.open, bytes(fn_with_NUL, 'ascii'), 'w') def test_raw_file_io(self): with self.open(support.TESTFN, "wb", buffering=0) as f: diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -147,7 +147,7 @@ re.escape(os.path.join(pth_dir, pth_fn))) # XXX: ditto previous XXX comment. self.assertRegex(err_out.getvalue(), 'Traceback') - self.assertRegex(err_out.getvalue(), 'TypeError') + self.assertRegex(err_out.getvalue(), 'ValueError') def test_addsitedir(self): # Same tests for test_addpackage since addsitedir() essentially just diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Core and Builtins ----------------- +- Issue #22215: Now ValueError is raised instead of TypeError when str or bytes + argument contains not permitted null character or byte. + - Issue #22258: Fix the internal function set_inheritable() on Illumos. This platform exposes the function ``ioctl(FIOCLEX)``, but calling it fails with errno is ENOTTY: "Inappropriate ioctl for device". set_inheritable() diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -256,7 +256,7 @@ int rv = _PyUnicode_HasNULChars(nameobj); if (rv) { if (rv != -1) - PyErr_SetString(PyExc_TypeError, "embedded NUL character"); + PyErr_SetString(PyExc_ValueError, "embedded null character"); return -1; } widename = PyUnicode_AsUnicode(nameobj); diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -1417,7 +1417,7 @@ } s = PyBytes_AsString(in); if (strlen(s) != (size_t)PyBytes_Size(in)) { - PyErr_SetString(PyExc_ValueError, "null byte in bytes object"); + PyErr_SetString(PyExc_ValueError, "embedded null byte"); return 0; } *out = s; @@ -1434,7 +1434,7 @@ return 0; } if (strlen(s) != (size_t)size) { - PyErr_SetString(PyExc_ValueError, "null character in string"); + PyErr_SetString(PyExc_ValueError, "embedded null character"); return 0; } *out = s; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -903,7 +903,7 @@ narrow = PyBytes_AS_STRING(bytes); if ((size_t)length != strlen(narrow)) { - FORMAT_EXCEPTION(PyExc_ValueError, "embedded NUL character in %s"); + FORMAT_EXCEPTION(PyExc_ValueError, "embedded null character in %s"); Py_DECREF(bytes); return 0; } diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -1273,7 +1273,7 @@ } if (strlen(data->buf) != len) { Py_CLEAR(data->obj); - PyErr_SetString(PyExc_TypeError, "host name must not contain NUL character"); + PyErr_SetString(PyExc_TypeError, "host name must not contain null character"); return 0; } return Py_CLEANUP_SUPPORTED; diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -593,8 +593,8 @@ if (len != NULL) *len = PyBytes_GET_SIZE(obj); else if (strlen(*s) != (size_t)PyBytes_GET_SIZE(obj)) { - PyErr_SetString(PyExc_TypeError, - "expected bytes with no null"); + PyErr_SetString(PyExc_ValueError, + "embedded null byte"); return -1; } return 0; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -3247,7 +3247,7 @@ wlen2 = wcslen(wstr); if (wlen2 != wlen) { PyMem_Free(wstr); - PyErr_SetString(PyExc_TypeError, "embedded null character"); + PyErr_SetString(PyExc_ValueError, "embedded null character"); return NULL; } @@ -3519,8 +3519,8 @@ if (locale_error_handler(errors, &surrogateescape) < 0) return NULL; - if (str[len] != '\0' || (size_t)len != strlen(str)) { - PyErr_SetString(PyExc_TypeError, "embedded null character"); + if (str[len] != '\0' || (size_t)len != strlen(str)) { + PyErr_SetString(PyExc_ValueError, "embedded null byte"); return NULL; } @@ -3697,7 +3697,7 @@ size = PyBytes_GET_SIZE(output); data = PyBytes_AS_STRING(output); if ((size_t)size != strlen(data)) { - PyErr_SetString(PyExc_TypeError, "embedded NUL character"); + PyErr_SetString(PyExc_ValueError, "embedded null byte"); Py_DECREF(output); return 0; } @@ -3741,7 +3741,7 @@ } if (findchar(PyUnicode_DATA(output), PyUnicode_KIND(output), PyUnicode_GET_LENGTH(output), 0, 1) >= 0) { - PyErr_SetString(PyExc_TypeError, "embedded NUL character"); + PyErr_SetString(PyExc_ValueError, "embedded null character"); Py_DECREF(output); return 0; } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -745,8 +745,8 @@ return NULL; } - if (strlen(str) != (size_t)size) { - PyErr_SetString(PyExc_TypeError, + if (strlen(str) != (size_t)size) { + PyErr_SetString(PyExc_ValueError, "source code string cannot contain null bytes"); return NULL; } diff --git a/Python/getargs.c b/Python/getargs.c --- a/Python/getargs.c +++ b/Python/getargs.c @@ -872,10 +872,10 @@ STORE_SIZE(count); format++; } else { - if (strlen(*p) != (size_t)count) - return converterr( - "bytes without null bytes", - arg, msgbuf, bufsize); + if (strlen(*p) != (size_t)count) { + PyErr_SetString(PyExc_ValueError, "embedded null byte"); + RETURN_ERR_OCCURRED; + } } break; } @@ -948,16 +948,15 @@ if (sarg == NULL) return converterr(CONV_UNICODE, arg, msgbuf, bufsize); + if (strlen(sarg) != (size_t)len) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + RETURN_ERR_OCCURRED; + } *p = sarg; } else return converterr(c == 'z' ? "str or None" : "str", arg, msgbuf, bufsize); - if (*p != NULL && sarg != NULL && (Py_ssize_t) strlen(*p) != len) - return converterr( - c == 'z' ? "str without null characters or None" - : "str without null characters", - arg, msgbuf, bufsize); } break; } @@ -994,10 +993,10 @@ *p = PyUnicode_AsUnicodeAndSize(arg, &len); if (*p == NULL) RETURN_ERR_OCCURRED; - if (Py_UNICODE_strlen(*p) != (size_t)len) - return converterr( - "str without null characters or None", - arg, msgbuf, bufsize); + if (Py_UNICODE_strlen(*p) != (size_t)len) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + RETURN_ERR_OCCURRED; + } } else return converterr(c == 'Z' ? "str or None" : "str", arg, msgbuf, bufsize); -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 6 20:46:06 2014 From: python-checkins at python.org (serhiy.storchaka) Date: Sat, 6 Sep 2014 20:46:06 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzE5NTI0?= =?utf-8?q?=3A_Fixed_resource_leak_in_the_HTTP_connection_when_an_invalid?= Message-ID: <3hr4TV6nbgz7Ljp@mail.python.org> http://hg.python.org/cpython/rev/c1fb19907cc4 changeset: 92366:c1fb19907cc4 branch: 3.4 parent: 92363:e8447da8791d user: Serhiy Storchaka date: Sat Sep 06 21:41:39 2014 +0300 summary: Issue #19524: Fixed resource leak in the HTTP connection when an invalid response is received. Patch by Martin Panter. files: Lib/test/test_urllib.py | 73 ++++++++++++++------------- Lib/test/test_urllib2.py | 29 +++++++++++ Lib/urllib/request.py | 25 +++++---- Misc/ACKS | 1 + Misc/NEWS | 3 + 5 files changed, 86 insertions(+), 45 deletions(-) diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -48,43 +48,48 @@ return opener.open(url, data) +def fakehttp(fakedata): + class FakeSocket(io.BytesIO): + io_refs = 1 + + def sendall(self, data): + FakeHTTPConnection.buf = data + + def makefile(self, *args, **kwds): + self.io_refs += 1 + return self + + def read(self, amt=None): + if self.closed: + return b"" + return io.BytesIO.read(self, amt) + + def readline(self, length=None): + if self.closed: + return b"" + return io.BytesIO.readline(self, length) + + def close(self): + self.io_refs -= 1 + if self.io_refs == 0: + io.BytesIO.close(self) + + class FakeHTTPConnection(http.client.HTTPConnection): + + # buffer to store data for verification in urlopen tests. + buf = None + fakesock = FakeSocket(fakedata) + + def connect(self): + self.sock = self.fakesock + + return FakeHTTPConnection + + class FakeHTTPMixin(object): def fakehttp(self, fakedata): - class FakeSocket(io.BytesIO): - io_refs = 1 - - def sendall(self, data): - FakeHTTPConnection.buf = data - - def makefile(self, *args, **kwds): - self.io_refs += 1 - return self - - def read(self, amt=None): - if self.closed: - return b"" - return io.BytesIO.read(self, amt) - - def readline(self, length=None): - if self.closed: - return b"" - return io.BytesIO.readline(self, length) - - def close(self): - self.io_refs -= 1 - if self.io_refs == 0: - io.BytesIO.close(self) - - class FakeHTTPConnection(http.client.HTTPConnection): - - # buffer to store data for verification in urlopen tests. - buf = None - - def connect(self): - self.sock = FakeSocket(fakedata) - self._connection_class = http.client.HTTPConnection - http.client.HTTPConnection = FakeHTTPConnection + http.client.HTTPConnection = fakehttp(fakedata) def unfakehttp(self): http.client.HTTPConnection = self._connection_class diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1,5 +1,6 @@ import unittest from test import support +from test import test_urllib import os import io @@ -13,6 +14,7 @@ from urllib.request import Request, OpenerDirector, _parse_proxy, _proxy_bypass_macosx_sysconf from urllib.parse import urlparse import urllib.error +import http.client # XXX # Request @@ -1393,6 +1395,33 @@ self.assertEqual(len(http_handler.requests), 1) self.assertFalse(http_handler.requests[0].has_header(auth_header)) + def test_http_closed(self): + """Test the connection is cleaned up when the response is closed""" + for (transfer, data) in ( + ("Connection: close", b"data"), + ("Transfer-Encoding: chunked", b"4\r\ndata\r\n0\r\n\r\n"), + ("Content-Length: 4", b"data"), + ): + header = "HTTP/1.1 200 OK\r\n{}\r\n\r\n".format(transfer) + conn = test_urllib.fakehttp(header.encode() + data) + handler = urllib.request.AbstractHTTPHandler() + req = Request("http://dummy/") + req.timeout = None + with handler.do_open(conn, req) as resp: + resp.read() + self.assertTrue(conn.fakesock.closed, + "Connection not closed with {!r}".format(transfer)) + + def test_invalid_closed(self): + """Test the connection is cleaned up after an invalid response""" + conn = test_urllib.fakehttp(b"") + handler = urllib.request.AbstractHTTPHandler() + req = Request("http://dummy/") + req.timeout = None + with self.assertRaises(http.client.BadStatusLine): + handler.do_open(conn, req) + self.assertTrue(conn.fakesock.closed, "Connection not closed") + class MiscTests(unittest.TestCase): diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1170,18 +1170,21 @@ h.set_tunnel(req._tunnel_host, headers=tunnel_headers) try: - h.request(req.get_method(), req.selector, req.data, headers) - except OSError as err: # timeout error + try: + h.request(req.get_method(), req.selector, req.data, headers) + except OSError as err: # timeout error + raise URLError(err) + r = h.getresponse() + except: h.close() - raise URLError(err) - else: - r = h.getresponse() - # If the server does not send us a 'Connection: close' header, - # HTTPConnection assumes the socket should be left open. Manually - # mark the socket to be closed when this response object goes away. - if h.sock: - h.sock.close() - h.sock = None + raise + + # If the server does not send us a 'Connection: close' header, + # HTTPConnection assumes the socket should be left open. Manually + # mark the socket to be closed when this response object goes away. + if h.sock: + h.sock.close() + h.sock = None r.url = req.get_full_url() # This line replaces the .msg attribute of the HTTPResponse diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1003,6 +1003,7 @@ Todd R. Palmer Juan David Ib??ez Palomar Jan Palus +Martin Panter Mathias Panzenb?ck M. Papillon Peter Parente diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Library ------- +- Issue #19524: Fixed resource leak in the HTTP connection when an invalid + response is received. Patch by Martin Panter. + - Issue #22051: turtledemo no longer reloads examples to re-run them. Initialization of variables and gui setup should be done in main(), which is called each time a demo is run, but not on import. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 6 20:46:08 2014 From: python-checkins at python.org (serhiy.storchaka) Date: Sat, 6 Sep 2014 20:46:08 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?q?=29=3A_Issue_=2319524=3A_Fixed_resource_leak_in_the_HTTP_connec?= =?utf-8?q?tion_when_an_invalid?= Message-ID: <3hr4TX2hCgz7LkN@mail.python.org> http://hg.python.org/cpython/rev/43bf95480c3c changeset: 92367:43bf95480c3c parent: 92365:25032ec29315 parent: 92366:c1fb19907cc4 user: Serhiy Storchaka date: Sat Sep 06 21:43:49 2014 +0300 summary: Issue #19524: Fixed resource leak in the HTTP connection when an invalid response is received. Patch by Martin Panter. files: Lib/test/test_urllib.py | 73 ++++++++++++++------------- Lib/test/test_urllib2.py | 29 +++++++++++ Lib/urllib/request.py | 25 +++++---- Misc/ACKS | 1 + Misc/NEWS | 3 + 5 files changed, 86 insertions(+), 45 deletions(-) diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -48,43 +48,48 @@ return opener.open(url, data) +def fakehttp(fakedata): + class FakeSocket(io.BytesIO): + io_refs = 1 + + def sendall(self, data): + FakeHTTPConnection.buf = data + + def makefile(self, *args, **kwds): + self.io_refs += 1 + return self + + def read(self, amt=None): + if self.closed: + return b"" + return io.BytesIO.read(self, amt) + + def readline(self, length=None): + if self.closed: + return b"" + return io.BytesIO.readline(self, length) + + def close(self): + self.io_refs -= 1 + if self.io_refs == 0: + io.BytesIO.close(self) + + class FakeHTTPConnection(http.client.HTTPConnection): + + # buffer to store data for verification in urlopen tests. + buf = None + fakesock = FakeSocket(fakedata) + + def connect(self): + self.sock = self.fakesock + + return FakeHTTPConnection + + class FakeHTTPMixin(object): def fakehttp(self, fakedata): - class FakeSocket(io.BytesIO): - io_refs = 1 - - def sendall(self, data): - FakeHTTPConnection.buf = data - - def makefile(self, *args, **kwds): - self.io_refs += 1 - return self - - def read(self, amt=None): - if self.closed: - return b"" - return io.BytesIO.read(self, amt) - - def readline(self, length=None): - if self.closed: - return b"" - return io.BytesIO.readline(self, length) - - def close(self): - self.io_refs -= 1 - if self.io_refs == 0: - io.BytesIO.close(self) - - class FakeHTTPConnection(http.client.HTTPConnection): - - # buffer to store data for verification in urlopen tests. - buf = None - - def connect(self): - self.sock = FakeSocket(fakedata) - self._connection_class = http.client.HTTPConnection - http.client.HTTPConnection = FakeHTTPConnection + http.client.HTTPConnection = fakehttp(fakedata) def unfakehttp(self): http.client.HTTPConnection = self._connection_class diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1,5 +1,6 @@ import unittest from test import support +from test import test_urllib import os import io @@ -13,6 +14,7 @@ from urllib.request import Request, OpenerDirector, _parse_proxy, _proxy_bypass_macosx_sysconf from urllib.parse import urlparse import urllib.error +import http.client # XXX # Request @@ -1393,6 +1395,33 @@ self.assertEqual(len(http_handler.requests), 1) self.assertFalse(http_handler.requests[0].has_header(auth_header)) + def test_http_closed(self): + """Test the connection is cleaned up when the response is closed""" + for (transfer, data) in ( + ("Connection: close", b"data"), + ("Transfer-Encoding: chunked", b"4\r\ndata\r\n0\r\n\r\n"), + ("Content-Length: 4", b"data"), + ): + header = "HTTP/1.1 200 OK\r\n{}\r\n\r\n".format(transfer) + conn = test_urllib.fakehttp(header.encode() + data) + handler = urllib.request.AbstractHTTPHandler() + req = Request("http://dummy/") + req.timeout = None + with handler.do_open(conn, req) as resp: + resp.read() + self.assertTrue(conn.fakesock.closed, + "Connection not closed with {!r}".format(transfer)) + + def test_invalid_closed(self): + """Test the connection is cleaned up after an invalid response""" + conn = test_urllib.fakehttp(b"") + handler = urllib.request.AbstractHTTPHandler() + req = Request("http://dummy/") + req.timeout = None + with self.assertRaises(http.client.BadStatusLine): + handler.do_open(conn, req) + self.assertTrue(conn.fakesock.closed, "Connection not closed") + class MiscTests(unittest.TestCase): diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1170,18 +1170,21 @@ h.set_tunnel(req._tunnel_host, headers=tunnel_headers) try: - h.request(req.get_method(), req.selector, req.data, headers) - except OSError as err: # timeout error + try: + h.request(req.get_method(), req.selector, req.data, headers) + except OSError as err: # timeout error + raise URLError(err) + r = h.getresponse() + except: h.close() - raise URLError(err) - else: - r = h.getresponse() - # If the server does not send us a 'Connection: close' header, - # HTTPConnection assumes the socket should be left open. Manually - # mark the socket to be closed when this response object goes away. - if h.sock: - h.sock.close() - h.sock = None + raise + + # If the server does not send us a 'Connection: close' header, + # HTTPConnection assumes the socket should be left open. Manually + # mark the socket to be closed when this response object goes away. + if h.sock: + h.sock.close() + h.sock = None r.url = req.get_full_url() # This line replaces the .msg attribute of the HTTPResponse diff --git a/Misc/ACKS b/Misc/ACKS --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1012,6 +1012,7 @@ Todd R. Palmer Juan David Ib??ez Palomar Jan Palus +Martin Panter Mathias Panzenb?ck M. Papillon Peter Parente diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -132,6 +132,9 @@ Library ------- +- Issue #19524: Fixed resource leak in the HTTP connection when an invalid + response is received. Patch by Martin Panter. + - Issue #20421: Add a .version() method to SSL sockets exposing the actual protocol version in use. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 6 21:21:26 2014 From: python-checkins at python.org (serhiy.storchaka) Date: Sat, 6 Sep 2014 21:21:26 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2322131=3A_Moderniz?= =?utf-8?q?ed_the_code_of_the_uuid_module=2E?= Message-ID: <3hr5GG3zyJz7Ljj@mail.python.org> http://hg.python.org/cpython/rev/f7b5038d3102 changeset: 92368:f7b5038d3102 user: Serhiy Storchaka date: Sat Sep 06 22:14:04 2014 +0300 summary: Issue #22131: Modernized the code of the uuid module. Optimized bytes and bytes_le properties of UUID and UUID constructor with bytes_le argument. Fixed a bug in handling an error occured during reading from a pipe in _ipconfig_getnode(). files: Lib/uuid.py | 33 ++++++++++++--------------------- 1 files changed, 12 insertions(+), 21 deletions(-) diff --git a/Lib/uuid.py b/Lib/uuid.py --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -139,10 +139,8 @@ if bytes_le is not None: if len(bytes_le) != 16: raise ValueError('bytes_le is not a 16-char string') - bytes = (bytes_(reversed(bytes_le[0:4])) + - bytes_(reversed(bytes_le[4:6])) + - bytes_(reversed(bytes_le[6:8])) + - bytes_le[8:]) + bytes = (bytes_le[4-1::-1] + bytes_le[6-1:4-1:-1] + + bytes_le[8-1:6-1:-1] + bytes_le[8:]) if bytes is not None: if len(bytes) != 16: raise ValueError('bytes is not a 16-char string') @@ -234,17 +232,12 @@ @property def bytes(self): - bytes = bytearray() - for shift in range(0, 128, 8): - bytes.insert(0, (self.int >> shift) & 0xff) - return bytes_(bytes) + return self.int.to_bytes(16, 'big') @property def bytes_le(self): bytes = self.bytes - return (bytes_(reversed(bytes[0:4])) + - bytes_(reversed(bytes[4:6])) + - bytes_(reversed(bytes[6:8])) + + return (bytes[4-1::-1] + bytes[6-1:4-1:-1] + bytes[8-1:6-1:-1] + bytes[8:]) @property @@ -383,13 +376,11 @@ pipe = os.popen(os.path.join(dir, 'ipconfig') + ' /all') except OSError: continue - else: + with pipe: for line in pipe: value = line.split(':')[-1].strip().lower() if re.match('([0-9a-f][0-9a-f]-){5}[0-9a-f][0-9a-f]', value): return int(value.replace('-', ''), 16) - finally: - pipe.close() def _netbios_getnode(): """Get the hardware address on Windows using NetBIOS calls. @@ -416,9 +407,10 @@ if win32wnet.Netbios(ncb) != 0: continue status._unpack() - bytes = status.adapter_address - return ((bytes[0]<<40) + (bytes[1]<<32) + (bytes[2]<<24) + - (bytes[3]<<16) + (bytes[4]<<8) + bytes[5]) + bytes = status.adapter_address[:6] + if len(bytes) != 6: + continue + return int.from_bytes(bytes, 'big') # Thanks to Thomas Heller for ctypes and for his help with its use here. @@ -487,7 +479,7 @@ def _random_getnode(): """Get a random node ID, with eighth bit set as suggested by RFC 4122.""" import random - return random.randrange(0, 1<<48) | 0x010000000000 + return random.getrandbits(48) | 0x010000000000 _node = None @@ -544,7 +536,7 @@ _last_timestamp = timestamp if clock_seq is None: import random - clock_seq = random.randrange(1<<14) # instead of stable storage + clock_seq = random.getrandbits(14) # instead of stable storage time_low = timestamp & 0xffffffff time_mid = (timestamp >> 32) & 0xffff time_hi_version = (timestamp >> 48) & 0x0fff @@ -576,8 +568,7 @@ return UUID(bytes=os.urandom(16), version=4) except: import random - bytes = bytes_(random.randrange(256) for i in range(16)) - return UUID(bytes=bytes, version=4) + return UUID(int=random.getrandbits(128), version=4) def uuid5(namespace, name): """Generate a UUID from the SHA-1 hash of a namespace UUID and a name.""" -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 6 21:21:27 2014 From: python-checkins at python.org (serhiy.storchaka) Date: Sat, 6 Sep 2014 21:21:27 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzIyMTMx?= =?utf-8?q?=3A_Fixed_a_bug_in_handling_an_error_occured_during_reading_fro?= =?utf-8?q?m?= Message-ID: <3hr5GH5WMCz7Lk8@mail.python.org> http://hg.python.org/cpython/rev/d8c6b15a2ae3 changeset: 92369:d8c6b15a2ae3 branch: 2.7 parent: 92357:322f77ee6d5a user: Serhiy Storchaka date: Sat Sep 06 22:17:06 2014 +0300 summary: Issue #22131: Fixed a bug in handling an error occured during reading from a pipe in _ipconfig_getnode(). files: Lib/uuid.py | 4 +--- 1 files changed, 1 insertions(+), 3 deletions(-) diff --git a/Lib/uuid.py b/Lib/uuid.py --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -366,13 +366,11 @@ pipe = os.popen(os.path.join(dir, 'ipconfig') + ' /all') except IOError: continue - else: + with pipe: for line in pipe: value = line.split(':')[-1].strip().lower() if re.match('([0-9a-f][0-9a-f]-){5}[0-9a-f][0-9a-f]', value): return int(value.replace('-', ''), 16) - finally: - pipe.close() def _netbios_getnode(): """Get the hardware address on Windows using NetBIOS calls. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 6 21:21:28 2014 From: python-checkins at python.org (serhiy.storchaka) Date: Sat, 6 Sep 2014 21:21:28 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzIyMTMx?= =?utf-8?q?=3A_Fixed_a_bug_in_handling_an_error_occured_during_reading_fro?= =?utf-8?q?m?= Message-ID: <3hr5GJ6w5Zz7LkT@mail.python.org> http://hg.python.org/cpython/rev/8a61a287776d changeset: 92370:8a61a287776d branch: 3.4 parent: 92366:c1fb19907cc4 user: Serhiy Storchaka date: Sat Sep 06 22:17:24 2014 +0300 summary: Issue #22131: Fixed a bug in handling an error occured during reading from a pipe in _ipconfig_getnode(). files: Lib/uuid.py | 4 +--- 1 files changed, 1 insertions(+), 3 deletions(-) diff --git a/Lib/uuid.py b/Lib/uuid.py --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -383,13 +383,11 @@ pipe = os.popen(os.path.join(dir, 'ipconfig') + ' /all') except OSError: continue - else: + with pipe: for line in pipe: value = line.split(':')[-1].strip().lower() if re.match('([0-9a-f][0-9a-f]-){5}[0-9a-f][0-9a-f]', value): return int(value.replace('-', ''), 16) - finally: - pipe.close() def _netbios_getnode(): """Get the hardware address on Windows using NetBIOS calls. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 6 21:21:30 2014 From: python-checkins at python.org (serhiy.storchaka) Date: Sat, 6 Sep 2014 21:21:30 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?q?=29=3A_Null_merge?= Message-ID: <3hr5GL1ffjz7LkR@mail.python.org> http://hg.python.org/cpython/rev/1f365cea77b8 changeset: 92371:1f365cea77b8 parent: 92368:f7b5038d3102 parent: 92370:8a61a287776d user: Serhiy Storchaka date: Sat Sep 06 22:18:35 2014 +0300 summary: Null merge files: -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 6 21:52:09 2014 From: python-checkins at python.org (serhiy.storchaka) Date: Sat, 6 Sep 2014 21:52:09 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzIyMjI2?= =?utf-8?q?=3A_Added_private_function_=5Fsplitdict=28=29_in_the_Tkinter_mo?= =?utf-8?q?dule=2E?= Message-ID: <3hr5xj6BzVz7LjY@mail.python.org> http://hg.python.org/cpython/rev/7b0fdc1e917a changeset: 92372:7b0fdc1e917a branch: 2.7 parent: 92369:d8c6b15a2ae3 user: Serhiy Storchaka date: Sat Sep 06 22:47:02 2014 +0300 summary: Issue #22226: Added private function _splitdict() in the Tkinter module. First letter no longer is stripped from the "status" key in the result of Treeview.heading(). files: Lib/lib-tk/Tkinter.py | 78 +++++----- Lib/lib-tk/test/test_ttk/test_functions.py | 27 +-- Lib/lib-tk/ttk.py | 62 +++---- Lib/test/test_tcl.py | 39 +++++ Misc/NEWS | 3 + 5 files changed, 116 insertions(+), 93 deletions(-) diff --git a/Lib/lib-tk/Tkinter.py b/Lib/lib-tk/Tkinter.py --- a/Lib/lib-tk/Tkinter.py +++ b/Lib/lib-tk/Tkinter.py @@ -123,6 +123,29 @@ try: _cnfmerge = _tkinter._cnfmerge except AttributeError: pass +def _splitdict(tk, v, cut_minus=True, conv=None): + """Return a properly formatted dict built from Tcl list pairs. + + If cut_minus is True, the supposed '-' prefix will be removed from + keys. If conv is specified, it is used to convert values. + + Tcl list is expected to contain an even number of elements. + """ + t = tk.splitlist(v) + if len(t) % 2: + raise RuntimeError('Tcl list representing a dict is expected ' + 'to contain an even number of elements') + it = iter(t) + dict = {} + for key, value in zip(it, it): + key = str(key) + if cut_minus and key[0] == '-': + key = key[1:] + if conv: + value = conv(value) + dict[key] = value + return dict + class Event: """Container for the properties of an event. @@ -1390,15 +1413,10 @@ else: options = self._options(cnf, kw) if not options: - res = self.tk.call('grid', - command, self._w, index) - words = self.tk.splitlist(res) - dict = {} - for i in range(0, len(words), 2): - key = words[i][1:] - value = words[i+1] - dict[key] = self._gridconvvalue(value) - return dict + return _splitdict( + self.tk, + self.tk.call('grid', command, self._w, index), + conv=self._gridconvvalue) res = self.tk.call( ('grid', command, self._w, index) + options) @@ -1921,16 +1939,10 @@ def pack_info(self): """Return information about the packing options for this widget.""" - words = self.tk.splitlist( - self.tk.call('pack', 'info', self._w)) - dict = {} - for i in range(0, len(words), 2): - key = words[i][1:] - value = words[i+1] - if str(value)[:1] == '.': - value = self._nametowidget(value) - dict[key] = value - return dict + d = _splitdict(self.tk, self.tk.call('pack', 'info', self._w)) + if 'in' in d: + d['in'] = self.nametowidget(d['in']) + return d info = pack_info propagate = pack_propagate = Misc.pack_propagate slaves = pack_slaves = Misc.pack_slaves @@ -1972,16 +1984,10 @@ def place_info(self): """Return information about the placing options for this widget.""" - words = self.tk.splitlist( - self.tk.call('place', 'info', self._w)) - dict = {} - for i in range(0, len(words), 2): - key = words[i][1:] - value = words[i+1] - if str(value)[:1] == '.': - value = self._nametowidget(value) - dict[key] = value - return dict + d = _splitdict(self.tk, self.tk.call('place', 'info', self._w)) + if 'in' in d: + d['in'] = self.nametowidget(d['in']) + return d info = place_info slaves = place_slaves = Misc.place_slaves @@ -2021,16 +2027,10 @@ def grid_info(self): """Return information about the options for positioning this widget in a grid.""" - words = self.tk.splitlist( - self.tk.call('grid', 'info', self._w)) - dict = {} - for i in range(0, len(words), 2): - key = words[i][1:] - value = words[i+1] - if str(value)[:1] == '.': - value = self._nametowidget(value) - dict[key] = value - return dict + d = _splitdict(self.tk, self.tk.call('grid', 'info', self._w)) + if 'in' in d: + d['in'] = self.nametowidget(d['in']) + return d info = grid_info location = grid_location = Misc.grid_location propagate = grid_propagate = Misc.grid_propagate diff --git a/Lib/lib-tk/test/test_ttk/test_functions.py b/Lib/lib-tk/test/test_ttk/test_functions.py --- a/Lib/lib-tk/test/test_ttk/test_functions.py +++ b/Lib/lib-tk/test/test_ttk/test_functions.py @@ -324,26 +324,13 @@ "-opt {3 2m}") - def test_dict_from_tcltuple(self): - fakettuple = ('-a', '{1 2 3}', '-something', 'foo') - - self.assertEqual(ttk._dict_from_tcltuple(fakettuple, False), - {'-a': '{1 2 3}', '-something': 'foo'}) - - self.assertEqual(ttk._dict_from_tcltuple(fakettuple), - {'a': '{1 2 3}', 'something': 'foo'}) - - # passing a tuple with a single item should return an empty dict, - # since it tries to break the tuple by pairs. - self.assertFalse(ttk._dict_from_tcltuple(('single', ))) - - sspec = MockStateSpec('a', 'b') - self.assertEqual(ttk._dict_from_tcltuple(('-a', (sspec, 'val'))), - {'a': [('a', 'b', 'val')]}) - - self.assertEqual(ttk._dict_from_tcltuple((MockTclObj('-padding'), - [MockTclObj('1'), 2, MockTclObj('3m')])), - {'padding': [1, 2, '3m']}) + def test_tclobj_to_py(self): + self.assertEqual( + ttk._tclobj_to_py((MockStateSpec('a', 'b'), 'val')), + [('a', 'b', 'val')]) + self.assertEqual( + ttk._tclobj_to_py([MockTclObj('1'), 2, MockTclObj('3m')]), + [1, 2, '3m']) def test_list_from_statespec(self): diff --git a/Lib/lib-tk/ttk.py b/Lib/lib-tk/ttk.py --- a/Lib/lib-tk/ttk.py +++ b/Lib/lib-tk/ttk.py @@ -26,7 +26,7 @@ "tclobjs_to_py", "setup_master"] import Tkinter -from Tkinter import _flatten, _join, _stringify +from Tkinter import _flatten, _join, _stringify, _splitdict # Verify if Tk is new enough to not need the Tile package _REQUIRE_TILE = True if Tkinter.TkVersion < 8.5 else False @@ -242,21 +242,6 @@ return '\n'.join(script) -def _dict_from_tcltuple(ttuple, cut_minus=True): - """Break tuple in pairs, format it properly, then build the return - dict. If cut_minus is True, the supposed '-' prefixing options will - be removed. - - ttuple is expected to contain an even number of elements.""" - opt_start = 1 if cut_minus else 0 - - retdict = {} - it = iter(ttuple) - for opt, val in zip(it, it): - retdict[str(opt)[opt_start:]] = val - - return tclobjs_to_py(retdict) - def _list_from_statespec(stuple): """Construct a list from the given statespec tuple according to the accepted statespec accepted by _format_mapdict.""" @@ -316,7 +301,7 @@ if len(options) % 2: # option specified without a value, return its value return res - return _dict_from_tcltuple(tk.splitlist(res)) + return _splitdict(tk, res, conv=_tclobj_to_py) def _convert_stringval(value): """Converts a value to, hopefully, a more appropriate Python object.""" @@ -336,20 +321,24 @@ x = int(x) return x +def _tclobj_to_py(val): + """Return value converted from Tcl object to Python object.""" + if val and hasattr(val, '__len__') and not isinstance(val, basestring): + if getattr(val[0], 'typename', None) == 'StateSpec': + val = _list_from_statespec(val) + else: + val = map(_convert_stringval, val) + + elif hasattr(val, 'typename'): # some other (single) Tcl object + val = _convert_stringval(val) + + return val + def tclobjs_to_py(adict): """Returns adict with its values converted from Tcl objects to Python objects.""" - for opt, val in adict.iteritems(): - if val and hasattr(val, '__len__') and not isinstance(val, basestring): - if getattr(val[0], 'typename', None) == 'StateSpec': - val = _list_from_statespec(val) - else: - val = map(_convert_stringval, val) - - elif hasattr(val, 'typename'): # some other (single) Tcl object - val = _convert_stringval(val) - - adict[opt] = val + for opt, val in adict.items(): + adict[opt] = _tclobj_to_py(val) return adict @@ -409,8 +398,10 @@ return _list_from_statespec(self.tk.splitlist( self.tk.call(self._name, "map", style, '-%s' % query_opt))) - return _dict_from_tcltuple(self.tk.splitlist( - self.tk.call(self._name, "map", style, *(_format_mapdict(kw))))) + return _splitdict( + self.tk, + self.tk.call(self._name, "map", style, *_format_mapdict(kw)), + conv=_tclobj_to_py) def lookup(self, style, option, state=None, default=None): @@ -1427,13 +1418,16 @@ def set(self, item, column=None, value=None): - """With one argument, returns a dictionary of column/value pairs - for the specified item. With two arguments, returns the current - value of the specified column. With three arguments, sets the + """Query or set the value of given item. + + With one argument, return a dictionary of column/value pairs + for the specified item. With two arguments, return the current + value of the specified column. With three arguments, set the value of given column in given item to the specified value.""" res = self.tk.call(self._w, "set", item, column, value) if column is None and value is None: - return _dict_from_tcltuple(self.tk.splitlist(res), False) + return _splitdict(self.tk, res, + cut_minus=False, conv=_tclobj_to_py) else: return res diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -7,6 +7,9 @@ # Skip this test if the _tkinter module wasn't built. _tkinter = test_support.import_module('_tkinter') +# Make sure tkinter._fix runs to set up the environment +tkinter = test_support.import_fresh_module('Tkinter') + from Tkinter import Tcl from _tkinter import TclError @@ -565,6 +568,42 @@ for arg, res in testcases: self.assertEqual(split(arg), res) + def test_splitdict(self): + splitdict = tkinter._splitdict + tcl = self.interp.tk + + arg = '-a {1 2 3} -something foo status {}' + self.assertEqual(splitdict(tcl, arg, False), + {'-a': '1 2 3', '-something': 'foo', 'status': ''}) + self.assertEqual(splitdict(tcl, arg), + {'a': '1 2 3', 'something': 'foo', 'status': ''}) + + arg = ('-a', (1, 2, 3), '-something', 'foo', 'status', '{}') + self.assertEqual(splitdict(tcl, arg, False), + {'-a': (1, 2, 3), '-something': 'foo', 'status': '{}'}) + self.assertEqual(splitdict(tcl, arg), + {'a': (1, 2, 3), 'something': 'foo', 'status': '{}'}) + + self.assertRaises(RuntimeError, splitdict, tcl, '-a b -c ') + self.assertRaises(RuntimeError, splitdict, tcl, ('-a', 'b', '-c')) + + arg = tcl.call('list', + '-a', (1, 2, 3), '-something', 'foo', 'status', ()) + self.assertEqual(splitdict(tcl, arg), + {'a': (1, 2, 3) if self.wantobjects else '1 2 3', + 'something': 'foo', 'status': ''}) + + if tcl_version >= (8, 5): + arg = tcl.call('dict', 'create', + '-a', (1, 2, 3), '-something', 'foo', 'status', ()) + if not self.wantobjects or get_tk_patchlevel() < (8, 5, 5): + # Before 8.5.5 dicts were converted to lists through string + expected = {'a': '1 2 3', 'something': 'foo', 'status': ''} + else: + expected = {'a': (1, 2, 3), 'something': 'foo', 'status': ''} + self.assertEqual(splitdict(tcl, arg), expected) + + character_size = 4 if sys.maxunicode > 0xFFFF else 2 class BigmemTclTest(unittest.TestCase): diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -22,6 +22,9 @@ Library ------- +- Issue #22226: First letter no longer is stripped from the "status" key in + the result of Treeview.heading(). + - Issue #22051: turtledemo no longer reloads examples to re-run them. Initialization of variables and gui setup should be done in main(), which is called each time a demo is run, but not on import. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 6 21:52:11 2014 From: python-checkins at python.org (serhiy.storchaka) Date: Sat, 6 Sep 2014 21:52:11 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzIyMjI2?= =?utf-8?q?=3A_Added_private_function_=5Fsplitdict=28=29_in_the_Tkinter_mo?= =?utf-8?q?dule=2E?= Message-ID: <3hr5xl2DQqz7Lk0@mail.python.org> http://hg.python.org/cpython/rev/f89995a4ec11 changeset: 92373:f89995a4ec11 branch: 3.4 parent: 92370:8a61a287776d user: Serhiy Storchaka date: Sat Sep 06 22:47:58 2014 +0300 summary: Issue #22226: Added private function _splitdict() in the Tkinter module. First letter no longer is stripped from the "status" key in the result of Treeview.heading(). files: Lib/test/test_tcl.py | 37 ++++- Lib/tkinter/__init__.py | 78 +++++----- Lib/tkinter/test/test_ttk/test_functions.py | 27 +-- Lib/tkinter/ttk.py | 60 +++---- Misc/NEWS | 3 + 5 files changed, 112 insertions(+), 93 deletions(-) diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -7,7 +7,7 @@ _tkinter = support.import_module('_tkinter') # Make sure tkinter._fix runs to set up the environment -support.import_fresh_module('tkinter') +tkinter = support.import_fresh_module('tkinter') from tkinter import Tcl from _tkinter import TclError @@ -554,6 +554,41 @@ for arg, res in testcases: self.assertEqual(split(arg), res, msg=arg) + def test_splitdict(self): + splitdict = tkinter._splitdict + tcl = self.interp.tk + + arg = '-a {1 2 3} -something foo status {}' + self.assertEqual(splitdict(tcl, arg, False), + {'-a': '1 2 3', '-something': 'foo', 'status': ''}) + self.assertEqual(splitdict(tcl, arg), + {'a': '1 2 3', 'something': 'foo', 'status': ''}) + + arg = ('-a', (1, 2, 3), '-something', 'foo', 'status', '{}') + self.assertEqual(splitdict(tcl, arg, False), + {'-a': (1, 2, 3), '-something': 'foo', 'status': '{}'}) + self.assertEqual(splitdict(tcl, arg), + {'a': (1, 2, 3), 'something': 'foo', 'status': '{}'}) + + self.assertRaises(RuntimeError, splitdict, tcl, '-a b -c ') + self.assertRaises(RuntimeError, splitdict, tcl, ('-a', 'b', '-c')) + + arg = tcl.call('list', + '-a', (1, 2, 3), '-something', 'foo', 'status', ()) + self.assertEqual(splitdict(tcl, arg), + {'a': (1, 2, 3) if self.wantobjects else '1 2 3', + 'something': 'foo', 'status': ''}) + + if tcl_version >= (8, 5): + arg = tcl.call('dict', 'create', + '-a', (1, 2, 3), '-something', 'foo', 'status', ()) + if not self.wantobjects or get_tk_patchlevel() < (8, 5, 5): + # Before 8.5.5 dicts were converted to lists through string + expected = {'a': '1 2 3', 'something': 'foo', 'status': ''} + else: + expected = {'a': (1, 2, 3), 'something': 'foo', 'status': ''} + self.assertEqual(splitdict(tcl, arg), expected) + class BigmemTclTest(unittest.TestCase): diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -112,6 +112,29 @@ try: _cnfmerge = _tkinter._cnfmerge except AttributeError: pass +def _splitdict(tk, v, cut_minus=True, conv=None): + """Return a properly formatted dict built from Tcl list pairs. + + If cut_minus is True, the supposed '-' prefix will be removed from + keys. If conv is specified, it is used to convert values. + + Tcl list is expected to contain an even number of elements. + """ + t = tk.splitlist(v) + if len(t) % 2: + raise RuntimeError('Tcl list representing a dict is expected ' + 'to contain an even number of elements') + it = iter(t) + dict = {} + for key, value in zip(it, it): + key = str(key) + if cut_minus and key[0] == '-': + key = key[1:] + if conv: + value = conv(value) + dict[key] = value + return dict + class Event: """Container for the properties of an event. @@ -1391,15 +1414,10 @@ else: options = self._options(cnf, kw) if not options: - res = self.tk.call('grid', - command, self._w, index) - words = self.tk.splitlist(res) - dict = {} - for i in range(0, len(words), 2): - key = words[i][1:] - value = words[i+1] - dict[key] = self._gridconvvalue(value) - return dict + return _splitdict( + self.tk, + self.tk.call('grid', command, self._w, index), + conv=self._gridconvvalue) res = self.tk.call( ('grid', command, self._w, index) + options) @@ -1959,16 +1977,10 @@ def pack_info(self): """Return information about the packing options for this widget.""" - words = self.tk.splitlist( - self.tk.call('pack', 'info', self._w)) - dict = {} - for i in range(0, len(words), 2): - key = words[i][1:] - value = words[i+1] - if str(value)[:1] == '.': - value = self._nametowidget(value) - dict[key] = value - return dict + d = _splitdict(self.tk, self.tk.call('pack', 'info', self._w)) + if 'in' in d: + d['in'] = self.nametowidget(d['in']) + return d info = pack_info propagate = pack_propagate = Misc.pack_propagate slaves = pack_slaves = Misc.pack_slaves @@ -2010,16 +2022,10 @@ def place_info(self): """Return information about the placing options for this widget.""" - words = self.tk.splitlist( - self.tk.call('place', 'info', self._w)) - dict = {} - for i in range(0, len(words), 2): - key = words[i][1:] - value = words[i+1] - if str(value)[:1] == '.': - value = self._nametowidget(value) - dict[key] = value - return dict + d = _splitdict(self.tk, self.tk.call('place', 'info', self._w)) + if 'in' in d: + d['in'] = self.nametowidget(d['in']) + return d info = place_info slaves = place_slaves = Misc.place_slaves @@ -2059,16 +2065,10 @@ def grid_info(self): """Return information about the options for positioning this widget in a grid.""" - words = self.tk.splitlist( - self.tk.call('grid', 'info', self._w)) - dict = {} - for i in range(0, len(words), 2): - key = words[i][1:] - value = words[i+1] - if str(value)[:1] == '.': - value = self._nametowidget(value) - dict[key] = value - return dict + d = _splitdict(self.tk, self.tk.call('grid', 'info', self._w)) + if 'in' in d: + d['in'] = self.nametowidget(d['in']) + return d info = grid_info location = grid_location = Misc.grid_location propagate = grid_propagate = Misc.grid_propagate diff --git a/Lib/tkinter/test/test_ttk/test_functions.py b/Lib/tkinter/test/test_ttk/test_functions.py --- a/Lib/tkinter/test/test_ttk/test_functions.py +++ b/Lib/tkinter/test/test_ttk/test_functions.py @@ -324,26 +324,13 @@ "-opt {3 2m}") - def test_dict_from_tcltuple(self): - fakettuple = ('-a', '{1 2 3}', '-something', 'foo') - - self.assertEqual(ttk._dict_from_tcltuple(fakettuple, False), - {'-a': '{1 2 3}', '-something': 'foo'}) - - self.assertEqual(ttk._dict_from_tcltuple(fakettuple), - {'a': '{1 2 3}', 'something': 'foo'}) - - # passing a tuple with a single item should return an empty dict, - # since it tries to break the tuple by pairs. - self.assertFalse(ttk._dict_from_tcltuple(('single', ))) - - sspec = MockStateSpec('a', 'b') - self.assertEqual(ttk._dict_from_tcltuple(('-a', (sspec, 'val'))), - {'a': [('a', 'b', 'val')]}) - - self.assertEqual(ttk._dict_from_tcltuple((MockTclObj('-padding'), - [MockTclObj('1'), 2, MockTclObj('3m')])), - {'padding': [1, 2, '3m']}) + def test_tclobj_to_py(self): + self.assertEqual( + ttk._tclobj_to_py((MockStateSpec('a', 'b'), 'val')), + [('a', 'b', 'val')]) + self.assertEqual( + ttk._tclobj_to_py([MockTclObj('1'), 2, MockTclObj('3m')]), + [1, 2, '3m']) def test_list_from_statespec(self): diff --git a/Lib/tkinter/ttk.py b/Lib/tkinter/ttk.py --- a/Lib/tkinter/ttk.py +++ b/Lib/tkinter/ttk.py @@ -26,7 +26,7 @@ "tclobjs_to_py", "setup_master"] import tkinter -from tkinter import _flatten, _join, _stringify +from tkinter import _flatten, _join, _stringify, _splitdict # Verify if Tk is new enough to not need the Tile package _REQUIRE_TILE = True if tkinter.TkVersion < 8.5 else False @@ -240,21 +240,6 @@ return '\n'.join(script) -def _dict_from_tcltuple(ttuple, cut_minus=True): - """Break tuple in pairs, format it properly, then build the return - dict. If cut_minus is True, the supposed '-' prefixing options will - be removed. - - ttuple is expected to contain an even number of elements.""" - opt_start = 1 if cut_minus else 0 - - retdict = {} - it = iter(ttuple) - for opt, val in zip(it, it): - retdict[str(opt)[opt_start:]] = val - - return tclobjs_to_py(retdict) - def _list_from_statespec(stuple): """Construct a list from the given statespec tuple according to the accepted statespec accepted by _format_mapdict.""" @@ -314,7 +299,7 @@ if len(options) % 2: # option specified without a value, return its value return res - return _dict_from_tcltuple(tk.splitlist(res)) + return _splitdict(tk, res, conv=_tclobj_to_py) def _convert_stringval(value): """Converts a value to, hopefully, a more appropriate Python object.""" @@ -334,20 +319,24 @@ x = int(x) return x +def _tclobj_to_py(val): + """Return value converted from Tcl object to Python object.""" + if val and hasattr(val, '__len__') and not isinstance(val, str): + if getattr(val[0], 'typename', None) == 'StateSpec': + val = _list_from_statespec(val) + else: + val = list(map(_convert_stringval, val)) + + elif hasattr(val, 'typename'): # some other (single) Tcl object + val = _convert_stringval(val) + + return val + def tclobjs_to_py(adict): """Returns adict with its values converted from Tcl objects to Python objects.""" for opt, val in adict.items(): - if val and hasattr(val, '__len__') and not isinstance(val, str): - if getattr(val[0], 'typename', None) == 'StateSpec': - val = _list_from_statespec(val) - else: - val = list(map(_convert_stringval, val)) - - elif hasattr(val, 'typename'): # some other (single) Tcl object - val = _convert_stringval(val) - - adict[opt] = val + adict[opt] = _tclobj_to_py(val) return adict @@ -407,8 +396,10 @@ return _list_from_statespec(self.tk.splitlist( self.tk.call(self._name, "map", style, '-%s' % query_opt))) - return _dict_from_tcltuple(self.tk.splitlist( - self.tk.call(self._name, "map", style, *(_format_mapdict(kw))))) + return _splitdict( + self.tk, + self.tk.call(self._name, "map", style, *_format_mapdict(kw)), + conv=_tclobj_to_py) def lookup(self, style, option, state=None, default=None): @@ -1425,13 +1416,16 @@ def set(self, item, column=None, value=None): - """With one argument, returns a dictionary of column/value pairs - for the specified item. With two arguments, returns the current - value of the specified column. With three arguments, sets the + """Query or set the value of given item. + + With one argument, return a dictionary of column/value pairs + for the specified item. With two arguments, return the current + value of the specified column. With three arguments, set the value of given column in given item to the specified value.""" res = self.tk.call(self._w, "set", item, column, value) if column is None and value is None: - return _dict_from_tcltuple(self.tk.splitlist(res), False) + return _splitdict(self.tk, res, + cut_minus=False, conv=_tclobj_to_py) else: return res diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,9 @@ Library ------- +- Issue #22226: First letter no longer is stripped from the "status" key in + the result of Treeview.heading(). + - Issue #19524: Fixed resource leak in the HTTP connection when an invalid response is received. Patch by Martin Panter. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 6 21:52:12 2014 From: python-checkins at python.org (serhiy.storchaka) Date: Sat, 6 Sep 2014 21:52:12 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?q?=29=3A_Issue_=2322226=3A_Added_private_function_=5Fsplitdict=28?= =?utf-8?q?=29_in_the_Tkinter_module=2E?= Message-ID: <3hr5xm5MG6z7LkY@mail.python.org> http://hg.python.org/cpython/rev/11cf18ec1900 changeset: 92374:11cf18ec1900 parent: 92371:1f365cea77b8 parent: 92373:f89995a4ec11 user: Serhiy Storchaka date: Sat Sep 06 22:49:07 2014 +0300 summary: Issue #22226: Added private function _splitdict() in the Tkinter module. First letter no longer is stripped from the "status" key in the result of Treeview.heading(). files: Lib/test/test_tcl.py | 37 ++++- Lib/tkinter/__init__.py | 78 +++++----- Lib/tkinter/test/test_ttk/test_functions.py | 27 +-- Lib/tkinter/ttk.py | 60 +++---- Misc/NEWS | 3 + 5 files changed, 112 insertions(+), 93 deletions(-) diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -7,7 +7,7 @@ _tkinter = support.import_module('_tkinter') # Make sure tkinter._fix runs to set up the environment -support.import_fresh_module('tkinter') +tkinter = support.import_fresh_module('tkinter') from tkinter import Tcl from _tkinter import TclError @@ -566,6 +566,41 @@ for arg, res in testcases: self.assertEqual(split(arg), res, msg=arg) + def test_splitdict(self): + splitdict = tkinter._splitdict + tcl = self.interp.tk + + arg = '-a {1 2 3} -something foo status {}' + self.assertEqual(splitdict(tcl, arg, False), + {'-a': '1 2 3', '-something': 'foo', 'status': ''}) + self.assertEqual(splitdict(tcl, arg), + {'a': '1 2 3', 'something': 'foo', 'status': ''}) + + arg = ('-a', (1, 2, 3), '-something', 'foo', 'status', '{}') + self.assertEqual(splitdict(tcl, arg, False), + {'-a': (1, 2, 3), '-something': 'foo', 'status': '{}'}) + self.assertEqual(splitdict(tcl, arg), + {'a': (1, 2, 3), 'something': 'foo', 'status': '{}'}) + + self.assertRaises(RuntimeError, splitdict, tcl, '-a b -c ') + self.assertRaises(RuntimeError, splitdict, tcl, ('-a', 'b', '-c')) + + arg = tcl.call('list', + '-a', (1, 2, 3), '-something', 'foo', 'status', ()) + self.assertEqual(splitdict(tcl, arg), + {'a': (1, 2, 3) if self.wantobjects else '1 2 3', + 'something': 'foo', 'status': ''}) + + if tcl_version >= (8, 5): + arg = tcl.call('dict', 'create', + '-a', (1, 2, 3), '-something', 'foo', 'status', ()) + if not self.wantobjects or get_tk_patchlevel() < (8, 5, 5): + # Before 8.5.5 dicts were converted to lists through string + expected = {'a': '1 2 3', 'something': 'foo', 'status': ''} + else: + expected = {'a': (1, 2, 3), 'something': 'foo', 'status': ''} + self.assertEqual(splitdict(tcl, arg), expected) + class BigmemTclTest(unittest.TestCase): diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -112,6 +112,29 @@ try: _cnfmerge = _tkinter._cnfmerge except AttributeError: pass +def _splitdict(tk, v, cut_minus=True, conv=None): + """Return a properly formatted dict built from Tcl list pairs. + + If cut_minus is True, the supposed '-' prefix will be removed from + keys. If conv is specified, it is used to convert values. + + Tcl list is expected to contain an even number of elements. + """ + t = tk.splitlist(v) + if len(t) % 2: + raise RuntimeError('Tcl list representing a dict is expected ' + 'to contain an even number of elements') + it = iter(t) + dict = {} + for key, value in zip(it, it): + key = str(key) + if cut_minus and key[0] == '-': + key = key[1:] + if conv: + value = conv(value) + dict[key] = value + return dict + class Event: """Container for the properties of an event. @@ -1393,15 +1416,10 @@ else: options = self._options(cnf, kw) if not options: - res = self.tk.call('grid', - command, self._w, index) - words = self.tk.splitlist(res) - dict = {} - for i in range(0, len(words), 2): - key = words[i][1:] - value = words[i+1] - dict[key] = self._gridconvvalue(value) - return dict + return _splitdict( + self.tk, + self.tk.call('grid', command, self._w, index), + conv=self._gridconvvalue) res = self.tk.call( ('grid', command, self._w, index) + options) @@ -1961,16 +1979,10 @@ def pack_info(self): """Return information about the packing options for this widget.""" - words = self.tk.splitlist( - self.tk.call('pack', 'info', self._w)) - dict = {} - for i in range(0, len(words), 2): - key = words[i][1:] - value = words[i+1] - if str(value)[:1] == '.': - value = self._nametowidget(value) - dict[key] = value - return dict + d = _splitdict(self.tk, self.tk.call('pack', 'info', self._w)) + if 'in' in d: + d['in'] = self.nametowidget(d['in']) + return d info = pack_info propagate = pack_propagate = Misc.pack_propagate slaves = pack_slaves = Misc.pack_slaves @@ -2012,16 +2024,10 @@ def place_info(self): """Return information about the placing options for this widget.""" - words = self.tk.splitlist( - self.tk.call('place', 'info', self._w)) - dict = {} - for i in range(0, len(words), 2): - key = words[i][1:] - value = words[i+1] - if str(value)[:1] == '.': - value = self._nametowidget(value) - dict[key] = value - return dict + d = _splitdict(self.tk, self.tk.call('place', 'info', self._w)) + if 'in' in d: + d['in'] = self.nametowidget(d['in']) + return d info = place_info slaves = place_slaves = Misc.place_slaves @@ -2061,16 +2067,10 @@ def grid_info(self): """Return information about the options for positioning this widget in a grid.""" - words = self.tk.splitlist( - self.tk.call('grid', 'info', self._w)) - dict = {} - for i in range(0, len(words), 2): - key = words[i][1:] - value = words[i+1] - if str(value)[:1] == '.': - value = self._nametowidget(value) - dict[key] = value - return dict + d = _splitdict(self.tk, self.tk.call('grid', 'info', self._w)) + if 'in' in d: + d['in'] = self.nametowidget(d['in']) + return d info = grid_info location = grid_location = Misc.grid_location propagate = grid_propagate = Misc.grid_propagate diff --git a/Lib/tkinter/test/test_ttk/test_functions.py b/Lib/tkinter/test/test_ttk/test_functions.py --- a/Lib/tkinter/test/test_ttk/test_functions.py +++ b/Lib/tkinter/test/test_ttk/test_functions.py @@ -324,26 +324,13 @@ "-opt {3 2m}") - def test_dict_from_tcltuple(self): - fakettuple = ('-a', '{1 2 3}', '-something', 'foo') - - self.assertEqual(ttk._dict_from_tcltuple(fakettuple, False), - {'-a': '{1 2 3}', '-something': 'foo'}) - - self.assertEqual(ttk._dict_from_tcltuple(fakettuple), - {'a': '{1 2 3}', 'something': 'foo'}) - - # passing a tuple with a single item should return an empty dict, - # since it tries to break the tuple by pairs. - self.assertFalse(ttk._dict_from_tcltuple(('single', ))) - - sspec = MockStateSpec('a', 'b') - self.assertEqual(ttk._dict_from_tcltuple(('-a', (sspec, 'val'))), - {'a': [('a', 'b', 'val')]}) - - self.assertEqual(ttk._dict_from_tcltuple((MockTclObj('-padding'), - [MockTclObj('1'), 2, MockTclObj('3m')])), - {'padding': [1, 2, '3m']}) + def test_tclobj_to_py(self): + self.assertEqual( + ttk._tclobj_to_py((MockStateSpec('a', 'b'), 'val')), + [('a', 'b', 'val')]) + self.assertEqual( + ttk._tclobj_to_py([MockTclObj('1'), 2, MockTclObj('3m')]), + [1, 2, '3m']) def test_list_from_statespec(self): diff --git a/Lib/tkinter/ttk.py b/Lib/tkinter/ttk.py --- a/Lib/tkinter/ttk.py +++ b/Lib/tkinter/ttk.py @@ -26,7 +26,7 @@ "tclobjs_to_py", "setup_master"] import tkinter -from tkinter import _flatten, _join, _stringify +from tkinter import _flatten, _join, _stringify, _splitdict # Verify if Tk is new enough to not need the Tile package _REQUIRE_TILE = True if tkinter.TkVersion < 8.5 else False @@ -240,21 +240,6 @@ return '\n'.join(script) -def _dict_from_tcltuple(ttuple, cut_minus=True): - """Break tuple in pairs, format it properly, then build the return - dict. If cut_minus is True, the supposed '-' prefixing options will - be removed. - - ttuple is expected to contain an even number of elements.""" - opt_start = 1 if cut_minus else 0 - - retdict = {} - it = iter(ttuple) - for opt, val in zip(it, it): - retdict[str(opt)[opt_start:]] = val - - return tclobjs_to_py(retdict) - def _list_from_statespec(stuple): """Construct a list from the given statespec tuple according to the accepted statespec accepted by _format_mapdict.""" @@ -314,7 +299,7 @@ if len(options) % 2: # option specified without a value, return its value return res - return _dict_from_tcltuple(tk.splitlist(res)) + return _splitdict(tk, res, conv=_tclobj_to_py) def _convert_stringval(value): """Converts a value to, hopefully, a more appropriate Python object.""" @@ -334,20 +319,24 @@ x = int(x) return x +def _tclobj_to_py(val): + """Return value converted from Tcl object to Python object.""" + if val and hasattr(val, '__len__') and not isinstance(val, str): + if getattr(val[0], 'typename', None) == 'StateSpec': + val = _list_from_statespec(val) + else: + val = list(map(_convert_stringval, val)) + + elif hasattr(val, 'typename'): # some other (single) Tcl object + val = _convert_stringval(val) + + return val + def tclobjs_to_py(adict): """Returns adict with its values converted from Tcl objects to Python objects.""" for opt, val in adict.items(): - if val and hasattr(val, '__len__') and not isinstance(val, str): - if getattr(val[0], 'typename', None) == 'StateSpec': - val = _list_from_statespec(val) - else: - val = list(map(_convert_stringval, val)) - - elif hasattr(val, 'typename'): # some other (single) Tcl object - val = _convert_stringval(val) - - adict[opt] = val + adict[opt] = _tclobj_to_py(val) return adict @@ -407,8 +396,10 @@ return _list_from_statespec(self.tk.splitlist( self.tk.call(self._name, "map", style, '-%s' % query_opt))) - return _dict_from_tcltuple(self.tk.splitlist( - self.tk.call(self._name, "map", style, *(_format_mapdict(kw))))) + return _splitdict( + self.tk, + self.tk.call(self._name, "map", style, *_format_mapdict(kw)), + conv=_tclobj_to_py) def lookup(self, style, option, state=None, default=None): @@ -1425,13 +1416,16 @@ def set(self, item, column=None, value=None): - """With one argument, returns a dictionary of column/value pairs - for the specified item. With two arguments, returns the current - value of the specified column. With three arguments, sets the + """Query or set the value of given item. + + With one argument, return a dictionary of column/value pairs + for the specified item. With two arguments, return the current + value of the specified column. With three arguments, set the value of given column in given item to the specified value.""" res = self.tk.call(self._w, "set", item, column, value) if column is None and value is None: - return _dict_from_tcltuple(self.tk.splitlist(res), False) + return _splitdict(self.tk, res, + cut_minus=False, conv=_tclobj_to_py) else: return res diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -132,6 +132,9 @@ Library ------- +- Issue #22226: First letter no longer is stripped from the "status" key in + the result of Treeview.heading(). + - Issue #19524: Fixed resource leak in the HTTP connection when an invalid response is received. Patch by Martin Panter. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 6 23:24:43 2014 From: python-checkins at python.org (benjamin.peterson) Date: Sat, 6 Sep 2014 23:24:43 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E4=29=3A_remove_various?= =?utf-8?q?_dead_version_checks_=28closes_=2322349=29?= Message-ID: <3hr80W3W09z7LlB@mail.python.org> http://hg.python.org/cpython/rev/7fece97e605d changeset: 92375:7fece97e605d branch: 3.4 parent: 92373:f89995a4ec11 user: Benjamin Peterson date: Sat Sep 06 17:24:12 2014 -0400 summary: remove various dead version checks (closes #22349) Patch from Thomas Kluyver. files: Lib/distutils/command/build_ext.py | 17 ++-------- Lib/distutils/sysconfig.py | 24 +------------- Lib/distutils/tests/test_build_ext.py | 20 +++++------ 3 files changed, 16 insertions(+), 45 deletions(-) diff --git a/Lib/distutils/command/build_ext.py b/Lib/distutils/command/build_ext.py --- a/Lib/distutils/command/build_ext.py +++ b/Lib/distutils/command/build_ext.py @@ -14,13 +14,7 @@ from distutils.util import get_platform from distutils import log -# this keeps compatibility from 2.3 to 2.5 -if sys.version < "2.6": - USER_BASE = None - HAS_USER_SITE = False -else: - from site import USER_BASE - HAS_USER_SITE = True +from site import USER_BASE if os.name == 'nt': from distutils.msvccompiler import get_build_version @@ -97,14 +91,11 @@ "list of SWIG command line options"), ('swig=', None, "path to the SWIG executable"), + ('user', None, + "add user include, library and rpath") ] - boolean_options = ['inplace', 'debug', 'force', 'swig-cpp'] - - if HAS_USER_SITE: - user_options.append(('user', None, - "add user include, library and rpath")) - boolean_options.append('user') + boolean_options = ['inplace', 'debug', 'force', 'swig-cpp', 'user'] help_options = [ ('help-compiler', None, diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -151,10 +151,7 @@ if standard_lib: return os.path.join(prefix, "Lib") else: - if get_python_version() < "2.2": - return prefix - else: - return os.path.join(prefix, "Lib", "site-packages") + return os.path.join(prefix, "Lib", "site-packages") else: raise DistutilsPlatformError( "I don't know where Python installs its library " @@ -244,12 +241,8 @@ inc_dir = _sys_home or project_base else: inc_dir = get_python_inc(plat_specific=1) - if get_python_version() < '2.2': - config_h = 'config.h' - else: - # The name of the config.h file changed in 2.2 - config_h = 'pyconfig.h' - return os.path.join(inc_dir, config_h) + + return os.path.join(inc_dir, 'pyconfig.h') def get_makefile_filename(): @@ -461,17 +454,6 @@ if python_build: g['LDSHARED'] = g['BLDSHARED'] - elif get_python_version() < '2.1': - # The following two branches are for 1.5.2 compatibility. - if sys.platform == 'aix4': # what about AIX 3.x ? - # Linker script is in the config directory, not in Modules as the - # Makefile says. - python_lib = get_python_lib(standard_lib=1) - ld_so_aix = os.path.join(python_lib, 'config', 'ld_so_aix') - python_exp = os.path.join(python_lib, 'config', 'python.exp') - - g['LDSHARED'] = "%s %s -bI:%s" % (ld_so_aix, g['CC'], python_exp) - global _config_vars _config_vars = g diff --git a/Lib/distutils/tests/test_build_ext.py b/Lib/distutils/tests/test_build_ext.py --- a/Lib/distutils/tests/test_build_ext.py +++ b/Lib/distutils/tests/test_build_ext.py @@ -31,12 +31,11 @@ self.tmp_dir = self.mkdtemp() self.sys_path = sys.path, sys.path[:] sys.path.append(self.tmp_dir) - if sys.version > "2.6": - import site - self.old_user_base = site.USER_BASE - site.USER_BASE = self.mkdtemp() - from distutils.command import build_ext - build_ext.USER_BASE = site.USER_BASE + import site + self.old_user_base = site.USER_BASE + site.USER_BASE = self.mkdtemp() + from distutils.command import build_ext + build_ext.USER_BASE = site.USER_BASE def test_build_ext(self): global ALREADY_TESTED @@ -84,11 +83,10 @@ support.unload('xx') sys.path = self.sys_path[0] sys.path[:] = self.sys_path[1] - if sys.version > "2.6": - import site - site.USER_BASE = self.old_user_base - from distutils.command import build_ext - build_ext.USER_BASE = self.old_user_base + import site + site.USER_BASE = self.old_user_base + from distutils.command import build_ext + build_ext.USER_BASE = self.old_user_base super(BuildExtTestCase, self).tearDown() def test_solaris_enable_shared(self): -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Sat Sep 6 23:24:44 2014 From: python-checkins at python.org (benjamin.peterson) Date: Sat, 6 Sep 2014 23:24:44 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?b?KTogbWVyZ2UgMy40ICgjMjIzNDkp?= Message-ID: <3hr80X5vP9z7Lkl@mail.python.org> http://hg.python.org/cpython/rev/688701337b1a changeset: 92376:688701337b1a parent: 92374:11cf18ec1900 parent: 92375:7fece97e605d user: Benjamin Peterson date: Sat Sep 06 17:24:35 2014 -0400 summary: merge 3.4 (#22349) files: Lib/distutils/command/build_ext.py | 17 ++-------- Lib/distutils/sysconfig.py | 24 +------------- Lib/distutils/tests/test_build_ext.py | 20 +++++------ 3 files changed, 16 insertions(+), 45 deletions(-) diff --git a/Lib/distutils/command/build_ext.py b/Lib/distutils/command/build_ext.py --- a/Lib/distutils/command/build_ext.py +++ b/Lib/distutils/command/build_ext.py @@ -14,13 +14,7 @@ from distutils.util import get_platform from distutils import log -# this keeps compatibility from 2.3 to 2.5 -if sys.version < "2.6": - USER_BASE = None - HAS_USER_SITE = False -else: - from site import USER_BASE - HAS_USER_SITE = True +from site import USER_BASE if os.name == 'nt': from distutils.msvccompiler import get_build_version @@ -97,14 +91,11 @@ "list of SWIG command line options"), ('swig=', None, "path to the SWIG executable"), + ('user', None, + "add user include, library and rpath") ] - boolean_options = ['inplace', 'debug', 'force', 'swig-cpp'] - - if HAS_USER_SITE: - user_options.append(('user', None, - "add user include, library and rpath")) - boolean_options.append('user') + boolean_options = ['inplace', 'debug', 'force', 'swig-cpp', 'user'] help_options = [ ('help-compiler', None, diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -151,10 +151,7 @@ if standard_lib: return os.path.join(prefix, "Lib") else: - if get_python_version() < "2.2": - return prefix - else: - return os.path.join(prefix, "Lib", "site-packages") + return os.path.join(prefix, "Lib", "site-packages") else: raise DistutilsPlatformError( "I don't know where Python installs its library " @@ -244,12 +241,8 @@ inc_dir = _sys_home or project_base else: inc_dir = get_python_inc(plat_specific=1) - if get_python_version() < '2.2': - config_h = 'config.h' - else: - # The name of the config.h file changed in 2.2 - config_h = 'pyconfig.h' - return os.path.join(inc_dir, config_h) + + return os.path.join(inc_dir, 'pyconfig.h') def get_makefile_filename(): @@ -461,17 +454,6 @@ if python_build: g['LDSHARED'] = g['BLDSHARED'] - elif get_python_version() < '2.1': - # The following two branches are for 1.5.2 compatibility. - if sys.platform == 'aix4': # what about AIX 3.x ? - # Linker script is in the config directory, not in Modules as the - # Makefile says. - python_lib = get_python_lib(standard_lib=1) - ld_so_aix = os.path.join(python_lib, 'config', 'ld_so_aix') - python_exp = os.path.join(python_lib, 'config', 'python.exp') - - g['LDSHARED'] = "%s %s -bI:%s" % (ld_so_aix, g['CC'], python_exp) - global _config_vars _config_vars = g diff --git a/Lib/distutils/tests/test_build_ext.py b/Lib/distutils/tests/test_build_ext.py --- a/Lib/distutils/tests/test_build_ext.py +++ b/Lib/distutils/tests/test_build_ext.py @@ -31,12 +31,11 @@ self.tmp_dir = self.mkdtemp() self.sys_path = sys.path, sys.path[:] sys.path.append(self.tmp_dir) - if sys.version > "2.6": - import site - self.old_user_base = site.USER_BASE - site.USER_BASE = self.mkdtemp() - from distutils.command import build_ext - build_ext.USER_BASE = site.USER_BASE + import site + self.old_user_base = site.USER_BASE + site.USER_BASE = self.mkdtemp() + from distutils.command import build_ext + build_ext.USER_BASE = site.USER_BASE def test_build_ext(self): global ALREADY_TESTED @@ -84,11 +83,10 @@ support.unload('xx') sys.path = self.sys_path[0] sys.path[:] = self.sys_path[1] - if sys.version > "2.6": - import site - site.USER_BASE = self.old_user_base - from distutils.command import build_ext - build_ext.USER_BASE = self.old_user_base + import site + site.USER_BASE = self.old_user_base + from distutils.command import build_ext + build_ext.USER_BASE = self.old_user_base super(BuildExtTestCase, self).tearDown() def test_solaris_enable_shared(self): -- Repository URL: http://hg.python.org/cpython From solipsis at pitrou.net Sun Sep 7 10:40:30 2014 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Sun, 07 Sep 2014 10:40:30 +0200 Subject: [Python-checkins] Daily reference leaks (688701337b1a): sum=151928 Message-ID: results for 688701337b1a on branch "default" -------------------------------------------- test_codecs leaked [5825, 5825, 5825] references, sum=17475 test_codecs leaked [1172, 1174, 1174] memory blocks, sum=3520 test_collections leaked [0, -2, 0] references, sum=-2 test_distutils leaked [37735, 37735, 37735] references, sum=113205 test_distutils leaked [5909, 5911, 5911] memory blocks, sum=17731 test_functools leaked [0, 0, 3] memory blocks, sum=3 test_site leaked [0, -2, 0] references, sum=-2 test_site leaked [0, -2, 0] memory blocks, sum=-2 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogummiCi', '-x'] From python-checkins at python.org Sun Sep 7 14:40:49 2014 From: python-checkins at python.org (larry.hastings) Date: Sun, 7 Sep 2014 14:40:49 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_Updated_3=2E4_release_schedul?= =?utf-8?q?e_for_all_extant_and_planned_releases=2E?= Message-ID: <3hrXKY0lM3z7LlJ@mail.python.org> http://hg.python.org/peps/rev/a20a9f37945f changeset: 5548:a20a9f37945f user: Larry Hastings date: Sun Sep 07 05:41:05 2014 -0700 summary: Updated 3.4 release schedule for all extant and planned releases. files: pep-0429.txt | 12 ++++++++++++ 1 files changed, 12 insertions(+), 0 deletions(-) diff --git a/pep-0429.txt b/pep-0429.txt --- a/pep-0429.txt +++ b/pep-0429.txt @@ -50,6 +50,18 @@ (Beta 1 was also "feature freeze"--no new features beyond this point.) +3.4.1 schedule +-------------- + +- 3.4.1 candidate 1: May 5, 2013 +- 3.4.1 final: May 18, 2013 + +3.4.2 schedule +-------------- + +- 3.4.2 candidate 1: May 5, 2013 +- 3.4.2 final: May 18, 2013 + Features for 3.4 -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Mon Sep 8 03:35:51 2014 From: python-checkins at python.org (larry.hastings) Date: Mon, 8 Sep 2014 03:35:51 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_Was_there_a_single_thing_in_m?= =?utf-8?q?y_3=2E4=2E1_/_3=2E4=2E2_schedules_I_didn=27t_screw_up=3F__Nope!?= Message-ID: <3hrsWq6mtWz7Ljn@mail.python.org> http://hg.python.org/peps/rev/a49c7ee793fa changeset: 5549:a49c7ee793fa user: Larry Hastings date: Sun Sep 07 18:36:08 2014 -0700 summary: Was there a single thing in my 3.4.1 / 3.4.2 schedules I didn't screw up? Nope! files: pep-0429.txt | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pep-0429.txt b/pep-0429.txt --- a/pep-0429.txt +++ b/pep-0429.txt @@ -53,14 +53,14 @@ 3.4.1 schedule -------------- -- 3.4.1 candidate 1: May 5, 2013 -- 3.4.1 final: May 18, 2013 +- 3.4.1 candidate 1: May 5, 2014 +- 3.4.1 final: May 18, 2014 3.4.2 schedule -------------- -- 3.4.2 candidate 1: May 5, 2013 -- 3.4.2 final: May 18, 2013 +- 3.4.2 candidate 1: September 15, 2014 +- 3.4.2 final: September 28, 2014 -- Repository URL: http://hg.python.org/peps From solipsis at pitrou.net Mon Sep 8 10:40:41 2014 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Mon, 08 Sep 2014 10:40:41 +0200 Subject: [Python-checkins] Daily reference leaks (688701337b1a): sum=151976 Message-ID: results for 688701337b1a on branch "default" -------------------------------------------- test_codecs leaked [5825, 5825, 5825] references, sum=17475 test_codecs leaked [1172, 1174, 1174] memory blocks, sum=3520 test_collections leaked [-2, -4, 0] references, sum=-6 test_collections leaked [-1, -2, 0] memory blocks, sum=-3 test_distutils leaked [37735, 37735, 37735] references, sum=113205 test_distutils leaked [5909, 5911, 5911] memory blocks, sum=17731 test_functools leaked [0, 0, 3] memory blocks, sum=3 test_multiprocessing_spawn leaked [0, 0, 38] references, sum=38 test_multiprocessing_spawn leaked [0, 0, 17] memory blocks, sum=17 test_site leaked [-2, 2, -2] references, sum=-2 test_site leaked [-2, 2, -2] memory blocks, sum=-2 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflog1YOlaw', '-x'] From python-checkins at python.org Mon Sep 8 20:24:28 2014 From: python-checkins at python.org (barry.warsaw) Date: Mon, 8 Sep 2014 20:24:28 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_-_Issue_=2316662=3A_load?= =?utf-8?q?=5Ftests=28=29_is_now_unconditionally_run_when_it_is_present_in?= Message-ID: <3hsHvc0BmGz7Ljb@mail.python.org> http://hg.python.org/cpython/rev/d0ff527c53da changeset: 92377:d0ff527c53da user: Barry Warsaw date: Mon Sep 08 14:21:37 2014 -0400 summary: - Issue #16662: load_tests() is now unconditionally run when it is present in a package's __init__.py. TestLoader.loadTestsFromModule() still accepts use_load_tests, but it is deprecated and ignored. A new keyword-only attribute `pattern` is added and documented. Patch given by Robert Collins, tweaked by Barry Warsaw. files: Doc/library/unittest.rst | 71 ++- Lib/unittest/loader.py | 53 ++- Lib/unittest/test/test_discovery.py | 254 +++++++++++++++- Lib/unittest/test/test_loader.py | 153 +++++++++- Misc/NEWS | 6 + 5 files changed, 474 insertions(+), 63 deletions(-) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1561,7 +1561,7 @@ :class:`testCaseClass`. - .. method:: loadTestsFromModule(module) + .. method:: loadTestsFromModule(module, pattern=None) Return a suite of all tests cases contained in the given module. This method searches *module* for classes derived from :class:`TestCase` and @@ -1578,11 +1578,18 @@ If a module provides a ``load_tests`` function it will be called to load the tests. This allows modules to customize test loading. - This is the `load_tests protocol`_. + This is the `load_tests protocol`_. The *pattern* argument is passed as + the third argument to ``load_tests``. .. versionchanged:: 3.2 Support for ``load_tests`` added. + .. versionchanged:: 3.5 + The undocumented and unofficial *use_load_tests* default argument is + deprecated and ignored, although it is still accepted for backward + compatibility. The method also now accepts a keyword-only argument + *pattern* which is passed to ``load_tests`` as the third argument. + .. method:: loadTestsFromName(name, module=None) @@ -1634,18 +1641,18 @@ the start directory is not the top level directory then the top level directory must be specified separately. - If importing a module fails, for example due to a syntax error, then this - will be recorded as a single error and discovery will continue. If the - import failure is due to :exc:`SkipTest` being raised, it will be recorded - as a skip instead of an error. - - If a test package name (directory with :file:`__init__.py`) matches the - pattern then the package will be checked for a ``load_tests`` - function. If this exists then it will be called with *loader*, *tests*, - *pattern*. - - If load_tests exists then discovery does *not* recurse into the package, - ``load_tests`` is responsible for loading all tests in the package. + If importing a module fails, for example due to a syntax error, then + this will be recorded as a single error and discovery will continue. If + the import failure is due to :exc:`SkipTest` being raised, it will be + recorded as a skip instead of an error. + + If a package (a directory containing a file named :file:`__init__.py`) is + found, the package will be checked for a ``load_tests`` function. If this + exists then it will be called with *loader*, *tests*, *pattern*. + + If ``load_tests`` exists then discovery does *not* recurse into the + package, ``load_tests`` is responsible for loading all tests in the + package. The pattern is deliberately not stored as a loader attribute so that packages can continue discovery themselves. *top_level_dir* is stored so @@ -1664,6 +1671,11 @@ the same even if the underlying file system's ordering is not dependent on file name. + .. versionchanged:: 3.5 + Found packages are now checked for ``load_tests`` regardless of + whether their path matches *pattern*, because it is impossible for + a package name to match the default pattern. + The following attributes of a :class:`TestLoader` can be configured either by subclassing or assignment on an instance: @@ -2032,7 +2044,10 @@ If a test module defines ``load_tests`` it will be called by :meth:`TestLoader.loadTestsFromModule` with the following arguments:: - load_tests(loader, standard_tests, None) + load_tests(loader, standard_tests, pattern) + +where *pattern* is passed straight through from ``loadTestsFromModule``. It +defaults to ``None``. It should return a :class:`TestSuite`. @@ -2054,21 +2069,12 @@ suite.addTests(tests) return suite -If discovery is started, either from the command line or by calling -:meth:`TestLoader.discover`, with a pattern that matches a package -name then the package :file:`__init__.py` will be checked for ``load_tests``. - -.. note:: - - The default pattern is ``'test*.py'``. This matches all Python files - that start with ``'test'`` but *won't* match any test directories. - - A pattern like ``'test*'`` will match test packages as well as - modules. - -If the package :file:`__init__.py` defines ``load_tests`` then it will be -called and discovery not continued into the package. ``load_tests`` -is called with the following arguments:: +If discovery is started in a directory containing a package, either from the +command line or by calling :meth:`TestLoader.discover`, then the package +:file:`__init__.py` will be checked for ``load_tests``. If that function does +not exist, discovery will recurse into the package as though it were just +another directory. Otherwise, discovery of the package's tests will be left up +to ``load_tests`` which is called with the following arguments:: load_tests(loader, standard_tests, pattern) @@ -2087,6 +2093,11 @@ standard_tests.addTests(package_tests) return standard_tests +.. versionchanged:: 3.5 + Discovery no longer checks package names for matching *pattern* due to the + impossibility of package names matching the default pattern. + + Class and Module Fixtures ------------------------- diff --git a/Lib/unittest/loader.py b/Lib/unittest/loader.py --- a/Lib/unittest/loader.py +++ b/Lib/unittest/loader.py @@ -6,6 +6,7 @@ import traceback import types import functools +import warnings from fnmatch import fnmatch @@ -70,8 +71,27 @@ loaded_suite = self.suiteClass(map(testCaseClass, testCaseNames)) return loaded_suite - def loadTestsFromModule(self, module, use_load_tests=True): + # XXX After Python 3.5, remove backward compatibility hacks for + # use_load_tests deprecation via *args and **kws. See issue 16662. + def loadTestsFromModule(self, module, *args, pattern=None, **kws): """Return a suite of all tests cases contained in the given module""" + # This method used to take an undocumented and unofficial + # use_load_tests argument. For backward compatibility, we still + # accept the argument (which can also be the first position) but we + # ignore it and issue a deprecation warning if it's present. + if len(args) == 1 or 'use_load_tests' in kws: + warnings.warn('use_load_tests is deprecated and ignored', + DeprecationWarning) + kws.pop('use_load_tests', None) + if len(args) > 1: + raise TypeError('loadTestsFromModule() takes 1 positional argument but {} were given'.format(len(args))) + if len(kws) != 0: + # Since the keyword arguments are unsorted (see PEP 468), just + # pick the alphabetically sorted first argument to complain about, + # if multiple were given. At least the error message will be + # predictable. + complaint = sorted(kws)[0] + raise TypeError("loadTestsFromModule() got an unexpected keyword argument '{}'".format(complaint)) tests = [] for name in dir(module): obj = getattr(module, name) @@ -80,9 +100,9 @@ load_tests = getattr(module, 'load_tests', None) tests = self.suiteClass(tests) - if use_load_tests and load_tests is not None: + if load_tests is not None: try: - return load_tests(self, tests, None) + return load_tests(self, tests, pattern) except Exception as e: return _make_failed_load_tests(module.__name__, e, self.suiteClass) @@ -325,7 +345,7 @@ msg = ("%r module incorrectly imported from %r. Expected %r. " "Is this module globally installed?") raise ImportError(msg % (mod_name, module_dir, expected_dir)) - yield self.loadTestsFromModule(module) + yield self.loadTestsFromModule(module, pattern=pattern) elif os.path.isdir(full_path): if (not namespace and not os.path.isfile(os.path.join(full_path, '__init__.py'))): @@ -333,26 +353,27 @@ load_tests = None tests = None - if fnmatch(path, pattern): - # only check load_tests if the package directory itself matches the filter - name = self._get_name_from_path(full_path) + name = self._get_name_from_path(full_path) + try: package = self._get_module_from_name(name) + except case.SkipTest as e: + yield _make_skipped_test(name, e, self.suiteClass) + except: + yield _make_failed_import_test(name, self.suiteClass) + else: load_tests = getattr(package, 'load_tests', None) - tests = self.loadTestsFromModule(package, use_load_tests=False) - - if load_tests is None: + tests = self.loadTestsFromModule(package, pattern=pattern) if tests is not None: # tests loaded from package file yield tests + + if load_tests is not None: + # loadTestsFromModule(package) has load_tests for us. + continue # recurse into the package yield from self._find_tests(full_path, pattern, namespace=namespace) - else: - try: - yield load_tests(self, tests, pattern) - except Exception as e: - yield _make_failed_load_tests(package.__name__, e, - self.suiteClass) + defaultTestLoader = TestLoader() diff --git a/Lib/unittest/test/test_discovery.py b/Lib/unittest/test/test_discovery.py --- a/Lib/unittest/test/test_discovery.py +++ b/Lib/unittest/test/test_discovery.py @@ -68,7 +68,13 @@ self.addCleanup(restore_isfile) loader._get_module_from_name = lambda path: path + ' module' - loader.loadTestsFromModule = lambda module: module + ' tests' + orig_load_tests = loader.loadTestsFromModule + def loadTestsFromModule(module, pattern=None): + # This is where load_tests is called. + base = orig_load_tests(module, pattern=pattern) + return base + [module + ' tests'] + loader.loadTestsFromModule = loadTestsFromModule + loader.suiteClass = lambda thing: thing top_level = os.path.abspath('/foo') loader._top_level_dir = top_level @@ -76,9 +82,9 @@ # The test suites found should be sorted alphabetically for reliable # execution order. - expected = [name + ' module tests' for name in - ('test1', 'test2')] - expected.extend([('test_dir.%s' % name) + ' module tests' for name in + expected = [[name + ' module tests'] for name in + ('test1', 'test2', 'test_dir')] + expected.extend([[('test_dir.%s' % name) + ' module tests'] for name in ('test3', 'test4')]) self.assertEqual(suite, expected) @@ -116,34 +122,204 @@ if os.path.basename(path) == 'test_directory': def load_tests(loader, tests, pattern): self.load_tests_args.append((loader, tests, pattern)) - return 'load_tests' + return [self.path + ' load_tests'] self.load_tests = load_tests def __eq__(self, other): return self.path == other.path loader._get_module_from_name = lambda name: Module(name) - def loadTestsFromModule(module, use_load_tests): - if use_load_tests: - raise self.failureException('use_load_tests should be False for packages') - return module.path + ' module tests' + orig_load_tests = loader.loadTestsFromModule + def loadTestsFromModule(module, pattern=None): + # This is where load_tests is called. + base = orig_load_tests(module, pattern=pattern) + return base + [module.path + ' module tests'] loader.loadTestsFromModule = loadTestsFromModule + loader.suiteClass = lambda thing: thing loader._top_level_dir = '/foo' # this time no '.py' on the pattern so that it can match # a test package suite = list(loader._find_tests('/foo', 'test*')) - # We should have loaded tests from the test_directory package by calling load_tests - # and directly from the test_directory2 package + # We should have loaded tests from the a_directory and test_directory2 + # directly and via load_tests for the test_directory package, which + # still calls the baseline module loader. self.assertEqual(suite, - ['load_tests', 'test_directory2' + ' module tests']) + [['a_directory module tests'], + ['test_directory load_tests', + 'test_directory module tests'], + ['test_directory2 module tests']]) + + # The test module paths should be sorted for reliable execution order - self.assertEqual(Module.paths, ['test_directory', 'test_directory2']) + self.assertEqual(Module.paths, + ['a_directory', 'test_directory', 'test_directory2']) + + # load_tests should have been called once with loader, tests and pattern + # (but there are no tests in our stub module itself, so thats [] at the + # time of call. + self.assertEqual(Module.load_tests_args, + [(loader, [], 'test*')]) + + def test_find_tests_default_calls_package_load_tests(self): + loader = unittest.TestLoader() + + original_listdir = os.listdir + def restore_listdir(): + os.listdir = original_listdir + original_isfile = os.path.isfile + def restore_isfile(): + os.path.isfile = original_isfile + original_isdir = os.path.isdir + def restore_isdir(): + os.path.isdir = original_isdir + + directories = ['a_directory', 'test_directory', 'test_directory2'] + path_lists = [directories, [], [], []] + os.listdir = lambda path: path_lists.pop(0) + self.addCleanup(restore_listdir) + + os.path.isdir = lambda path: True + self.addCleanup(restore_isdir) + + os.path.isfile = lambda path: os.path.basename(path) not in directories + self.addCleanup(restore_isfile) + + class Module(object): + paths = [] + load_tests_args = [] + + def __init__(self, path): + self.path = path + self.paths.append(path) + if os.path.basename(path) == 'test_directory': + def load_tests(loader, tests, pattern): + self.load_tests_args.append((loader, tests, pattern)) + return [self.path + ' load_tests'] + self.load_tests = load_tests + + def __eq__(self, other): + return self.path == other.path + + loader._get_module_from_name = lambda name: Module(name) + orig_load_tests = loader.loadTestsFromModule + def loadTestsFromModule(module, pattern=None): + # This is where load_tests is called. + base = orig_load_tests(module, pattern=pattern) + return base + [module.path + ' module tests'] + loader.loadTestsFromModule = loadTestsFromModule + loader.suiteClass = lambda thing: thing + + loader._top_level_dir = '/foo' + # this time no '.py' on the pattern so that it can match + # a test package + suite = list(loader._find_tests('/foo', 'test*.py')) + + # We should have loaded tests from the a_directory and test_directory2 + # directly and via load_tests for the test_directory package, which + # still calls the baseline module loader. + self.assertEqual(suite, + [['a_directory module tests'], + ['test_directory load_tests', + 'test_directory module tests'], + ['test_directory2 module tests']]) + # The test module paths should be sorted for reliable execution order + self.assertEqual(Module.paths, + ['a_directory', 'test_directory', 'test_directory2']) + # load_tests should have been called once with loader, tests and pattern self.assertEqual(Module.load_tests_args, - [(loader, 'test_directory' + ' module tests', 'test*')]) + [(loader, [], 'test*.py')]) + + def test_find_tests_customise_via_package_pattern(self): + # This test uses the example 'do-nothing' load_tests from + # https://docs.python.org/3/library/unittest.html#load-tests-protocol + # to make sure that that actually works. + # Housekeeping + original_listdir = os.listdir + def restore_listdir(): + os.listdir = original_listdir + self.addCleanup(restore_listdir) + original_isfile = os.path.isfile + def restore_isfile(): + os.path.isfile = original_isfile + self.addCleanup(restore_isfile) + original_isdir = os.path.isdir + def restore_isdir(): + os.path.isdir = original_isdir + self.addCleanup(restore_isdir) + self.addCleanup(sys.path.remove, '/foo') + + # Test data: we expect the following: + # a listdir to find our package, and a isfile and isdir check on it. + # a module-from-name call to turn that into a module + # followed by load_tests. + # then our load_tests will call discover() which is messy + # but that finally chains into find_tests again for the child dir - + # which is why we don't have a infinite loop. + # We expect to see: + # the module load tests for both package and plain module called, + # and the plain module result nested by the package module load_tests + # indicating that it was processed and could have been mutated. + vfs = {'/foo': ['my_package'], + '/foo/my_package': ['__init__.py', 'test_module.py']} + def list_dir(path): + return list(vfs[path]) + os.listdir = list_dir + os.path.isdir = lambda path: not path.endswith('.py') + os.path.isfile = lambda path: path.endswith('.py') + + class Module(object): + paths = [] + load_tests_args = [] + + def __init__(self, path): + self.path = path + self.paths.append(path) + if path.endswith('test_module'): + def load_tests(loader, tests, pattern): + self.load_tests_args.append((loader, tests, pattern)) + return [self.path + ' load_tests'] + else: + def load_tests(loader, tests, pattern): + self.load_tests_args.append((loader, tests, pattern)) + # top level directory cached on loader instance + __file__ = '/foo/my_package/__init__.py' + this_dir = os.path.dirname(__file__) + pkg_tests = loader.discover( + start_dir=this_dir, pattern=pattern) + return [self.path + ' load_tests', tests + ] + pkg_tests + self.load_tests = load_tests + + def __eq__(self, other): + return self.path == other.path + + loader = unittest.TestLoader() + loader._get_module_from_name = lambda name: Module(name) + loader.suiteClass = lambda thing: thing + + loader._top_level_dir = '/foo' + # this time no '.py' on the pattern so that it can match + # a test package + suite = list(loader._find_tests('/foo', 'test*.py')) + + # We should have loaded tests from both my_package and + # my_pacakge.test_module, and also run the load_tests hook in both. + # (normally this would be nested TestSuites.) + self.assertEqual(suite, + [['my_package load_tests', [], + ['my_package.test_module load_tests']]]) + # Parents before children. + self.assertEqual(Module.paths, + ['my_package', 'my_package.test_module']) + + # load_tests should have been called twice with loader, tests and pattern + self.assertEqual(Module.load_tests_args, + [(loader, [], 'test*.py'), + (loader, [], 'test*.py')]) def test_discover(self): loader = unittest.TestLoader() @@ -203,6 +379,17 @@ sys.path[:] = orig_sys_path self.addCleanup(restore) + def setup_import_issue_package_tests(self, vfs): + self.addCleanup(setattr, os, 'listdir', os.listdir) + self.addCleanup(setattr, os.path, 'isfile', os.path.isfile) + self.addCleanup(setattr, os.path, 'isdir', os.path.isdir) + self.addCleanup(sys.path.__setitem__, slice(None), list(sys.path)) + def list_dir(path): + return list(vfs[path]) + os.listdir = list_dir + os.path.isdir = lambda path: not path.endswith('.py') + os.path.isfile = lambda path: path.endswith('.py') + def test_discover_with_modules_that_fail_to_import(self): loader = unittest.TestLoader() @@ -216,6 +403,25 @@ with self.assertRaises(ImportError): test.test_this_does_not_exist() + def test_discover_with_init_modules_that_fail_to_import(self): + vfs = {'/foo': ['my_package'], + '/foo/my_package': ['__init__.py', 'test_module.py']} + self.setup_import_issue_package_tests(vfs) + import_calls = [] + def _get_module_from_name(name): + import_calls.append(name) + raise ImportError("Cannot import Name") + loader = unittest.TestLoader() + loader._get_module_from_name = _get_module_from_name + suite = loader.discover('/foo') + + self.assertIn('/foo', sys.path) + self.assertEqual(suite.countTestCases(), 1) + test = list(list(suite)[0])[0] # extract test from suite + with self.assertRaises(ImportError): + test.my_package() + self.assertEqual(import_calls, ['my_package']) + def test_discover_with_module_that_raises_SkipTest_on_import(self): loader = unittest.TestLoader() @@ -232,6 +438,26 @@ suite.run(result) self.assertEqual(len(result.skipped), 1) + def test_discover_with_init_module_that_raises_SkipTest_on_import(self): + vfs = {'/foo': ['my_package'], + '/foo/my_package': ['__init__.py', 'test_module.py']} + self.setup_import_issue_package_tests(vfs) + import_calls = [] + def _get_module_from_name(name): + import_calls.append(name) + raise unittest.SkipTest('skipperoo') + loader = unittest.TestLoader() + loader._get_module_from_name = _get_module_from_name + suite = loader.discover('/foo') + + self.assertIn('/foo', sys.path) + self.assertEqual(suite.countTestCases(), 1) + result = unittest.TestResult() + suite.run(result) + self.assertEqual(len(result.skipped), 1) + self.assertEqual(result.testsRun, 1) + self.assertEqual(import_calls, ['my_package']) + def test_command_line_handling_parseArgs(self): program = TestableTestProgram() diff --git a/Lib/unittest/test/test_loader.py b/Lib/unittest/test/test_loader.py --- a/Lib/unittest/test/test_loader.py +++ b/Lib/unittest/test/test_loader.py @@ -1,9 +1,26 @@ import sys import types - +import warnings import unittest +# Decorator used in the deprecation tests to reset the warning registry for +# test isolation and reproducibility. +def warningregistry(func): + def wrapper(*args, **kws): + missing = object() + saved = getattr(warnings, '__warningregistry__', missing).copy() + try: + return func(*args, **kws) + finally: + if saved is missing: + try: + del warnings.__warningregistry__ + except AttributeError: + pass + else: + warnings.__warningregistry__ = saved + class Test_TestLoader(unittest.TestCase): @@ -150,6 +167,7 @@ # Check that loadTestsFromModule honors (or not) a module # with a load_tests function. + @warningregistry def test_loadTestsFromModule__load_tests(self): m = types.ModuleType('m') class MyTestCase(unittest.TestCase): @@ -168,10 +186,139 @@ suite = loader.loadTestsFromModule(m) self.assertIsInstance(suite, unittest.TestSuite) self.assertEqual(load_tests_args, [loader, suite, None]) + # With Python 3.5, the undocumented and unofficial use_load_tests is + # ignored (and deprecated). + load_tests_args = [] + with warnings.catch_warnings(record=False): + warnings.simplefilter('never') + suite = loader.loadTestsFromModule(m, use_load_tests=False) + self.assertEqual(load_tests_args, [loader, suite, None]) + + @warningregistry + def test_loadTestsFromModule__use_load_tests_deprecated_positional(self): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + m = types.ModuleType('m') + class MyTestCase(unittest.TestCase): + def test(self): + pass + m.testcase_1 = MyTestCase + + load_tests_args = [] + def load_tests(loader, tests, pattern): + self.assertIsInstance(tests, unittest.TestSuite) + load_tests_args.extend((loader, tests, pattern)) + return tests + m.load_tests = load_tests + # The method still works. + loader = unittest.TestLoader() + # use_load_tests=True as a positional argument. + suite = loader.loadTestsFromModule(m, False) + self.assertIsInstance(suite, unittest.TestSuite) + # load_tests was still called because use_load_tests is deprecated + # and ignored. + self.assertEqual(load_tests_args, [loader, suite, None]) + # We got a warning. + self.assertIs(w[-1].category, DeprecationWarning) + self.assertEqual(str(w[-1].message), + 'use_load_tests is deprecated and ignored') + + @warningregistry + def test_loadTestsFromModule__use_load_tests_deprecated_keyword(self): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + m = types.ModuleType('m') + class MyTestCase(unittest.TestCase): + def test(self): + pass + m.testcase_1 = MyTestCase + + load_tests_args = [] + def load_tests(loader, tests, pattern): + self.assertIsInstance(tests, unittest.TestSuite) + load_tests_args.extend((loader, tests, pattern)) + return tests + m.load_tests = load_tests + # The method still works. + loader = unittest.TestLoader() + suite = loader.loadTestsFromModule(m, use_load_tests=False) + self.assertIsInstance(suite, unittest.TestSuite) + # load_tests was still called because use_load_tests is deprecated + # and ignored. + self.assertEqual(load_tests_args, [loader, suite, None]) + # We got a warning. + self.assertIs(w[-1].category, DeprecationWarning) + self.assertEqual(str(w[-1].message), + 'use_load_tests is deprecated and ignored') + + def test_loadTestsFromModule__too_many_positional_args(self): + m = types.ModuleType('m') + class MyTestCase(unittest.TestCase): + def test(self): + pass + m.testcase_1 = MyTestCase load_tests_args = [] - suite = loader.loadTestsFromModule(m, use_load_tests=False) - self.assertEqual(load_tests_args, []) + def load_tests(loader, tests, pattern): + self.assertIsInstance(tests, unittest.TestSuite) + load_tests_args.extend((loader, tests, pattern)) + return tests + m.load_tests = load_tests + loader = unittest.TestLoader() + with self.assertRaises(TypeError) as cm: + loader.loadTestsFromModule(m, False, 'testme.*') + self.assertEqual(type(cm.exception), TypeError) + # The error message names the first bad argument alphabetically, + # however use_load_tests (which sorts first) is ignored. + self.assertEqual( + str(cm.exception), + 'loadTestsFromModule() takes 1 positional argument but 2 were given') + + @warningregistry + def test_loadTestsFromModule__use_load_tests_other_bad_keyword(self): + m = types.ModuleType('m') + class MyTestCase(unittest.TestCase): + def test(self): + pass + m.testcase_1 = MyTestCase + + load_tests_args = [] + def load_tests(loader, tests, pattern): + self.assertIsInstance(tests, unittest.TestSuite) + load_tests_args.extend((loader, tests, pattern)) + return tests + m.load_tests = load_tests + loader = unittest.TestLoader() + with warnings.catch_warnings(): + warnings.simplefilter('never') + with self.assertRaises(TypeError) as cm: + loader.loadTestsFromModule( + m, use_load_tests=False, very_bad=True, worse=False) + self.assertEqual(type(cm.exception), TypeError) + # The error message names the first bad argument alphabetically, + # however use_load_tests (which sorts first) is ignored. + self.assertEqual( + str(cm.exception), + "loadTestsFromModule() got an unexpected keyword argument 'very_bad'") + + def test_loadTestsFromModule__pattern(self): + m = types.ModuleType('m') + class MyTestCase(unittest.TestCase): + def test(self): + pass + m.testcase_1 = MyTestCase + + load_tests_args = [] + def load_tests(loader, tests, pattern): + self.assertIsInstance(tests, unittest.TestSuite) + load_tests_args.extend((loader, tests, pattern)) + return tests + m.load_tests = load_tests + + loader = unittest.TestLoader() + suite = loader.loadTestsFromModule(m, pattern='testme.*') + self.assertIsInstance(suite, unittest.TestSuite) + self.assertEqual(load_tests_args, [loader, suite, 'testme.*']) def test_loadTestsFromModule__faulty_load_tests(self): m = types.ModuleType('m') diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -132,6 +132,12 @@ Library ------- +- Issue #16662: load_tests() is now unconditionally run when it is present in + a package's __init__.py. TestLoader.loadTestsFromModule() still accepts + use_load_tests, but it is deprecated and ignored. A new keyword-only + attribute `pattern` is added and documented. Patch given by Robert Collins, + tweaked by Barry Warsaw. + - Issue #22226: First letter no longer is stripped from the "status" key in the result of Treeview.heading(). -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Mon Sep 8 23:29:10 2014 From: python-checkins at python.org (barry.warsaw) Date: Mon, 8 Sep 2014 23:29:10 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_A_few_tweaks_for_issue1666?= =?utf-8?q?2_based_on_feedback_from_Robert_Collins=2E?= Message-ID: <3hsN0k6yJPz7LkJ@mail.python.org> http://hg.python.org/cpython/rev/92b292d68104 changeset: 92378:92b292d68104 user: Barry Warsaw date: Mon Sep 08 17:29:02 2014 -0400 summary: A few tweaks for issue16662 based on feedback from Robert Collins. files: Lib/unittest/loader.py | 7 +- Lib/unittest/test/test_loader.py | 77 ++++++++++--------- 2 files changed, 46 insertions(+), 38 deletions(-) diff --git a/Lib/unittest/loader.py b/Lib/unittest/loader.py --- a/Lib/unittest/loader.py +++ b/Lib/unittest/loader.py @@ -79,12 +79,15 @@ # use_load_tests argument. For backward compatibility, we still # accept the argument (which can also be the first position) but we # ignore it and issue a deprecation warning if it's present. - if len(args) == 1 or 'use_load_tests' in kws: + if len(args) > 0 or 'use_load_tests' in kws: warnings.warn('use_load_tests is deprecated and ignored', DeprecationWarning) kws.pop('use_load_tests', None) if len(args) > 1: - raise TypeError('loadTestsFromModule() takes 1 positional argument but {} were given'.format(len(args))) + # Complain about the number of arguments, but don't forget the + # required `module` argument. + complaint = len(args) + 1 + raise TypeError('loadTestsFromModule() takes 1 positional argument but {} were given'.format(complaint)) if len(kws) != 0: # Since the keyword arguments are unsorted (see PEP 468), just # pick the alphabetically sorted first argument to complain about, diff --git a/Lib/unittest/test/test_loader.py b/Lib/unittest/test/test_loader.py --- a/Lib/unittest/test/test_loader.py +++ b/Lib/unittest/test/test_loader.py @@ -196,23 +196,23 @@ @warningregistry def test_loadTestsFromModule__use_load_tests_deprecated_positional(self): + m = types.ModuleType('m') + class MyTestCase(unittest.TestCase): + def test(self): + pass + m.testcase_1 = MyTestCase + + load_tests_args = [] + def load_tests(loader, tests, pattern): + self.assertIsInstance(tests, unittest.TestSuite) + load_tests_args.extend((loader, tests, pattern)) + return tests + m.load_tests = load_tests + # The method still works. + loader = unittest.TestLoader() + # use_load_tests=True as a positional argument. with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') - m = types.ModuleType('m') - class MyTestCase(unittest.TestCase): - def test(self): - pass - m.testcase_1 = MyTestCase - - load_tests_args = [] - def load_tests(loader, tests, pattern): - self.assertIsInstance(tests, unittest.TestSuite) - load_tests_args.extend((loader, tests, pattern)) - return tests - m.load_tests = load_tests - # The method still works. - loader = unittest.TestLoader() - # use_load_tests=True as a positional argument. suite = loader.loadTestsFromModule(m, False) self.assertIsInstance(suite, unittest.TestSuite) # load_tests was still called because use_load_tests is deprecated @@ -225,22 +225,22 @@ @warningregistry def test_loadTestsFromModule__use_load_tests_deprecated_keyword(self): + m = types.ModuleType('m') + class MyTestCase(unittest.TestCase): + def test(self): + pass + m.testcase_1 = MyTestCase + + load_tests_args = [] + def load_tests(loader, tests, pattern): + self.assertIsInstance(tests, unittest.TestSuite) + load_tests_args.extend((loader, tests, pattern)) + return tests + m.load_tests = load_tests + # The method still works. + loader = unittest.TestLoader() with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') - m = types.ModuleType('m') - class MyTestCase(unittest.TestCase): - def test(self): - pass - m.testcase_1 = MyTestCase - - load_tests_args = [] - def load_tests(loader, tests, pattern): - self.assertIsInstance(tests, unittest.TestSuite) - load_tests_args.extend((loader, tests, pattern)) - return tests - m.load_tests = load_tests - # The method still works. - loader = unittest.TestLoader() suite = loader.loadTestsFromModule(m, use_load_tests=False) self.assertIsInstance(suite, unittest.TestSuite) # load_tests was still called because use_load_tests is deprecated @@ -251,6 +251,7 @@ self.assertEqual(str(w[-1].message), 'use_load_tests is deprecated and ignored') + @warningregistry def test_loadTestsFromModule__too_many_positional_args(self): m = types.ModuleType('m') class MyTestCase(unittest.TestCase): @@ -265,14 +266,18 @@ return tests m.load_tests = load_tests loader = unittest.TestLoader() - with self.assertRaises(TypeError) as cm: + with self.assertRaises(TypeError) as cm, \ + warnings.catch_warning(record=True) as w: loader.loadTestsFromModule(m, False, 'testme.*') - self.assertEqual(type(cm.exception), TypeError) - # The error message names the first bad argument alphabetically, - # however use_load_tests (which sorts first) is ignored. - self.assertEqual( - str(cm.exception), - 'loadTestsFromModule() takes 1 positional argument but 2 were given') + # We still got the deprecation warning. + self.assertIs(w[-1].category, DeprecationWarning) + self.assertEqual(str(w[-1].message), + 'use_load_tests is deprecated and ignored') + # We also got a TypeError for too many positional arguments. + self.assertEqual(type(cm.exception), TypeError) + self.assertEqual( + str(cm.exception), + 'loadTestsFromModule() takes 1 positional argument but 3 were given') @warningregistry def test_loadTestsFromModule__use_load_tests_other_bad_keyword(self): -- Repository URL: http://hg.python.org/cpython From solipsis at pitrou.net Tue Sep 9 09:38:27 2014 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Tue, 09 Sep 2014 09:38:27 +0200 Subject: [Python-checkins] Daily reference leaks (92b292d68104): sum=151934 Message-ID: results for 92b292d68104 on branch "default" -------------------------------------------- test_codecs leaked [5825, 5825, 5825] references, sum=17475 test_codecs leaked [1172, 1174, 1174] memory blocks, sum=3520 test_distutils leaked [37735, 37735, 37735] references, sum=113205 test_distutils leaked [5909, 5911, 5911] memory blocks, sum=17731 test_functools leaked [0, 0, 3] memory blocks, sum=3 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogIMQ681', '-x'] From python-checkins at python.org Tue Sep 9 17:47:15 2014 From: python-checkins at python.org (benjamin.peterson) Date: Tue, 9 Sep 2014 17:47:15 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_show_how_to_purge_CDN?= Message-ID: <3hsrMl2ht0z7Ljc@mail.python.org> http://hg.python.org/peps/rev/b8b1cad3aacd changeset: 5550:b8b1cad3aacd user: Benjamin Peterson date: Tue Sep 09 11:47:10 2014 -0400 summary: show how to purge CDN files: pep-0101.txt | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diff --git a/pep-0101.txt b/pep-0101.txt --- a/pep-0101.txt +++ b/pep-0101.txt @@ -439,6 +439,12 @@ checkout. The Doc's version_switcher.js script also needs to be updated. + ___ Note both the documentation and downloads are behind a caching CDN. If + you change archives after downloading them through the website, you'll + need to purge the stale data in the CDN like this: + + $ curl -X PURGE https://www.python.org/ftp/python/2.7.5/Python-2.7.5.tar.xz + ___ For the extra paranoid, do a completely clean test of the release. This includes downloading the tarball from www.python.org. -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Tue Sep 9 18:09:20 2014 From: python-checkins at python.org (serhiy.storchaka) Date: Tue, 9 Sep 2014 18:09:20 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_imaplib=2EIMAP4_now_suppor?= =?utf-8?q?ts_the_context_manager_protocol=2E?= Message-ID: <3hsrsD12PNz7LjR@mail.python.org> http://hg.python.org/cpython/rev/e1b1be597736 changeset: 92379:e1b1be597736 user: Serhiy Storchaka date: Tue Sep 09 19:07:49 2014 +0300 summary: imaplib.IMAP4 now supports the context manager protocol. Original patch by Tarek Ziad?. files: Doc/library/imaplib.rst | 13 ++++++++++ Doc/whatsnew/3.5.rst | 8 ++++++ Lib/imaplib.py | 8 ++++++ Lib/test/test_imaplib.py | 35 ++++++++++++++++++++++++++++ Misc/NEWS | 3 ++ 5 files changed, 67 insertions(+), 0 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -37,6 +37,19 @@ initialized. If *host* is not specified, ``''`` (the local host) is used. If *port* is omitted, the standard IMAP4 port (143) is used. + The :class:`IMAP4` class supports the :keyword:`with` statement. When used + like this, the IMAP4 ``LOGOUT`` command is issued automatically when the + :keyword:`with` statement exits. E.g.:: + + >>> from imaplib import IMAP4 + >>> with IMAP4("domain.org") as M: + ... M.noop() + ... + ('OK', [b'Nothing Accomplished. d25if65hy903weo.87']) + + .. versionchanged:: 3.5 + Support for the :keyword:`with` statement was added. + Three exceptions are defined as attributes of the :class:`IMAP4` class: diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -141,6 +141,14 @@ *module* contains no docstrings instead of raising :exc:`ValueError` (contributed by Glenn Jones in :issue:`15916`). +imaplib +------- + +* :class:`IMAP4` now supports the context management protocol. When used in a + :keyword:`with` statement, the IMAP4 ``LOGOUT`` command will be called + automatically at the end of the block. (Contributed by Tarek Ziad? and + Serhiy Storchaka in :issue:`4972`). + imghdr ------ diff --git a/Lib/imaplib.py b/Lib/imaplib.py --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -238,6 +238,14 @@ return getattr(self, attr.lower()) raise AttributeError("Unknown IMAP4 command: '%s'" % attr) + def __enter__(self): + return self + + def __exit__(self, *args): + try: + self.logout() + except OSError: + pass # Overridable methods diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -98,6 +98,10 @@ continuation = None capabilities = '' + def setup(self): + super().setup() + self.server.logged = None + def _send(self, message): if verbose: print("SENT: %r" % message.strip()) @@ -162,9 +166,14 @@ self._send_tagged(tag, 'OK', 'CAPABILITY completed') def cmd_LOGOUT(self, tag, args): + self.server.logged = None self._send_textline('* BYE IMAP4ref1 Server logging out') self._send_tagged(tag, 'OK', 'LOGOUT completed') + def cmd_LOGIN(self, tag, args): + self.server.logged = args[0] + self._send_tagged(tag, 'OK', 'LOGIN completed') + class ThreadedNetworkedTests(unittest.TestCase): server_class = socketserver.TCPServer @@ -345,6 +354,32 @@ self.assertRaises(imaplib.IMAP4.error, self.imap_class, *server.server_address) + @reap_threads + def test_simple_with_statement(self): + # simplest call + with self.reaped_server(SimpleIMAPHandler) as server: + with self.imap_class(*server.server_address): + pass + + @reap_threads + def test_with_statement(self): + with self.reaped_server(SimpleIMAPHandler) as server: + with self.imap_class(*server.server_address) as imap: + imap.login('user', 'pass') + self.assertEqual(server.logged, 'user') + self.assertIsNone(server.logged) + + @reap_threads + def test_with_statement_logout(self): + # what happens if already logout in the block? + with self.reaped_server(SimpleIMAPHandler) as server: + with self.imap_class(*server.server_address) as imap: + imap.login('user', 'pass') + self.assertEqual(server.logged, 'user') + imap.logout() + self.assertIsNone(server.logged) + self.assertIsNone(server.logged) + @unittest.skipUnless(ssl, "SSL not available") class ThreadedNetworkedTestsSSL(ThreadedNetworkedTests): diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -132,6 +132,9 @@ Library ------- +- Issue #12410: imaplib.IMAP4 now supports the context manager protocol. + Original patch by Tarek Ziad?. + - Issue #16662: load_tests() is now unconditionally run when it is present in a package's __init__.py. TestLoader.loadTestsFromModule() still accepts use_load_tests, but it is deprecated and ignored. A new keyword-only -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Tue Sep 9 19:59:10 2014 From: python-checkins at python.org (stefan.krah) Date: Tue, 9 Sep 2014 19:59:10 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzIyMjg0?= =?utf-8?b?OiBVcGRhdGUgZGVjaW1hbC5fX2FsbF9f?= Message-ID: <3hsvHy1VFNz7Ljc@mail.python.org> http://hg.python.org/cpython/rev/2b3dbbd2bd92 changeset: 92380:2b3dbbd2bd92 branch: 3.4 parent: 92375:7fece97e605d user: Stefan Krah date: Tue Sep 09 19:56:56 2014 +0200 summary: Issue #22284: Update decimal.__all__ files: Lib/decimal.py | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diff --git a/Lib/decimal.py b/Lib/decimal.py --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -116,6 +116,9 @@ # Two major classes 'Decimal', 'Context', + # Named tuple representation + 'DecimalTuple', + # Contexts 'DefaultContext', 'BasicContext', 'ExtendedContext', @@ -124,6 +127,9 @@ 'Inexact', 'Rounded', 'Subnormal', 'Overflow', 'Underflow', 'FloatOperation', + # Exceptional conditions that trigger InvalidOperation + 'DivisionImpossible', 'InvalidContext', 'ConversionSyntax', 'DivisionUndefined', + # Constants for use in setting up contexts 'ROUND_DOWN', 'ROUND_HALF_UP', 'ROUND_HALF_EVEN', 'ROUND_CEILING', 'ROUND_FLOOR', 'ROUND_UP', 'ROUND_HALF_DOWN', 'ROUND_05UP', -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Tue Sep 9 19:59:11 2014 From: python-checkins at python.org (stefan.krah) Date: Tue, 9 Sep 2014 19:59:11 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?q?=29=3A_Issue_=2322284=3A_Merge_3=2E4?= Message-ID: <3hsvHz49Lmz7Lks@mail.python.org> http://hg.python.org/cpython/rev/5bc23c111de1 changeset: 92381:5bc23c111de1 parent: 92379:e1b1be597736 parent: 92380:2b3dbbd2bd92 user: Stefan Krah date: Tue Sep 09 19:57:59 2014 +0200 summary: Issue #22284: Merge 3.4 files: Lib/decimal.py | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diff --git a/Lib/decimal.py b/Lib/decimal.py --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -116,6 +116,9 @@ # Two major classes 'Decimal', 'Context', + # Named tuple representation + 'DecimalTuple', + # Contexts 'DefaultContext', 'BasicContext', 'ExtendedContext', @@ -124,6 +127,9 @@ 'Inexact', 'Rounded', 'Subnormal', 'Overflow', 'Underflow', 'FloatOperation', + # Exceptional conditions that trigger InvalidOperation + 'DivisionImpossible', 'InvalidContext', 'ConversionSyntax', 'DivisionUndefined', + # Constants for use in setting up contexts 'ROUND_DOWN', 'ROUND_HALF_UP', 'ROUND_HALF_EVEN', 'ROUND_CEILING', 'ROUND_FLOOR', 'ROUND_UP', 'ROUND_HALF_DOWN', 'ROUND_05UP', -- Repository URL: http://hg.python.org/cpython From solipsis at pitrou.net Wed Sep 10 10:40:12 2014 From: solipsis at pitrou.net (solipsis at pitrou.net) Date: Wed, 10 Sep 2014 10:40:12 +0200 Subject: [Python-checkins] Daily reference leaks (5bc23c111de1): sum=151934 Message-ID: results for 5bc23c111de1 on branch "default" -------------------------------------------- test_codecs leaked [5825, 5825, 5825] references, sum=17475 test_codecs leaked [1172, 1174, 1174] memory blocks, sum=3520 test_distutils leaked [37735, 37735, 37735] references, sum=113205 test_distutils leaked [5909, 5911, 5911] memory blocks, sum=17731 test_functools leaked [0, 0, 3] memory blocks, sum=3 test_site leaked [2, 0, -2] references, sum=0 test_site leaked [2, 0, -2] memory blocks, sum=0 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogw4ILWE', '-x'] From python-checkins at python.org Wed Sep 10 18:01:22 2014 From: python-checkins at python.org (stefan.krah) Date: Wed, 10 Sep 2014 18:01:22 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Issue_=2319232=3A_Speed_up?= =?utf-8?q?_decimal_import=2E__Additionally=2C_since_=5Fdecimal_is?= Message-ID: <3htSdZ6Z4vz7Lk6@mail.python.org> http://hg.python.org/cpython/rev/8bf51cf94405 changeset: 92382:8bf51cf94405 user: Stefan Krah date: Wed Sep 10 17:58:15 2014 +0200 summary: Issue #19232: Speed up decimal import. Additionally, since _decimal is self-contained, this change facilitates maintenance and the Python version can be easily imported for experimentation. files: Lib/decimal.py | 17 +- Lib/decimal.py | 6406 +--------------- Lib/test/test_decimal.py | 4 +- Modules/_decimal/tests/deccheck.py | 7 +- 4 files changed, 14 insertions(+), 6420 deletions(-) diff --git a/Lib/decimal.py b/Lib/_pydecimal.py copy from Lib/decimal.py copy to Lib/_pydecimal.py --- a/Lib/decimal.py +++ b/Lib/_pydecimal.py @@ -144,6 +144,7 @@ 'HAVE_THREADS' ] +__name__ = 'decimal' # For pickling __version__ = '1.70' # Highest version of the spec this complies with # See http://speleotrove.com/decimal/ __libmpdec_version__ = "2.4.1" # compatible libmpdec version @@ -6386,19 +6387,3 @@ # _PyHASH_10INV is the inverse of 10 modulo the prime _PyHASH_MODULUS _PyHASH_10INV = pow(10, _PyHASH_MODULUS - 2, _PyHASH_MODULUS) del sys - -try: - import _decimal -except ImportError: - pass -else: - s1 = set(dir()) - s2 = set(dir(_decimal)) - for name in s1 - s2: - del globals()[name] - del s1, s2, name - from _decimal import * - -if __name__ == '__main__': - import doctest, decimal - doctest.testmod(decimal) diff --git a/Lib/decimal.py b/Lib/decimal.py --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -1,6404 +1,14 @@ -# Copyright (c) 2004 Python Software Foundation. -# All rights reserved. - -# Written by Eric Price -# and Facundo Batista -# and Raymond Hettinger -# and Aahz -# and Tim Peters - -# This module should be kept in sync with the latest updates of the -# IBM specification as it evolves. Those updates will be treated -# as bug fixes (deviation from the spec is a compatibility, usability -# bug) and will be backported. At this point the spec is stabilizing -# and the updates are becoming fewer, smaller, and less significant. - -""" -This is an implementation of decimal floating point arithmetic based on -the General Decimal Arithmetic Specification: - - http://speleotrove.com/decimal/decarith.html - -and IEEE standard 854-1987: - - http://en.wikipedia.org/wiki/IEEE_854-1987 - -Decimal floating point has finite precision with arbitrarily large bounds. - -The purpose of this module is to support arithmetic using familiar -"schoolhouse" rules and to avoid some of the tricky representation -issues associated with binary floating point. The package is especially -useful for financial applications or for contexts where users have -expectations that are at odds with binary floating point (for instance, -in binary floating point, 1.00 % 0.1 gives 0.09999999999999995 instead -of 0.0; Decimal('1.00') % Decimal('0.1') returns the expected -Decimal('0.00')). - -Here are some examples of using the decimal module: - ->>> from decimal import * ->>> setcontext(ExtendedContext) ->>> Decimal(0) -Decimal('0') ->>> Decimal('1') -Decimal('1') ->>> Decimal('-.0123') -Decimal('-0.0123') ->>> Decimal(123456) -Decimal('123456') ->>> Decimal('123.45e12345678') -Decimal('1.2345E+12345680') ->>> Decimal('1.33') + Decimal('1.27') -Decimal('2.60') ->>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41') -Decimal('-2.20') ->>> dig = Decimal(1) ->>> print(dig / Decimal(3)) -0.333333333 ->>> getcontext().prec = 18 ->>> print(dig / Decimal(3)) -0.333333333333333333 ->>> print(dig.sqrt()) -1 ->>> print(Decimal(3).sqrt()) -1.73205080756887729 ->>> print(Decimal(3) ** 123) -4.85192780976896427E+58 ->>> inf = Decimal(1) / Decimal(0) ->>> print(inf) -Infinity ->>> neginf = Decimal(-1) / Decimal(0) ->>> print(neginf) --Infinity ->>> print(neginf + inf) -NaN ->>> print(neginf * inf) --Infinity ->>> print(dig / 0) -Infinity ->>> getcontext().traps[DivisionByZero] = 1 ->>> print(dig / 0) -Traceback (most recent call last): - ... - ... - ... -decimal.DivisionByZero: x / 0 ->>> c = Context() ->>> c.traps[InvalidOperation] = 0 ->>> print(c.flags[InvalidOperation]) -0 ->>> c.divide(Decimal(0), Decimal(0)) -Decimal('NaN') ->>> c.traps[InvalidOperation] = 1 ->>> print(c.flags[InvalidOperation]) -1 ->>> c.flags[InvalidOperation] = 0 ->>> print(c.flags[InvalidOperation]) -0 ->>> print(c.divide(Decimal(0), Decimal(0))) -Traceback (most recent call last): - ... - ... - ... -decimal.InvalidOperation: 0 / 0 ->>> print(c.flags[InvalidOperation]) -1 ->>> c.flags[InvalidOperation] = 0 ->>> c.traps[InvalidOperation] = 0 ->>> print(c.divide(Decimal(0), Decimal(0))) -NaN ->>> print(c.flags[InvalidOperation]) -1 ->>> -""" - -__all__ = [ - # Two major classes - 'Decimal', 'Context', - - # Named tuple representation - 'DecimalTuple', - - # Contexts - 'DefaultContext', 'BasicContext', 'ExtendedContext', - - # Exceptions - 'DecimalException', 'Clamped', 'InvalidOperation', 'DivisionByZero', - 'Inexact', 'Rounded', 'Subnormal', 'Overflow', 'Underflow', - 'FloatOperation', - - # Exceptional conditions that trigger InvalidOperation - 'DivisionImpossible', 'InvalidContext', 'ConversionSyntax', 'DivisionUndefined', - - # Constants for use in setting up contexts - 'ROUND_DOWN', 'ROUND_HALF_UP', 'ROUND_HALF_EVEN', 'ROUND_CEILING', - 'ROUND_FLOOR', 'ROUND_UP', 'ROUND_HALF_DOWN', 'ROUND_05UP', - - # Functions for manipulating contexts - 'setcontext', 'getcontext', 'localcontext', - - # Limits for the C version for compatibility - 'MAX_PREC', 'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY', - - # C version: compile time choice that enables the thread local context - 'HAVE_THREADS' -] - -__version__ = '1.70' # Highest version of the spec this complies with - # See http://speleotrove.com/decimal/ -__libmpdec_version__ = "2.4.1" # compatible libmpdec version - -import math as _math -import numbers as _numbers -import sys try: - from collections import namedtuple as _namedtuple - DecimalTuple = _namedtuple('DecimalTuple', 'sign digits exponent') + from _decimal import * + from _decimal import __doc__ + from _decimal import __version__ + from _decimal import __libmpdec_version__ except ImportError: - DecimalTuple = lambda *args: args + from _pydecimal import * + from _pydecimal import __doc__ + from _pydecimal import __version__ + from _pydecimal import __libmpdec_version__ -# Rounding -ROUND_DOWN = 'ROUND_DOWN' -ROUND_HALF_UP = 'ROUND_HALF_UP' -ROUND_HALF_EVEN = 'ROUND_HALF_EVEN' -ROUND_CEILING = 'ROUND_CEILING' -ROUND_FLOOR = 'ROUND_FLOOR' -ROUND_UP = 'ROUND_UP' -ROUND_HALF_DOWN = 'ROUND_HALF_DOWN' -ROUND_05UP = 'ROUND_05UP' -# Compatibility with the C version -HAVE_THREADS = True -if sys.maxsize == 2**63-1: - MAX_PREC = 999999999999999999 - MAX_EMAX = 999999999999999999 - MIN_EMIN = -999999999999999999 -else: - MAX_PREC = 425000000 - MAX_EMAX = 425000000 - MIN_EMIN = -425000000 -MIN_ETINY = MIN_EMIN - (MAX_PREC-1) - -# Errors - -class DecimalException(ArithmeticError): - """Base exception class. - - Used exceptions derive from this. - If an exception derives from another exception besides this (such as - Underflow (Inexact, Rounded, Subnormal) that indicates that it is only - called if the others are present. This isn't actually used for - anything, though. - - handle -- Called when context._raise_error is called and the - trap_enabler is not set. First argument is self, second is the - context. More arguments can be given, those being after - the explanation in _raise_error (For example, - context._raise_error(NewError, '(-x)!', self._sign) would - call NewError().handle(context, self._sign).) - - To define a new exception, it should be sufficient to have it derive - from DecimalException. - """ - def handle(self, context, *args): - pass - - -class Clamped(DecimalException): - """Exponent of a 0 changed to fit bounds. - - This occurs and signals clamped if the exponent of a result has been - altered in order to fit the constraints of a specific concrete - representation. This may occur when the exponent of a zero result would - be outside the bounds of a representation, or when a large normal - number would have an encoded exponent that cannot be represented. In - this latter case, the exponent is reduced to fit and the corresponding - number of zero digits are appended to the coefficient ("fold-down"). - """ - -class InvalidOperation(DecimalException): - """An invalid operation was performed. - - Various bad things cause this: - - Something creates a signaling NaN - -INF + INF - 0 * (+-)INF - (+-)INF / (+-)INF - x % 0 - (+-)INF % x - x._rescale( non-integer ) - sqrt(-x) , x > 0 - 0 ** 0 - x ** (non-integer) - x ** (+-)INF - An operand is invalid - - The result of the operation after these is a quiet positive NaN, - except when the cause is a signaling NaN, in which case the result is - also a quiet NaN, but with the original sign, and an optional - diagnostic information. - """ - def handle(self, context, *args): - if args: - ans = _dec_from_triple(args[0]._sign, args[0]._int, 'n', True) - return ans._fix_nan(context) - return _NaN - -class ConversionSyntax(InvalidOperation): - """Trying to convert badly formed string. - - This occurs and signals invalid-operation if an string is being - converted to a number and it does not conform to the numeric string - syntax. The result is [0,qNaN]. - """ - def handle(self, context, *args): - return _NaN - -class DivisionByZero(DecimalException, ZeroDivisionError): - """Division by 0. - - This occurs and signals division-by-zero if division of a finite number - by zero was attempted (during a divide-integer or divide operation, or a - power operation with negative right-hand operand), and the dividend was - not zero. - - The result of the operation is [sign,inf], where sign is the exclusive - or of the signs of the operands for divide, or is 1 for an odd power of - -0, for power. - """ - - def handle(self, context, sign, *args): - return _SignedInfinity[sign] - -class DivisionImpossible(InvalidOperation): - """Cannot perform the division adequately. - - This occurs and signals invalid-operation if the integer result of a - divide-integer or remainder operation had too many digits (would be - longer than precision). The result is [0,qNaN]. - """ - - def handle(self, context, *args): - return _NaN - -class DivisionUndefined(InvalidOperation, ZeroDivisionError): - """Undefined result of division. - - This occurs and signals invalid-operation if division by zero was - attempted (during a divide-integer, divide, or remainder operation), and - the dividend is also zero. The result is [0,qNaN]. - """ - - def handle(self, context, *args): - return _NaN - -class Inexact(DecimalException): - """Had to round, losing information. - - This occurs and signals inexact whenever the result of an operation is - not exact (that is, it needed to be rounded and any discarded digits - were non-zero), or if an overflow or underflow condition occurs. The - result in all cases is unchanged. - - The inexact signal may be tested (or trapped) to determine if a given - operation (or sequence of operations) was inexact. - """ - -class InvalidContext(InvalidOperation): - """Invalid context. Unknown rounding, for example. - - This occurs and signals invalid-operation if an invalid context was - detected during an operation. This can occur if contexts are not checked - on creation and either the precision exceeds the capability of the - underlying concrete representation or an unknown or unsupported rounding - was specified. These aspects of the context need only be checked when - the values are required to be used. The result is [0,qNaN]. - """ - - def handle(self, context, *args): - return _NaN - -class Rounded(DecimalException): - """Number got rounded (not necessarily changed during rounding). - - This occurs and signals rounded whenever the result of an operation is - rounded (that is, some zero or non-zero digits were discarded from the - coefficient), or if an overflow or underflow condition occurs. The - result in all cases is unchanged. - - The rounded signal may be tested (or trapped) to determine if a given - operation (or sequence of operations) caused a loss of precision. - """ - -class Subnormal(DecimalException): - """Exponent < Emin before rounding. - - This occurs and signals subnormal whenever the result of a conversion or - operation is subnormal (that is, its adjusted exponent is less than - Emin, before any rounding). The result in all cases is unchanged. - - The subnormal signal may be tested (or trapped) to determine if a given - or operation (or sequence of operations) yielded a subnormal result. - """ - -class Overflow(Inexact, Rounded): - """Numerical overflow. - - This occurs and signals overflow if the adjusted exponent of a result - (from a conversion or from an operation that is not an attempt to divide - by zero), after rounding, would be greater than the largest value that - can be handled by the implementation (the value Emax). - - The result depends on the rounding mode: - - For round-half-up and round-half-even (and for round-half-down and - round-up, if implemented), the result of the operation is [sign,inf], - where sign is the sign of the intermediate result. For round-down, the - result is the largest finite number that can be represented in the - current precision, with the sign of the intermediate result. For - round-ceiling, the result is the same as for round-down if the sign of - the intermediate result is 1, or is [0,inf] otherwise. For round-floor, - the result is the same as for round-down if the sign of the intermediate - result is 0, or is [1,inf] otherwise. In all cases, Inexact and Rounded - will also be raised. - """ - - def handle(self, context, sign, *args): - if context.rounding in (ROUND_HALF_UP, ROUND_HALF_EVEN, - ROUND_HALF_DOWN, ROUND_UP): - return _SignedInfinity[sign] - if sign == 0: - if context.rounding == ROUND_CEILING: - return _SignedInfinity[sign] - return _dec_from_triple(sign, '9'*context.prec, - context.Emax-context.prec+1) - if sign == 1: - if context.rounding == ROUND_FLOOR: - return _SignedInfinity[sign] - return _dec_from_triple(sign, '9'*context.prec, - context.Emax-context.prec+1) - - -class Underflow(Inexact, Rounded, Subnormal): - """Numerical underflow with result rounded to 0. - - This occurs and signals underflow if a result is inexact and the - adjusted exponent of the result would be smaller (more negative) than - the smallest value that can be handled by the implementation (the value - Emin). That is, the result is both inexact and subnormal. - - The result after an underflow will be a subnormal number rounded, if - necessary, so that its exponent is not less than Etiny. This may result - in 0 with the sign of the intermediate result and an exponent of Etiny. - - In all cases, Inexact, Rounded, and Subnormal will also be raised. - """ - -class FloatOperation(DecimalException, TypeError): - """Enable stricter semantics for mixing floats and Decimals. - - If the signal is not trapped (default), mixing floats and Decimals is - permitted in the Decimal() constructor, context.create_decimal() and - all comparison operators. Both conversion and comparisons are exact. - Any occurrence of a mixed operation is silently recorded by setting - FloatOperation in the context flags. Explicit conversions with - Decimal.from_float() or context.create_decimal_from_float() do not - set the flag. - - Otherwise (the signal is trapped), only equality comparisons and explicit - conversions are silent. All other mixed operations raise FloatOperation. - """ - -# List of public traps and flags -_signals = [Clamped, DivisionByZero, Inexact, Overflow, Rounded, - Underflow, InvalidOperation, Subnormal, FloatOperation] - -# Map conditions (per the spec) to signals -_condition_map = {ConversionSyntax:InvalidOperation, - DivisionImpossible:InvalidOperation, - DivisionUndefined:InvalidOperation, - InvalidContext:InvalidOperation} - -# Valid rounding modes -_rounding_modes = (ROUND_DOWN, ROUND_HALF_UP, ROUND_HALF_EVEN, ROUND_CEILING, - ROUND_FLOOR, ROUND_UP, ROUND_HALF_DOWN, ROUND_05UP) - -##### Context Functions ################################################## - -# The getcontext() and setcontext() function manage access to a thread-local -# current context. Py2.4 offers direct support for thread locals. If that -# is not available, use threading.current_thread() which is slower but will -# work for older Pythons. If threads are not part of the build, create a -# mock threading object with threading.local() returning the module namespace. - -try: - import threading -except ImportError: - # Python was compiled without threads; create a mock object instead - class MockThreading(object): - def local(self, sys=sys): - return sys.modules[__name__] - threading = MockThreading() - del MockThreading - -try: - threading.local - -except AttributeError: - - # To fix reloading, force it to create a new context - # Old contexts have different exceptions in their dicts, making problems. - if hasattr(threading.current_thread(), '__decimal_context__'): - del threading.current_thread().__decimal_context__ - - def setcontext(context): - """Set this thread's context to context.""" - if context in (DefaultContext, BasicContext, ExtendedContext): - context = context.copy() - context.clear_flags() - threading.current_thread().__decimal_context__ = context - - def getcontext(): - """Returns this thread's context. - - If this thread does not yet have a context, returns - a new context and sets this thread's context. - New contexts are copies of DefaultContext. - """ - try: - return threading.current_thread().__decimal_context__ - except AttributeError: - context = Context() - threading.current_thread().__decimal_context__ = context - return context - -else: - - local = threading.local() - if hasattr(local, '__decimal_context__'): - del local.__decimal_context__ - - def getcontext(_local=local): - """Returns this thread's context. - - If this thread does not yet have a context, returns - a new context and sets this thread's context. - New contexts are copies of DefaultContext. - """ - try: - return _local.__decimal_context__ - except AttributeError: - context = Context() - _local.__decimal_context__ = context - return context - - def setcontext(context, _local=local): - """Set this thread's context to context.""" - if context in (DefaultContext, BasicContext, ExtendedContext): - context = context.copy() - context.clear_flags() - _local.__decimal_context__ = context - - del threading, local # Don't contaminate the namespace - -def localcontext(ctx=None): - """Return a context manager for a copy of the supplied context - - Uses a copy of the current context if no context is specified - The returned context manager creates a local decimal context - in a with statement: - def sin(x): - with localcontext() as ctx: - ctx.prec += 2 - # Rest of sin calculation algorithm - # uses a precision 2 greater than normal - return +s # Convert result to normal precision - - def sin(x): - with localcontext(ExtendedContext): - # Rest of sin calculation algorithm - # uses the Extended Context from the - # General Decimal Arithmetic Specification - return +s # Convert result to normal context - - >>> setcontext(DefaultContext) - >>> print(getcontext().prec) - 28 - >>> with localcontext(): - ... ctx = getcontext() - ... ctx.prec += 2 - ... print(ctx.prec) - ... - 30 - >>> with localcontext(ExtendedContext): - ... print(getcontext().prec) - ... - 9 - >>> print(getcontext().prec) - 28 - """ - if ctx is None: ctx = getcontext() - return _ContextManager(ctx) - - -##### Decimal class ####################################################### - -# Do not subclass Decimal from numbers.Real and do not register it as such -# (because Decimals are not interoperable with floats). See the notes in -# numbers.py for more detail. - -class Decimal(object): - """Floating point class for decimal arithmetic.""" - - __slots__ = ('_exp','_int','_sign', '_is_special') - # Generally, the value of the Decimal instance is given by - # (-1)**_sign * _int * 10**_exp - # Special values are signified by _is_special == True - - # We're immutable, so use __new__ not __init__ - def __new__(cls, value="0", context=None): - """Create a decimal point instance. - - >>> Decimal('3.14') # string input - Decimal('3.14') - >>> Decimal((0, (3, 1, 4), -2)) # tuple (sign, digit_tuple, exponent) - Decimal('3.14') - >>> Decimal(314) # int - Decimal('314') - >>> Decimal(Decimal(314)) # another decimal instance - Decimal('314') - >>> Decimal(' 3.14 \\n') # leading and trailing whitespace okay - Decimal('3.14') - """ - - # Note that the coefficient, self._int, is actually stored as - # a string rather than as a tuple of digits. This speeds up - # the "digits to integer" and "integer to digits" conversions - # that are used in almost every arithmetic operation on - # Decimals. This is an internal detail: the as_tuple function - # and the Decimal constructor still deal with tuples of - # digits. - - self = object.__new__(cls) - - # From a string - # REs insist on real strings, so we can too. - if isinstance(value, str): - m = _parser(value.strip()) - if m is None: - if context is None: - context = getcontext() - return context._raise_error(ConversionSyntax, - "Invalid literal for Decimal: %r" % value) - - if m.group('sign') == "-": - self._sign = 1 - else: - self._sign = 0 - intpart = m.group('int') - if intpart is not None: - # finite number - fracpart = m.group('frac') or '' - exp = int(m.group('exp') or '0') - self._int = str(int(intpart+fracpart)) - self._exp = exp - len(fracpart) - self._is_special = False - else: - diag = m.group('diag') - if diag is not None: - # NaN - self._int = str(int(diag or '0')).lstrip('0') - if m.group('signal'): - self._exp = 'N' - else: - self._exp = 'n' - else: - # infinity - self._int = '0' - self._exp = 'F' - self._is_special = True - return self - - # From an integer - if isinstance(value, int): - if value >= 0: - self._sign = 0 - else: - self._sign = 1 - self._exp = 0 - self._int = str(abs(value)) - self._is_special = False - return self - - # From another decimal - if isinstance(value, Decimal): - self._exp = value._exp - self._sign = value._sign - self._int = value._int - self._is_special = value._is_special - return self - - # From an internal working value - if isinstance(value, _WorkRep): - self._sign = value.sign - self._int = str(value.int) - self._exp = int(value.exp) - self._is_special = False - return self - - # tuple/list conversion (possibly from as_tuple()) - if isinstance(value, (list,tuple)): - if len(value) != 3: - raise ValueError('Invalid tuple size in creation of Decimal ' - 'from list or tuple. The list or tuple ' - 'should have exactly three elements.') - # process sign. The isinstance test rejects floats - if not (isinstance(value[0], int) and value[0] in (0,1)): - raise ValueError("Invalid sign. The first value in the tuple " - "should be an integer; either 0 for a " - "positive number or 1 for a negative number.") - self._sign = value[0] - if value[2] == 'F': - # infinity: value[1] is ignored - self._int = '0' - self._exp = value[2] - self._is_special = True - else: - # process and validate the digits in value[1] - digits = [] - for digit in value[1]: - if isinstance(digit, int) and 0 <= digit <= 9: - # skip leading zeros - if digits or digit != 0: - digits.append(digit) - else: - raise ValueError("The second value in the tuple must " - "be composed of integers in the range " - "0 through 9.") - if value[2] in ('n', 'N'): - # NaN: digits form the diagnostic - self._int = ''.join(map(str, digits)) - self._exp = value[2] - self._is_special = True - elif isinstance(value[2], int): - # finite number: digits give the coefficient - self._int = ''.join(map(str, digits or [0])) - self._exp = value[2] - self._is_special = False - else: - raise ValueError("The third value in the tuple must " - "be an integer, or one of the " - "strings 'F', 'n', 'N'.") - return self - - if isinstance(value, float): - if context is None: - context = getcontext() - context._raise_error(FloatOperation, - "strict semantics for mixing floats and Decimals are " - "enabled") - value = Decimal.from_float(value) - self._exp = value._exp - self._sign = value._sign - self._int = value._int - self._is_special = value._is_special - return self - - raise TypeError("Cannot convert %r to Decimal" % value) - - @classmethod - def from_float(cls, f): - """Converts a float to a decimal number, exactly. - - Note that Decimal.from_float(0.1) is not the same as Decimal('0.1'). - Since 0.1 is not exactly representable in binary floating point, the - value is stored as the nearest representable value which is - 0x1.999999999999ap-4. The exact equivalent of the value in decimal - is 0.1000000000000000055511151231257827021181583404541015625. - - >>> Decimal.from_float(0.1) - Decimal('0.1000000000000000055511151231257827021181583404541015625') - >>> Decimal.from_float(float('nan')) - Decimal('NaN') - >>> Decimal.from_float(float('inf')) - Decimal('Infinity') - >>> Decimal.from_float(-float('inf')) - Decimal('-Infinity') - >>> Decimal.from_float(-0.0) - Decimal('-0') - - """ - if isinstance(f, int): # handle integer inputs - return cls(f) - if not isinstance(f, float): - raise TypeError("argument must be int or float.") - if _math.isinf(f) or _math.isnan(f): - return cls(repr(f)) - if _math.copysign(1.0, f) == 1.0: - sign = 0 - else: - sign = 1 - n, d = abs(f).as_integer_ratio() - k = d.bit_length() - 1 - result = _dec_from_triple(sign, str(n*5**k), -k) - if cls is Decimal: - return result - else: - return cls(result) - - def _isnan(self): - """Returns whether the number is not actually one. - - 0 if a number - 1 if NaN - 2 if sNaN - """ - if self._is_special: - exp = self._exp - if exp == 'n': - return 1 - elif exp == 'N': - return 2 - return 0 - - def _isinfinity(self): - """Returns whether the number is infinite - - 0 if finite or not a number - 1 if +INF - -1 if -INF - """ - if self._exp == 'F': - if self._sign: - return -1 - return 1 - return 0 - - def _check_nans(self, other=None, context=None): - """Returns whether the number is not actually one. - - if self, other are sNaN, signal - if self, other are NaN return nan - return 0 - - Done before operations. - """ - - self_is_nan = self._isnan() - if other is None: - other_is_nan = False - else: - other_is_nan = other._isnan() - - if self_is_nan or other_is_nan: - if context is None: - context = getcontext() - - if self_is_nan == 2: - return context._raise_error(InvalidOperation, 'sNaN', - self) - if other_is_nan == 2: - return context._raise_error(InvalidOperation, 'sNaN', - other) - if self_is_nan: - return self._fix_nan(context) - - return other._fix_nan(context) - return 0 - - def _compare_check_nans(self, other, context): - """Version of _check_nans used for the signaling comparisons - compare_signal, __le__, __lt__, __ge__, __gt__. - - Signal InvalidOperation if either self or other is a (quiet - or signaling) NaN. Signaling NaNs take precedence over quiet - NaNs. - - Return 0 if neither operand is a NaN. - - """ - if context is None: - context = getcontext() - - if self._is_special or other._is_special: - if self.is_snan(): - return context._raise_error(InvalidOperation, - 'comparison involving sNaN', - self) - elif other.is_snan(): - return context._raise_error(InvalidOperation, - 'comparison involving sNaN', - other) - elif self.is_qnan(): - return context._raise_error(InvalidOperation, - 'comparison involving NaN', - self) - elif other.is_qnan(): - return context._raise_error(InvalidOperation, - 'comparison involving NaN', - other) - return 0 - - def __bool__(self): - """Return True if self is nonzero; otherwise return False. - - NaNs and infinities are considered nonzero. - """ - return self._is_special or self._int != '0' - - def _cmp(self, other): - """Compare the two non-NaN decimal instances self and other. - - Returns -1 if self < other, 0 if self == other and 1 - if self > other. This routine is for internal use only.""" - - if self._is_special or other._is_special: - self_inf = self._isinfinity() - other_inf = other._isinfinity() - if self_inf == other_inf: - return 0 - elif self_inf < other_inf: - return -1 - else: - return 1 - - # check for zeros; Decimal('0') == Decimal('-0') - if not self: - if not other: - return 0 - else: - return -((-1)**other._sign) - if not other: - return (-1)**self._sign - - # If different signs, neg one is less - if other._sign < self._sign: - return -1 - if self._sign < other._sign: - return 1 - - self_adjusted = self.adjusted() - other_adjusted = other.adjusted() - if self_adjusted == other_adjusted: - self_padded = self._int + '0'*(self._exp - other._exp) - other_padded = other._int + '0'*(other._exp - self._exp) - if self_padded == other_padded: - return 0 - elif self_padded < other_padded: - return -(-1)**self._sign - else: - return (-1)**self._sign - elif self_adjusted > other_adjusted: - return (-1)**self._sign - else: # self_adjusted < other_adjusted - return -((-1)**self._sign) - - # Note: The Decimal standard doesn't cover rich comparisons for - # Decimals. In particular, the specification is silent on the - # subject of what should happen for a comparison involving a NaN. - # We take the following approach: - # - # == comparisons involving a quiet NaN always return False - # != comparisons involving a quiet NaN always return True - # == or != comparisons involving a signaling NaN signal - # InvalidOperation, and return False or True as above if the - # InvalidOperation is not trapped. - # <, >, <= and >= comparisons involving a (quiet or signaling) - # NaN signal InvalidOperation, and return False if the - # InvalidOperation is not trapped. - # - # This behavior is designed to conform as closely as possible to - # that specified by IEEE 754. - - def __eq__(self, other, context=None): - self, other = _convert_for_comparison(self, other, equality_op=True) - if other is NotImplemented: - return other - if self._check_nans(other, context): - return False - return self._cmp(other) == 0 - - def __ne__(self, other, context=None): - self, other = _convert_for_comparison(self, other, equality_op=True) - if other is NotImplemented: - return other - if self._check_nans(other, context): - return True - return self._cmp(other) != 0 - - - def __lt__(self, other, context=None): - self, other = _convert_for_comparison(self, other) - if other is NotImplemented: - return other - ans = self._compare_check_nans(other, context) - if ans: - return False - return self._cmp(other) < 0 - - def __le__(self, other, context=None): - self, other = _convert_for_comparison(self, other) - if other is NotImplemented: - return other - ans = self._compare_check_nans(other, context) - if ans: - return False - return self._cmp(other) <= 0 - - def __gt__(self, other, context=None): - self, other = _convert_for_comparison(self, other) - if other is NotImplemented: - return other - ans = self._compare_check_nans(other, context) - if ans: - return False - return self._cmp(other) > 0 - - def __ge__(self, other, context=None): - self, other = _convert_for_comparison(self, other) - if other is NotImplemented: - return other - ans = self._compare_check_nans(other, context) - if ans: - return False - return self._cmp(other) >= 0 - - def compare(self, other, context=None): - """Compares one to another. - - -1 => a < b - 0 => a = b - 1 => a > b - NaN => one is NaN - Like __cmp__, but returns Decimal instances. - """ - other = _convert_other(other, raiseit=True) - - # Compare(NaN, NaN) = NaN - if (self._is_special or other and other._is_special): - ans = self._check_nans(other, context) - if ans: - return ans - - return Decimal(self._cmp(other)) - - def __hash__(self): - """x.__hash__() <==> hash(x)""" - - # In order to make sure that the hash of a Decimal instance - # agrees with the hash of a numerically equal integer, float - # or Fraction, we follow the rules for numeric hashes outlined - # in the documentation. (See library docs, 'Built-in Types'). - if self._is_special: - if self.is_snan(): - raise TypeError('Cannot hash a signaling NaN value.') - elif self.is_nan(): - return _PyHASH_NAN - else: - if self._sign: - return -_PyHASH_INF - else: - return _PyHASH_INF - - if self._exp >= 0: - exp_hash = pow(10, self._exp, _PyHASH_MODULUS) - else: - exp_hash = pow(_PyHASH_10INV, -self._exp, _PyHASH_MODULUS) - hash_ = int(self._int) * exp_hash % _PyHASH_MODULUS - ans = hash_ if self >= 0 else -hash_ - return -2 if ans == -1 else ans - - def as_tuple(self): - """Represents the number as a triple tuple. - - To show the internals exactly as they are. - """ - return DecimalTuple(self._sign, tuple(map(int, self._int)), self._exp) - - def __repr__(self): - """Represents the number as an instance of Decimal.""" - # Invariant: eval(repr(d)) == d - return "Decimal('%s')" % str(self) - - def __str__(self, eng=False, context=None): - """Return string representation of the number in scientific notation. - - Captures all of the information in the underlying representation. - """ - - sign = ['', '-'][self._sign] - if self._is_special: - if self._exp == 'F': - return sign + 'Infinity' - elif self._exp == 'n': - return sign + 'NaN' + self._int - else: # self._exp == 'N' - return sign + 'sNaN' + self._int - - # number of digits of self._int to left of decimal point - leftdigits = self._exp + len(self._int) - - # dotplace is number of digits of self._int to the left of the - # decimal point in the mantissa of the output string (that is, - # after adjusting the exponent) - if self._exp <= 0 and leftdigits > -6: - # no exponent required - dotplace = leftdigits - elif not eng: - # usual scientific notation: 1 digit on left of the point - dotplace = 1 - elif self._int == '0': - # engineering notation, zero - dotplace = (leftdigits + 1) % 3 - 1 - else: - # engineering notation, nonzero - dotplace = (leftdigits - 1) % 3 + 1 - - if dotplace <= 0: - intpart = '0' - fracpart = '.' + '0'*(-dotplace) + self._int - elif dotplace >= len(self._int): - intpart = self._int+'0'*(dotplace-len(self._int)) - fracpart = '' - else: - intpart = self._int[:dotplace] - fracpart = '.' + self._int[dotplace:] - if leftdigits == dotplace: - exp = '' - else: - if context is None: - context = getcontext() - exp = ['e', 'E'][context.capitals] + "%+d" % (leftdigits-dotplace) - - return sign + intpart + fracpart + exp - - def to_eng_string(self, context=None): - """Convert to engineering-type string. - - Engineering notation has an exponent which is a multiple of 3, so there - are up to 3 digits left of the decimal place. - - Same rules for when in exponential and when as a value as in __str__. - """ - return self.__str__(eng=True, context=context) - - def __neg__(self, context=None): - """Returns a copy with the sign switched. - - Rounds, if it has reason. - """ - if self._is_special: - ans = self._check_nans(context=context) - if ans: - return ans - - if context is None: - context = getcontext() - - if not self and context.rounding != ROUND_FLOOR: - # -Decimal('0') is Decimal('0'), not Decimal('-0'), except - # in ROUND_FLOOR rounding mode. - ans = self.copy_abs() - else: - ans = self.copy_negate() - - return ans._fix(context) - - def __pos__(self, context=None): - """Returns a copy, unless it is a sNaN. - - Rounds the number (if more then precision digits) - """ - if self._is_special: - ans = self._check_nans(context=context) - if ans: - return ans - - if context is None: - context = getcontext() - - if not self and context.rounding != ROUND_FLOOR: - # + (-0) = 0, except in ROUND_FLOOR rounding mode. - ans = self.copy_abs() - else: - ans = Decimal(self) - - return ans._fix(context) - - def __abs__(self, round=True, context=None): - """Returns the absolute value of self. - - If the keyword argument 'round' is false, do not round. The - expression self.__abs__(round=False) is equivalent to - self.copy_abs(). - """ - if not round: - return self.copy_abs() - - if self._is_special: - ans = self._check_nans(context=context) - if ans: - return ans - - if self._sign: - ans = self.__neg__(context=context) - else: - ans = self.__pos__(context=context) - - return ans - - def __add__(self, other, context=None): - """Returns self + other. - - -INF + INF (or the reverse) cause InvalidOperation errors. - """ - other = _convert_other(other) - if other is NotImplemented: - return other - - if context is None: - context = getcontext() - - if self._is_special or other._is_special: - ans = self._check_nans(other, context) - if ans: - return ans - - if self._isinfinity(): - # If both INF, same sign => same as both, opposite => error. - if self._sign != other._sign and other._isinfinity(): - return context._raise_error(InvalidOperation, '-INF + INF') - return Decimal(self) - if other._isinfinity(): - return Decimal(other) # Can't both be infinity here - - exp = min(self._exp, other._exp) - negativezero = 0 - if context.rounding == ROUND_FLOOR and self._sign != other._sign: - # If the answer is 0, the sign should be negative, in this case. - negativezero = 1 - - if not self and not other: - sign = min(self._sign, other._sign) - if negativezero: - sign = 1 - ans = _dec_from_triple(sign, '0', exp) - ans = ans._fix(context) - return ans - if not self: - exp = max(exp, other._exp - context.prec-1) - ans = other._rescale(exp, context.rounding) - ans = ans._fix(context) - return ans - if not other: - exp = max(exp, self._exp - context.prec-1) - ans = self._rescale(exp, context.rounding) - ans = ans._fix(context) - return ans - - op1 = _WorkRep(self) - op2 = _WorkRep(other) - op1, op2 = _normalize(op1, op2, context.prec) - - result = _WorkRep() - if op1.sign != op2.sign: - # Equal and opposite - if op1.int == op2.int: - ans = _dec_from_triple(negativezero, '0', exp) - ans = ans._fix(context) - return ans - if op1.int < op2.int: - op1, op2 = op2, op1 - # OK, now abs(op1) > abs(op2) - if op1.sign == 1: - result.sign = 1 - op1.sign, op2.sign = op2.sign, op1.sign - else: - result.sign = 0 - # So we know the sign, and op1 > 0. - elif op1.sign == 1: - result.sign = 1 - op1.sign, op2.sign = (0, 0) - else: - result.sign = 0 - # Now, op1 > abs(op2) > 0 - - if op2.sign == 0: - result.int = op1.int + op2.int - else: - result.int = op1.int - op2.int - - result.exp = op1.exp - ans = Decimal(result) - ans = ans._fix(context) - return ans - - __radd__ = __add__ - - def __sub__(self, other, context=None): - """Return self - other""" - other = _convert_other(other) - if other is NotImplemented: - return other - - if self._is_special or other._is_special: - ans = self._check_nans(other, context=context) - if ans: - return ans - - # self - other is computed as self + other.copy_negate() - return self.__add__(other.copy_negate(), context=context) - - def __rsub__(self, other, context=None): - """Return other - self""" - other = _convert_other(other) - if other is NotImplemented: - return other - - return other.__sub__(self, context=context) - - def __mul__(self, other, context=None): - """Return self * other. - - (+-) INF * 0 (or its reverse) raise InvalidOperation. - """ - other = _convert_other(other) - if other is NotImplemented: - return other - - if context is None: - context = getcontext() - - resultsign = self._sign ^ other._sign - - if self._is_special or other._is_special: - ans = self._check_nans(other, context) - if ans: - return ans - - if self._isinfinity(): - if not other: - return context._raise_error(InvalidOperation, '(+-)INF * 0') - return _SignedInfinity[resultsign] - - if other._isinfinity(): - if not self: - return context._raise_error(InvalidOperation, '0 * (+-)INF') - return _SignedInfinity[resultsign] - - resultexp = self._exp + other._exp - - # Special case for multiplying by zero - if not self or not other: - ans = _dec_from_triple(resultsign, '0', resultexp) - # Fixing in case the exponent is out of bounds - ans = ans._fix(context) - return ans - - # Special case for multiplying by power of 10 - if self._int == '1': - ans = _dec_from_triple(resultsign, other._int, resultexp) - ans = ans._fix(context) - return ans - if other._int == '1': - ans = _dec_from_triple(resultsign, self._int, resultexp) - ans = ans._fix(context) - return ans - - op1 = _WorkRep(self) - op2 = _WorkRep(other) - - ans = _dec_from_triple(resultsign, str(op1.int * op2.int), resultexp) - ans = ans._fix(context) - - return ans - __rmul__ = __mul__ - - def __truediv__(self, other, context=None): - """Return self / other.""" - other = _convert_other(other) - if other is NotImplemented: - return NotImplemented - - if context is None: - context = getcontext() - - sign = self._sign ^ other._sign - - if self._is_special or other._is_special: - ans = self._check_nans(other, context) - if ans: - return ans - - if self._isinfinity() and other._isinfinity(): - return context._raise_error(InvalidOperation, '(+-)INF/(+-)INF') - - if self._isinfinity(): - return _SignedInfinity[sign] - - if other._isinfinity(): - context._raise_error(Clamped, 'Division by infinity') - return _dec_from_triple(sign, '0', context.Etiny()) - - # Special cases for zeroes - if not other: - if not self: - return context._raise_error(DivisionUndefined, '0 / 0') - return context._raise_error(DivisionByZero, 'x / 0', sign) - - if not self: - exp = self._exp - other._exp - coeff = 0 - else: - # OK, so neither = 0, INF or NaN - shift = len(other._int) - len(self._int) + context.prec + 1 - exp = self._exp - other._exp - shift - op1 = _WorkRep(self) - op2 = _WorkRep(other) - if shift >= 0: - coeff, remainder = divmod(op1.int * 10**shift, op2.int) - else: - coeff, remainder = divmod(op1.int, op2.int * 10**-shift) - if remainder: - # result is not exact; adjust to ensure correct rounding - if coeff % 5 == 0: - coeff += 1 - else: - # result is exact; get as close to ideal exponent as possible - ideal_exp = self._exp - other._exp - while exp < ideal_exp and coeff % 10 == 0: - coeff //= 10 - exp += 1 - - ans = _dec_from_triple(sign, str(coeff), exp) - return ans._fix(context) - - def _divide(self, other, context): - """Return (self // other, self % other), to context.prec precision. - - Assumes that neither self nor other is a NaN, that self is not - infinite and that other is nonzero. - """ - sign = self._sign ^ other._sign - if other._isinfinity(): - ideal_exp = self._exp - else: - ideal_exp = min(self._exp, other._exp) - - expdiff = self.adjusted() - other.adjusted() - if not self or other._isinfinity() or expdiff <= -2: - return (_dec_from_triple(sign, '0', 0), - self._rescale(ideal_exp, context.rounding)) - if expdiff <= context.prec: - op1 = _WorkRep(self) - op2 = _WorkRep(other) - if op1.exp >= op2.exp: - op1.int *= 10**(op1.exp - op2.exp) - else: - op2.int *= 10**(op2.exp - op1.exp) - q, r = divmod(op1.int, op2.int) - if q < 10**context.prec: - return (_dec_from_triple(sign, str(q), 0), - _dec_from_triple(self._sign, str(r), ideal_exp)) - - # Here the quotient is too large to be representable - ans = context._raise_error(DivisionImpossible, - 'quotient too large in //, % or divmod') - return ans, ans - - def __rtruediv__(self, other, context=None): - """Swaps self/other and returns __truediv__.""" - other = _convert_other(other) - if other is NotImplemented: - return other - return other.__truediv__(self, context=context) - - def __divmod__(self, other, context=None): - """ - Return (self // other, self % other) - """ - other = _convert_other(other) - if other is NotImplemented: - return other - - if context is None: - context = getcontext() - - ans = self._check_nans(other, context) - if ans: - return (ans, ans) - - sign = self._sign ^ other._sign - if self._isinfinity(): - if other._isinfinity(): - ans = context._raise_error(InvalidOperation, 'divmod(INF, INF)') - return ans, ans - else: - return (_SignedInfinity[sign], - context._raise_error(InvalidOperation, 'INF % x')) - - if not other: - if not self: - ans = context._raise_error(DivisionUndefined, 'divmod(0, 0)') - return ans, ans - else: - return (context._raise_error(DivisionByZero, 'x // 0', sign), - context._raise_error(InvalidOperation, 'x % 0')) - - quotient, remainder = self._divide(other, context) - remainder = remainder._fix(context) - return quotient, remainder - - def __rdivmod__(self, other, context=None): - """Swaps self/other and returns __divmod__.""" - other = _convert_other(other) - if other is NotImplemented: - return other - return other.__divmod__(self, context=context) - - def __mod__(self, other, context=None): - """ - self % other - """ - other = _convert_other(other) - if other is NotImplemented: - return other - - if context is None: - context = getcontext() - - ans = self._check_nans(other, context) - if ans: - return ans - - if self._isinfinity(): - return context._raise_error(InvalidOperation, 'INF % x') - elif not other: - if self: - return context._raise_error(InvalidOperation, 'x % 0') - else: - return context._raise_error(DivisionUndefined, '0 % 0') - - remainder = self._divide(other, context)[1] - remainder = remainder._fix(context) - return remainder - - def __rmod__(self, other, context=None): - """Swaps self/other and returns __mod__.""" - other = _convert_other(other) - if other is NotImplemented: - return other - return other.__mod__(self, context=context) - - def remainder_near(self, other, context=None): - """ - Remainder nearest to 0- abs(remainder-near) <= other/2 - """ - if context is None: - context = getcontext() - - other = _convert_other(other, raiseit=True) - - ans = self._check_nans(other, context) - if ans: - return ans - - # self == +/-infinity -> InvalidOperation - if self._isinfinity(): - return context._raise_error(InvalidOperation, - 'remainder_near(infinity, x)') - - # other == 0 -> either InvalidOperation or DivisionUndefined - if not other: - if self: - return context._raise_error(InvalidOperation, - 'remainder_near(x, 0)') - else: - return context._raise_error(DivisionUndefined, - 'remainder_near(0, 0)') - - # other = +/-infinity -> remainder = self - if other._isinfinity(): - ans = Decimal(self) - return ans._fix(context) - - # self = 0 -> remainder = self, with ideal exponent - ideal_exponent = min(self._exp, other._exp) - if not self: - ans = _dec_from_triple(self._sign, '0', ideal_exponent) - return ans._fix(context) - - # catch most cases of large or small quotient - expdiff = self.adjusted() - other.adjusted() - if expdiff >= context.prec + 1: - # expdiff >= prec+1 => abs(self/other) > 10**prec - return context._raise_error(DivisionImpossible) - if expdiff <= -2: - # expdiff <= -2 => abs(self/other) < 0.1 - ans = self._rescale(ideal_exponent, context.rounding) - return ans._fix(context) - - # adjust both arguments to have the same exponent, then divide - op1 = _WorkRep(self) - op2 = _WorkRep(other) - if op1.exp >= op2.exp: - op1.int *= 10**(op1.exp - op2.exp) - else: - op2.int *= 10**(op2.exp - op1.exp) - q, r = divmod(op1.int, op2.int) - # remainder is r*10**ideal_exponent; other is +/-op2.int * - # 10**ideal_exponent. Apply correction to ensure that - # abs(remainder) <= abs(other)/2 - if 2*r + (q&1) > op2.int: - r -= op2.int - q += 1 - - if q >= 10**context.prec: - return context._raise_error(DivisionImpossible) - - # result has same sign as self unless r is negative - sign = self._sign - if r < 0: - sign = 1-sign - r = -r - - ans = _dec_from_triple(sign, str(r), ideal_exponent) - return ans._fix(context) - - def __floordiv__(self, other, context=None): - """self // other""" - other = _convert_other(other) - if other is NotImplemented: - return other - - if context is None: - context = getcontext() - - ans = self._check_nans(other, context) - if ans: - return ans - - if self._isinfinity(): - if other._isinfinity(): - return context._raise_error(InvalidOperation, 'INF // INF') - else: - return _SignedInfinity[self._sign ^ other._sign] - - if not other: - if self: - return context._raise_error(DivisionByZero, 'x // 0', - self._sign ^ other._sign) - else: - return context._raise_error(DivisionUndefined, '0 // 0') - - return self._divide(other, context)[0] - - def __rfloordiv__(self, other, context=None): - """Swaps self/other and returns __floordiv__.""" - other = _convert_other(other) - if other is NotImplemented: - return other - return other.__floordiv__(self, context=context) - - def __float__(self): - """Float representation.""" - if self._isnan(): - if self.is_snan(): - raise ValueError("Cannot convert signaling NaN to float") - s = "-nan" if self._sign else "nan" - else: - s = str(self) - return float(s) - - def __int__(self): - """Converts self to an int, truncating if necessary.""" - if self._is_special: - if self._isnan(): - raise ValueError("Cannot convert NaN to integer") - elif self._isinfinity(): - raise OverflowError("Cannot convert infinity to integer") - s = (-1)**self._sign - if self._exp >= 0: - return s*int(self._int)*10**self._exp - else: - return s*int(self._int[:self._exp] or '0') - - __trunc__ = __int__ - - def real(self): - return self - real = property(real) - - def imag(self): - return Decimal(0) - imag = property(imag) - - def conjugate(self): - return self - - def __complex__(self): - return complex(float(self)) - - def _fix_nan(self, context): - """Decapitate the payload of a NaN to fit the context""" - payload = self._int - - # maximum length of payload is precision if clamp=0, - # precision-1 if clamp=1. - max_payload_len = context.prec - context.clamp - if len(payload) > max_payload_len: - payload = payload[len(payload)-max_payload_len:].lstrip('0') - return _dec_from_triple(self._sign, payload, self._exp, True) - return Decimal(self) - - def _fix(self, context): - """Round if it is necessary to keep self within prec precision. - - Rounds and fixes the exponent. Does not raise on a sNaN. - - Arguments: - self - Decimal instance - context - context used. - """ - - if self._is_special: - if self._isnan(): - # decapitate payload if necessary - return self._fix_nan(context) - else: - # self is +/-Infinity; return unaltered - return Decimal(self) - - # if self is zero then exponent should be between Etiny and - # Emax if clamp==0, and between Etiny and Etop if clamp==1. - Etiny = context.Etiny() - Etop = context.Etop() - if not self: - exp_max = [context.Emax, Etop][context.clamp] - new_exp = min(max(self._exp, Etiny), exp_max) - if new_exp != self._exp: - context._raise_error(Clamped) - return _dec_from_triple(self._sign, '0', new_exp) - else: - return Decimal(self) - - # exp_min is the smallest allowable exponent of the result, - # equal to max(self.adjusted()-context.prec+1, Etiny) - exp_min = len(self._int) + self._exp - context.prec - if exp_min > Etop: - # overflow: exp_min > Etop iff self.adjusted() > Emax - ans = context._raise_error(Overflow, 'above Emax', self._sign) - context._raise_error(Inexact) - context._raise_error(Rounded) - return ans - - self_is_subnormal = exp_min < Etiny - if self_is_subnormal: - exp_min = Etiny - - # round if self has too many digits - if self._exp < exp_min: - digits = len(self._int) + self._exp - exp_min - if digits < 0: - self = _dec_from_triple(self._sign, '1', exp_min-1) - digits = 0 - rounding_method = self._pick_rounding_function[context.rounding] - changed = rounding_method(self, digits) - coeff = self._int[:digits] or '0' - if changed > 0: - coeff = str(int(coeff)+1) - if len(coeff) > context.prec: - coeff = coeff[:-1] - exp_min += 1 - - # check whether the rounding pushed the exponent out of range - if exp_min > Etop: - ans = context._raise_error(Overflow, 'above Emax', self._sign) - else: - ans = _dec_from_triple(self._sign, coeff, exp_min) - - # raise the appropriate signals, taking care to respect - # the precedence described in the specification - if changed and self_is_subnormal: - context._raise_error(Underflow) - if self_is_subnormal: - context._raise_error(Subnormal) - if changed: - context._raise_error(Inexact) - context._raise_error(Rounded) - if not ans: - # raise Clamped on underflow to 0 - context._raise_error(Clamped) - return ans - - if self_is_subnormal: - context._raise_error(Subnormal) - - # fold down if clamp == 1 and self has too few digits - if context.clamp == 1 and self._exp > Etop: - context._raise_error(Clamped) - self_padded = self._int + '0'*(self._exp - Etop) - return _dec_from_triple(self._sign, self_padded, Etop) - - # here self was representable to begin with; return unchanged - return Decimal(self) - - # for each of the rounding functions below: - # self is a finite, nonzero Decimal - # prec is an integer satisfying 0 <= prec < len(self._int) - # - # each function returns either -1, 0, or 1, as follows: - # 1 indicates that self should be rounded up (away from zero) - # 0 indicates that self should be truncated, and that all the - # digits to be truncated are zeros (so the value is unchanged) - # -1 indicates that there are nonzero digits to be truncated - - def _round_down(self, prec): - """Also known as round-towards-0, truncate.""" - if _all_zeros(self._int, prec): - return 0 - else: - return -1 - - def _round_up(self, prec): - """Rounds away from 0.""" - return -self._round_down(prec) - - def _round_half_up(self, prec): - """Rounds 5 up (away from 0)""" - if self._int[prec] in '56789': - return 1 - elif _all_zeros(self._int, prec): - return 0 - else: - return -1 - - def _round_half_down(self, prec): - """Round 5 down""" - if _exact_half(self._int, prec): - return -1 - else: - return self._round_half_up(prec) - - def _round_half_even(self, prec): - """Round 5 to even, rest to nearest.""" - if _exact_half(self._int, prec) and \ - (prec == 0 or self._int[prec-1] in '02468'): - return -1 - else: - return self._round_half_up(prec) - - def _round_ceiling(self, prec): - """Rounds up (not away from 0 if negative.)""" - if self._sign: - return self._round_down(prec) - else: - return -self._round_down(prec) - - def _round_floor(self, prec): - """Rounds down (not towards 0 if negative)""" - if not self._sign: - return self._round_down(prec) - else: - return -self._round_down(prec) - - def _round_05up(self, prec): - """Round down unless digit prec-1 is 0 or 5.""" - if prec and self._int[prec-1] not in '05': - return self._round_down(prec) - else: - return -self._round_down(prec) - - _pick_rounding_function = dict( - ROUND_DOWN = _round_down, - ROUND_UP = _round_up, - ROUND_HALF_UP = _round_half_up, - ROUND_HALF_DOWN = _round_half_down, - ROUND_HALF_EVEN = _round_half_even, - ROUND_CEILING = _round_ceiling, - ROUND_FLOOR = _round_floor, - ROUND_05UP = _round_05up, - ) - - def __round__(self, n=None): - """Round self to the nearest integer, or to a given precision. - - If only one argument is supplied, round a finite Decimal - instance self to the nearest integer. If self is infinite or - a NaN then a Python exception is raised. If self is finite - and lies exactly halfway between two integers then it is - rounded to the integer with even last digit. - - >>> round(Decimal('123.456')) - 123 - >>> round(Decimal('-456.789')) - -457 - >>> round(Decimal('-3.0')) - -3 - >>> round(Decimal('2.5')) - 2 - >>> round(Decimal('3.5')) - 4 - >>> round(Decimal('Inf')) - Traceback (most recent call last): - ... - OverflowError: cannot round an infinity - >>> round(Decimal('NaN')) - Traceback (most recent call last): - ... - ValueError: cannot round a NaN - - If a second argument n is supplied, self is rounded to n - decimal places using the rounding mode for the current - context. - - For an integer n, round(self, -n) is exactly equivalent to - self.quantize(Decimal('1En')). - - >>> round(Decimal('123.456'), 0) - Decimal('123') - >>> round(Decimal('123.456'), 2) - Decimal('123.46') - >>> round(Decimal('123.456'), -2) - Decimal('1E+2') - >>> round(Decimal('-Infinity'), 37) - Decimal('NaN') - >>> round(Decimal('sNaN123'), 0) - Decimal('NaN123') - - """ - if n is not None: - # two-argument form: use the equivalent quantize call - if not isinstance(n, int): - raise TypeError('Second argument to round should be integral') - exp = _dec_from_triple(0, '1', -n) - return self.quantize(exp) - - # one-argument form - if self._is_special: - if self.is_nan(): - raise ValueError("cannot round a NaN") - else: - raise OverflowError("cannot round an infinity") - return int(self._rescale(0, ROUND_HALF_EVEN)) - - def __floor__(self): - """Return the floor of self, as an integer. - - For a finite Decimal instance self, return the greatest - integer n such that n <= self. If self is infinite or a NaN - then a Python exception is raised. - - """ - if self._is_special: - if self.is_nan(): - raise ValueError("cannot round a NaN") - else: - raise OverflowError("cannot round an infinity") - return int(self._rescale(0, ROUND_FLOOR)) - - def __ceil__(self): - """Return the ceiling of self, as an integer. - - For a finite Decimal instance self, return the least integer n - such that n >= self. If self is infinite or a NaN then a - Python exception is raised. - - """ - if self._is_special: - if self.is_nan(): - raise ValueError("cannot round a NaN") - else: - raise OverflowError("cannot round an infinity") - return int(self._rescale(0, ROUND_CEILING)) - - def fma(self, other, third, context=None): - """Fused multiply-add. - - Returns self*other+third with no rounding of the intermediate - product self*other. - - self and other are multiplied together, with no rounding of - the result. The third operand is then added to the result, - and a single final rounding is performed. - """ - - other = _convert_other(other, raiseit=True) - third = _convert_other(third, raiseit=True) - - # compute product; raise InvalidOperation if either operand is - # a signaling NaN or if the product is zero times infinity. - if self._is_special or other._is_special: - if context is None: - context = getcontext() - if self._exp == 'N': - return context._raise_error(InvalidOperation, 'sNaN', self) - if other._exp == 'N': - return context._raise_error(InvalidOperation, 'sNaN', other) - if self._exp == 'n': - product = self - elif other._exp == 'n': - product = other - elif self._exp == 'F': - if not other: - return context._raise_error(InvalidOperation, - 'INF * 0 in fma') - product = _SignedInfinity[self._sign ^ other._sign] - elif other._exp == 'F': - if not self: - return context._raise_error(InvalidOperation, - '0 * INF in fma') - product = _SignedInfinity[self._sign ^ other._sign] - else: - product = _dec_from_triple(self._sign ^ other._sign, - str(int(self._int) * int(other._int)), - self._exp + other._exp) - - return product.__add__(third, context) - - def _power_modulo(self, other, modulo, context=None): - """Three argument version of __pow__""" - - other = _convert_other(other) - if other is NotImplemented: - return other - modulo = _convert_other(modulo) - if modulo is NotImplemented: - return modulo - - if context is None: - context = getcontext() - - # deal with NaNs: if there are any sNaNs then first one wins, - # (i.e. behaviour for NaNs is identical to that of fma) - self_is_nan = self._isnan() - other_is_nan = other._isnan() - modulo_is_nan = modulo._isnan() - if self_is_nan or other_is_nan or modulo_is_nan: - if self_is_nan == 2: - return context._raise_error(InvalidOperation, 'sNaN', - self) - if other_is_nan == 2: - return context._raise_error(InvalidOperation, 'sNaN', - other) - if modulo_is_nan == 2: - return context._raise_error(InvalidOperation, 'sNaN', - modulo) - if self_is_nan: - return self._fix_nan(context) - if other_is_nan: - return other._fix_nan(context) - return modulo._fix_nan(context) - - # check inputs: we apply same restrictions as Python's pow() - if not (self._isinteger() and - other._isinteger() and - modulo._isinteger()): - return context._raise_error(InvalidOperation, - 'pow() 3rd argument not allowed ' - 'unless all arguments are integers') - if other < 0: - return context._raise_error(InvalidOperation, - 'pow() 2nd argument cannot be ' - 'negative when 3rd argument specified') - if not modulo: - return context._raise_error(InvalidOperation, - 'pow() 3rd argument cannot be 0') - - # additional restriction for decimal: the modulus must be less - # than 10**prec in absolute value - if modulo.adjusted() >= context.prec: - return context._raise_error(InvalidOperation, - 'insufficient precision: pow() 3rd ' - 'argument must not have more than ' - 'precision digits') - - # define 0**0 == NaN, for consistency with two-argument pow - # (even though it hurts!) - if not other and not self: - return context._raise_error(InvalidOperation, - 'at least one of pow() 1st argument ' - 'and 2nd argument must be nonzero ;' - '0**0 is not defined') - - # compute sign of result - if other._iseven(): - sign = 0 - else: - sign = self._sign - - # convert modulo to a Python integer, and self and other to - # Decimal integers (i.e. force their exponents to be >= 0) - modulo = abs(int(modulo)) - base = _WorkRep(self.to_integral_value()) - exponent = _WorkRep(other.to_integral_value()) - - # compute result using integer pow() - base = (base.int % modulo * pow(10, base.exp, modulo)) % modulo - for i in range(exponent.exp): - base = pow(base, 10, modulo) - base = pow(base, exponent.int, modulo) - - return _dec_from_triple(sign, str(base), 0) - - def _power_exact(self, other, p): - """Attempt to compute self**other exactly. - - Given Decimals self and other and an integer p, attempt to - compute an exact result for the power self**other, with p - digits of precision. Return None if self**other is not - exactly representable in p digits. - - Assumes that elimination of special cases has already been - performed: self and other must both be nonspecial; self must - be positive and not numerically equal to 1; other must be - nonzero. For efficiency, other._exp should not be too large, - so that 10**abs(other._exp) is a feasible calculation.""" - - # In the comments below, we write x for the value of self and y for the - # value of other. Write x = xc*10**xe and abs(y) = yc*10**ye, with xc - # and yc positive integers not divisible by 10. - - # The main purpose of this method is to identify the *failure* - # of x**y to be exactly representable with as little effort as - # possible. So we look for cheap and easy tests that - # eliminate the possibility of x**y being exact. Only if all - # these tests are passed do we go on to actually compute x**y. - - # Here's the main idea. Express y as a rational number m/n, with m and - # n relatively prime and n>0. Then for x**y to be exactly - # representable (at *any* precision), xc must be the nth power of a - # positive integer and xe must be divisible by n. If y is negative - # then additionally xc must be a power of either 2 or 5, hence a power - # of 2**n or 5**n. - # - # There's a limit to how small |y| can be: if y=m/n as above - # then: - # - # (1) if xc != 1 then for the result to be representable we - # need xc**(1/n) >= 2, and hence also xc**|y| >= 2. So - # if |y| <= 1/nbits(xc) then xc < 2**nbits(xc) <= - # 2**(1/|y|), hence xc**|y| < 2 and the result is not - # representable. - # - # (2) if xe != 0, |xe|*(1/n) >= 1, so |xe|*|y| >= 1. Hence if - # |y| < 1/|xe| then the result is not representable. - # - # Note that since x is not equal to 1, at least one of (1) and - # (2) must apply. Now |y| < 1/nbits(xc) iff |yc|*nbits(xc) < - # 10**-ye iff len(str(|yc|*nbits(xc)) <= -ye. - # - # There's also a limit to how large y can be, at least if it's - # positive: the normalized result will have coefficient xc**y, - # so if it's representable then xc**y < 10**p, and y < - # p/log10(xc). Hence if y*log10(xc) >= p then the result is - # not exactly representable. - - # if len(str(abs(yc*xe)) <= -ye then abs(yc*xe) < 10**-ye, - # so |y| < 1/xe and the result is not representable. - # Similarly, len(str(abs(yc)*xc_bits)) <= -ye implies |y| - # < 1/nbits(xc). - - x = _WorkRep(self) - xc, xe = x.int, x.exp - while xc % 10 == 0: - xc //= 10 - xe += 1 - - y = _WorkRep(other) - yc, ye = y.int, y.exp - while yc % 10 == 0: - yc //= 10 - ye += 1 - - # case where xc == 1: result is 10**(xe*y), with xe*y - # required to be an integer - if xc == 1: - xe *= yc - # result is now 10**(xe * 10**ye); xe * 10**ye must be integral - while xe % 10 == 0: - xe //= 10 - ye += 1 - if ye < 0: - return None - exponent = xe * 10**ye - if y.sign == 1: - exponent = -exponent - # if other is a nonnegative integer, use ideal exponent - if other._isinteger() and other._sign == 0: - ideal_exponent = self._exp*int(other) - zeros = min(exponent-ideal_exponent, p-1) - else: - zeros = 0 - return _dec_from_triple(0, '1' + '0'*zeros, exponent-zeros) - - # case where y is negative: xc must be either a power - # of 2 or a power of 5. - if y.sign == 1: - last_digit = xc % 10 - if last_digit in (2,4,6,8): - # quick test for power of 2 - if xc & -xc != xc: - return None - # now xc is a power of 2; e is its exponent - e = _nbits(xc)-1 - - # We now have: - # - # x = 2**e * 10**xe, e > 0, and y < 0. - # - # The exact result is: - # - # x**y = 5**(-e*y) * 10**(e*y + xe*y) - # - # provided that both e*y and xe*y are integers. Note that if - # 5**(-e*y) >= 10**p, then the result can't be expressed - # exactly with p digits of precision. - # - # Using the above, we can guard against large values of ye. - # 93/65 is an upper bound for log(10)/log(5), so if - # - # ye >= len(str(93*p//65)) - # - # then - # - # -e*y >= -y >= 10**ye > 93*p/65 > p*log(10)/log(5), - # - # so 5**(-e*y) >= 10**p, and the coefficient of the result - # can't be expressed in p digits. - - # emax >= largest e such that 5**e < 10**p. - emax = p*93//65 - if ye >= len(str(emax)): - return None - - # Find -e*y and -xe*y; both must be integers - e = _decimal_lshift_exact(e * yc, ye) - xe = _decimal_lshift_exact(xe * yc, ye) - if e is None or xe is None: - return None - - if e > emax: - return None - xc = 5**e - - elif last_digit == 5: - # e >= log_5(xc) if xc is a power of 5; we have - # equality all the way up to xc=5**2658 - e = _nbits(xc)*28//65 - xc, remainder = divmod(5**e, xc) - if remainder: - return None - while xc % 5 == 0: - xc //= 5 - e -= 1 - - # Guard against large values of ye, using the same logic as in - # the 'xc is a power of 2' branch. 10/3 is an upper bound for - # log(10)/log(2). - emax = p*10//3 - if ye >= len(str(emax)): - return None - - e = _decimal_lshift_exact(e * yc, ye) - xe = _decimal_lshift_exact(xe * yc, ye) - if e is None or xe is None: - return None - - if e > emax: - return None - xc = 2**e - else: - return None - - if xc >= 10**p: - return None - xe = -e-xe - return _dec_from_triple(0, str(xc), xe) - - # now y is positive; find m and n such that y = m/n - if ye >= 0: - m, n = yc*10**ye, 1 - else: - if xe != 0 and len(str(abs(yc*xe))) <= -ye: - return None - xc_bits = _nbits(xc) - if xc != 1 and len(str(abs(yc)*xc_bits)) <= -ye: - return None - m, n = yc, 10**(-ye) - while m % 2 == n % 2 == 0: - m //= 2 - n //= 2 - while m % 5 == n % 5 == 0: - m //= 5 - n //= 5 - - # compute nth root of xc*10**xe - if n > 1: - # if 1 < xc < 2**n then xc isn't an nth power - if xc != 1 and xc_bits <= n: - return None - - xe, rem = divmod(xe, n) - if rem != 0: - return None - - # compute nth root of xc using Newton's method - a = 1 << -(-_nbits(xc)//n) # initial estimate - while True: - q, r = divmod(xc, a**(n-1)) - if a <= q: - break - else: - a = (a*(n-1) + q)//n - if not (a == q and r == 0): - return None - xc = a - - # now xc*10**xe is the nth root of the original xc*10**xe - # compute mth power of xc*10**xe - - # if m > p*100//_log10_lb(xc) then m > p/log10(xc), hence xc**m > - # 10**p and the result is not representable. - if xc > 1 and m > p*100//_log10_lb(xc): - return None - xc = xc**m - xe *= m - if xc > 10**p: - return None - - # by this point the result *is* exactly representable - # adjust the exponent to get as close as possible to the ideal - # exponent, if necessary - str_xc = str(xc) - if other._isinteger() and other._sign == 0: - ideal_exponent = self._exp*int(other) - zeros = min(xe-ideal_exponent, p-len(str_xc)) - else: - zeros = 0 - return _dec_from_triple(0, str_xc+'0'*zeros, xe-zeros) - - def __pow__(self, other, modulo=None, context=None): - """Return self ** other [ % modulo]. - - With two arguments, compute self**other. - - With three arguments, compute (self**other) % modulo. For the - three argument form, the following restrictions on the - arguments hold: - - - all three arguments must be integral - - other must be nonnegative - - either self or other (or both) must be nonzero - - modulo must be nonzero and must have at most p digits, - where p is the context precision. - - If any of these restrictions is violated the InvalidOperation - flag is raised. - - The result of pow(self, other, modulo) is identical to the - result that would be obtained by computing (self**other) % - modulo with unbounded precision, but is computed more - efficiently. It is always exact. - """ - - if modulo is not None: - return self._power_modulo(other, modulo, context) - - other = _convert_other(other) - if other is NotImplemented: - return other - - if context is None: - context = getcontext() - - # either argument is a NaN => result is NaN - ans = self._check_nans(other, context) - if ans: - return ans - - # 0**0 = NaN (!), x**0 = 1 for nonzero x (including +/-Infinity) - if not other: - if not self: - return context._raise_error(InvalidOperation, '0 ** 0') - else: - return _One - - # result has sign 1 iff self._sign is 1 and other is an odd integer - result_sign = 0 - if self._sign == 1: - if other._isinteger(): - if not other._iseven(): - result_sign = 1 - else: - # -ve**noninteger = NaN - # (-0)**noninteger = 0**noninteger - if self: - return context._raise_error(InvalidOperation, - 'x ** y with x negative and y not an integer') - # negate self, without doing any unwanted rounding - self = self.copy_negate() - - # 0**(+ve or Inf)= 0; 0**(-ve or -Inf) = Infinity - if not self: - if other._sign == 0: - return _dec_from_triple(result_sign, '0', 0) - else: - return _SignedInfinity[result_sign] - - # Inf**(+ve or Inf) = Inf; Inf**(-ve or -Inf) = 0 - if self._isinfinity(): - if other._sign == 0: - return _SignedInfinity[result_sign] - else: - return _dec_from_triple(result_sign, '0', 0) - - # 1**other = 1, but the choice of exponent and the flags - # depend on the exponent of self, and on whether other is a - # positive integer, a negative integer, or neither - if self == _One: - if other._isinteger(): - # exp = max(self._exp*max(int(other), 0), - # 1-context.prec) but evaluating int(other) directly - # is dangerous until we know other is small (other - # could be 1e999999999) - if other._sign == 1: - multiplier = 0 - elif other > context.prec: - multiplier = context.prec - else: - multiplier = int(other) - - exp = self._exp * multiplier - if exp < 1-context.prec: - exp = 1-context.prec - context._raise_error(Rounded) - else: - context._raise_error(Inexact) - context._raise_error(Rounded) - exp = 1-context.prec - - return _dec_from_triple(result_sign, '1'+'0'*-exp, exp) - - # compute adjusted exponent of self - self_adj = self.adjusted() - - # self ** infinity is infinity if self > 1, 0 if self < 1 - # self ** -infinity is infinity if self < 1, 0 if self > 1 - if other._isinfinity(): - if (other._sign == 0) == (self_adj < 0): - return _dec_from_triple(result_sign, '0', 0) - else: - return _SignedInfinity[result_sign] - - # from here on, the result always goes through the call - # to _fix at the end of this function. - ans = None - exact = False - - # crude test to catch cases of extreme overflow/underflow. If - # log10(self)*other >= 10**bound and bound >= len(str(Emax)) - # then 10**bound >= 10**len(str(Emax)) >= Emax+1 and hence - # self**other >= 10**(Emax+1), so overflow occurs. The test - # for underflow is similar. - bound = self._log10_exp_bound() + other.adjusted() - if (self_adj >= 0) == (other._sign == 0): - # self > 1 and other +ve, or self < 1 and other -ve - # possibility of overflow - if bound >= len(str(context.Emax)): - ans = _dec_from_triple(result_sign, '1', context.Emax+1) - else: - # self > 1 and other -ve, or self < 1 and other +ve - # possibility of underflow to 0 - Etiny = context.Etiny() - if bound >= len(str(-Etiny)): - ans = _dec_from_triple(result_sign, '1', Etiny-1) - - # try for an exact result with precision +1 - if ans is None: - ans = self._power_exact(other, context.prec + 1) - if ans is not None: - if result_sign == 1: - ans = _dec_from_triple(1, ans._int, ans._exp) - exact = True - - # usual case: inexact result, x**y computed directly as exp(y*log(x)) - if ans is None: - p = context.prec - x = _WorkRep(self) - xc, xe = x.int, x.exp - y = _WorkRep(other) - yc, ye = y.int, y.exp - if y.sign == 1: - yc = -yc - - # compute correctly rounded result: start with precision +3, - # then increase precision until result is unambiguously roundable - extra = 3 - while True: - coeff, exp = _dpower(xc, xe, yc, ye, p+extra) - if coeff % (5*10**(len(str(coeff))-p-1)): - break - extra += 3 - - ans = _dec_from_triple(result_sign, str(coeff), exp) - - # unlike exp, ln and log10, the power function respects the - # rounding mode; no need to switch to ROUND_HALF_EVEN here - - # There's a difficulty here when 'other' is not an integer and - # the result is exact. In this case, the specification - # requires that the Inexact flag be raised (in spite of - # exactness), but since the result is exact _fix won't do this - # for us. (Correspondingly, the Underflow signal should also - # be raised for subnormal results.) We can't directly raise - # these signals either before or after calling _fix, since - # that would violate the precedence for signals. So we wrap - # the ._fix call in a temporary context, and reraise - # afterwards. - if exact and not other._isinteger(): - # pad with zeros up to length context.prec+1 if necessary; this - # ensures that the Rounded signal will be raised. - if len(ans._int) <= context.prec: - expdiff = context.prec + 1 - len(ans._int) - ans = _dec_from_triple(ans._sign, ans._int+'0'*expdiff, - ans._exp-expdiff) - - # create a copy of the current context, with cleared flags/traps - newcontext = context.copy() - newcontext.clear_flags() - for exception in _signals: - newcontext.traps[exception] = 0 - - # round in the new context - ans = ans._fix(newcontext) - - # raise Inexact, and if necessary, Underflow - newcontext._raise_error(Inexact) - if newcontext.flags[Subnormal]: - newcontext._raise_error(Underflow) - - # propagate signals to the original context; _fix could - # have raised any of Overflow, Underflow, Subnormal, - # Inexact, Rounded, Clamped. Overflow needs the correct - # arguments. Note that the order of the exceptions is - # important here. - if newcontext.flags[Overflow]: - context._raise_error(Overflow, 'above Emax', ans._sign) - for exception in Underflow, Subnormal, Inexact, Rounded, Clamped: - if newcontext.flags[exception]: - context._raise_error(exception) - - else: - ans = ans._fix(context) - - return ans - - def __rpow__(self, other, context=None): - """Swaps self/other and returns __pow__.""" - other = _convert_other(other) - if other is NotImplemented: - return other - return other.__pow__(self, context=context) - - def normalize(self, context=None): - """Normalize- strip trailing 0s, change anything equal to 0 to 0e0""" - - if context is None: - context = getcontext() - - if self._is_special: - ans = self._check_nans(context=context) - if ans: - return ans - - dup = self._fix(context) - if dup._isinfinity(): - return dup - - if not dup: - return _dec_from_triple(dup._sign, '0', 0) - exp_max = [context.Emax, context.Etop()][context.clamp] - end = len(dup._int) - exp = dup._exp - while dup._int[end-1] == '0' and exp < exp_max: - exp += 1 - end -= 1 - return _dec_from_triple(dup._sign, dup._int[:end], exp) - - def quantize(self, exp, rounding=None, context=None): - """Quantize self so its exponent is the same as that of exp. - - Similar to self._rescale(exp._exp) but with error checking. - """ - exp = _convert_other(exp, raiseit=True) - - if context is None: - context = getcontext() - if rounding is None: - rounding = context.rounding - - if self._is_special or exp._is_special: - ans = self._check_nans(exp, context) - if ans: - return ans - - if exp._isinfinity() or self._isinfinity(): - if exp._isinfinity() and self._isinfinity(): - return Decimal(self) # if both are inf, it is OK - return context._raise_error(InvalidOperation, - 'quantize with one INF') - - # exp._exp should be between Etiny and Emax - if not (context.Etiny() <= exp._exp <= context.Emax): - return context._raise_error(InvalidOperation, - 'target exponent out of bounds in quantize') - - if not self: - ans = _dec_from_triple(self._sign, '0', exp._exp) - return ans._fix(context) - - self_adjusted = self.adjusted() - if self_adjusted > context.Emax: - return context._raise_error(InvalidOperation, - 'exponent of quantize result too large for current context') - if self_adjusted - exp._exp + 1 > context.prec: - return context._raise_error(InvalidOperation, - 'quantize result has too many digits for current context') - - ans = self._rescale(exp._exp, rounding) - if ans.adjusted() > context.Emax: - return context._raise_error(InvalidOperation, - 'exponent of quantize result too large for current context') - if len(ans._int) > context.prec: - return context._raise_error(InvalidOperation, - 'quantize result has too many digits for current context') - - # raise appropriate flags - if ans and ans.adjusted() < context.Emin: - context._raise_error(Subnormal) - if ans._exp > self._exp: - if ans != self: - context._raise_error(Inexact) - context._raise_error(Rounded) - - # call to fix takes care of any necessary folddown, and - # signals Clamped if necessary - ans = ans._fix(context) - return ans - - def same_quantum(self, other, context=None): - """Return True if self and other have the same exponent; otherwise - return False. - - If either operand is a special value, the following rules are used: - * return True if both operands are infinities - * return True if both operands are NaNs - * otherwise, return False. - """ - other = _convert_other(other, raiseit=True) - if self._is_special or other._is_special: - return (self.is_nan() and other.is_nan() or - self.is_infinite() and other.is_infinite()) - return self._exp == other._exp - - def _rescale(self, exp, rounding): - """Rescale self so that the exponent is exp, either by padding with zeros - or by truncating digits, using the given rounding mode. - - Specials are returned without change. This operation is - quiet: it raises no flags, and uses no information from the - context. - - exp = exp to scale to (an integer) - rounding = rounding mode - """ - if self._is_special: - return Decimal(self) - if not self: - return _dec_from_triple(self._sign, '0', exp) - - if self._exp >= exp: - # pad answer with zeros if necessary - return _dec_from_triple(self._sign, - self._int + '0'*(self._exp - exp), exp) - - # too many digits; round and lose data. If self.adjusted() < - # exp-1, replace self by 10**(exp-1) before rounding - digits = len(self._int) + self._exp - exp - if digits < 0: - self = _dec_from_triple(self._sign, '1', exp-1) - digits = 0 - this_function = self._pick_rounding_function[rounding] - changed = this_function(self, digits) - coeff = self._int[:digits] or '0' - if changed == 1: - coeff = str(int(coeff)+1) - return _dec_from_triple(self._sign, coeff, exp) - - def _round(self, places, rounding): - """Round a nonzero, nonspecial Decimal to a fixed number of - significant figures, using the given rounding mode. - - Infinities, NaNs and zeros are returned unaltered. - - This operation is quiet: it raises no flags, and uses no - information from the context. - - """ - if places <= 0: - raise ValueError("argument should be at least 1 in _round") - if self._is_special or not self: - return Decimal(self) - ans = self._rescale(self.adjusted()+1-places, rounding) - # it can happen that the rescale alters the adjusted exponent; - # for example when rounding 99.97 to 3 significant figures. - # When this happens we end up with an extra 0 at the end of - # the number; a second rescale fixes this. - if ans.adjusted() != self.adjusted(): - ans = ans._rescale(ans.adjusted()+1-places, rounding) - return ans - - def to_integral_exact(self, rounding=None, context=None): - """Rounds to a nearby integer. - - If no rounding mode is specified, take the rounding mode from - the context. This method raises the Rounded and Inexact flags - when appropriate. - - See also: to_integral_value, which does exactly the same as - this method except that it doesn't raise Inexact or Rounded. - """ - if self._is_special: - ans = self._check_nans(context=context) - if ans: - return ans - return Decimal(self) - if self._exp >= 0: - return Decimal(self) - if not self: - return _dec_from_triple(self._sign, '0', 0) - if context is None: - context = getcontext() - if rounding is None: - rounding = context.rounding - ans = self._rescale(0, rounding) - if ans != self: - context._raise_error(Inexact) - context._raise_error(Rounded) - return ans - - def to_integral_value(self, rounding=None, context=None): - """Rounds to the nearest integer, without raising inexact, rounded.""" - if context is None: - context = getcontext() - if rounding is None: - rounding = context.rounding - if self._is_special: - ans = self._check_nans(context=context) - if ans: - return ans - return Decimal(self) - if self._exp >= 0: - return Decimal(self) - else: - return self._rescale(0, rounding) - - # the method name changed, but we provide also the old one, for compatibility - to_integral = to_integral_value - - def sqrt(self, context=None): - """Return the square root of self.""" - if context is None: - context = getcontext() - - if self._is_special: - ans = self._check_nans(context=context) - if ans: - return ans - - if self._isinfinity() and self._sign == 0: - return Decimal(self) - - if not self: - # exponent = self._exp // 2. sqrt(-0) = -0 - ans = _dec_from_triple(self._sign, '0', self._exp // 2) - return ans._fix(context) - - if self._sign == 1: - return context._raise_error(InvalidOperation, 'sqrt(-x), x > 0') - - # At this point self represents a positive number. Let p be - # the desired precision and express self in the form c*100**e - # with c a positive real number and e an integer, c and e - # being chosen so that 100**(p-1) <= c < 100**p. Then the - # (exact) square root of self is sqrt(c)*10**e, and 10**(p-1) - # <= sqrt(c) < 10**p, so the closest representable Decimal at - # precision p is n*10**e where n = round_half_even(sqrt(c)), - # the closest integer to sqrt(c) with the even integer chosen - # in the case of a tie. - # - # To ensure correct rounding in all cases, we use the - # following trick: we compute the square root to an extra - # place (precision p+1 instead of precision p), rounding down. - # Then, if the result is inexact and its last digit is 0 or 5, - # we increase the last digit to 1 or 6 respectively; if it's - # exact we leave the last digit alone. Now the final round to - # p places (or fewer in the case of underflow) will round - # correctly and raise the appropriate flags. - - # use an extra digit of precision - prec = context.prec+1 - - # write argument in the form c*100**e where e = self._exp//2 - # is the 'ideal' exponent, to be used if the square root is - # exactly representable. l is the number of 'digits' of c in - # base 100, so that 100**(l-1) <= c < 100**l. - op = _WorkRep(self) - e = op.exp >> 1 - if op.exp & 1: - c = op.int * 10 - l = (len(self._int) >> 1) + 1 - else: - c = op.int - l = len(self._int)+1 >> 1 - - # rescale so that c has exactly prec base 100 'digits' - shift = prec-l - if shift >= 0: - c *= 100**shift - exact = True - else: - c, remainder = divmod(c, 100**-shift) - exact = not remainder - e -= shift - - # find n = floor(sqrt(c)) using Newton's method - n = 10**prec - while True: - q = c//n - if n <= q: - break - else: - n = n + q >> 1 - exact = exact and n*n == c - - if exact: - # result is exact; rescale to use ideal exponent e - if shift >= 0: - # assert n % 10**shift == 0 - n //= 10**shift - else: - n *= 10**-shift - e += shift - else: - # result is not exact; fix last digit as described above - if n % 5 == 0: - n += 1 - - ans = _dec_from_triple(0, str(n), e) - - # round, and fit to current context - context = context._shallow_copy() - rounding = context._set_rounding(ROUND_HALF_EVEN) - ans = ans._fix(context) - context.rounding = rounding - - return ans - - def max(self, other, context=None): - """Returns the larger value. - - Like max(self, other) except if one is not a number, returns - NaN (and signals if one is sNaN). Also rounds. - """ - other = _convert_other(other, raiseit=True) - - if context is None: - context = getcontext() - - if self._is_special or other._is_special: - # If one operand is a quiet NaN and the other is number, then the - # number is always returned - sn = self._isnan() - on = other._isnan() - if sn or on: - if on == 1 and sn == 0: - return self._fix(context) - if sn == 1 and on == 0: - return other._fix(context) - return self._check_nans(other, context) - - c = self._cmp(other) - if c == 0: - # If both operands are finite and equal in numerical value - # then an ordering is applied: - # - # If the signs differ then max returns the operand with the - # positive sign and min returns the operand with the negative sign - # - # If the signs are the same then the exponent is used to select - # the result. This is exactly the ordering used in compare_total. - c = self.compare_total(other) - - if c == -1: - ans = other - else: - ans = self - - return ans._fix(context) - - def min(self, other, context=None): - """Returns the smaller value. - - Like min(self, other) except if one is not a number, returns - NaN (and signals if one is sNaN). Also rounds. - """ - other = _convert_other(other, raiseit=True) - - if context is None: - context = getcontext() - - if self._is_special or other._is_special: - # If one operand is a quiet NaN and the other is number, then the - # number is always returned - sn = self._isnan() - on = other._isnan() - if sn or on: - if on == 1 and sn == 0: - return self._fix(context) - if sn == 1 and on == 0: - return other._fix(context) - return self._check_nans(other, context) - - c = self._cmp(other) - if c == 0: - c = self.compare_total(other) - - if c == -1: - ans = self - else: - ans = other - - return ans._fix(context) - - def _isinteger(self): - """Returns whether self is an integer""" - if self._is_special: - return False - if self._exp >= 0: - return True - rest = self._int[self._exp:] - return rest == '0'*len(rest) - - def _iseven(self): - """Returns True if self is even. Assumes self is an integer.""" - if not self or self._exp > 0: - return True - return self._int[-1+self._exp] in '02468' - - def adjusted(self): - """Return the adjusted exponent of self""" - try: - return self._exp + len(self._int) - 1 - # If NaN or Infinity, self._exp is string - except TypeError: - return 0 - - def canonical(self): - """Returns the same Decimal object. - - As we do not have different encodings for the same number, the - received object already is in its canonical form. - """ - return self - - def compare_signal(self, other, context=None): - """Compares self to the other operand numerically. - - It's pretty much like compare(), but all NaNs signal, with signaling - NaNs taking precedence over quiet NaNs. - """ - other = _convert_other(other, raiseit = True) - ans = self._compare_check_nans(other, context) - if ans: - return ans - return self.compare(other, context=context) - - def compare_total(self, other, context=None): - """Compares self to other using the abstract representations. - - This is not like the standard compare, which use their numerical - value. Note that a total ordering is defined for all possible abstract - representations. - """ - other = _convert_other(other, raiseit=True) - - # if one is negative and the other is positive, it's easy - if self._sign and not other._sign: - return _NegativeOne - if not self._sign and other._sign: - return _One - sign = self._sign - - # let's handle both NaN types - self_nan = self._isnan() - other_nan = other._isnan() - if self_nan or other_nan: - if self_nan == other_nan: - # compare payloads as though they're integers - self_key = len(self._int), self._int - other_key = len(other._int), other._int - if self_key < other_key: - if sign: - return _One - else: - return _NegativeOne - if self_key > other_key: - if sign: - return _NegativeOne - else: - return _One - return _Zero - - if sign: - if self_nan == 1: - return _NegativeOne - if other_nan == 1: - return _One - if self_nan == 2: - return _NegativeOne - if other_nan == 2: - return _One - else: - if self_nan == 1: - return _One - if other_nan == 1: - return _NegativeOne - if self_nan == 2: - return _One - if other_nan == 2: - return _NegativeOne - - if self < other: - return _NegativeOne - if self > other: - return _One - - if self._exp < other._exp: - if sign: - return _One - else: - return _NegativeOne - if self._exp > other._exp: - if sign: - return _NegativeOne - else: - return _One - return _Zero - - - def compare_total_mag(self, other, context=None): - """Compares self to other using abstract repr., ignoring sign. - - Like compare_total, but with operand's sign ignored and assumed to be 0. - """ - other = _convert_other(other, raiseit=True) - - s = self.copy_abs() - o = other.copy_abs() - return s.compare_total(o) - - def copy_abs(self): - """Returns a copy with the sign set to 0. """ - return _dec_from_triple(0, self._int, self._exp, self._is_special) - - def copy_negate(self): - """Returns a copy with the sign inverted.""" - if self._sign: - return _dec_from_triple(0, self._int, self._exp, self._is_special) - else: - return _dec_from_triple(1, self._int, self._exp, self._is_special) - - def copy_sign(self, other, context=None): - """Returns self with the sign of other.""" - other = _convert_other(other, raiseit=True) - return _dec_from_triple(other._sign, self._int, - self._exp, self._is_special) - - def exp(self, context=None): - """Returns e ** self.""" - - if context is None: - context = getcontext() - - # exp(NaN) = NaN - ans = self._check_nans(context=context) - if ans: - return ans - - # exp(-Infinity) = 0 - if self._isinfinity() == -1: - return _Zero - - # exp(0) = 1 - if not self: - return _One - - # exp(Infinity) = Infinity - if self._isinfinity() == 1: - return Decimal(self) - - # the result is now guaranteed to be inexact (the true - # mathematical result is transcendental). There's no need to - # raise Rounded and Inexact here---they'll always be raised as - # a result of the call to _fix. - p = context.prec - adj = self.adjusted() - - # we only need to do any computation for quite a small range - # of adjusted exponents---for example, -29 <= adj <= 10 for - # the default context. For smaller exponent the result is - # indistinguishable from 1 at the given precision, while for - # larger exponent the result either overflows or underflows. - if self._sign == 0 and adj > len(str((context.Emax+1)*3)): - # overflow - ans = _dec_from_triple(0, '1', context.Emax+1) - elif self._sign == 1 and adj > len(str((-context.Etiny()+1)*3)): - # underflow to 0 - ans = _dec_from_triple(0, '1', context.Etiny()-1) - elif self._sign == 0 and adj < -p: - # p+1 digits; final round will raise correct flags - ans = _dec_from_triple(0, '1' + '0'*(p-1) + '1', -p) - elif self._sign == 1 and adj < -p-1: - # p+1 digits; final round will raise correct flags - ans = _dec_from_triple(0, '9'*(p+1), -p-1) - # general case - else: - op = _WorkRep(self) - c, e = op.int, op.exp - if op.sign == 1: - c = -c - - # compute correctly rounded result: increase precision by - # 3 digits at a time until we get an unambiguously - # roundable result - extra = 3 - while True: - coeff, exp = _dexp(c, e, p+extra) - if coeff % (5*10**(len(str(coeff))-p-1)): - break - extra += 3 - - ans = _dec_from_triple(0, str(coeff), exp) - - # at this stage, ans should round correctly with *any* - # rounding mode, not just with ROUND_HALF_EVEN - context = context._shallow_copy() - rounding = context._set_rounding(ROUND_HALF_EVEN) - ans = ans._fix(context) - context.rounding = rounding - - return ans - - def is_canonical(self): - """Return True if self is canonical; otherwise return False. - - Currently, the encoding of a Decimal instance is always - canonical, so this method returns True for any Decimal. - """ - return True - - def is_finite(self): - """Return True if self is finite; otherwise return False. - - A Decimal instance is considered finite if it is neither - infinite nor a NaN. - """ - return not self._is_special - - def is_infinite(self): - """Return True if self is infinite; otherwise return False.""" - return self._exp == 'F' - - def is_nan(self): - """Return True if self is a qNaN or sNaN; otherwise return False.""" - return self._exp in ('n', 'N') - - def is_normal(self, context=None): - """Return True if self is a normal number; otherwise return False.""" - if self._is_special or not self: - return False - if context is None: - context = getcontext() - return context.Emin <= self.adjusted() - - def is_qnan(self): - """Return True if self is a quiet NaN; otherwise return False.""" - return self._exp == 'n' - - def is_signed(self): - """Return True if self is negative; otherwise return False.""" - return self._sign == 1 - - def is_snan(self): - """Return True if self is a signaling NaN; otherwise return False.""" - return self._exp == 'N' - - def is_subnormal(self, context=None): - """Return True if self is subnormal; otherwise return False.""" - if self._is_special or not self: - return False - if context is None: - context = getcontext() - return self.adjusted() < context.Emin - - def is_zero(self): - """Return True if self is a zero; otherwise return False.""" - return not self._is_special and self._int == '0' - - def _ln_exp_bound(self): - """Compute a lower bound for the adjusted exponent of self.ln(). - In other words, compute r such that self.ln() >= 10**r. Assumes - that self is finite and positive and that self != 1. - """ - - # for 0.1 <= x <= 10 we use the inequalities 1-1/x <= ln(x) <= x-1 - adj = self._exp + len(self._int) - 1 - if adj >= 1: - # argument >= 10; we use 23/10 = 2.3 as a lower bound for ln(10) - return len(str(adj*23//10)) - 1 - if adj <= -2: - # argument <= 0.1 - return len(str((-1-adj)*23//10)) - 1 - op = _WorkRep(self) - c, e = op.int, op.exp - if adj == 0: - # 1 < self < 10 - num = str(c-10**-e) - den = str(c) - return len(num) - len(den) - (num < den) - # adj == -1, 0.1 <= self < 1 - return e + len(str(10**-e - c)) - 1 - - - def ln(self, context=None): - """Returns the natural (base e) logarithm of self.""" - - if context is None: - context = getcontext() - - # ln(NaN) = NaN - ans = self._check_nans(context=context) - if ans: - return ans - - # ln(0.0) == -Infinity - if not self: - return _NegativeInfinity - - # ln(Infinity) = Infinity - if self._isinfinity() == 1: - return _Infinity - - # ln(1.0) == 0.0 - if self == _One: - return _Zero - - # ln(negative) raises InvalidOperation - if self._sign == 1: - return context._raise_error(InvalidOperation, - 'ln of a negative value') - - # result is irrational, so necessarily inexact - op = _WorkRep(self) - c, e = op.int, op.exp - p = context.prec - - # correctly rounded result: repeatedly increase precision by 3 - # until we get an unambiguously roundable result - places = p - self._ln_exp_bound() + 2 # at least p+3 places - while True: - coeff = _dlog(c, e, places) - # assert len(str(abs(coeff)))-p >= 1 - if coeff % (5*10**(len(str(abs(coeff)))-p-1)): - break - places += 3 - ans = _dec_from_triple(int(coeff<0), str(abs(coeff)), -places) - - context = context._shallow_copy() - rounding = context._set_rounding(ROUND_HALF_EVEN) - ans = ans._fix(context) - context.rounding = rounding - return ans - - def _log10_exp_bound(self): - """Compute a lower bound for the adjusted exponent of self.log10(). - In other words, find r such that self.log10() >= 10**r. - Assumes that self is finite and positive and that self != 1. - """ - - # For x >= 10 or x < 0.1 we only need a bound on the integer - # part of log10(self), and this comes directly from the - # exponent of x. For 0.1 <= x <= 10 we use the inequalities - # 1-1/x <= log(x) <= x-1. If x > 1 we have |log10(x)| > - # (1-1/x)/2.31 > 0. If x < 1 then |log10(x)| > (1-x)/2.31 > 0 - - adj = self._exp + len(self._int) - 1 - if adj >= 1: - # self >= 10 - return len(str(adj))-1 - if adj <= -2: - # self < 0.1 - return len(str(-1-adj))-1 - op = _WorkRep(self) - c, e = op.int, op.exp - if adj == 0: - # 1 < self < 10 - num = str(c-10**-e) - den = str(231*c) - return len(num) - len(den) - (num < den) + 2 - # adj == -1, 0.1 <= self < 1 - num = str(10**-e-c) - return len(num) + e - (num < "231") - 1 - - def log10(self, context=None): - """Returns the base 10 logarithm of self.""" - - if context is None: - context = getcontext() - - # log10(NaN) = NaN - ans = self._check_nans(context=context) - if ans: - return ans - - # log10(0.0) == -Infinity - if not self: - return _NegativeInfinity - - # log10(Infinity) = Infinity - if self._isinfinity() == 1: - return _Infinity - - # log10(negative or -Infinity) raises InvalidOperation - if self._sign == 1: - return context._raise_error(InvalidOperation, - 'log10 of a negative value') - - # log10(10**n) = n - if self._int[0] == '1' and self._int[1:] == '0'*(len(self._int) - 1): - # answer may need rounding - ans = Decimal(self._exp + len(self._int) - 1) - else: - # result is irrational, so necessarily inexact - op = _WorkRep(self) - c, e = op.int, op.exp - p = context.prec - - # correctly rounded result: repeatedly increase precision - # until result is unambiguously roundable - places = p-self._log10_exp_bound()+2 - while True: - coeff = _dlog10(c, e, places) - # assert len(str(abs(coeff)))-p >= 1 - if coeff % (5*10**(len(str(abs(coeff)))-p-1)): - break - places += 3 - ans = _dec_from_triple(int(coeff<0), str(abs(coeff)), -places) - - context = context._shallow_copy() - rounding = context._set_rounding(ROUND_HALF_EVEN) - ans = ans._fix(context) - context.rounding = rounding - return ans - - def logb(self, context=None): - """ Returns the exponent of the magnitude of self's MSD. - - The result is the integer which is the exponent of the magnitude - of the most significant digit of self (as though it were truncated - to a single digit while maintaining the value of that digit and - without limiting the resulting exponent). - """ - # logb(NaN) = NaN - ans = self._check_nans(context=context) - if ans: - return ans - - if context is None: - context = getcontext() - - # logb(+/-Inf) = +Inf - if self._isinfinity(): - return _Infinity - - # logb(0) = -Inf, DivisionByZero - if not self: - return context._raise_error(DivisionByZero, 'logb(0)', 1) - - # otherwise, simply return the adjusted exponent of self, as a - # Decimal. Note that no attempt is made to fit the result - # into the current context. - ans = Decimal(self.adjusted()) - return ans._fix(context) - - def _islogical(self): - """Return True if self is a logical operand. - - For being logical, it must be a finite number with a sign of 0, - an exponent of 0, and a coefficient whose digits must all be - either 0 or 1. - """ - if self._sign != 0 or self._exp != 0: - return False - for dig in self._int: - if dig not in '01': - return False - return True - - def _fill_logical(self, context, opa, opb): - dif = context.prec - len(opa) - if dif > 0: - opa = '0'*dif + opa - elif dif < 0: - opa = opa[-context.prec:] - dif = context.prec - len(opb) - if dif > 0: - opb = '0'*dif + opb - elif dif < 0: - opb = opb[-context.prec:] - return opa, opb - - def logical_and(self, other, context=None): - """Applies an 'and' operation between self and other's digits.""" - if context is None: - context = getcontext() - - other = _convert_other(other, raiseit=True) - - if not self._islogical() or not other._islogical(): - return context._raise_error(InvalidOperation) - - # fill to context.prec - (opa, opb) = self._fill_logical(context, self._int, other._int) - - # make the operation, and clean starting zeroes - result = "".join([str(int(a)&int(b)) for a,b in zip(opa,opb)]) - return _dec_from_triple(0, result.lstrip('0') or '0', 0) - - def logical_invert(self, context=None): - """Invert all its digits.""" - if context is None: - context = getcontext() - return self.logical_xor(_dec_from_triple(0,'1'*context.prec,0), - context) - - def logical_or(self, other, context=None): - """Applies an 'or' operation between self and other's digits.""" - if context is None: - context = getcontext() - - other = _convert_other(other, raiseit=True) - - if not self._islogical() or not other._islogical(): - return context._raise_error(InvalidOperation) - - # fill to context.prec - (opa, opb) = self._fill_logical(context, self._int, other._int) - - # make the operation, and clean starting zeroes - result = "".join([str(int(a)|int(b)) for a,b in zip(opa,opb)]) - return _dec_from_triple(0, result.lstrip('0') or '0', 0) - - def logical_xor(self, other, context=None): - """Applies an 'xor' operation between self and other's digits.""" - if context is None: - context = getcontext() - - other = _convert_other(other, raiseit=True) - - if not self._islogical() or not other._islogical(): - return context._raise_error(InvalidOperation) - - # fill to context.prec - (opa, opb) = self._fill_logical(context, self._int, other._int) - - # make the operation, and clean starting zeroes - result = "".join([str(int(a)^int(b)) for a,b in zip(opa,opb)]) - return _dec_from_triple(0, result.lstrip('0') or '0', 0) - - def max_mag(self, other, context=None): - """Compares the values numerically with their sign ignored.""" - other = _convert_other(other, raiseit=True) - - if context is None: - context = getcontext() - - if self._is_special or other._is_special: - # If one operand is a quiet NaN and the other is number, then the - # number is always returned - sn = self._isnan() - on = other._isnan() - if sn or on: - if on == 1 and sn == 0: - return self._fix(context) - if sn == 1 and on == 0: - return other._fix(context) - return self._check_nans(other, context) - - c = self.copy_abs()._cmp(other.copy_abs()) - if c == 0: - c = self.compare_total(other) - - if c == -1: - ans = other - else: - ans = self - - return ans._fix(context) - - def min_mag(self, other, context=None): - """Compares the values numerically with their sign ignored.""" - other = _convert_other(other, raiseit=True) - - if context is None: - context = getcontext() - - if self._is_special or other._is_special: - # If one operand is a quiet NaN and the other is number, then the - # number is always returned - sn = self._isnan() - on = other._isnan() - if sn or on: - if on == 1 and sn == 0: - return self._fix(context) - if sn == 1 and on == 0: - return other._fix(context) - return self._check_nans(other, context) - - c = self.copy_abs()._cmp(other.copy_abs()) - if c == 0: - c = self.compare_total(other) - - if c == -1: - ans = self - else: - ans = other - - return ans._fix(context) - - def next_minus(self, context=None): - """Returns the largest representable number smaller than itself.""" - if context is None: - context = getcontext() - - ans = self._check_nans(context=context) - if ans: - return ans - - if self._isinfinity() == -1: - return _NegativeInfinity - if self._isinfinity() == 1: - return _dec_from_triple(0, '9'*context.prec, context.Etop()) - - context = context.copy() - context._set_rounding(ROUND_FLOOR) - context._ignore_all_flags() - new_self = self._fix(context) - if new_self != self: - return new_self - return self.__sub__(_dec_from_triple(0, '1', context.Etiny()-1), - context) - - def next_plus(self, context=None): - """Returns the smallest representable number larger than itself.""" - if context is None: - context = getcontext() - - ans = self._check_nans(context=context) - if ans: - return ans - - if self._isinfinity() == 1: - return _Infinity - if self._isinfinity() == -1: - return _dec_from_triple(1, '9'*context.prec, context.Etop()) - - context = context.copy() - context._set_rounding(ROUND_CEILING) - context._ignore_all_flags() - new_self = self._fix(context) - if new_self != self: - return new_self - return self.__add__(_dec_from_triple(0, '1', context.Etiny()-1), - context) - - def next_toward(self, other, context=None): - """Returns the number closest to self, in the direction towards other. - - The result is the closest representable number to self - (excluding self) that is in the direction towards other, - unless both have the same value. If the two operands are - numerically equal, then the result is a copy of self with the - sign set to be the same as the sign of other. - """ - other = _convert_other(other, raiseit=True) - - if context is None: - context = getcontext() - - ans = self._check_nans(other, context) - if ans: - return ans - - comparison = self._cmp(other) - if comparison == 0: - return self.copy_sign(other) - - if comparison == -1: - ans = self.next_plus(context) - else: # comparison == 1 - ans = self.next_minus(context) - - # decide which flags to raise using value of ans - if ans._isinfinity(): - context._raise_error(Overflow, - 'Infinite result from next_toward', - ans._sign) - context._raise_error(Inexact) - context._raise_error(Rounded) - elif ans.adjusted() < context.Emin: - context._raise_error(Underflow) - context._raise_error(Subnormal) - context._raise_error(Inexact) - context._raise_error(Rounded) - # if precision == 1 then we don't raise Clamped for a - # result 0E-Etiny. - if not ans: - context._raise_error(Clamped) - - return ans - - def number_class(self, context=None): - """Returns an indication of the class of self. - - The class is one of the following strings: - sNaN - NaN - -Infinity - -Normal - -Subnormal - -Zero - +Zero - +Subnormal - +Normal - +Infinity - """ - if self.is_snan(): - return "sNaN" - if self.is_qnan(): - return "NaN" - inf = self._isinfinity() - if inf == 1: - return "+Infinity" - if inf == -1: - return "-Infinity" - if self.is_zero(): - if self._sign: - return "-Zero" - else: - return "+Zero" - if context is None: - context = getcontext() - if self.is_subnormal(context=context): - if self._sign: - return "-Subnormal" - else: - return "+Subnormal" - # just a normal, regular, boring number, :) - if self._sign: - return "-Normal" - else: - return "+Normal" - - def radix(self): - """Just returns 10, as this is Decimal, :)""" - return Decimal(10) - - def rotate(self, other, context=None): - """Returns a rotated copy of self, value-of-other times.""" - if context is None: - context = getcontext() - - other = _convert_other(other, raiseit=True) - - ans = self._check_nans(other, context) - if ans: - return ans - - if other._exp != 0: - return context._raise_error(InvalidOperation) - if not (-context.prec <= int(other) <= context.prec): - return context._raise_error(InvalidOperation) - - if self._isinfinity(): - return Decimal(self) - - # get values, pad if necessary - torot = int(other) - rotdig = self._int - topad = context.prec - len(rotdig) - if topad > 0: - rotdig = '0'*topad + rotdig - elif topad < 0: - rotdig = rotdig[-topad:] - - # let's rotate! - rotated = rotdig[torot:] + rotdig[:torot] - return _dec_from_triple(self._sign, - rotated.lstrip('0') or '0', self._exp) - - def scaleb(self, other, context=None): - """Returns self operand after adding the second value to its exp.""" - if context is None: - context = getcontext() - - other = _convert_other(other, raiseit=True) - - ans = self._check_nans(other, context) - if ans: - return ans - - if other._exp != 0: - return context._raise_error(InvalidOperation) - liminf = -2 * (context.Emax + context.prec) - limsup = 2 * (context.Emax + context.prec) - if not (liminf <= int(other) <= limsup): - return context._raise_error(InvalidOperation) - - if self._isinfinity(): - return Decimal(self) - - d = _dec_from_triple(self._sign, self._int, self._exp + int(other)) - d = d._fix(context) - return d - - def shift(self, other, context=None): - """Returns a shifted copy of self, value-of-other times.""" - if context is None: - context = getcontext() - - other = _convert_other(other, raiseit=True) - - ans = self._check_nans(other, context) - if ans: - return ans - - if other._exp != 0: - return context._raise_error(InvalidOperation) - if not (-context.prec <= int(other) <= context.prec): - return context._raise_error(InvalidOperation) - - if self._isinfinity(): - return Decimal(self) - - # get values, pad if necessary - torot = int(other) - rotdig = self._int - topad = context.prec - len(rotdig) - if topad > 0: - rotdig = '0'*topad + rotdig - elif topad < 0: - rotdig = rotdig[-topad:] - - # let's shift! - if torot < 0: - shifted = rotdig[:torot] - else: - shifted = rotdig + '0'*torot - shifted = shifted[-context.prec:] - - return _dec_from_triple(self._sign, - shifted.lstrip('0') or '0', self._exp) - - # Support for pickling, copy, and deepcopy - def __reduce__(self): - return (self.__class__, (str(self),)) - - def __copy__(self): - if type(self) is Decimal: - return self # I'm immutable; therefore I am my own clone - return self.__class__(str(self)) - - def __deepcopy__(self, memo): - if type(self) is Decimal: - return self # My components are also immutable - return self.__class__(str(self)) - - # PEP 3101 support. the _localeconv keyword argument should be - # considered private: it's provided for ease of testing only. - def __format__(self, specifier, context=None, _localeconv=None): - """Format a Decimal instance according to the given specifier. - - The specifier should be a standard format specifier, with the - form described in PEP 3101. Formatting types 'e', 'E', 'f', - 'F', 'g', 'G', 'n' and '%' are supported. If the formatting - type is omitted it defaults to 'g' or 'G', depending on the - value of context.capitals. - """ - - # Note: PEP 3101 says that if the type is not present then - # there should be at least one digit after the decimal point. - # We take the liberty of ignoring this requirement for - # Decimal---it's presumably there to make sure that - # format(float, '') behaves similarly to str(float). - if context is None: - context = getcontext() - - spec = _parse_format_specifier(specifier, _localeconv=_localeconv) - - # special values don't care about the type or precision - if self._is_special: - sign = _format_sign(self._sign, spec) - body = str(self.copy_abs()) - if spec['type'] == '%': - body += '%' - return _format_align(sign, body, spec) - - # a type of None defaults to 'g' or 'G', depending on context - if spec['type'] is None: - spec['type'] = ['g', 'G'][context.capitals] - - # if type is '%', adjust exponent of self accordingly - if spec['type'] == '%': - self = _dec_from_triple(self._sign, self._int, self._exp+2) - - # round if necessary, taking rounding mode from the context - rounding = context.rounding - precision = spec['precision'] - if precision is not None: - if spec['type'] in 'eE': - self = self._round(precision+1, rounding) - elif spec['type'] in 'fF%': - self = self._rescale(-precision, rounding) - elif spec['type'] in 'gG' and len(self._int) > precision: - self = self._round(precision, rounding) - # special case: zeros with a positive exponent can't be - # represented in fixed point; rescale them to 0e0. - if not self and self._exp > 0 and spec['type'] in 'fF%': - self = self._rescale(0, rounding) - - # figure out placement of the decimal point - leftdigits = self._exp + len(self._int) - if spec['type'] in 'eE': - if not self and precision is not None: - dotplace = 1 - precision - else: - dotplace = 1 - elif spec['type'] in 'fF%': - dotplace = leftdigits - elif spec['type'] in 'gG': - if self._exp <= 0 and leftdigits > -6: - dotplace = leftdigits - else: - dotplace = 1 - - # find digits before and after decimal point, and get exponent - if dotplace < 0: - intpart = '0' - fracpart = '0'*(-dotplace) + self._int - elif dotplace > len(self._int): - intpart = self._int + '0'*(dotplace-len(self._int)) - fracpart = '' - else: - intpart = self._int[:dotplace] or '0' - fracpart = self._int[dotplace:] - exp = leftdigits-dotplace - - # done with the decimal-specific stuff; hand over the rest - # of the formatting to the _format_number function - return _format_number(self._sign, intpart, fracpart, exp, spec) - -def _dec_from_triple(sign, coefficient, exponent, special=False): - """Create a decimal instance directly, without any validation, - normalization (e.g. removal of leading zeros) or argument - conversion. - - This function is for *internal use only*. - """ - - self = object.__new__(Decimal) - self._sign = sign - self._int = coefficient - self._exp = exponent - self._is_special = special - - return self - -# Register Decimal as a kind of Number (an abstract base class). -# However, do not register it as Real (because Decimals are not -# interoperable with floats). -_numbers.Number.register(Decimal) - - -##### Context class ####################################################### - -class _ContextManager(object): - """Context manager class to support localcontext(). - - Sets a copy of the supplied context in __enter__() and restores - the previous decimal context in __exit__() - """ - def __init__(self, new_context): - self.new_context = new_context.copy() - def __enter__(self): - self.saved_context = getcontext() - setcontext(self.new_context) - return self.new_context - def __exit__(self, t, v, tb): - setcontext(self.saved_context) - -class Context(object): - """Contains the context for a Decimal instance. - - Contains: - prec - precision (for use in rounding, division, square roots..) - rounding - rounding type (how you round) - traps - If traps[exception] = 1, then the exception is - raised when it is caused. Otherwise, a value is - substituted in. - flags - When an exception is caused, flags[exception] is set. - (Whether or not the trap_enabler is set) - Should be reset by user of Decimal instance. - Emin - Minimum exponent - Emax - Maximum exponent - capitals - If 1, 1*10^1 is printed as 1E+1. - If 0, printed as 1e1 - clamp - If 1, change exponents if too high (Default 0) - """ - - def __init__(self, prec=None, rounding=None, Emin=None, Emax=None, - capitals=None, clamp=None, flags=None, traps=None, - _ignored_flags=None): - # Set defaults; for everything except flags and _ignored_flags, - # inherit from DefaultContext. - try: - dc = DefaultContext - except NameError: - pass - - self.prec = prec if prec is not None else dc.prec - self.rounding = rounding if rounding is not None else dc.rounding - self.Emin = Emin if Emin is not None else dc.Emin - self.Emax = Emax if Emax is not None else dc.Emax - self.capitals = capitals if capitals is not None else dc.capitals - self.clamp = clamp if clamp is not None else dc.clamp - - if _ignored_flags is None: - self._ignored_flags = [] - else: - self._ignored_flags = _ignored_flags - - if traps is None: - self.traps = dc.traps.copy() - elif not isinstance(traps, dict): - self.traps = dict((s, int(s in traps)) for s in _signals + traps) - else: - self.traps = traps - - if flags is None: - self.flags = dict.fromkeys(_signals, 0) - elif not isinstance(flags, dict): - self.flags = dict((s, int(s in flags)) for s in _signals + flags) - else: - self.flags = flags - - def _set_integer_check(self, name, value, vmin, vmax): - if not isinstance(value, int): - raise TypeError("%s must be an integer" % name) - if vmin == '-inf': - if value > vmax: - raise ValueError("%s must be in [%s, %d]. got: %s" % (name, vmin, vmax, value)) - elif vmax == 'inf': - if value < vmin: - raise ValueError("%s must be in [%d, %s]. got: %s" % (name, vmin, vmax, value)) - else: - if value < vmin or value > vmax: - raise ValueError("%s must be in [%d, %d]. got %s" % (name, vmin, vmax, value)) - return object.__setattr__(self, name, value) - - def _set_signal_dict(self, name, d): - if not isinstance(d, dict): - raise TypeError("%s must be a signal dict" % d) - for key in d: - if not key in _signals: - raise KeyError("%s is not a valid signal dict" % d) - for key in _signals: - if not key in d: - raise KeyError("%s is not a valid signal dict" % d) - return object.__setattr__(self, name, d) - - def __setattr__(self, name, value): - if name == 'prec': - return self._set_integer_check(name, value, 1, 'inf') - elif name == 'Emin': - return self._set_integer_check(name, value, '-inf', 0) - elif name == 'Emax': - return self._set_integer_check(name, value, 0, 'inf') - elif name == 'capitals': - return self._set_integer_check(name, value, 0, 1) - elif name == 'clamp': - return self._set_integer_check(name, value, 0, 1) - elif name == 'rounding': - if not value in _rounding_modes: - # raise TypeError even for strings to have consistency - # among various implementations. - raise TypeError("%s: invalid rounding mode" % value) - return object.__setattr__(self, name, value) - elif name == 'flags' or name == 'traps': - return self._set_signal_dict(name, value) - elif name == '_ignored_flags': - return object.__setattr__(self, name, value) - else: - raise AttributeError( - "'decimal.Context' object has no attribute '%s'" % name) - - def __delattr__(self, name): - raise AttributeError("%s cannot be deleted" % name) - - # Support for pickling, copy, and deepcopy - def __reduce__(self): - flags = [sig for sig, v in self.flags.items() if v] - traps = [sig for sig, v in self.traps.items() if v] - return (self.__class__, - (self.prec, self.rounding, self.Emin, self.Emax, - self.capitals, self.clamp, flags, traps)) - - def __repr__(self): - """Show the current context.""" - s = [] - s.append('Context(prec=%(prec)d, rounding=%(rounding)s, ' - 'Emin=%(Emin)d, Emax=%(Emax)d, capitals=%(capitals)d, ' - 'clamp=%(clamp)d' - % vars(self)) - names = [f.__name__ for f, v in self.flags.items() if v] - s.append('flags=[' + ', '.join(names) + ']') - names = [t.__name__ for t, v in self.traps.items() if v] - s.append('traps=[' + ', '.join(names) + ']') - return ', '.join(s) + ')' - - def clear_flags(self): - """Reset all flags to zero""" - for flag in self.flags: - self.flags[flag] = 0 - - def clear_traps(self): - """Reset all traps to zero""" - for flag in self.traps: - self.traps[flag] = 0 - - def _shallow_copy(self): - """Returns a shallow copy from self.""" - nc = Context(self.prec, self.rounding, self.Emin, self.Emax, - self.capitals, self.clamp, self.flags, self.traps, - self._ignored_flags) - return nc - - def copy(self): - """Returns a deep copy from self.""" - nc = Context(self.prec, self.rounding, self.Emin, self.Emax, - self.capitals, self.clamp, - self.flags.copy(), self.traps.copy(), - self._ignored_flags) - return nc - __copy__ = copy - - def _raise_error(self, condition, explanation = None, *args): - """Handles an error - - If the flag is in _ignored_flags, returns the default response. - Otherwise, it sets the flag, then, if the corresponding - trap_enabler is set, it reraises the exception. Otherwise, it returns - the default value after setting the flag. - """ - error = _condition_map.get(condition, condition) - if error in self._ignored_flags: - # Don't touch the flag - return error().handle(self, *args) - - self.flags[error] = 1 - if not self.traps[error]: - # The errors define how to handle themselves. - return condition().handle(self, *args) - - # Errors should only be risked on copies of the context - # self._ignored_flags = [] - raise error(explanation) - - def _ignore_all_flags(self): - """Ignore all flags, if they are raised""" - return self._ignore_flags(*_signals) - - def _ignore_flags(self, *flags): - """Ignore the flags, if they are raised""" - # Do not mutate-- This way, copies of a context leave the original - # alone. - self._ignored_flags = (self._ignored_flags + list(flags)) - return list(flags) - - def _regard_flags(self, *flags): - """Stop ignoring the flags, if they are raised""" - if flags and isinstance(flags[0], (tuple,list)): - flags = flags[0] - for flag in flags: - self._ignored_flags.remove(flag) - - # We inherit object.__hash__, so we must deny this explicitly - __hash__ = None - - def Etiny(self): - """Returns Etiny (= Emin - prec + 1)""" - return int(self.Emin - self.prec + 1) - - def Etop(self): - """Returns maximum exponent (= Emax - prec + 1)""" - return int(self.Emax - self.prec + 1) - - def _set_rounding(self, type): - """Sets the rounding type. - - Sets the rounding type, and returns the current (previous) - rounding type. Often used like: - - context = context.copy() - # so you don't change the calling context - # if an error occurs in the middle. - rounding = context._set_rounding(ROUND_UP) - val = self.__sub__(other, context=context) - context._set_rounding(rounding) - - This will make it round up for that operation. - """ - rounding = self.rounding - self.rounding= type - return rounding - - def create_decimal(self, num='0'): - """Creates a new Decimal instance but using self as context. - - This method implements the to-number operation of the - IBM Decimal specification.""" - - if isinstance(num, str) and num != num.strip(): - return self._raise_error(ConversionSyntax, - "no trailing or leading whitespace is " - "permitted.") - - d = Decimal(num, context=self) - if d._isnan() and len(d._int) > self.prec - self.clamp: - return self._raise_error(ConversionSyntax, - "diagnostic info too long in NaN") - return d._fix(self) - - def create_decimal_from_float(self, f): - """Creates a new Decimal instance from a float but rounding using self - as the context. - - >>> context = Context(prec=5, rounding=ROUND_DOWN) - >>> context.create_decimal_from_float(3.1415926535897932) - Decimal('3.1415') - >>> context = Context(prec=5, traps=[Inexact]) - >>> context.create_decimal_from_float(3.1415926535897932) - Traceback (most recent call last): - ... - decimal.Inexact: None - - """ - d = Decimal.from_float(f) # An exact conversion - return d._fix(self) # Apply the context rounding - - # Methods - def abs(self, a): - """Returns the absolute value of the operand. - - If the operand is negative, the result is the same as using the minus - operation on the operand. Otherwise, the result is the same as using - the plus operation on the operand. - - >>> ExtendedContext.abs(Decimal('2.1')) - Decimal('2.1') - >>> ExtendedContext.abs(Decimal('-100')) - Decimal('100') - >>> ExtendedContext.abs(Decimal('101.5')) - Decimal('101.5') - >>> ExtendedContext.abs(Decimal('-101.5')) - Decimal('101.5') - >>> ExtendedContext.abs(-1) - Decimal('1') - """ - a = _convert_other(a, raiseit=True) - return a.__abs__(context=self) - - def add(self, a, b): - """Return the sum of the two operands. - - >>> ExtendedContext.add(Decimal('12'), Decimal('7.00')) - Decimal('19.00') - >>> ExtendedContext.add(Decimal('1E+2'), Decimal('1.01E+4')) - Decimal('1.02E+4') - >>> ExtendedContext.add(1, Decimal(2)) - Decimal('3') - >>> ExtendedContext.add(Decimal(8), 5) - Decimal('13') - >>> ExtendedContext.add(5, 5) - Decimal('10') - """ - a = _convert_other(a, raiseit=True) - r = a.__add__(b, context=self) - if r is NotImplemented: - raise TypeError("Unable to convert %s to Decimal" % b) - else: - return r - - def _apply(self, a): - return str(a._fix(self)) - - def canonical(self, a): - """Returns the same Decimal object. - - As we do not have different encodings for the same number, the - received object already is in its canonical form. - - >>> ExtendedContext.canonical(Decimal('2.50')) - Decimal('2.50') - """ - if not isinstance(a, Decimal): - raise TypeError("canonical requires a Decimal as an argument.") - return a.canonical() - - def compare(self, a, b): - """Compares values numerically. - - If the signs of the operands differ, a value representing each operand - ('-1' if the operand is less than zero, '0' if the operand is zero or - negative zero, or '1' if the operand is greater than zero) is used in - place of that operand for the comparison instead of the actual - operand. - - The comparison is then effected by subtracting the second operand from - the first and then returning a value according to the result of the - subtraction: '-1' if the result is less than zero, '0' if the result is - zero or negative zero, or '1' if the result is greater than zero. - - >>> ExtendedContext.compare(Decimal('2.1'), Decimal('3')) - Decimal('-1') - >>> ExtendedContext.compare(Decimal('2.1'), Decimal('2.1')) - Decimal('0') - >>> ExtendedContext.compare(Decimal('2.1'), Decimal('2.10')) - Decimal('0') - >>> ExtendedContext.compare(Decimal('3'), Decimal('2.1')) - Decimal('1') - >>> ExtendedContext.compare(Decimal('2.1'), Decimal('-3')) - Decimal('1') - >>> ExtendedContext.compare(Decimal('-3'), Decimal('2.1')) - Decimal('-1') - >>> ExtendedContext.compare(1, 2) - Decimal('-1') - >>> ExtendedContext.compare(Decimal(1), 2) - Decimal('-1') - >>> ExtendedContext.compare(1, Decimal(2)) - Decimal('-1') - """ - a = _convert_other(a, raiseit=True) - return a.compare(b, context=self) - - def compare_signal(self, a, b): - """Compares the values of the two operands numerically. - - It's pretty much like compare(), but all NaNs signal, with signaling - NaNs taking precedence over quiet NaNs. - - >>> c = ExtendedContext - >>> c.compare_signal(Decimal('2.1'), Decimal('3')) - Decimal('-1') - >>> c.compare_signal(Decimal('2.1'), Decimal('2.1')) - Decimal('0') - >>> c.flags[InvalidOperation] = 0 - >>> print(c.flags[InvalidOperation]) - 0 - >>> c.compare_signal(Decimal('NaN'), Decimal('2.1')) - Decimal('NaN') - >>> print(c.flags[InvalidOperation]) - 1 - >>> c.flags[InvalidOperation] = 0 - >>> print(c.flags[InvalidOperation]) - 0 - >>> c.compare_signal(Decimal('sNaN'), Decimal('2.1')) - Decimal('NaN') - >>> print(c.flags[InvalidOperation]) - 1 - >>> c.compare_signal(-1, 2) - Decimal('-1') - >>> c.compare_signal(Decimal(-1), 2) - Decimal('-1') - >>> c.compare_signal(-1, Decimal(2)) - Decimal('-1') - """ - a = _convert_other(a, raiseit=True) - return a.compare_signal(b, context=self) - - def compare_total(self, a, b): - """Compares two operands using their abstract representation. - - This is not like the standard compare, which use their numerical - value. Note that a total ordering is defined for all possible abstract - representations. - - >>> ExtendedContext.compare_total(Decimal('12.73'), Decimal('127.9')) - Decimal('-1') - >>> ExtendedContext.compare_total(Decimal('-127'), Decimal('12')) - Decimal('-1') - >>> ExtendedContext.compare_total(Decimal('12.30'), Decimal('12.3')) - Decimal('-1') - >>> ExtendedContext.compare_total(Decimal('12.30'), Decimal('12.30')) - Decimal('0') - >>> ExtendedContext.compare_total(Decimal('12.3'), Decimal('12.300')) - Decimal('1') - >>> ExtendedContext.compare_total(Decimal('12.3'), Decimal('NaN')) - Decimal('-1') - >>> ExtendedContext.compare_total(1, 2) - Decimal('-1') - >>> ExtendedContext.compare_total(Decimal(1), 2) - Decimal('-1') - >>> ExtendedContext.compare_total(1, Decimal(2)) - Decimal('-1') - """ - a = _convert_other(a, raiseit=True) - return a.compare_total(b) - - def compare_total_mag(self, a, b): - """Compares two operands using their abstract representation ignoring sign. - - Like compare_total, but with operand's sign ignored and assumed to be 0. - """ - a = _convert_other(a, raiseit=True) - return a.compare_total_mag(b) - - def copy_abs(self, a): - """Returns a copy of the operand with the sign set to 0. - - >>> ExtendedContext.copy_abs(Decimal('2.1')) - Decimal('2.1') - >>> ExtendedContext.copy_abs(Decimal('-100')) - Decimal('100') - >>> ExtendedContext.copy_abs(-1) - Decimal('1') - """ - a = _convert_other(a, raiseit=True) - return a.copy_abs() - - def copy_decimal(self, a): - """Returns a copy of the decimal object. - - >>> ExtendedContext.copy_decimal(Decimal('2.1')) - Decimal('2.1') - >>> ExtendedContext.copy_decimal(Decimal('-1.00')) - Decimal('-1.00') - >>> ExtendedContext.copy_decimal(1) - Decimal('1') - """ - a = _convert_other(a, raiseit=True) - return Decimal(a) - - def copy_negate(self, a): - """Returns a copy of the operand with the sign inverted. - - >>> ExtendedContext.copy_negate(Decimal('101.5')) - Decimal('-101.5') - >>> ExtendedContext.copy_negate(Decimal('-101.5')) - Decimal('101.5') - >>> ExtendedContext.copy_negate(1) - Decimal('-1') - """ - a = _convert_other(a, raiseit=True) - return a.copy_negate() - - def copy_sign(self, a, b): - """Copies the second operand's sign to the first one. - - In detail, it returns a copy of the first operand with the sign - equal to the sign of the second operand. - - >>> ExtendedContext.copy_sign(Decimal( '1.50'), Decimal('7.33')) - Decimal('1.50') - >>> ExtendedContext.copy_sign(Decimal('-1.50'), Decimal('7.33')) - Decimal('1.50') - >>> ExtendedContext.copy_sign(Decimal( '1.50'), Decimal('-7.33')) - Decimal('-1.50') - >>> ExtendedContext.copy_sign(Decimal('-1.50'), Decimal('-7.33')) - Decimal('-1.50') - >>> ExtendedContext.copy_sign(1, -2) - Decimal('-1') - >>> ExtendedContext.copy_sign(Decimal(1), -2) - Decimal('-1') - >>> ExtendedContext.copy_sign(1, Decimal(-2)) - Decimal('-1') - """ - a = _convert_other(a, raiseit=True) - return a.copy_sign(b) - - def divide(self, a, b): - """Decimal division in a specified context. - - >>> ExtendedContext.divide(Decimal('1'), Decimal('3')) - Decimal('0.333333333') - >>> ExtendedContext.divide(Decimal('2'), Decimal('3')) - Decimal('0.666666667') - >>> ExtendedContext.divide(Decimal('5'), Decimal('2')) - Decimal('2.5') - >>> ExtendedContext.divide(Decimal('1'), Decimal('10')) - Decimal('0.1') - >>> ExtendedContext.divide(Decimal('12'), Decimal('12')) - Decimal('1') - >>> ExtendedContext.divide(Decimal('8.00'), Decimal('2')) - Decimal('4.00') - >>> ExtendedContext.divide(Decimal('2.400'), Decimal('2.0')) - Decimal('1.20') - >>> ExtendedContext.divide(Decimal('1000'), Decimal('100')) - Decimal('10') - >>> ExtendedContext.divide(Decimal('1000'), Decimal('1')) - Decimal('1000') - >>> ExtendedContext.divide(Decimal('2.40E+6'), Decimal('2')) - Decimal('1.20E+6') - >>> ExtendedContext.divide(5, 5) - Decimal('1') - >>> ExtendedContext.divide(Decimal(5), 5) - Decimal('1') - >>> ExtendedContext.divide(5, Decimal(5)) - Decimal('1') - """ - a = _convert_other(a, raiseit=True) - r = a.__truediv__(b, context=self) - if r is NotImplemented: - raise TypeError("Unable to convert %s to Decimal" % b) - else: - return r - - def divide_int(self, a, b): - """Divides two numbers and returns the integer part of the result. - - >>> ExtendedContext.divide_int(Decimal('2'), Decimal('3')) - Decimal('0') - >>> ExtendedContext.divide_int(Decimal('10'), Decimal('3')) - Decimal('3') - >>> ExtendedContext.divide_int(Decimal('1'), Decimal('0.3')) - Decimal('3') - >>> ExtendedContext.divide_int(10, 3) - Decimal('3') - >>> ExtendedContext.divide_int(Decimal(10), 3) - Decimal('3') - >>> ExtendedContext.divide_int(10, Decimal(3)) - Decimal('3') - """ - a = _convert_other(a, raiseit=True) - r = a.__floordiv__(b, context=self) - if r is NotImplemented: - raise TypeError("Unable to convert %s to Decimal" % b) - else: - return r - - def divmod(self, a, b): - """Return (a // b, a % b). - - >>> ExtendedContext.divmod(Decimal(8), Decimal(3)) - (Decimal('2'), Decimal('2')) - >>> ExtendedContext.divmod(Decimal(8), Decimal(4)) - (Decimal('2'), Decimal('0')) - >>> ExtendedContext.divmod(8, 4) - (Decimal('2'), Decimal('0')) - >>> ExtendedContext.divmod(Decimal(8), 4) - (Decimal('2'), Decimal('0')) - >>> ExtendedContext.divmod(8, Decimal(4)) - (Decimal('2'), Decimal('0')) - """ - a = _convert_other(a, raiseit=True) - r = a.__divmod__(b, context=self) - if r is NotImplemented: - raise TypeError("Unable to convert %s to Decimal" % b) - else: - return r - - def exp(self, a): - """Returns e ** a. - - >>> c = ExtendedContext.copy() - >>> c.Emin = -999 - >>> c.Emax = 999 - >>> c.exp(Decimal('-Infinity')) - Decimal('0') - >>> c.exp(Decimal('-1')) - Decimal('0.367879441') - >>> c.exp(Decimal('0')) - Decimal('1') - >>> c.exp(Decimal('1')) - Decimal('2.71828183') - >>> c.exp(Decimal('0.693147181')) - Decimal('2.00000000') - >>> c.exp(Decimal('+Infinity')) - Decimal('Infinity') - >>> c.exp(10) - Decimal('22026.4658') - """ - a =_convert_other(a, raiseit=True) - return a.exp(context=self) - - def fma(self, a, b, c): - """Returns a multiplied by b, plus c. - - The first two operands are multiplied together, using multiply, - the third operand is then added to the result of that - multiplication, using add, all with only one final rounding. - - >>> ExtendedContext.fma(Decimal('3'), Decimal('5'), Decimal('7')) - Decimal('22') - >>> ExtendedContext.fma(Decimal('3'), Decimal('-5'), Decimal('7')) - Decimal('-8') - >>> ExtendedContext.fma(Decimal('888565290'), Decimal('1557.96930'), Decimal('-86087.7578')) - Decimal('1.38435736E+12') - >>> ExtendedContext.fma(1, 3, 4) - Decimal('7') - >>> ExtendedContext.fma(1, Decimal(3), 4) - Decimal('7') - >>> ExtendedContext.fma(1, 3, Decimal(4)) - Decimal('7') - """ - a = _convert_other(a, raiseit=True) - return a.fma(b, c, context=self) - - def is_canonical(self, a): - """Return True if the operand is canonical; otherwise return False. - - Currently, the encoding of a Decimal instance is always - canonical, so this method returns True for any Decimal. - - >>> ExtendedContext.is_canonical(Decimal('2.50')) - True - """ - if not isinstance(a, Decimal): - raise TypeError("is_canonical requires a Decimal as an argument.") - return a.is_canonical() - - def is_finite(self, a): - """Return True if the operand is finite; otherwise return False. - - A Decimal instance is considered finite if it is neither - infinite nor a NaN. - - >>> ExtendedContext.is_finite(Decimal('2.50')) - True - >>> ExtendedContext.is_finite(Decimal('-0.3')) - True - >>> ExtendedContext.is_finite(Decimal('0')) - True - >>> ExtendedContext.is_finite(Decimal('Inf')) - False - >>> ExtendedContext.is_finite(Decimal('NaN')) - False - >>> ExtendedContext.is_finite(1) - True - """ - a = _convert_other(a, raiseit=True) - return a.is_finite() - - def is_infinite(self, a): - """Return True if the operand is infinite; otherwise return False. - - >>> ExtendedContext.is_infinite(Decimal('2.50')) - False - >>> ExtendedContext.is_infinite(Decimal('-Inf')) - True - >>> ExtendedContext.is_infinite(Decimal('NaN')) - False - >>> ExtendedContext.is_infinite(1) - False - """ - a = _convert_other(a, raiseit=True) - return a.is_infinite() - - def is_nan(self, a): - """Return True if the operand is a qNaN or sNaN; - otherwise return False. - - >>> ExtendedContext.is_nan(Decimal('2.50')) - False - >>> ExtendedContext.is_nan(Decimal('NaN')) - True - >>> ExtendedContext.is_nan(Decimal('-sNaN')) - True - >>> ExtendedContext.is_nan(1) - False - """ - a = _convert_other(a, raiseit=True) - return a.is_nan() - - def is_normal(self, a): - """Return True if the operand is a normal number; - otherwise return False. - - >>> c = ExtendedContext.copy() - >>> c.Emin = -999 - >>> c.Emax = 999 - >>> c.is_normal(Decimal('2.50')) - True - >>> c.is_normal(Decimal('0.1E-999')) - False - >>> c.is_normal(Decimal('0.00')) - False - >>> c.is_normal(Decimal('-Inf')) - False - >>> c.is_normal(Decimal('NaN')) - False - >>> c.is_normal(1) - True - """ - a = _convert_other(a, raiseit=True) - return a.is_normal(context=self) - - def is_qnan(self, a): - """Return True if the operand is a quiet NaN; otherwise return False. - - >>> ExtendedContext.is_qnan(Decimal('2.50')) - False - >>> ExtendedContext.is_qnan(Decimal('NaN')) - True - >>> ExtendedContext.is_qnan(Decimal('sNaN')) - False - >>> ExtendedContext.is_qnan(1) - False - """ - a = _convert_other(a, raiseit=True) - return a.is_qnan() - - def is_signed(self, a): - """Return True if the operand is negative; otherwise return False. - - >>> ExtendedContext.is_signed(Decimal('2.50')) - False - >>> ExtendedContext.is_signed(Decimal('-12')) - True - >>> ExtendedContext.is_signed(Decimal('-0')) - True - >>> ExtendedContext.is_signed(8) - False - >>> ExtendedContext.is_signed(-8) - True - """ - a = _convert_other(a, raiseit=True) - return a.is_signed() - - def is_snan(self, a): - """Return True if the operand is a signaling NaN; - otherwise return False. - - >>> ExtendedContext.is_snan(Decimal('2.50')) - False - >>> ExtendedContext.is_snan(Decimal('NaN')) - False - >>> ExtendedContext.is_snan(Decimal('sNaN')) - True - >>> ExtendedContext.is_snan(1) - False - """ - a = _convert_other(a, raiseit=True) - return a.is_snan() - - def is_subnormal(self, a): - """Return True if the operand is subnormal; otherwise return False. - - >>> c = ExtendedContext.copy() - >>> c.Emin = -999 - >>> c.Emax = 999 - >>> c.is_subnormal(Decimal('2.50')) - False - >>> c.is_subnormal(Decimal('0.1E-999')) - True - >>> c.is_subnormal(Decimal('0.00')) - False - >>> c.is_subnormal(Decimal('-Inf')) - False - >>> c.is_subnormal(Decimal('NaN')) - False - >>> c.is_subnormal(1) - False - """ - a = _convert_other(a, raiseit=True) - return a.is_subnormal(context=self) - - def is_zero(self, a): - """Return True if the operand is a zero; otherwise return False. - - >>> ExtendedContext.is_zero(Decimal('0')) - True - >>> ExtendedContext.is_zero(Decimal('2.50')) - False - >>> ExtendedContext.is_zero(Decimal('-0E+2')) - True - >>> ExtendedContext.is_zero(1) - False - >>> ExtendedContext.is_zero(0) - True - """ - a = _convert_other(a, raiseit=True) - return a.is_zero() - - def ln(self, a): - """Returns the natural (base e) logarithm of the operand. - - >>> c = ExtendedContext.copy() - >>> c.Emin = -999 - >>> c.Emax = 999 - >>> c.ln(Decimal('0')) - Decimal('-Infinity') - >>> c.ln(Decimal('1.000')) - Decimal('0') - >>> c.ln(Decimal('2.71828183')) - Decimal('1.00000000') - >>> c.ln(Decimal('10')) - Decimal('2.30258509') - >>> c.ln(Decimal('+Infinity')) - Decimal('Infinity') - >>> c.ln(1) - Decimal('0') - """ - a = _convert_other(a, raiseit=True) - return a.ln(context=self) - - def log10(self, a): - """Returns the base 10 logarithm of the operand. - - >>> c = ExtendedContext.copy() - >>> c.Emin = -999 - >>> c.Emax = 999 - >>> c.log10(Decimal('0')) - Decimal('-Infinity') - >>> c.log10(Decimal('0.001')) - Decimal('-3') - >>> c.log10(Decimal('1.000')) - Decimal('0') - >>> c.log10(Decimal('2')) - Decimal('0.301029996') - >>> c.log10(Decimal('10')) - Decimal('1') - >>> c.log10(Decimal('70')) - Decimal('1.84509804') - >>> c.log10(Decimal('+Infinity')) - Decimal('Infinity') - >>> c.log10(0) - Decimal('-Infinity') - >>> c.log10(1) - Decimal('0') - """ - a = _convert_other(a, raiseit=True) - return a.log10(context=self) - - def logb(self, a): - """ Returns the exponent of the magnitude of the operand's MSD. - - The result is the integer which is the exponent of the magnitude - of the most significant digit of the operand (as though the - operand were truncated to a single digit while maintaining the - value of that digit and without limiting the resulting exponent). - - >>> ExtendedContext.logb(Decimal('250')) - Decimal('2') - >>> ExtendedContext.logb(Decimal('2.50')) - Decimal('0') - >>> ExtendedContext.logb(Decimal('0.03')) - Decimal('-2') - >>> ExtendedContext.logb(Decimal('0')) - Decimal('-Infinity') - >>> ExtendedContext.logb(1) - Decimal('0') - >>> ExtendedContext.logb(10) - Decimal('1') - >>> ExtendedContext.logb(100) - Decimal('2') - """ - a = _convert_other(a, raiseit=True) - return a.logb(context=self) - - def logical_and(self, a, b): - """Applies the logical operation 'and' between each operand's digits. - - The operands must be both logical numbers. - - >>> ExtendedContext.logical_and(Decimal('0'), Decimal('0')) - Decimal('0') - >>> ExtendedContext.logical_and(Decimal('0'), Decimal('1')) - Decimal('0') - >>> ExtendedContext.logical_and(Decimal('1'), Decimal('0')) - Decimal('0') - >>> ExtendedContext.logical_and(Decimal('1'), Decimal('1')) - Decimal('1') - >>> ExtendedContext.logical_and(Decimal('1100'), Decimal('1010')) - Decimal('1000') - >>> ExtendedContext.logical_and(Decimal('1111'), Decimal('10')) - Decimal('10') - >>> ExtendedContext.logical_and(110, 1101) - Decimal('100') - >>> ExtendedContext.logical_and(Decimal(110), 1101) - Decimal('100') - >>> ExtendedContext.logical_and(110, Decimal(1101)) - Decimal('100') - """ - a = _convert_other(a, raiseit=True) - return a.logical_and(b, context=self) - - def logical_invert(self, a): - """Invert all the digits in the operand. - - The operand must be a logical number. - - >>> ExtendedContext.logical_invert(Decimal('0')) - Decimal('111111111') - >>> ExtendedContext.logical_invert(Decimal('1')) - Decimal('111111110') - >>> ExtendedContext.logical_invert(Decimal('111111111')) - Decimal('0') - >>> ExtendedContext.logical_invert(Decimal('101010101')) - Decimal('10101010') - >>> ExtendedContext.logical_invert(1101) - Decimal('111110010') - """ - a = _convert_other(a, raiseit=True) - return a.logical_invert(context=self) - - def logical_or(self, a, b): - """Applies the logical operation 'or' between each operand's digits. - - The operands must be both logical numbers. - - >>> ExtendedContext.logical_or(Decimal('0'), Decimal('0')) - Decimal('0') - >>> ExtendedContext.logical_or(Decimal('0'), Decimal('1')) - Decimal('1') - >>> ExtendedContext.logical_or(Decimal('1'), Decimal('0')) - Decimal('1') - >>> ExtendedContext.logical_or(Decimal('1'), Decimal('1')) - Decimal('1') - >>> ExtendedContext.logical_or(Decimal('1100'), Decimal('1010')) - Decimal('1110') - >>> ExtendedContext.logical_or(Decimal('1110'), Decimal('10')) - Decimal('1110') - >>> ExtendedContext.logical_or(110, 1101) - Decimal('1111') - >>> ExtendedContext.logical_or(Decimal(110), 1101) - Decimal('1111') - >>> ExtendedContext.logical_or(110, Decimal(1101)) - Decimal('1111') - """ - a = _convert_other(a, raiseit=True) - return a.logical_or(b, context=self) - - def logical_xor(self, a, b): - """Applies the logical operation 'xor' between each operand's digits. - - The operands must be both logical numbers. - - >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('0')) - Decimal('0') - >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('1')) - Decimal('1') - >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('0')) - Decimal('1') - >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('1')) - Decimal('0') - >>> ExtendedContext.logical_xor(Decimal('1100'), Decimal('1010')) - Decimal('110') - >>> ExtendedContext.logical_xor(Decimal('1111'), Decimal('10')) - Decimal('1101') - >>> ExtendedContext.logical_xor(110, 1101) - Decimal('1011') - >>> ExtendedContext.logical_xor(Decimal(110), 1101) - Decimal('1011') - >>> ExtendedContext.logical_xor(110, Decimal(1101)) - Decimal('1011') - """ - a = _convert_other(a, raiseit=True) - return a.logical_xor(b, context=self) - - def max(self, a, b): - """max compares two values numerically and returns the maximum. - - If either operand is a NaN then the general rules apply. - Otherwise, the operands are compared as though by the compare - operation. If they are numerically equal then the left-hand operand - is chosen as the result. Otherwise the maximum (closer to positive - infinity) of the two operands is chosen as the result. - - >>> ExtendedContext.max(Decimal('3'), Decimal('2')) - Decimal('3') - >>> ExtendedContext.max(Decimal('-10'), Decimal('3')) - Decimal('3') - >>> ExtendedContext.max(Decimal('1.0'), Decimal('1')) - Decimal('1') - >>> ExtendedContext.max(Decimal('7'), Decimal('NaN')) - Decimal('7') - >>> ExtendedContext.max(1, 2) - Decimal('2') - >>> ExtendedContext.max(Decimal(1), 2) - Decimal('2') - >>> ExtendedContext.max(1, Decimal(2)) - Decimal('2') - """ - a = _convert_other(a, raiseit=True) - return a.max(b, context=self) - - def max_mag(self, a, b): - """Compares the values numerically with their sign ignored. - - >>> ExtendedContext.max_mag(Decimal('7'), Decimal('NaN')) - Decimal('7') - >>> ExtendedContext.max_mag(Decimal('7'), Decimal('-10')) - Decimal('-10') - >>> ExtendedContext.max_mag(1, -2) - Decimal('-2') - >>> ExtendedContext.max_mag(Decimal(1), -2) - Decimal('-2') - >>> ExtendedContext.max_mag(1, Decimal(-2)) - Decimal('-2') - """ - a = _convert_other(a, raiseit=True) - return a.max_mag(b, context=self) - - def min(self, a, b): - """min compares two values numerically and returns the minimum. - - If either operand is a NaN then the general rules apply. - Otherwise, the operands are compared as though by the compare - operation. If they are numerically equal then the left-hand operand - is chosen as the result. Otherwise the minimum (closer to negative - infinity) of the two operands is chosen as the result. - - >>> ExtendedContext.min(Decimal('3'), Decimal('2')) - Decimal('2') - >>> ExtendedContext.min(Decimal('-10'), Decimal('3')) - Decimal('-10') - >>> ExtendedContext.min(Decimal('1.0'), Decimal('1')) - Decimal('1.0') - >>> ExtendedContext.min(Decimal('7'), Decimal('NaN')) - Decimal('7') - >>> ExtendedContext.min(1, 2) - Decimal('1') - >>> ExtendedContext.min(Decimal(1), 2) - Decimal('1') - >>> ExtendedContext.min(1, Decimal(29)) - Decimal('1') - """ - a = _convert_other(a, raiseit=True) - return a.min(b, context=self) - - def min_mag(self, a, b): - """Compares the values numerically with their sign ignored. - - >>> ExtendedContext.min_mag(Decimal('3'), Decimal('-2')) - Decimal('-2') - >>> ExtendedContext.min_mag(Decimal('-3'), Decimal('NaN')) - Decimal('-3') - >>> ExtendedContext.min_mag(1, -2) - Decimal('1') - >>> ExtendedContext.min_mag(Decimal(1), -2) - Decimal('1') - >>> ExtendedContext.min_mag(1, Decimal(-2)) - Decimal('1') - """ - a = _convert_other(a, raiseit=True) - return a.min_mag(b, context=self) - - def minus(self, a): - """Minus corresponds to unary prefix minus in Python. - - The operation is evaluated using the same rules as subtract; the - operation minus(a) is calculated as subtract('0', a) where the '0' - has the same exponent as the operand. - - >>> ExtendedContext.minus(Decimal('1.3')) - Decimal('-1.3') - >>> ExtendedContext.minus(Decimal('-1.3')) - Decimal('1.3') - >>> ExtendedContext.minus(1) - Decimal('-1') - """ - a = _convert_other(a, raiseit=True) - return a.__neg__(context=self) - - def multiply(self, a, b): - """multiply multiplies two operands. - - If either operand is a special value then the general rules apply. - Otherwise, the operands are multiplied together - ('long multiplication'), resulting in a number which may be as long as - the sum of the lengths of the two operands. - - >>> ExtendedContext.multiply(Decimal('1.20'), Decimal('3')) - Decimal('3.60') - >>> ExtendedContext.multiply(Decimal('7'), Decimal('3')) - Decimal('21') - >>> ExtendedContext.multiply(Decimal('0.9'), Decimal('0.8')) - Decimal('0.72') - >>> ExtendedContext.multiply(Decimal('0.9'), Decimal('-0')) - Decimal('-0.0') - >>> ExtendedContext.multiply(Decimal('654321'), Decimal('654321')) - Decimal('4.28135971E+11') - >>> ExtendedContext.multiply(7, 7) - Decimal('49') - >>> ExtendedContext.multiply(Decimal(7), 7) - Decimal('49') - >>> ExtendedContext.multiply(7, Decimal(7)) - Decimal('49') - """ - a = _convert_other(a, raiseit=True) - r = a.__mul__(b, context=self) - if r is NotImplemented: - raise TypeError("Unable to convert %s to Decimal" % b) - else: - return r - - def next_minus(self, a): - """Returns the largest representable number smaller than a. - - >>> c = ExtendedContext.copy() - >>> c.Emin = -999 - >>> c.Emax = 999 - >>> ExtendedContext.next_minus(Decimal('1')) - Decimal('0.999999999') - >>> c.next_minus(Decimal('1E-1007')) - Decimal('0E-1007') - >>> ExtendedContext.next_minus(Decimal('-1.00000003')) - Decimal('-1.00000004') - >>> c.next_minus(Decimal('Infinity')) - Decimal('9.99999999E+999') - >>> c.next_minus(1) - Decimal('0.999999999') - """ - a = _convert_other(a, raiseit=True) - return a.next_minus(context=self) - - def next_plus(self, a): - """Returns the smallest representable number larger than a. - - >>> c = ExtendedContext.copy() - >>> c.Emin = -999 - >>> c.Emax = 999 - >>> ExtendedContext.next_plus(Decimal('1')) - Decimal('1.00000001') - >>> c.next_plus(Decimal('-1E-1007')) - Decimal('-0E-1007') - >>> ExtendedContext.next_plus(Decimal('-1.00000003')) - Decimal('-1.00000002') - >>> c.next_plus(Decimal('-Infinity')) - Decimal('-9.99999999E+999') - >>> c.next_plus(1) - Decimal('1.00000001') - """ - a = _convert_other(a, raiseit=True) - return a.next_plus(context=self) - - def next_toward(self, a, b): - """Returns the number closest to a, in direction towards b. - - The result is the closest representable number from the first - operand (but not the first operand) that is in the direction - towards the second operand, unless the operands have the same - value. - - >>> c = ExtendedContext.copy() - >>> c.Emin = -999 - >>> c.Emax = 999 - >>> c.next_toward(Decimal('1'), Decimal('2')) - Decimal('1.00000001') - >>> c.next_toward(Decimal('-1E-1007'), Decimal('1')) - Decimal('-0E-1007') - >>> c.next_toward(Decimal('-1.00000003'), Decimal('0')) - Decimal('-1.00000002') - >>> c.next_toward(Decimal('1'), Decimal('0')) - Decimal('0.999999999') - >>> c.next_toward(Decimal('1E-1007'), Decimal('-100')) - Decimal('0E-1007') - >>> c.next_toward(Decimal('-1.00000003'), Decimal('-10')) - Decimal('-1.00000004') - >>> c.next_toward(Decimal('0.00'), Decimal('-0.0000')) - Decimal('-0.00') - >>> c.next_toward(0, 1) - Decimal('1E-1007') - >>> c.next_toward(Decimal(0), 1) - Decimal('1E-1007') - >>> c.next_toward(0, Decimal(1)) - Decimal('1E-1007') - """ - a = _convert_other(a, raiseit=True) - return a.next_toward(b, context=self) - - def normalize(self, a): - """normalize reduces an operand to its simplest form. - - Essentially a plus operation with all trailing zeros removed from the - result. - - >>> ExtendedContext.normalize(Decimal('2.1')) - Decimal('2.1') - >>> ExtendedContext.normalize(Decimal('-2.0')) - Decimal('-2') - >>> ExtendedContext.normalize(Decimal('1.200')) - Decimal('1.2') - >>> ExtendedContext.normalize(Decimal('-120')) - Decimal('-1.2E+2') - >>> ExtendedContext.normalize(Decimal('120.00')) - Decimal('1.2E+2') - >>> ExtendedContext.normalize(Decimal('0.00')) - Decimal('0') - >>> ExtendedContext.normalize(6) - Decimal('6') - """ - a = _convert_other(a, raiseit=True) - return a.normalize(context=self) - - def number_class(self, a): - """Returns an indication of the class of the operand. - - The class is one of the following strings: - -sNaN - -NaN - -Infinity - -Normal - -Subnormal - -Zero - +Zero - +Subnormal - +Normal - +Infinity - - >>> c = ExtendedContext.copy() - >>> c.Emin = -999 - >>> c.Emax = 999 - >>> c.number_class(Decimal('Infinity')) - '+Infinity' - >>> c.number_class(Decimal('1E-10')) - '+Normal' - >>> c.number_class(Decimal('2.50')) - '+Normal' - >>> c.number_class(Decimal('0.1E-999')) - '+Subnormal' - >>> c.number_class(Decimal('0')) - '+Zero' - >>> c.number_class(Decimal('-0')) - '-Zero' - >>> c.number_class(Decimal('-0.1E-999')) - '-Subnormal' - >>> c.number_class(Decimal('-1E-10')) - '-Normal' - >>> c.number_class(Decimal('-2.50')) - '-Normal' - >>> c.number_class(Decimal('-Infinity')) - '-Infinity' - >>> c.number_class(Decimal('NaN')) - 'NaN' - >>> c.number_class(Decimal('-NaN')) - 'NaN' - >>> c.number_class(Decimal('sNaN')) - 'sNaN' - >>> c.number_class(123) - '+Normal' - """ - a = _convert_other(a, raiseit=True) - return a.number_class(context=self) - - def plus(self, a): - """Plus corresponds to unary prefix plus in Python. - - The operation is evaluated using the same rules as add; the - operation plus(a) is calculated as add('0', a) where the '0' - has the same exponent as the operand. - - >>> ExtendedContext.plus(Decimal('1.3')) - Decimal('1.3') - >>> ExtendedContext.plus(Decimal('-1.3')) - Decimal('-1.3') - >>> ExtendedContext.plus(-1) - Decimal('-1') - """ - a = _convert_other(a, raiseit=True) - return a.__pos__(context=self) - - def power(self, a, b, modulo=None): - """Raises a to the power of b, to modulo if given. - - With two arguments, compute a**b. If a is negative then b - must be integral. The result will be inexact unless b is - integral and the result is finite and can be expressed exactly - in 'precision' digits. - - With three arguments, compute (a**b) % modulo. For the - three argument form, the following restrictions on the - arguments hold: - - - all three arguments must be integral - - b must be nonnegative - - at least one of a or b must be nonzero - - modulo must be nonzero and have at most 'precision' digits - - The result of pow(a, b, modulo) is identical to the result - that would be obtained by computing (a**b) % modulo with - unbounded precision, but is computed more efficiently. It is - always exact. - - >>> c = ExtendedContext.copy() - >>> c.Emin = -999 - >>> c.Emax = 999 - >>> c.power(Decimal('2'), Decimal('3')) - Decimal('8') - >>> c.power(Decimal('-2'), Decimal('3')) - Decimal('-8') - >>> c.power(Decimal('2'), Decimal('-3')) - Decimal('0.125') - >>> c.power(Decimal('1.7'), Decimal('8')) - Decimal('69.7575744') - >>> c.power(Decimal('10'), Decimal('0.301029996')) - Decimal('2.00000000') - >>> c.power(Decimal('Infinity'), Decimal('-1')) - Decimal('0') - >>> c.power(Decimal('Infinity'), Decimal('0')) - Decimal('1') - >>> c.power(Decimal('Infinity'), Decimal('1')) - Decimal('Infinity') - >>> c.power(Decimal('-Infinity'), Decimal('-1')) - Decimal('-0') - >>> c.power(Decimal('-Infinity'), Decimal('0')) - Decimal('1') - >>> c.power(Decimal('-Infinity'), Decimal('1')) - Decimal('-Infinity') - >>> c.power(Decimal('-Infinity'), Decimal('2')) - Decimal('Infinity') - >>> c.power(Decimal('0'), Decimal('0')) - Decimal('NaN') - - >>> c.power(Decimal('3'), Decimal('7'), Decimal('16')) - Decimal('11') - >>> c.power(Decimal('-3'), Decimal('7'), Decimal('16')) - Decimal('-11') - >>> c.power(Decimal('-3'), Decimal('8'), Decimal('16')) - Decimal('1') - >>> c.power(Decimal('3'), Decimal('7'), Decimal('-16')) - Decimal('11') - >>> c.power(Decimal('23E12345'), Decimal('67E189'), Decimal('123456789')) - Decimal('11729830') - >>> c.power(Decimal('-0'), Decimal('17'), Decimal('1729')) - Decimal('-0') - >>> c.power(Decimal('-23'), Decimal('0'), Decimal('65537')) - Decimal('1') - >>> ExtendedContext.power(7, 7) - Decimal('823543') - >>> ExtendedContext.power(Decimal(7), 7) - Decimal('823543') - >>> ExtendedContext.power(7, Decimal(7), 2) - Decimal('1') - """ - a = _convert_other(a, raiseit=True) - r = a.__pow__(b, modulo, context=self) - if r is NotImplemented: - raise TypeError("Unable to convert %s to Decimal" % b) - else: - return r - - def quantize(self, a, b): - """Returns a value equal to 'a' (rounded), having the exponent of 'b'. - - The coefficient of the result is derived from that of the left-hand - operand. It may be rounded using the current rounding setting (if the - exponent is being increased), multiplied by a positive power of ten (if - the exponent is being decreased), or is unchanged (if the exponent is - already equal to that of the right-hand operand). - - Unlike other operations, if the length of the coefficient after the - quantize operation would be greater than precision then an Invalid - operation condition is raised. This guarantees that, unless there is - an error condition, the exponent of the result of a quantize is always - equal to that of the right-hand operand. - - Also unlike other operations, quantize will never raise Underflow, even - if the result is subnormal and inexact. - - >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.001')) - Decimal('2.170') - >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.01')) - Decimal('2.17') - >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.1')) - Decimal('2.2') - >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('1e+0')) - Decimal('2') - >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('1e+1')) - Decimal('0E+1') - >>> ExtendedContext.quantize(Decimal('-Inf'), Decimal('Infinity')) - Decimal('-Infinity') - >>> ExtendedContext.quantize(Decimal('2'), Decimal('Infinity')) - Decimal('NaN') - >>> ExtendedContext.quantize(Decimal('-0.1'), Decimal('1')) - Decimal('-0') - >>> ExtendedContext.quantize(Decimal('-0'), Decimal('1e+5')) - Decimal('-0E+5') - >>> ExtendedContext.quantize(Decimal('+35236450.6'), Decimal('1e-2')) - Decimal('NaN') - >>> ExtendedContext.quantize(Decimal('-35236450.6'), Decimal('1e-2')) - Decimal('NaN') - >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e-1')) - Decimal('217.0') - >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e-0')) - Decimal('217') - >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e+1')) - Decimal('2.2E+2') - >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e+2')) - Decimal('2E+2') - >>> ExtendedContext.quantize(1, 2) - Decimal('1') - >>> ExtendedContext.quantize(Decimal(1), 2) - Decimal('1') - >>> ExtendedContext.quantize(1, Decimal(2)) - Decimal('1') - """ - a = _convert_other(a, raiseit=True) - return a.quantize(b, context=self) - - def radix(self): - """Just returns 10, as this is Decimal, :) - - >>> ExtendedContext.radix() - Decimal('10') - """ - return Decimal(10) - - def remainder(self, a, b): - """Returns the remainder from integer division. - - The result is the residue of the dividend after the operation of - calculating integer division as described for divide-integer, rounded - to precision digits if necessary. The sign of the result, if - non-zero, is the same as that of the original dividend. - - This operation will fail under the same conditions as integer division - (that is, if integer division on the same two operands would fail, the - remainder cannot be calculated). - - >>> ExtendedContext.remainder(Decimal('2.1'), Decimal('3')) - Decimal('2.1') - >>> ExtendedContext.remainder(Decimal('10'), Decimal('3')) - Decimal('1') - >>> ExtendedContext.remainder(Decimal('-10'), Decimal('3')) - Decimal('-1') - >>> ExtendedContext.remainder(Decimal('10.2'), Decimal('1')) - Decimal('0.2') - >>> ExtendedContext.remainder(Decimal('10'), Decimal('0.3')) - Decimal('0.1') - >>> ExtendedContext.remainder(Decimal('3.6'), Decimal('1.3')) - Decimal('1.0') - >>> ExtendedContext.remainder(22, 6) - Decimal('4') - >>> ExtendedContext.remainder(Decimal(22), 6) - Decimal('4') - >>> ExtendedContext.remainder(22, Decimal(6)) - Decimal('4') - """ - a = _convert_other(a, raiseit=True) - r = a.__mod__(b, context=self) - if r is NotImplemented: - raise TypeError("Unable to convert %s to Decimal" % b) - else: - return r - - def remainder_near(self, a, b): - """Returns to be "a - b * n", where n is the integer nearest the exact - value of "x / b" (if two integers are equally near then the even one - is chosen). If the result is equal to 0 then its sign will be the - sign of a. - - This operation will fail under the same conditions as integer division - (that is, if integer division on the same two operands would fail, the - remainder cannot be calculated). - - >>> ExtendedContext.remainder_near(Decimal('2.1'), Decimal('3')) - Decimal('-0.9') - >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('6')) - Decimal('-2') - >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('3')) - Decimal('1') - >>> ExtendedContext.remainder_near(Decimal('-10'), Decimal('3')) - Decimal('-1') - >>> ExtendedContext.remainder_near(Decimal('10.2'), Decimal('1')) - Decimal('0.2') - >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('0.3')) - Decimal('0.1') - >>> ExtendedContext.remainder_near(Decimal('3.6'), Decimal('1.3')) - Decimal('-0.3') - >>> ExtendedContext.remainder_near(3, 11) - Decimal('3') - >>> ExtendedContext.remainder_near(Decimal(3), 11) - Decimal('3') - >>> ExtendedContext.remainder_near(3, Decimal(11)) - Decimal('3') - """ - a = _convert_other(a, raiseit=True) - return a.remainder_near(b, context=self) - - def rotate(self, a, b): - """Returns a rotated copy of a, b times. - - The coefficient of the result is a rotated copy of the digits in - the coefficient of the first operand. The number of places of - rotation is taken from the absolute value of the second operand, - with the rotation being to the left if the second operand is - positive or to the right otherwise. - - >>> ExtendedContext.rotate(Decimal('34'), Decimal('8')) - Decimal('400000003') - >>> ExtendedContext.rotate(Decimal('12'), Decimal('9')) - Decimal('12') - >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('-2')) - Decimal('891234567') - >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('0')) - Decimal('123456789') - >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('+2')) - Decimal('345678912') - >>> ExtendedContext.rotate(1333333, 1) - Decimal('13333330') - >>> ExtendedContext.rotate(Decimal(1333333), 1) - Decimal('13333330') - >>> ExtendedContext.rotate(1333333, Decimal(1)) - Decimal('13333330') - """ - a = _convert_other(a, raiseit=True) - return a.rotate(b, context=self) - - def same_quantum(self, a, b): - """Returns True if the two operands have the same exponent. - - The result is never affected by either the sign or the coefficient of - either operand. - - >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('0.001')) - False - >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('0.01')) - True - >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('1')) - False - >>> ExtendedContext.same_quantum(Decimal('Inf'), Decimal('-Inf')) - True - >>> ExtendedContext.same_quantum(10000, -1) - True - >>> ExtendedContext.same_quantum(Decimal(10000), -1) - True - >>> ExtendedContext.same_quantum(10000, Decimal(-1)) - True - """ - a = _convert_other(a, raiseit=True) - return a.same_quantum(b) - - def scaleb (self, a, b): - """Returns the first operand after adding the second value its exp. - - >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('-2')) - Decimal('0.0750') - >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('0')) - Decimal('7.50') - >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('3')) - Decimal('7.50E+3') - >>> ExtendedContext.scaleb(1, 4) - Decimal('1E+4') - >>> ExtendedContext.scaleb(Decimal(1), 4) - Decimal('1E+4') - >>> ExtendedContext.scaleb(1, Decimal(4)) - Decimal('1E+4') - """ - a = _convert_other(a, raiseit=True) - return a.scaleb(b, context=self) - - def shift(self, a, b): - """Returns a shifted copy of a, b times. - - The coefficient of the result is a shifted copy of the digits - in the coefficient of the first operand. The number of places - to shift is taken from the absolute value of the second operand, - with the shift being to the left if the second operand is - positive or to the right otherwise. Digits shifted into the - coefficient are zeros. - - >>> ExtendedContext.shift(Decimal('34'), Decimal('8')) - Decimal('400000000') - >>> ExtendedContext.shift(Decimal('12'), Decimal('9')) - Decimal('0') - >>> ExtendedContext.shift(Decimal('123456789'), Decimal('-2')) - Decimal('1234567') - >>> ExtendedContext.shift(Decimal('123456789'), Decimal('0')) - Decimal('123456789') - >>> ExtendedContext.shift(Decimal('123456789'), Decimal('+2')) - Decimal('345678900') - >>> ExtendedContext.shift(88888888, 2) - Decimal('888888800') - >>> ExtendedContext.shift(Decimal(88888888), 2) - Decimal('888888800') - >>> ExtendedContext.shift(88888888, Decimal(2)) - Decimal('888888800') - """ - a = _convert_other(a, raiseit=True) - return a.shift(b, context=self) - - def sqrt(self, a): - """Square root of a non-negative number to context precision. - - If the result must be inexact, it is rounded using the round-half-even - algorithm. - - >>> ExtendedContext.sqrt(Decimal('0')) - Decimal('0') - >>> ExtendedContext.sqrt(Decimal('-0')) - Decimal('-0') - >>> ExtendedContext.sqrt(Decimal('0.39')) - Decimal('0.624499800') - >>> ExtendedContext.sqrt(Decimal('100')) - Decimal('10') - >>> ExtendedContext.sqrt(Decimal('1')) - Decimal('1') - >>> ExtendedContext.sqrt(Decimal('1.0')) - Decimal('1.0') - >>> ExtendedContext.sqrt(Decimal('1.00')) - Decimal('1.0') - >>> ExtendedContext.sqrt(Decimal('7')) - Decimal('2.64575131') - >>> ExtendedContext.sqrt(Decimal('10')) - Decimal('3.16227766') - >>> ExtendedContext.sqrt(2) - Decimal('1.41421356') - >>> ExtendedContext.prec - 9 - """ - a = _convert_other(a, raiseit=True) - return a.sqrt(context=self) - - def subtract(self, a, b): - """Return the difference between the two operands. - - >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('1.07')) - Decimal('0.23') - >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('1.30')) - Decimal('0.00') - >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('2.07')) - Decimal('-0.77') - >>> ExtendedContext.subtract(8, 5) - Decimal('3') - >>> ExtendedContext.subtract(Decimal(8), 5) - Decimal('3') - >>> ExtendedContext.subtract(8, Decimal(5)) - Decimal('3') - """ - a = _convert_other(a, raiseit=True) - r = a.__sub__(b, context=self) - if r is NotImplemented: - raise TypeError("Unable to convert %s to Decimal" % b) - else: - return r - - def to_eng_string(self, a): - """Converts a number to a string, using scientific notation. - - The operation is not affected by the context. - """ - a = _convert_other(a, raiseit=True) - return a.to_eng_string(context=self) - - def to_sci_string(self, a): - """Converts a number to a string, using scientific notation. - - The operation is not affected by the context. - """ - a = _convert_other(a, raiseit=True) - return a.__str__(context=self) - - def to_integral_exact(self, a): - """Rounds to an integer. - - When the operand has a negative exponent, the result is the same - as using the quantize() operation using the given operand as the - left-hand-operand, 1E+0 as the right-hand-operand, and the precision - of the operand as the precision setting; Inexact and Rounded flags - are allowed in this operation. The rounding mode is taken from the - context. - - >>> ExtendedContext.to_integral_exact(Decimal('2.1')) - Decimal('2') - >>> ExtendedContext.to_integral_exact(Decimal('100')) - Decimal('100') - >>> ExtendedContext.to_integral_exact(Decimal('100.0')) - Decimal('100') - >>> ExtendedContext.to_integral_exact(Decimal('101.5')) - Decimal('102') - >>> ExtendedContext.to_integral_exact(Decimal('-101.5')) - Decimal('-102') - >>> ExtendedContext.to_integral_exact(Decimal('10E+5')) - Decimal('1.0E+6') - >>> ExtendedContext.to_integral_exact(Decimal('7.89E+77')) - Decimal('7.89E+77') - >>> ExtendedContext.to_integral_exact(Decimal('-Inf')) - Decimal('-Infinity') - """ - a = _convert_other(a, raiseit=True) - return a.to_integral_exact(context=self) - - def to_integral_value(self, a): - """Rounds to an integer. - - When the operand has a negative exponent, the result is the same - as using the quantize() operation using the given operand as the - left-hand-operand, 1E+0 as the right-hand-operand, and the precision - of the operand as the precision setting, except that no flags will - be set. The rounding mode is taken from the context. - - >>> ExtendedContext.to_integral_value(Decimal('2.1')) - Decimal('2') - >>> ExtendedContext.to_integral_value(Decimal('100')) - Decimal('100') - >>> ExtendedContext.to_integral_value(Decimal('100.0')) - Decimal('100') - >>> ExtendedContext.to_integral_value(Decimal('101.5')) - Decimal('102') - >>> ExtendedContext.to_integral_value(Decimal('-101.5')) - Decimal('-102') - >>> ExtendedContext.to_integral_value(Decimal('10E+5')) - Decimal('1.0E+6') - >>> ExtendedContext.to_integral_value(Decimal('7.89E+77')) - Decimal('7.89E+77') - >>> ExtendedContext.to_integral_value(Decimal('-Inf')) - Decimal('-Infinity') - """ - a = _convert_other(a, raiseit=True) - return a.to_integral_value(context=self) - - # the method name changed, but we provide also the old one, for compatibility - to_integral = to_integral_value - -class _WorkRep(object): - __slots__ = ('sign','int','exp') - # sign: 0 or 1 - # int: int - # exp: None, int, or string - - def __init__(self, value=None): - if value is None: - self.sign = None - self.int = 0 - self.exp = None - elif isinstance(value, Decimal): - self.sign = value._sign - self.int = int(value._int) - self.exp = value._exp - else: - # assert isinstance(value, tuple) - self.sign = value[0] - self.int = value[1] - self.exp = value[2] - - def __repr__(self): - return "(%r, %r, %r)" % (self.sign, self.int, self.exp) - - __str__ = __repr__ - - - -def _normalize(op1, op2, prec = 0): - """Normalizes op1, op2 to have the same exp and length of coefficient. - - Done during addition. - """ - if op1.exp < op2.exp: - tmp = op2 - other = op1 - else: - tmp = op1 - other = op2 - - # Let exp = min(tmp.exp - 1, tmp.adjusted() - precision - 1). - # Then adding 10**exp to tmp has the same effect (after rounding) - # as adding any positive quantity smaller than 10**exp; similarly - # for subtraction. So if other is smaller than 10**exp we replace - # it with 10**exp. This avoids tmp.exp - other.exp getting too large. - tmp_len = len(str(tmp.int)) - other_len = len(str(other.int)) - exp = tmp.exp + min(-1, tmp_len - prec - 2) - if other_len + other.exp - 1 < exp: - other.int = 1 - other.exp = exp - - tmp.int *= 10 ** (tmp.exp - other.exp) - tmp.exp = other.exp - return op1, op2 - -##### Integer arithmetic functions used by ln, log10, exp and __pow__ ##### - -_nbits = int.bit_length - -def _decimal_lshift_exact(n, e): - """ Given integers n and e, return n * 10**e if it's an integer, else None. - - The computation is designed to avoid computing large powers of 10 - unnecessarily. - - >>> _decimal_lshift_exact(3, 4) - 30000 - >>> _decimal_lshift_exact(300, -999999999) # returns None - - """ - if n == 0: - return 0 - elif e >= 0: - return n * 10**e - else: - # val_n = largest power of 10 dividing n. - str_n = str(abs(n)) - val_n = len(str_n) - len(str_n.rstrip('0')) - return None if val_n < -e else n // 10**-e - -def _sqrt_nearest(n, a): - """Closest integer to the square root of the positive integer n. a is - an initial approximation to the square root. Any positive integer - will do for a, but the closer a is to the square root of n the - faster convergence will be. - - """ - if n <= 0 or a <= 0: - raise ValueError("Both arguments to _sqrt_nearest should be positive.") - - b=0 - while a != b: - b, a = a, a--n//a>>1 - return a - -def _rshift_nearest(x, shift): - """Given an integer x and a nonnegative integer shift, return closest - integer to x / 2**shift; use round-to-even in case of a tie. - - """ - b, q = 1 << shift, x >> shift - return q + (2*(x & (b-1)) + (q&1) > b) - -def _div_nearest(a, b): - """Closest integer to a/b, a and b positive integers; rounds to even - in the case of a tie. - - """ - q, r = divmod(a, b) - return q + (2*r + (q&1) > b) - -def _ilog(x, M, L = 8): - """Integer approximation to M*log(x/M), with absolute error boundable - in terms only of x/M. - - Given positive integers x and M, return an integer approximation to - M * log(x/M). For L = 8 and 0.1 <= x/M <= 10 the difference - between the approximation and the exact result is at most 22. For - L = 8 and 1.0 <= x/M <= 10.0 the difference is at most 15. In - both cases these are upper bounds on the error; it will usually be - much smaller.""" - - # The basic algorithm is the following: let log1p be the function - # log1p(x) = log(1+x). Then log(x/M) = log1p((x-M)/M). We use - # the reduction - # - # log1p(y) = 2*log1p(y/(1+sqrt(1+y))) - # - # repeatedly until the argument to log1p is small (< 2**-L in - # absolute value). For small y we can use the Taylor series - # expansion - # - # log1p(y) ~ y - y**2/2 + y**3/3 - ... - (-y)**T/T - # - # truncating at T such that y**T is small enough. The whole - # computation is carried out in a form of fixed-point arithmetic, - # with a real number z being represented by an integer - # approximation to z*M. To avoid loss of precision, the y below - # is actually an integer approximation to 2**R*y*M, where R is the - # number of reductions performed so far. - - y = x-M - # argument reduction; R = number of reductions performed - R = 0 - while (R <= L and abs(y) << L-R >= M or - R > L and abs(y) >> R-L >= M): - y = _div_nearest((M*y) << 1, - M + _sqrt_nearest(M*(M+_rshift_nearest(y, R)), M)) - R += 1 - - # Taylor series with T terms - T = -int(-10*len(str(M))//(3*L)) - yshift = _rshift_nearest(y, R) - w = _div_nearest(M, T) - for k in range(T-1, 0, -1): - w = _div_nearest(M, k) - _div_nearest(yshift*w, M) - - return _div_nearest(w*y, M) - -def _dlog10(c, e, p): - """Given integers c, e and p with c > 0, p >= 0, compute an integer - approximation to 10**p * log10(c*10**e), with an absolute error of - at most 1. Assumes that c*10**e is not exactly 1.""" - - # increase precision by 2; compensate for this by dividing - # final result by 100 - p += 2 - - # write c*10**e as d*10**f with either: - # f >= 0 and 1 <= d <= 10, or - # f <= 0 and 0.1 <= d <= 1. - # Thus for c*10**e close to 1, f = 0 - l = len(str(c)) - f = e+l - (e+l >= 1) - - if p > 0: - M = 10**p - k = e+p-f - if k >= 0: - c *= 10**k - else: - c = _div_nearest(c, 10**-k) - - log_d = _ilog(c, M) # error < 5 + 22 = 27 - log_10 = _log10_digits(p) # error < 1 - log_d = _div_nearest(log_d*M, log_10) - log_tenpower = f*M # exact - else: - log_d = 0 # error < 2.31 - log_tenpower = _div_nearest(f, 10**-p) # error < 0.5 - - return _div_nearest(log_tenpower+log_d, 100) - -def _dlog(c, e, p): - """Given integers c, e and p with c > 0, compute an integer - approximation to 10**p * log(c*10**e), with an absolute error of - at most 1. Assumes that c*10**e is not exactly 1.""" - - # Increase precision by 2. The precision increase is compensated - # for at the end with a division by 100. - p += 2 - - # rewrite c*10**e as d*10**f with either f >= 0 and 1 <= d <= 10, - # or f <= 0 and 0.1 <= d <= 1. Then we can compute 10**p * log(c*10**e) - # as 10**p * log(d) + 10**p*f * log(10). - l = len(str(c)) - f = e+l - (e+l >= 1) - - # compute approximation to 10**p*log(d), with error < 27 - if p > 0: - k = e+p-f - if k >= 0: - c *= 10**k - else: - c = _div_nearest(c, 10**-k) # error of <= 0.5 in c - - # _ilog magnifies existing error in c by a factor of at most 10 - log_d = _ilog(c, 10**p) # error < 5 + 22 = 27 - else: - # p <= 0: just approximate the whole thing by 0; error < 2.31 - log_d = 0 - - # compute approximation to f*10**p*log(10), with error < 11. - if f: - extra = len(str(abs(f)))-1 - if p + extra >= 0: - # error in f * _log10_digits(p+extra) < |f| * 1 = |f| - # after division, error < |f|/10**extra + 0.5 < 10 + 0.5 < 11 - f_log_ten = _div_nearest(f*_log10_digits(p+extra), 10**extra) - else: - f_log_ten = 0 - else: - f_log_ten = 0 - - # error in sum < 11+27 = 38; error after division < 0.38 + 0.5 < 1 - return _div_nearest(f_log_ten + log_d, 100) - -class _Log10Memoize(object): - """Class to compute, store, and allow retrieval of, digits of the - constant log(10) = 2.302585.... This constant is needed by - Decimal.ln, Decimal.log10, Decimal.exp and Decimal.__pow__.""" - def __init__(self): - self.digits = "23025850929940456840179914546843642076011014886" - - def getdigits(self, p): - """Given an integer p >= 0, return floor(10**p)*log(10). - - For example, self.getdigits(3) returns 2302. - """ - # digits are stored as a string, for quick conversion to - # integer in the case that we've already computed enough - # digits; the stored digits should always be correct - # (truncated, not rounded to nearest). - if p < 0: - raise ValueError("p should be nonnegative") - - if p >= len(self.digits): - # compute p+3, p+6, p+9, ... digits; continue until at - # least one of the extra digits is nonzero - extra = 3 - while True: - # compute p+extra digits, correct to within 1ulp - M = 10**(p+extra+2) - digits = str(_div_nearest(_ilog(10*M, M), 100)) - if digits[-extra:] != '0'*extra: - break - extra += 3 - # keep all reliable digits so far; remove trailing zeros - # and next nonzero digit - self.digits = digits.rstrip('0')[:-1] - return int(self.digits[:p+1]) - -_log10_digits = _Log10Memoize().getdigits - -def _iexp(x, M, L=8): - """Given integers x and M, M > 0, such that x/M is small in absolute - value, compute an integer approximation to M*exp(x/M). For 0 <= - x/M <= 2.4, the absolute error in the result is bounded by 60 (and - is usually much smaller).""" - - # Algorithm: to compute exp(z) for a real number z, first divide z - # by a suitable power R of 2 so that |z/2**R| < 2**-L. Then - # compute expm1(z/2**R) = exp(z/2**R) - 1 using the usual Taylor - # series - # - # expm1(x) = x + x**2/2! + x**3/3! + ... - # - # Now use the identity - # - # expm1(2x) = expm1(x)*(expm1(x)+2) - # - # R times to compute the sequence expm1(z/2**R), - # expm1(z/2**(R-1)), ... , exp(z/2), exp(z). - - # Find R such that x/2**R/M <= 2**-L - R = _nbits((x< M - T = -int(-10*len(str(M))//(3*L)) - y = _div_nearest(x, T) - Mshift = M<= 0: - cshift = c*10**shift - else: - cshift = c//10**-shift - quot, rem = divmod(cshift, _log10_digits(q)) - - # reduce remainder back to original precision - rem = _div_nearest(rem, 10**extra) - - # error in result of _iexp < 120; error after division < 0.62 - return _div_nearest(_iexp(rem, 10**p), 1000), quot - p + 3 - -def _dpower(xc, xe, yc, ye, p): - """Given integers xc, xe, yc and ye representing Decimals x = xc*10**xe and - y = yc*10**ye, compute x**y. Returns a pair of integers (c, e) such that: - - 10**(p-1) <= c <= 10**p, and - (c-1)*10**e < x**y < (c+1)*10**e - - in other words, c*10**e is an approximation to x**y with p digits - of precision, and with an error in c of at most 1. (This is - almost, but not quite, the same as the error being < 1ulp: when c - == 10**(p-1) we can only guarantee error < 10ulp.) - - We assume that: x is positive and not equal to 1, and y is nonzero. - """ - - # Find b such that 10**(b-1) <= |y| <= 10**b - b = len(str(abs(yc))) + ye - - # log(x) = lxc*10**(-p-b-1), to p+b+1 places after the decimal point - lxc = _dlog(xc, xe, p+b+1) - - # compute product y*log(x) = yc*lxc*10**(-p-b-1+ye) = pc*10**(-p-1) - shift = ye-b - if shift >= 0: - pc = lxc*yc*10**shift - else: - pc = _div_nearest(lxc*yc, 10**-shift) - - if pc == 0: - # we prefer a result that isn't exactly 1; this makes it - # easier to compute a correctly rounded result in __pow__ - if ((len(str(xc)) + xe >= 1) == (yc > 0)): # if x**y > 1: - coeff, exp = 10**(p-1)+1, 1-p - else: - coeff, exp = 10**p-1, -p - else: - coeff, exp = _dexp(pc, -(p+1), p+1) - coeff = _div_nearest(coeff, 10) - exp += 1 - - return coeff, exp - -def _log10_lb(c, correction = { - '1': 100, '2': 70, '3': 53, '4': 40, '5': 31, - '6': 23, '7': 16, '8': 10, '9': 5}): - """Compute a lower bound for 100*log10(c) for a positive integer c.""" - if c <= 0: - raise ValueError("The argument to _log10_lb should be nonnegative.") - str_c = str(c) - return 100*len(str_c) - correction[str_c[0]] - -##### Helper Functions #################################################### - -def _convert_other(other, raiseit=False, allow_float=False): - """Convert other to Decimal. - - Verifies that it's ok to use in an implicit construction. - If allow_float is true, allow conversion from float; this - is used in the comparison methods (__eq__ and friends). - - """ - if isinstance(other, Decimal): - return other - if isinstance(other, int): - return Decimal(other) - if allow_float and isinstance(other, float): - return Decimal.from_float(other) - - if raiseit: - raise TypeError("Unable to convert %s to Decimal" % other) - return NotImplemented - -def _convert_for_comparison(self, other, equality_op=False): - """Given a Decimal instance self and a Python object other, return - a pair (s, o) of Decimal instances such that "s op o" is - equivalent to "self op other" for any of the 6 comparison - operators "op". - - """ - if isinstance(other, Decimal): - return self, other - - # Comparison with a Rational instance (also includes integers): - # self op n/d <=> self*d op n (for n and d integers, d positive). - # A NaN or infinity can be left unchanged without affecting the - # comparison result. - if isinstance(other, _numbers.Rational): - if not self._is_special: - self = _dec_from_triple(self._sign, - str(int(self._int) * other.denominator), - self._exp) - return self, Decimal(other.numerator) - - # Comparisons with float and complex types. == and != comparisons - # with complex numbers should succeed, returning either True or False - # as appropriate. Other comparisons return NotImplemented. - if equality_op and isinstance(other, _numbers.Complex) and other.imag == 0: - other = other.real - if isinstance(other, float): - context = getcontext() - if equality_op: - context.flags[FloatOperation] = 1 - else: - context._raise_error(FloatOperation, - "strict semantics for mixing floats and Decimals are enabled") - return self, Decimal.from_float(other) - return NotImplemented, NotImplemented - - -##### Setup Specific Contexts ############################################ - -# The default context prototype used by Context() -# Is mutable, so that new contexts can have different default values - -DefaultContext = Context( - prec=28, rounding=ROUND_HALF_EVEN, - traps=[DivisionByZero, Overflow, InvalidOperation], - flags=[], - Emax=999999, - Emin=-999999, - capitals=1, - clamp=0 -) - -# Pre-made alternate contexts offered by the specification -# Don't change these; the user should be able to select these -# contexts and be able to reproduce results from other implementations -# of the spec. - -BasicContext = Context( - prec=9, rounding=ROUND_HALF_UP, - traps=[DivisionByZero, Overflow, InvalidOperation, Clamped, Underflow], - flags=[], -) - -ExtendedContext = Context( - prec=9, rounding=ROUND_HALF_EVEN, - traps=[], - flags=[], -) - - -##### crud for parsing strings ############################################# -# -# Regular expression used for parsing numeric strings. Additional -# comments: -# -# 1. Uncomment the two '\s*' lines to allow leading and/or trailing -# whitespace. But note that the specification disallows whitespace in -# a numeric string. -# -# 2. For finite numbers (not infinities and NaNs) the body of the -# number between the optional sign and the optional exponent must have -# at least one decimal digit, possibly after the decimal point. The -# lookahead expression '(?=\d|\.\d)' checks this. - -import re -_parser = re.compile(r""" # A numeric string consists of: -# \s* - (?P[-+])? # an optional sign, followed by either... - ( - (?=\d|\.\d) # ...a number (with at least one digit) - (?P\d*) # having a (possibly empty) integer part - (\.(?P\d*))? # followed by an optional fractional part - (E(?P[-+]?\d+))? # followed by an optional exponent, or... - | - Inf(inity)? # ...an infinity, or... - | - (?Ps)? # ...an (optionally signaling) - NaN # NaN - (?P\d*) # with (possibly empty) diagnostic info. - ) -# \s* - \Z -""", re.VERBOSE | re.IGNORECASE).match - -_all_zeros = re.compile('0*$').match -_exact_half = re.compile('50*$').match - -##### PEP3101 support functions ############################################## -# The functions in this section have little to do with the Decimal -# class, and could potentially be reused or adapted for other pure -# Python numeric classes that want to implement __format__ -# -# A format specifier for Decimal looks like: -# -# [[fill]align][sign][#][0][minimumwidth][,][.precision][type] - -_parse_format_specifier_regex = re.compile(r"""\A -(?: - (?P.)? - (?P[<>=^]) -)? -(?P[-+ ])? -(?P\#)? -(?P0)? -(?P(?!0)\d+)? -(?P,)? -(?:\.(?P0|(?!0)\d+))? -(?P[eEfFgGn%])? -\Z -""", re.VERBOSE|re.DOTALL) - -del re - -# The locale module is only needed for the 'n' format specifier. The -# rest of the PEP 3101 code functions quite happily without it, so we -# don't care too much if locale isn't present. -try: - import locale as _locale -except ImportError: - pass - -def _parse_format_specifier(format_spec, _localeconv=None): - """Parse and validate a format specifier. - - Turns a standard numeric format specifier into a dict, with the - following entries: - - fill: fill character to pad field to minimum width - align: alignment type, either '<', '>', '=' or '^' - sign: either '+', '-' or ' ' - minimumwidth: nonnegative integer giving minimum width - zeropad: boolean, indicating whether to pad with zeros - thousands_sep: string to use as thousands separator, or '' - grouping: grouping for thousands separators, in format - used by localeconv - decimal_point: string to use for decimal point - precision: nonnegative integer giving precision, or None - type: one of the characters 'eEfFgG%', or None - - """ - m = _parse_format_specifier_regex.match(format_spec) - if m is None: - raise ValueError("Invalid format specifier: " + format_spec) - - # get the dictionary - format_dict = m.groupdict() - - # zeropad; defaults for fill and alignment. If zero padding - # is requested, the fill and align fields should be absent. - fill = format_dict['fill'] - align = format_dict['align'] - format_dict['zeropad'] = (format_dict['zeropad'] is not None) - if format_dict['zeropad']: - if fill is not None: - raise ValueError("Fill character conflicts with '0'" - " in format specifier: " + format_spec) - if align is not None: - raise ValueError("Alignment conflicts with '0' in " - "format specifier: " + format_spec) - format_dict['fill'] = fill or ' ' - # PEP 3101 originally specified that the default alignment should - # be left; it was later agreed that right-aligned makes more sense - # for numeric types. See http://bugs.python.org/issue6857. - format_dict['align'] = align or '>' - - # default sign handling: '-' for negative, '' for positive - if format_dict['sign'] is None: - format_dict['sign'] = '-' - - # minimumwidth defaults to 0; precision remains None if not given - format_dict['minimumwidth'] = int(format_dict['minimumwidth'] or '0') - if format_dict['precision'] is not None: - format_dict['precision'] = int(format_dict['precision']) - - # if format type is 'g' or 'G' then a precision of 0 makes little - # sense; convert it to 1. Same if format type is unspecified. - if format_dict['precision'] == 0: - if format_dict['type'] is None or format_dict['type'] in 'gGn': - format_dict['precision'] = 1 - - # determine thousands separator, grouping, and decimal separator, and - # add appropriate entries to format_dict - if format_dict['type'] == 'n': - # apart from separators, 'n' behaves just like 'g' - format_dict['type'] = 'g' - if _localeconv is None: - _localeconv = _locale.localeconv() - if format_dict['thousands_sep'] is not None: - raise ValueError("Explicit thousands separator conflicts with " - "'n' type in format specifier: " + format_spec) - format_dict['thousands_sep'] = _localeconv['thousands_sep'] - format_dict['grouping'] = _localeconv['grouping'] - format_dict['decimal_point'] = _localeconv['decimal_point'] - else: - if format_dict['thousands_sep'] is None: - format_dict['thousands_sep'] = '' - format_dict['grouping'] = [3, 0] - format_dict['decimal_point'] = '.' - - return format_dict - -def _format_align(sign, body, spec): - """Given an unpadded, non-aligned numeric string 'body' and sign - string 'sign', add padding and alignment conforming to the given - format specifier dictionary 'spec' (as produced by - parse_format_specifier). - - """ - # how much extra space do we have to play with? - minimumwidth = spec['minimumwidth'] - fill = spec['fill'] - padding = fill*(minimumwidth - len(sign) - len(body)) - - align = spec['align'] - if align == '<': - result = sign + body + padding - elif align == '>': - result = padding + sign + body - elif align == '=': - result = sign + padding + body - elif align == '^': - half = len(padding)//2 - result = padding[:half] + sign + body + padding[half:] - else: - raise ValueError('Unrecognised alignment field') - - return result - -def _group_lengths(grouping): - """Convert a localeconv-style grouping into a (possibly infinite) - iterable of integers representing group lengths. - - """ - # The result from localeconv()['grouping'], and the input to this - # function, should be a list of integers in one of the - # following three forms: - # - # (1) an empty list, or - # (2) nonempty list of positive integers + [0] - # (3) list of positive integers + [locale.CHAR_MAX], or - - from itertools import chain, repeat - if not grouping: - return [] - elif grouping[-1] == 0 and len(grouping) >= 2: - return chain(grouping[:-1], repeat(grouping[-2])) - elif grouping[-1] == _locale.CHAR_MAX: - return grouping[:-1] - else: - raise ValueError('unrecognised format for grouping') - -def _insert_thousands_sep(digits, spec, min_width=1): - """Insert thousands separators into a digit string. - - spec is a dictionary whose keys should include 'thousands_sep' and - 'grouping'; typically it's the result of parsing the format - specifier using _parse_format_specifier. - - The min_width keyword argument gives the minimum length of the - result, which will be padded on the left with zeros if necessary. - - If necessary, the zero padding adds an extra '0' on the left to - avoid a leading thousands separator. For example, inserting - commas every three digits in '123456', with min_width=8, gives - '0,123,456', even though that has length 9. - - """ - - sep = spec['thousands_sep'] - grouping = spec['grouping'] - - groups = [] - for l in _group_lengths(grouping): - if l <= 0: - raise ValueError("group length should be positive") - # max(..., 1) forces at least 1 digit to the left of a separator - l = min(max(len(digits), min_width, 1), l) - groups.append('0'*(l - len(digits)) + digits[-l:]) - digits = digits[:-l] - min_width -= l - if not digits and min_width <= 0: - break - min_width -= len(sep) - else: - l = max(len(digits), min_width, 1) - groups.append('0'*(l - len(digits)) + digits[-l:]) - return sep.join(reversed(groups)) - -def _format_sign(is_negative, spec): - """Determine sign character.""" - - if is_negative: - return '-' - elif spec['sign'] in ' +': - return spec['sign'] - else: - return '' - -def _format_number(is_negative, intpart, fracpart, exp, spec): - """Format a number, given the following data: - - is_negative: true if the number is negative, else false - intpart: string of digits that must appear before the decimal point - fracpart: string of digits that must come after the point - exp: exponent, as an integer - spec: dictionary resulting from parsing the format specifier - - This function uses the information in spec to: - insert separators (decimal separator and thousands separators) - format the sign - format the exponent - add trailing '%' for the '%' type - zero-pad if necessary - fill and align if necessary - """ - - sign = _format_sign(is_negative, spec) - - if fracpart or spec['alt']: - fracpart = spec['decimal_point'] + fracpart - - if exp != 0 or spec['type'] in 'eE': - echar = {'E': 'E', 'e': 'e', 'G': 'E', 'g': 'e'}[spec['type']] - fracpart += "{0}{1:+}".format(echar, exp) - if spec['type'] == '%': - fracpart += '%' - - if spec['zeropad']: - min_width = spec['minimumwidth'] - len(fracpart) - len(sign) - else: - min_width = 0 - intpart = _insert_thousands_sep(intpart, spec, min_width) - - return _format_align(sign, intpart+fracpart, spec) - - -##### Useful Constants (internal use only) ################################ - -# Reusable defaults -_Infinity = Decimal('Inf') -_NegativeInfinity = Decimal('-Inf') -_NaN = Decimal('NaN') -_Zero = Decimal(0) -_One = Decimal(1) -_NegativeOne = Decimal(-1) - -# _SignedInfinity[sign] is infinity w/ that sign -_SignedInfinity = (_Infinity, _NegativeInfinity) - -# Constants related to the hash implementation; hash(x) is based -# on the reduction of x modulo _PyHASH_MODULUS -_PyHASH_MODULUS = sys.hash_info.modulus -# hash values to use for positive and negative infinities, and nans -_PyHASH_INF = sys.hash_info.inf -_PyHASH_NAN = sys.hash_info.nan - -# _PyHASH_10INV is the inverse of 10 modulo the prime _PyHASH_MODULUS -_PyHASH_10INV = pow(10, _PyHASH_MODULUS - 2, _PyHASH_MODULUS) -del sys - -try: - import _decimal -except ImportError: - pass -else: - s1 = set(dir()) - s2 = set(dir(_decimal)) - for name in s1 - s2: - del globals()[name] - del s1, s2, name - from _decimal import * - -if __name__ == '__main__': - import doctest, decimal - doctest.testmod(decimal) diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -4173,9 +4173,7 @@ self.assertEqual(C.__version__, P.__version__) self.assertEqual(C.__libmpdec_version__, P.__libmpdec_version__) - x = dir(C) - y = [s for s in dir(P) if '__' in s or not s.startswith('_')] - self.assertEqual(set(x) - set(y), set()) + self.assertEqual(dir(C), dir(P)) def test_context_attributes(self): diff --git a/Modules/_decimal/tests/deccheck.py b/Modules/_decimal/tests/deccheck.py --- a/Modules/_decimal/tests/deccheck.py +++ b/Modules/_decimal/tests/deccheck.py @@ -36,6 +36,7 @@ from randdec import randfloat, all_unary, all_binary, all_ternary from randdec import unary_optarg, binary_optarg, ternary_optarg from formathelper import rand_format, rand_locale +from _pydecimal import _dec_from_triple C = import_fresh_module('decimal', fresh=['_decimal']) P = import_fresh_module('decimal', blocked=['_decimal']) @@ -370,7 +371,7 @@ return abs(a - b) def standard_ulp(self, dec, prec): - return P._dec_from_triple(0, '1', dec._exp+len(dec._int)-prec) + return _dec_from_triple(0, '1', dec._exp+len(dec._int)-prec) def rounding_direction(self, x, mode): """Determine the effective direction of the rounding when @@ -401,10 +402,10 @@ # Convert infinities to the largest representable number + 1. x = exact if exact.is_infinite(): - x = P._dec_from_triple(exact._sign, '10', context.p.Emax) + x = _dec_from_triple(exact._sign, '10', context.p.Emax) y = rounded if rounded.is_infinite(): - y = P._dec_from_triple(rounded._sign, '10', context.p.Emax) + y = _dec_from_triple(rounded._sign, '10', context.p.Emax) # err = (rounded - exact) / ulp(rounded) self.maxctx.prec = p * 2 -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 10 18:01:24 2014 From: python-checkins at python.org (stefan.krah) Date: Wed, 10 Sep 2014 18:01:24 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython=3A_Whitespace=2E?= Message-ID: <3htSdc17t3z7LkC@mail.python.org> http://hg.python.org/cpython/rev/81c4225fa994 changeset: 92383:81c4225fa994 user: Stefan Krah date: Wed Sep 10 18:00:46 2014 +0200 summary: Whitespace. files: Lib/decimal.py | 3 --- 1 files changed, 0 insertions(+), 3 deletions(-) diff --git a/Lib/decimal.py b/Lib/decimal.py --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -9,6 +9,3 @@ from _pydecimal import __doc__ from _pydecimal import __version__ from _pydecimal import __libmpdec_version__ - - - -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 10 22:34:26 2014 From: python-checkins at python.org (larry.hastings) Date: Wed, 10 Sep 2014 22:34:26 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?peps=3A_Updated_dates_for_3=2E4=2E2_s?= =?utf-8?q?chedule=2E?= Message-ID: <3htZhf084Mz7Lk7@mail.python.org> http://hg.python.org/peps/rev/370cd63d1c88 changeset: 5551:370cd63d1c88 user: Larry Hastings date: Wed Sep 10 13:34:46 2014 -0700 summary: Updated dates for 3.4.2 schedule. files: pep-0429.txt | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pep-0429.txt b/pep-0429.txt --- a/pep-0429.txt +++ b/pep-0429.txt @@ -59,8 +59,8 @@ 3.4.2 schedule -------------- -- 3.4.2 candidate 1: September 15, 2014 -- 3.4.2 final: September 28, 2014 +- 3.4.2 candidate 1: September 22, 2014 +- 3.4.2 final: October 6, 2014 -- Repository URL: http://hg.python.org/peps From python-checkins at python.org Wed Sep 10 22:59:49 2014 From: python-checkins at python.org (serhiy.storchaka) Date: Wed, 10 Sep 2014 22:59:49 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzIyMzY5?= =?utf-8?q?=3A_Change_=22context_manager_protocol=22_to_=22context_managem?= =?utf-8?q?ent?= Message-ID: <3htbFx6vKCz7Ll3@mail.python.org> http://hg.python.org/cpython/rev/cc5b183a2ad4 changeset: 92384:cc5b183a2ad4 branch: 3.4 parent: 92380:2b3dbbd2bd92 user: Serhiy Storchaka date: Wed Sep 10 23:43:41 2014 +0300 summary: Issue #22369: Change "context manager protocol" to "context management protocol". files: Doc/library/asyncio-sync.rst | 4 ++-- Doc/library/mailbox.rst | 2 +- Doc/library/multiprocessing.rst | 8 ++++---- Doc/library/ossaudiodev.rst | 4 ++-- Doc/library/tarfile.rst | 2 +- Doc/library/threading.rst | 8 ++++---- Doc/whatsnew/2.7.rst | 2 +- Doc/whatsnew/3.1.rst | 2 +- Doc/whatsnew/3.2.rst | 6 +++--- Doc/whatsnew/3.3.rst | 4 ++-- Doc/whatsnew/3.4.rst | 6 +++--- Lib/asyncio/locks.py | 4 ++-- Lib/mailbox.py | 2 +- Misc/HISTORY | 18 +++++++++--------- Misc/NEWS | 6 +++--- Modules/ossaudiodev.c | 4 ++-- 16 files changed, 41 insertions(+), 41 deletions(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -34,7 +34,7 @@ :meth:`acquire` is a coroutine and should be called with ``yield from``. - Locks also support the context manager protocol. ``(yield from lock)`` + Locks also support the context management protocol. ``(yield from lock)`` should be used as context manager expression. Usage:: @@ -229,7 +229,7 @@ counter can never go below zero; when :meth:`acquire` finds that it is zero, it blocks, waiting until some other thread calls :meth:`release`. - Semaphores also support the context manager protocol. + Semaphores also support the context management protocol. The optional argument gives the initial value for the internal counter; it defaults to ``1``. If the value given is less than ``0``, :exc:`ValueError` diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -202,7 +202,7 @@ .. versionchanged:: 3.2 The file object really is a binary file; previously it was incorrectly returned in text mode. Also, the file-like object now supports the - context manager protocol: you can use a :keyword:`with` statement to + context management protocol: you can use a :keyword:`with` statement to automatically close it. .. note:: diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1046,7 +1046,7 @@ using :meth:`Connection.send` and :meth:`Connection.recv`. .. versionadded:: 3.3 - Connection objects now support the context manager protocol -- see + Connection objects now support the context management protocol -- see :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the connection object, and :meth:`~contextmanager.__exit__` calls :meth:`close`. @@ -1503,7 +1503,7 @@ The address used by the manager. .. versionchanged:: 3.3 - Manager objects support the context manager protocol -- see + Manager objects support the context management protocol -- see :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` starts the server process (if it has not already started) and then returns the manager object. :meth:`~contextmanager.__exit__` calls :meth:`shutdown`. @@ -1995,7 +1995,7 @@ :meth:`terminate` before using :meth:`join`. .. versionadded:: 3.3 - Pool objects now support the context manager protocol -- see + Pool objects now support the context management protocol -- see :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the pool object, and :meth:`~contextmanager.__exit__` calls :meth:`terminate`. @@ -2168,7 +2168,7 @@ unavailable then it is ``None``. .. versionadded:: 3.3 - Listener objects now support the context manager protocol -- see + Listener objects now support the context management protocol -- see :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the listener object, and :meth:`~contextmanager.__exit__` calls :meth:`close`. diff --git a/Doc/library/ossaudiodev.rst b/Doc/library/ossaudiodev.rst --- a/Doc/library/ossaudiodev.rst +++ b/Doc/library/ossaudiodev.rst @@ -165,7 +165,7 @@ data written is always equal to the amount of data supplied. .. versionchanged:: 3.2 - Audio device objects also support the context manager protocol, i.e. they can + Audio device objects also support the context management protocol, i.e. they can be used in a :keyword:`with` statement. @@ -357,7 +357,7 @@ Returns the file handle number of the open mixer device file. .. versionchanged:: 3.2 - Mixer objects also support the context manager protocol. + Mixer objects also support the context management protocol. The remaining methods are specific to audio mixing: diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -238,7 +238,7 @@ :ref:`tar-examples` section for a use case. .. versionadded:: 3.2 - Added support for the context manager protocol. + Added support for the context management protocol. .. class:: TarFile(name=None, mode='r', fileobj=None, format=DEFAULT_FORMAT, tarinfo=TarInfo, dereference=False, ignore_zeros=False, encoding=ENCODING, errors='surrogateescape', pax_headers=None, debug=0, errorlevel=0) diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -354,7 +354,7 @@ immediately. If an attempt is made to release an unlocked lock, a :exc:`RuntimeError` will be raised. -Locks also support the :ref:`context manager protocol `. +Locks also support the :ref:`context management protocol `. When more than one thread is blocked in :meth:`~Lock.acquire` waiting for the state to turn to unlocked, only one thread proceeds when a :meth:`~Lock.release` @@ -433,7 +433,7 @@ :meth:`~Lock.release` of the outermost pair) resets the lock to unlocked and allows another thread blocked in :meth:`~Lock.acquire` to proceed. -Reentrant locks also support the :ref:`context manager protocol `. +Reentrant locks also support the :ref:`context management protocol `. .. class:: RLock() @@ -501,7 +501,7 @@ several condition variables must share the same lock. The lock is part of the condition object: you don't have to track it separately. -A condition variable obeys the :ref:`context manager protocol `: +A condition variable obeys the :ref:`context management protocol `: using the ``with`` statement acquires the associated lock for the duration of the enclosed block. The :meth:`~Condition.acquire` and :meth:`~Condition.release` methods also call the corresponding methods of @@ -677,7 +677,7 @@ finds that it is zero, it blocks, waiting until some other thread calls :meth:`~Semaphore.release`. -Semaphores also support the :ref:`context manager protocol `. +Semaphores also support the :ref:`context management protocol `. .. class:: Semaphore(value=1) diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -1612,7 +1612,7 @@ resulting archive. This is more powerful than the existing *exclude* argument, which has therefore been deprecated. (Added by Lars Gust?bel; :issue:`6856`.) - The :class:`~tarfile.TarFile` class also now supports the context manager protocol. + The :class:`~tarfile.TarFile` class also now supports the context management protocol. (Added by Lars Gust?bel; :issue:`7232`.) * The :meth:`~threading.Event.wait` method of the :class:`threading.Event` class diff --git a/Doc/whatsnew/3.1.rst b/Doc/whatsnew/3.1.rst --- a/Doc/whatsnew/3.1.rst +++ b/Doc/whatsnew/3.1.rst @@ -238,7 +238,7 @@ (Contributed by Guilherme Polo; :issue:`2983`.) * The :class:`gzip.GzipFile` and :class:`bz2.BZ2File` classes now support - the context manager protocol:: + the context management protocol:: >>> # Automatically close file after writing >>> with gzip.GzipFile(filename, "wb") as f: diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -522,7 +522,7 @@ (Proposed and implemented by Mark Dickinson; :issue:`9337`.) * :class:`memoryview` objects now have a :meth:`~memoryview.release()` method - and they also now support the context manager protocol. This allows timely + and they also now support the context management protocol. This allows timely release of any resources that were acquired when requesting a buffer from the original object. @@ -1315,7 +1315,7 @@ ftp --- -The :class:`ftplib.FTP` class now supports the context manager protocol to +The :class:`ftplib.FTP` class now supports the context management protocol to unconditionally consume :exc:`socket.error` exceptions and to close the FTP connection when done:: @@ -1595,7 +1595,7 @@ descriptor. The latter can then be reused for other purposes. (Added by Antoine Pitrou; :issue:`8524`.) -* :func:`socket.create_connection` now supports the context manager protocol +* :func:`socket.create_connection` now supports the context management protocol to unconditionally consume :exc:`socket.error` exceptions and to close the socket when done. (Contributed by Giampaolo Rodol?; :issue:`9794`.) diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -1556,7 +1556,7 @@ nntplib ------- -The :class:`nntplib.NNTP` class now supports the context manager protocol to +The :class:`nntplib.NNTP` class now supports the context management protocol to unconditionally consume :exc:`socket.error` exceptions and to close the NNTP connection when done:: @@ -1861,7 +1861,7 @@ when creating the outgoing socket. (Contributed by Paulo Scardine in :issue:`11281`.) -:class:`~smtplib.SMTP` now supports the context manager protocol, allowing an +:class:`~smtplib.SMTP` now supports the context management protocol, allowing an ``SMTP`` instance to be used in a ``with`` statement. (Contributed by Giampaolo Rodol? in :issue:`11289`.) diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -608,7 +608,7 @@ The :meth:`~aifc.aifc.getparams` method now returns a namedtuple rather than a plain tuple. (Contributed by Claudiu Popa in :issue:`17818`.) -:func:`aifc.open` now supports the context manager protocol: when used in a +:func:`aifc.open` now supports the context management protocol: when used in a :keyword:`with` block, the :meth:`~aifc.aifc.close` method of the returned object will be called automatically at the end of the block. (Contributed by Serhiy Storchacha in :issue:`16486`.) @@ -1521,7 +1521,7 @@ The :meth:`~sunau.getparams` method now returns a namedtuple rather than a plain tuple. (Contributed by Claudiu Popa in :issue:`18901`.) -:meth:`sunau.open` now supports the context manager protocol: when used in a +:meth:`sunau.open` now supports the context management protocol: when used in a :keyword:`with` block, the ``close`` method of the returned object will be called automatically at the end of the block. (Contributed by Serhiy Storchaka in :issue:`18878`.) @@ -1723,7 +1723,7 @@ The :meth:`~wave.getparams` method now returns a namedtuple rather than a plain tuple. (Contributed by Claudiu Popa in :issue:`17487`.) -:meth:`wave.open` now supports the context manager protocol. (Contributed +:meth:`wave.open` now supports the context management protocol. (Contributed by Claudiu Popa in :issue:`17616`.) :mod:`wave` can now :ref:`write output to unseekable files diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -63,7 +63,7 @@ acquire() is a coroutine and should be called with 'yield from'. - Locks also support the context manager protocol. '(yield from lock)' + Locks also support the context management protocol. '(yield from lock)' should be used as context manager expression. Usage: @@ -376,7 +376,7 @@ can never go below zero; when acquire() finds that it is zero, it blocks, waiting until some other thread calls release(). - Semaphores also support the context manager protocol. + Semaphores also support the context management protocol. The optional argument gives the initial value for the internal counter; it defaults to 1. If the value given is less than 0, diff --git a/Lib/mailbox.py b/Lib/mailbox.py --- a/Lib/mailbox.py +++ b/Lib/mailbox.py @@ -1980,7 +1980,7 @@ return result def __enter__(self): - """Context manager protocol support.""" + """Context management protocol support.""" return self def __exit__(self, *exc): diff --git a/Misc/HISTORY b/Misc/HISTORY --- a/Misc/HISTORY +++ b/Misc/HISTORY @@ -811,7 +811,7 @@ - Issue #14772: Return destination values from some shutil functions. -- Issue #15064: Implement context manager protocol for multiprocessing types +- Issue #15064: Implement context management protocol for multiprocessing types - Issue #15101: Make pool finalizer avoid joining current thread. @@ -3606,7 +3606,7 @@ ``mmap.PROT_READ|mmap.PROT_EXEC`` would segfault instead of raising a TypeError. Patch by Charles-Fran?ois Natali. -- Issue #9795: add context manager protocol support for nntplib.NNTP class. +- Issue #9795: add context management protocol support for nntplib.NNTP class. - Issue #11306: mailbox in certain cases adapts to an inability to open certain files in read-write mode. Previously it detected this by @@ -4804,7 +4804,7 @@ - Issue #1486713: HTMLParser now has an optional tolerant mode where it tries to guess at the correct parsing of invalid html. -- Issue #10554: Add context manager support to subprocess.Popen objects. +- Issue #10554: Add context management protocol support to subprocess.Popen objects. - Issue #8989: email.utils.make_msgid now has a domain parameter that can override the domain name used in the generated msgid. @@ -5166,7 +5166,7 @@ - Issue #10253: FileIO leaks a file descriptor when trying to open a file for append that isn't seekable. Patch by Brian Brazil. -- Support context manager protocol for file-like objects returned by mailbox +- Support context management protocol for file-like objects returned by mailbox ``get_file()`` methods. - Issue #10246: uu.encode didn't close file objects explicitly when filenames @@ -5300,7 +5300,7 @@ - Issue #10143: Update "os.pathconf" values. -- Issue #6518: Support context manager protcol for ossaudiodev types. +- Issue #6518: Support context management protocol for ossaudiodev types. - Issue #678250: Make mmap flush a noop on ACCESS_READ and ACCESS_COPY. @@ -5909,7 +5909,7 @@ - Issue #8105: Validate file descriptor passed to mmap.mmap on Windows. -- Issue #8046: Add context manager protocol support and .closed property to mmap +- Issue #8046: Add context management protocol support and .closed property to mmap objects. Library @@ -6916,7 +6916,7 @@ - The audioop module now supports sound fragments of length greater than 2**31 bytes on 64-bit machines, and is PY_SSIZE_T_CLEAN. -- Issue #4972: Add support for the context manager protocol to the ftplib.FTP +- Issue #4972: Add support for the context management protocol to the ftplib.FTP class. - Issue #8664: In py_compile, create __pycache__ when the compiled path is @@ -7204,7 +7204,7 @@ - Issue #7494: fix a crash in _lsprof (cProfile) after clearing the profiler, reset also the pointer to the current pointer context. -- Issue #7232: Add support for the context manager protocol to the TarFile +- Issue #7232: Add support for the context management protocol to the TarFile class. - Issue #7250: Fix info leak of os.environ across multi-run uses of @@ -9242,7 +9242,7 @@ - Issue #1696199: Add collections.Counter() for rapid and convenient counting. -- Issue #3860: GzipFile and BZ2File now support the context manager protocol. +- Issue #3860: GzipFile and BZ2File now support the context management protocol. - Issue #4867: Fixed a crash in ctypes when passing a string to a function without defining argtypes. diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -1958,7 +1958,7 @@ - Issue #19448: Add private API to SSL module to lookup ASN.1 objects by OID, NID, short name and long name. -- Issue #19282: dbm.open now supports the context manager protocol. (Inital +- Issue #19282: dbm.open now supports the context management protocol. (Inital patch by Claudiu Popa) - Issue #8311: Added support for writing any bytes-like objects in the aifc, @@ -2667,7 +2667,7 @@ - Issue #18830: inspect.getclasstree() no longer produces duplicate entries even when input list contains duplicates. -- Issue #18878: sunau.open now supports the context manager protocol. Based on +- Issue #18878: sunau.open now supports the context management protocol. Based on patches by Claudiu Popa and R. David Murray. - Issue #18909: Fix _tkinter.tkapp.interpaddr() on Windows 64-bit, don't cast @@ -3307,7 +3307,7 @@ initialization, so as to reclaim allocated resources (Python callbacks) at shutdown. Original patch by Robin Schreiber. -- Issue #17616: wave.open now supports the context manager protocol. +- Issue #17616: wave.open now supports the context management protocol. - Issue #18599: Fix name attribute of _sha1.sha1() object. It now returns 'SHA1' instead of 'SHA'. diff --git a/Modules/ossaudiodev.c b/Modules/ossaudiodev.c --- a/Modules/ossaudiodev.c +++ b/Modules/ossaudiodev.c @@ -894,7 +894,7 @@ /* Aliases for backwards compatibility */ { "flush", (PyCFunction)oss_sync, METH_VARARGS }, - /* Support for the context manager protocol */ + /* Support for the context management protocol */ { "__enter__", oss_self, METH_NOARGS }, { "__exit__", oss_exit, METH_VARARGS }, @@ -906,7 +906,7 @@ { "close", (PyCFunction)oss_mixer_close, METH_NOARGS }, { "fileno", (PyCFunction)oss_mixer_fileno, METH_NOARGS }, - /* Support for the context manager protocol */ + /* Support for the context management protocol */ { "__enter__", oss_self, METH_NOARGS }, { "__exit__", oss_exit, METH_VARARGS }, -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 10 22:59:51 2014 From: python-checkins at python.org (serhiy.storchaka) Date: Wed, 10 Sep 2014 22:59:51 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?q?=29=3A_Issue_=2322369=3A_Change_=22context_manager_protocol=22_?= =?utf-8?q?to_=22context_management?= Message-ID: <3htbFz51hqz7Ll8@mail.python.org> http://hg.python.org/cpython/rev/ad568d52af4b changeset: 92385:ad568d52af4b parent: 92383:81c4225fa994 parent: 92384:cc5b183a2ad4 user: Serhiy Storchaka date: Wed Sep 10 23:45:42 2014 +0300 summary: Issue #22369: Change "context manager protocol" to "context management protocol". files: Doc/library/asyncio-sync.rst | 4 ++-- Doc/library/mailbox.rst | 2 +- Doc/library/multiprocessing.rst | 8 ++++---- Doc/library/ossaudiodev.rst | 4 ++-- Doc/library/tarfile.rst | 2 +- Doc/library/threading.rst | 8 ++++---- Doc/whatsnew/2.7.rst | 2 +- Doc/whatsnew/3.1.rst | 2 +- Doc/whatsnew/3.2.rst | 6 +++--- Doc/whatsnew/3.3.rst | 4 ++-- Doc/whatsnew/3.4.rst | 6 +++--- Lib/asyncio/locks.py | 4 ++-- Lib/mailbox.py | 2 +- Misc/HISTORY | 18 +++++++++--------- Misc/NEWS | 8 ++++---- Modules/ossaudiodev.c | 4 ++-- 16 files changed, 42 insertions(+), 42 deletions(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -34,7 +34,7 @@ :meth:`acquire` is a coroutine and should be called with ``yield from``. - Locks also support the context manager protocol. ``(yield from lock)`` + Locks also support the context management protocol. ``(yield from lock)`` should be used as context manager expression. Usage:: @@ -229,7 +229,7 @@ counter can never go below zero; when :meth:`acquire` finds that it is zero, it blocks, waiting until some other thread calls :meth:`release`. - Semaphores also support the context manager protocol. + Semaphores also support the context management protocol. The optional argument gives the initial value for the internal counter; it defaults to ``1``. If the value given is less than ``0``, :exc:`ValueError` diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -202,7 +202,7 @@ .. versionchanged:: 3.2 The file object really is a binary file; previously it was incorrectly returned in text mode. Also, the file-like object now supports the - context manager protocol: you can use a :keyword:`with` statement to + context management protocol: you can use a :keyword:`with` statement to automatically close it. .. note:: diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1046,7 +1046,7 @@ using :meth:`Connection.send` and :meth:`Connection.recv`. .. versionadded:: 3.3 - Connection objects now support the context manager protocol -- see + Connection objects now support the context management protocol -- see :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the connection object, and :meth:`~contextmanager.__exit__` calls :meth:`close`. @@ -1506,7 +1506,7 @@ The address used by the manager. .. versionchanged:: 3.3 - Manager objects support the context manager protocol -- see + Manager objects support the context management protocol -- see :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` starts the server process (if it has not already started) and then returns the manager object. :meth:`~contextmanager.__exit__` calls :meth:`shutdown`. @@ -1998,7 +1998,7 @@ :meth:`terminate` before using :meth:`join`. .. versionadded:: 3.3 - Pool objects now support the context manager protocol -- see + Pool objects now support the context management protocol -- see :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the pool object, and :meth:`~contextmanager.__exit__` calls :meth:`terminate`. @@ -2171,7 +2171,7 @@ unavailable then it is ``None``. .. versionadded:: 3.3 - Listener objects now support the context manager protocol -- see + Listener objects now support the context management protocol -- see :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the listener object, and :meth:`~contextmanager.__exit__` calls :meth:`close`. diff --git a/Doc/library/ossaudiodev.rst b/Doc/library/ossaudiodev.rst --- a/Doc/library/ossaudiodev.rst +++ b/Doc/library/ossaudiodev.rst @@ -165,7 +165,7 @@ data written is always equal to the amount of data supplied. .. versionchanged:: 3.2 - Audio device objects also support the context manager protocol, i.e. they can + Audio device objects also support the context management protocol, i.e. they can be used in a :keyword:`with` statement. @@ -357,7 +357,7 @@ Returns the file handle number of the open mixer device file. .. versionchanged:: 3.2 - Mixer objects also support the context manager protocol. + Mixer objects also support the context management protocol. The remaining methods are specific to audio mixing: diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -238,7 +238,7 @@ :ref:`tar-examples` section for a use case. .. versionadded:: 3.2 - Added support for the context manager protocol. + Added support for the context management protocol. .. class:: TarFile(name=None, mode='r', fileobj=None, format=DEFAULT_FORMAT, tarinfo=TarInfo, dereference=False, ignore_zeros=False, encoding=ENCODING, errors='surrogateescape', pax_headers=None, debug=0, errorlevel=0) diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -354,7 +354,7 @@ immediately. If an attempt is made to release an unlocked lock, a :exc:`RuntimeError` will be raised. -Locks also support the :ref:`context manager protocol `. +Locks also support the :ref:`context management protocol `. When more than one thread is blocked in :meth:`~Lock.acquire` waiting for the state to turn to unlocked, only one thread proceeds when a :meth:`~Lock.release` @@ -433,7 +433,7 @@ :meth:`~Lock.release` of the outermost pair) resets the lock to unlocked and allows another thread blocked in :meth:`~Lock.acquire` to proceed. -Reentrant locks also support the :ref:`context manager protocol `. +Reentrant locks also support the :ref:`context management protocol `. .. class:: RLock() @@ -501,7 +501,7 @@ several condition variables must share the same lock. The lock is part of the condition object: you don't have to track it separately. -A condition variable obeys the :ref:`context manager protocol `: +A condition variable obeys the :ref:`context management protocol `: using the ``with`` statement acquires the associated lock for the duration of the enclosed block. The :meth:`~Condition.acquire` and :meth:`~Condition.release` methods also call the corresponding methods of @@ -677,7 +677,7 @@ finds that it is zero, it blocks, waiting until some other thread calls :meth:`~Semaphore.release`. -Semaphores also support the :ref:`context manager protocol `. +Semaphores also support the :ref:`context management protocol `. .. class:: Semaphore(value=1) diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -1612,7 +1612,7 @@ resulting archive. This is more powerful than the existing *exclude* argument, which has therefore been deprecated. (Added by Lars Gust?bel; :issue:`6856`.) - The :class:`~tarfile.TarFile` class also now supports the context manager protocol. + The :class:`~tarfile.TarFile` class also now supports the context management protocol. (Added by Lars Gust?bel; :issue:`7232`.) * The :meth:`~threading.Event.wait` method of the :class:`threading.Event` class diff --git a/Doc/whatsnew/3.1.rst b/Doc/whatsnew/3.1.rst --- a/Doc/whatsnew/3.1.rst +++ b/Doc/whatsnew/3.1.rst @@ -238,7 +238,7 @@ (Contributed by Guilherme Polo; :issue:`2983`.) * The :class:`gzip.GzipFile` and :class:`bz2.BZ2File` classes now support - the context manager protocol:: + the context management protocol:: >>> # Automatically close file after writing >>> with gzip.GzipFile(filename, "wb") as f: diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -522,7 +522,7 @@ (Proposed and implemented by Mark Dickinson; :issue:`9337`.) * :class:`memoryview` objects now have a :meth:`~memoryview.release()` method - and they also now support the context manager protocol. This allows timely + and they also now support the context management protocol. This allows timely release of any resources that were acquired when requesting a buffer from the original object. @@ -1315,7 +1315,7 @@ ftp --- -The :class:`ftplib.FTP` class now supports the context manager protocol to +The :class:`ftplib.FTP` class now supports the context management protocol to unconditionally consume :exc:`socket.error` exceptions and to close the FTP connection when done:: @@ -1595,7 +1595,7 @@ descriptor. The latter can then be reused for other purposes. (Added by Antoine Pitrou; :issue:`8524`.) -* :func:`socket.create_connection` now supports the context manager protocol +* :func:`socket.create_connection` now supports the context management protocol to unconditionally consume :exc:`socket.error` exceptions and to close the socket when done. (Contributed by Giampaolo Rodol?; :issue:`9794`.) diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -1556,7 +1556,7 @@ nntplib ------- -The :class:`nntplib.NNTP` class now supports the context manager protocol to +The :class:`nntplib.NNTP` class now supports the context management protocol to unconditionally consume :exc:`socket.error` exceptions and to close the NNTP connection when done:: @@ -1861,7 +1861,7 @@ when creating the outgoing socket. (Contributed by Paulo Scardine in :issue:`11281`.) -:class:`~smtplib.SMTP` now supports the context manager protocol, allowing an +:class:`~smtplib.SMTP` now supports the context management protocol, allowing an ``SMTP`` instance to be used in a ``with`` statement. (Contributed by Giampaolo Rodol? in :issue:`11289`.) diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -608,7 +608,7 @@ The :meth:`~aifc.aifc.getparams` method now returns a namedtuple rather than a plain tuple. (Contributed by Claudiu Popa in :issue:`17818`.) -:func:`aifc.open` now supports the context manager protocol: when used in a +:func:`aifc.open` now supports the context management protocol: when used in a :keyword:`with` block, the :meth:`~aifc.aifc.close` method of the returned object will be called automatically at the end of the block. (Contributed by Serhiy Storchacha in :issue:`16486`.) @@ -1521,7 +1521,7 @@ The :meth:`~sunau.getparams` method now returns a namedtuple rather than a plain tuple. (Contributed by Claudiu Popa in :issue:`18901`.) -:meth:`sunau.open` now supports the context manager protocol: when used in a +:meth:`sunau.open` now supports the context management protocol: when used in a :keyword:`with` block, the ``close`` method of the returned object will be called automatically at the end of the block. (Contributed by Serhiy Storchaka in :issue:`18878`.) @@ -1723,7 +1723,7 @@ The :meth:`~wave.getparams` method now returns a namedtuple rather than a plain tuple. (Contributed by Claudiu Popa in :issue:`17487`.) -:meth:`wave.open` now supports the context manager protocol. (Contributed +:meth:`wave.open` now supports the context management protocol. (Contributed by Claudiu Popa in :issue:`17616`.) :mod:`wave` can now :ref:`write output to unseekable files diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -63,7 +63,7 @@ acquire() is a coroutine and should be called with 'yield from'. - Locks also support the context manager protocol. '(yield from lock)' + Locks also support the context management protocol. '(yield from lock)' should be used as context manager expression. Usage: @@ -376,7 +376,7 @@ can never go below zero; when acquire() finds that it is zero, it blocks, waiting until some other thread calls release(). - Semaphores also support the context manager protocol. + Semaphores also support the context management protocol. The optional argument gives the initial value for the internal counter; it defaults to 1. If the value given is less than 0, diff --git a/Lib/mailbox.py b/Lib/mailbox.py --- a/Lib/mailbox.py +++ b/Lib/mailbox.py @@ -1980,7 +1980,7 @@ return result def __enter__(self): - """Context manager protocol support.""" + """Context management protocol support.""" return self def __exit__(self, *exc): diff --git a/Misc/HISTORY b/Misc/HISTORY --- a/Misc/HISTORY +++ b/Misc/HISTORY @@ -806,7 +806,7 @@ - Issue #14772: Return destination values from some shutil functions. -- Issue #15064: Implement context manager protocol for multiprocessing types +- Issue #15064: Implement context management protocol for multiprocessing types - Issue #15101: Make pool finalizer avoid joining current thread. @@ -3527,7 +3527,7 @@ ``mmap.PROT_READ|mmap.PROT_EXEC`` would segfault instead of raising a TypeError. Patch by Charles-Fran?ois Natali. -- Issue #9795: add context manager protocol support for nntplib.NNTP class. +- Issue #9795: add context management protocol support for nntplib.NNTP class. - Issue #11306: mailbox in certain cases adapts to an inability to open certain files in read-write mode. Previously it detected this by @@ -4837,7 +4837,7 @@ - Issue #1486713: HTMLParser now has an optional tolerant mode where it tries to guess at the correct parsing of invalid html. -- Issue #10554: Add context manager support to subprocess.Popen objects. +- Issue #10554: Add context management protocol support to subprocess.Popen objects. - Issue #8989: email.utils.make_msgid now has a domain parameter that can override the domain name used in the generated msgid. @@ -5199,7 +5199,7 @@ - Issue #10253: FileIO leaks a file descriptor when trying to open a file for append that isn't seekable. Patch by Brian Brazil. -- Support context manager protocol for file-like objects returned by mailbox +- Support context management protocol for file-like objects returned by mailbox ``get_file()`` methods. - Issue #10246: uu.encode didn't close file objects explicitly when filenames @@ -5333,7 +5333,7 @@ - Issue #10143: Update "os.pathconf" values. -- Issue #6518: Support context manager protcol for ossaudiodev types. +- Issue #6518: Support context management protocol for ossaudiodev types. - Issue #678250: Make mmap flush a noop on ACCESS_READ and ACCESS_COPY. @@ -5942,7 +5942,7 @@ - Issue #8105: Validate file descriptor passed to mmap.mmap on Windows. -- Issue #8046: Add context manager protocol support and .closed property to mmap +- Issue #8046: Add context management protocol support and .closed property to mmap objects. Library @@ -6949,7 +6949,7 @@ - The audioop module now supports sound fragments of length greater than 2**31 bytes on 64-bit machines, and is PY_SSIZE_T_CLEAN. -- Issue #4972: Add support for the context manager protocol to the ftplib.FTP +- Issue #4972: Add support for the context management protocol to the ftplib.FTP class. - Issue #8664: In py_compile, create __pycache__ when the compiled path is @@ -7237,7 +7237,7 @@ - Issue #7494: fix a crash in _lsprof (cProfile) after clearing the profiler, reset also the pointer to the current pointer context. -- Issue #7232: Add support for the context manager protocol to the TarFile +- Issue #7232: Add support for the context management protocol to the TarFile class. - Issue #7250: Fix info leak of os.environ across multi-run uses of @@ -9275,7 +9275,7 @@ - Issue #1696199: Add collections.Counter() for rapid and convenient counting. -- Issue #3860: GzipFile and BZ2File now support the context manager protocol. +- Issue #3860: GzipFile and BZ2File now support the context management protocol. - Issue #4867: Fixed a crash in ctypes when passing a string to a function without defining argtypes. diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -132,7 +132,7 @@ Library ------- -- Issue #12410: imaplib.IMAP4 now supports the context manager protocol. +- Issue #12410: imaplib.IMAP4 now supports the context management protocol. Original patch by Tarek Ziad?. - Issue #16662: load_tests() is now unconditionally run when it is present in @@ -2286,7 +2286,7 @@ - Issue #19448: Add private API to SSL module to lookup ASN.1 objects by OID, NID, short name and long name. -- Issue #19282: dbm.open now supports the context manager protocol. (Inital +- Issue #19282: dbm.open now supports the context management protocol. (Inital patch by Claudiu Popa) - Issue #8311: Added support for writing any bytes-like objects in the aifc, @@ -2995,7 +2995,7 @@ - Issue #18830: inspect.getclasstree() no longer produces duplicate entries even when input list contains duplicates. -- Issue #18878: sunau.open now supports the context manager protocol. Based on +- Issue #18878: sunau.open now supports the context management protocol. Based on patches by Claudiu Popa and R. David Murray. - Issue #18909: Fix _tkinter.tkapp.interpaddr() on Windows 64-bit, don't cast @@ -3635,7 +3635,7 @@ initialization, so as to reclaim allocated resources (Python callbacks) at shutdown. Original patch by Robin Schreiber. -- Issue #17616: wave.open now supports the context manager protocol. +- Issue #17616: wave.open now supports the context management protocol. - Issue #18599: Fix name attribute of _sha1.sha1() object. It now returns 'SHA1' instead of 'SHA'. diff --git a/Modules/ossaudiodev.c b/Modules/ossaudiodev.c --- a/Modules/ossaudiodev.c +++ b/Modules/ossaudiodev.c @@ -894,7 +894,7 @@ /* Aliases for backwards compatibility */ { "flush", (PyCFunction)oss_sync, METH_VARARGS }, - /* Support for the context manager protocol */ + /* Support for the context management protocol */ { "__enter__", oss_self, METH_NOARGS }, { "__exit__", oss_exit, METH_VARARGS }, @@ -906,7 +906,7 @@ { "close", (PyCFunction)oss_mixer_close, METH_NOARGS }, { "fileno", (PyCFunction)oss_mixer_fileno, METH_NOARGS }, - /* Support for the context manager protocol */ + /* Support for the context management protocol */ { "__enter__", oss_self, METH_NOARGS }, { "__exit__", oss_exit, METH_VARARGS }, -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 10 22:59:53 2014 From: python-checkins at python.org (serhiy.storchaka) Date: Wed, 10 Sep 2014 22:59:53 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogSXNzdWUgIzIyMzY5?= =?utf-8?q?=3A_Change_=22context_manager_protocol=22_to_=22context_managem?= =?utf-8?q?ent?= Message-ID: <3htbG106tvz7Lkk@mail.python.org> http://hg.python.org/cpython/rev/e62082a70b6b changeset: 92386:e62082a70b6b branch: 2.7 parent: 92372:7b0fdc1e917a user: Serhiy Storchaka date: Wed Sep 10 23:46:14 2014 +0300 summary: Issue #22369: Change "context manager protocol" to "context management protocol". files: Doc/library/tarfile.rst | 2 +- Doc/whatsnew/2.7.rst | 2 +- Misc/NEWS | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -247,7 +247,7 @@ :ref:`tar-examples` section for a use case. .. versionadded:: 2.7 - Added support for the context manager protocol. + Added support for the context management protocol. .. class:: TarFile(name=None, mode='r', fileobj=None, format=DEFAULT_FORMAT, tarinfo=TarInfo, dereference=False, ignore_zeros=False, encoding=ENCODING, errors=None, pax_headers=None, debug=0, errorlevel=0) diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -1612,7 +1612,7 @@ resulting archive. This is more powerful than the existing *exclude* argument, which has therefore been deprecated. (Added by Lars Gust?bel; :issue:`6856`.) - The :class:`~tarfile.TarFile` class also now supports the context manager protocol. + The :class:`~tarfile.TarFile` class also now supports the context management protocol. (Added by Lars Gust?bel; :issue:`7232`.) * The :meth:`~threading.Event.wait` method of the :class:`threading.Event` class diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -5120,7 +5120,7 @@ - Issue #7494: Fix a crash in ``_lsprof`` (cProfile) after clearing the profiler, reset also the pointer to the current pointer context. -- Issue #7232: Add support for the context manager protocol to the +- Issue #7232: Add support for the context management protocol to the ``tarfile.TarFile`` class. - Issue #7250: Fix info leak of os.environ across multi-run uses of @@ -6870,7 +6870,7 @@ - Issue #1696199: Add collections.Counter() for rapid and convenient counting. -- Issue #3860: GzipFile and BZ2File now support the context manager protocol. +- Issue #3860: GzipFile and BZ2File now support the context management protocol. - Issue #4272: Add an optional argument to the GzipFile constructor to override the timestamp in the gzip stream. The default value remains the current time. @@ -10039,7 +10039,7 @@ parameter. - _winreg's HKEY object has gained __enter__ and __exit__ methods to - support the context manager protocol. The _winreg module also + support the context management protocol. The _winreg module also gained a new function ``ExpandEnvironmentStrings`` to expand REG_EXPAND_SZ keys. -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 10 23:13:51 2014 From: python-checkins at python.org (benjamin.peterson) Date: Wed, 10 Sep 2014 23:13:51 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=283=2E4=29=3A_y2k_compliance?= =?utf-8?b?LCBsb2wgKGNsb3NlcyAjMjIzODAp?= Message-ID: <3htbZ70NC0z7Lk5@mail.python.org> http://hg.python.org/cpython/rev/071a2620917f changeset: 92387:071a2620917f branch: 3.4 parent: 92384:cc5b183a2ad4 user: Benjamin Peterson date: Wed Sep 10 17:13:06 2014 -0400 summary: y2k compliance, lol (closes #22380) files: Doc/faq/general.rst | 37 --------------------------------- 1 files changed, 0 insertions(+), 37 deletions(-) diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -377,43 +377,6 @@ changes while minimizing disruption for users. -Is Python Y2K (Year 2000) Compliant? ------------------------------------- - -.. remove this question? - -As of August, 2003 no major problems have been reported and Y2K compliance seems -to be a non-issue. - -Python does very few date calculations and for those it does perform relies on -the C library functions. Python generally represents times either as seconds -since 1970 or as a ``(year, month, day, ...)`` tuple where the year is expressed -with four digits, which makes Y2K bugs unlikely. So as long as your C library -is okay, Python should be okay. Of course, it's possible that a particular -application written in Python makes assumptions about 2-digit years. - -Because Python is available free of charge, there are no absolute guarantees. -If there *are* unforeseen problems, liability is the user's problem rather than -the developers', and there is nobody you can sue for damages. The Python -copyright notice contains the following disclaimer: - - 4. PSF is making Python 2.3 available to Licensee on an "AS IS" - basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY - WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY - REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR - PURPOSE OR THAT THE USE OF PYTHON 2.3 WILL NOT INFRINGE ANY THIRD PARTY - RIGHTS. - - 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON - 2.3 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS - A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 2.3, - OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -The good news is that *if* you encounter a problem, you have full source -available to track it down and fix it. This is one advantage of an open source -programming environment. - - Is Python a good language for beginning programmers? ---------------------------------------------------- -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 10 23:13:52 2014 From: python-checkins at python.org (benjamin.peterson) Date: Wed, 10 Sep 2014 23:13:52 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=282=2E7=29=3A_y2k_compliance?= =?utf-8?b?LCBsb2wgKGNsb3NlcyAjMjIzODAp?= Message-ID: <3htbZ82NFLz7Lkq@mail.python.org> http://hg.python.org/cpython/rev/02c94b9451f8 changeset: 92388:02c94b9451f8 branch: 2.7 parent: 92386:e62082a70b6b user: Benjamin Peterson date: Wed Sep 10 17:13:06 2014 -0400 summary: y2k compliance, lol (closes #22380) files: Doc/faq/general.rst | 37 --------------------------------- 1 files changed, 0 insertions(+), 37 deletions(-) diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -377,43 +377,6 @@ changes while minimizing disruption for users. -Is Python Y2K (Year 2000) Compliant? ------------------------------------- - -.. remove this question? - -As of August, 2003 no major problems have been reported and Y2K compliance seems -to be a non-issue. - -Python does very few date calculations and for those it does perform relies on -the C library functions. Python generally represents times either as seconds -since 1970 or as a ``(year, month, day, ...)`` tuple where the year is expressed -with four digits, which makes Y2K bugs unlikely. So as long as your C library -is okay, Python should be okay. Of course, it's possible that a particular -application written in Python makes assumptions about 2-digit years. - -Because Python is available free of charge, there are no absolute guarantees. -If there *are* unforeseen problems, liability is the user's problem rather than -the developers', and there is nobody you can sue for damages. The Python -copyright notice contains the following disclaimer: - - 4. PSF is making Python 2.3 available to Licensee on an "AS IS" - basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY - WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY - REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR - PURPOSE OR THAT THE USE OF PYTHON 2.3 WILL NOT INFRINGE ANY THIRD PARTY - RIGHTS. - - 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON - 2.3 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS - A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 2.3, - OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -The good news is that *if* you encounter a problem, you have full source -available to track it down and fix it. This is one advantage of an open source -programming environment. - - Is Python a good language for beginning programmers? ---------------------------------------------------- -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 10 23:13:53 2014 From: python-checkins at python.org (benjamin.peterson) Date: Wed, 10 Sep 2014 23:13:53 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?b?KTogbWVyZ2UgMy40ICgjMjIzODAp?= Message-ID: <3htbZ94DMSz7Lkn@mail.python.org> http://hg.python.org/cpython/rev/e3a3c8809d09 changeset: 92389:e3a3c8809d09 parent: 92385:ad568d52af4b parent: 92387:071a2620917f user: Benjamin Peterson date: Wed Sep 10 17:13:36 2014 -0400 summary: merge 3.4 (#22380) files: Doc/faq/general.rst | 37 --------------------------------- 1 files changed, 0 insertions(+), 37 deletions(-) diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -377,43 +377,6 @@ changes while minimizing disruption for users. -Is Python Y2K (Year 2000) Compliant? ------------------------------------- - -.. remove this question? - -As of August, 2003 no major problems have been reported and Y2K compliance seems -to be a non-issue. - -Python does very few date calculations and for those it does perform relies on -the C library functions. Python generally represents times either as seconds -since 1970 or as a ``(year, month, day, ...)`` tuple where the year is expressed -with four digits, which makes Y2K bugs unlikely. So as long as your C library -is okay, Python should be okay. Of course, it's possible that a particular -application written in Python makes assumptions about 2-digit years. - -Because Python is available free of charge, there are no absolute guarantees. -If there *are* unforeseen problems, liability is the user's problem rather than -the developers', and there is nobody you can sue for damages. The Python -copyright notice contains the following disclaimer: - - 4. PSF is making Python 2.3 available to Licensee on an "AS IS" - basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY - WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY - REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR - PURPOSE OR THAT THE USE OF PYTHON 2.3 WILL NOT INFRINGE ANY THIRD PARTY - RIGHTS. - - 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON - 2.3 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS - A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 2.3, - OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -The good news is that *if* you encounter a problem, you have full source -available to track it down and fix it. This is one advantage of an open source -programming environment. - - Is Python a good language for beginning programmers? ---------------------------------------------------- -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 10 23:34:54 2014 From: python-checkins at python.org (victor.stinner) Date: Wed, 10 Sep 2014 23:34:54 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMy40KTogSXNzdWUgIzIyMzM4?= =?utf-8?q?=3A_Fix_a_crash_in_the_json_module_on_memory_allocation_failure?= =?utf-8?q?=2E?= Message-ID: <3htc2Q3JChz7Lk6@mail.python.org> http://hg.python.org/cpython/rev/3ac9f9576ce6 changeset: 92390:3ac9f9576ce6 branch: 3.4 parent: 92387:071a2620917f user: Victor Stinner date: Wed Sep 10 23:31:42 2014 +0200 summary: Issue #22338: Fix a crash in the json module on memory allocation failure. files: Misc/NEWS | 2 ++ Modules/_json.c | 3 ++- 2 files changed, 4 insertions(+), 1 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -32,6 +32,8 @@ Library ------- +- Issue #22338: Fix a crash in the json module on memory allocation failure. + - Issue #22226: First letter no longer is stripped from the "status" key in the result of Treeview.heading(). diff --git a/Modules/_json.c b/Modules/_json.c --- a/Modules/_json.c +++ b/Modules/_json.c @@ -287,7 +287,7 @@ } \ } \ if (PyList_Append(chunks, chunk)) { \ - Py_DECREF(chunk); \ + Py_CLEAR(chunk); \ goto bail; \ } \ Py_CLEAR(chunk); \ @@ -1555,6 +1555,7 @@ if (item == NULL) goto bail; PyList_SET_ITEM(items, i, item); + item = NULL; Py_DECREF(key); } } -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Wed Sep 10 23:34:55 2014 From: python-checkins at python.org (victor.stinner) Date: Wed, 10 Sep 2014 23:34:55 +0200 (CEST) Subject: [Python-checkins] =?utf-8?q?cpython_=28merge_3=2E4_-=3E_default?= =?utf-8?q?=29=3A_=28Merge_3=2E4=29_Issue_=2322338=3A_Fix_a_crash_in_the_j?= =?utf-8?q?son_module_on_memory_allocation?= Message-ID: <3htc2R5Tkzz7Lkq@mail.python.org> http://hg.python.org/cpython/rev/135fc23e475c changeset: 92391:135fc23e475c parent: 92389:e3a3c8809d09 parent: 92390:3ac9f9576ce6 user: Victor Stinner date: Wed Sep 10 23:32:36 2014 +0200 summary: (Merge 3.4) Issue #22338: Fix a crash in the json module on memory allocation failure. files: Misc/NEWS | 2 ++ Modules/_json.c | 3 ++- 2 files changed, 4 insertions(+), 1 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -132,6 +132,8 @@ Library ------- +- Issue #22338: Fix a crash in the json module on memory allocation failure. + - Issue #12410: imaplib.IMAP4 now supports the context management protocol. Original patch by Tarek Ziad?. diff --git a/Modules/_json.c b/Modules/_json.c --- a/Modules/_json.c +++ b/Modules/_json.c @@ -287,7 +287,7 @@ } \ } \ if (PyList_Append(chunks, chunk)) { \ - Py_DECREF(chunk); \ + Py_CLEAR(chunk); \ goto bail; \ } \ Py_CLEAR(chunk); \ @@ -1555,6 +1555,7 @@ if (item == NULL) goto bail; PyList_SET_ITEM(items, i, item); + item = NULL; Py_DECREF(key); } } -- Repository URL: http://hg.python.org/cpython From python-checkins at python.org Thu Sep 11 08:33:15 2014 From: python-checkins at python.org (matthias.klose) Date: Thu, 11 Sep 2014 08:33:15 +0200 (CEST) Subject: [Python-checkins] =?utf-8?b?Y3B5dGhvbiAoMi43KTogLSBJc3N1ZSAjMjIz?= =?utf-8?q?81=3A_Update_zlib_to_1=2E2=2E8=2E?= Message-ID: <3htqzb1gdmz7LkK@mail.python.org> http://hg.python.org/cpython/rev/3c343588f6a6 changeset: 92392:3c343588f6a6 branch: 2.7 parent: 92388:02c94b9451f8 user: doko at ubuntu.com date: Thu Sep 11 08:32:46 2014 +0200 summary: - Issue #22381: Update zlib to 1.2.8. files: Misc/NEWS | 2 + Modules/zlib/ChangeLog | 619 ++++++++++- Modules/zlib/FAQ | 267 ++- Modules/zlib/INDEX | 41 +- Modules/zlib/Makefile | 157 +-- Modules/zlib/Makefile.in | 260 +++- Modules/zlib/README | 94 +- Modules/zlib/adler32.c | 120 +- Modules/zlib/algorithm.txt | 4 +- Modules/zlib/compress.c | 7 +- Modules/zlib/configure | 786 +++++++++--- Modules/zlib/crc32.c | 112 +- Modules/zlib/crc32.h | 2 +- Modules/zlib/deflate.c | 521 ++++++-- Modules/zlib/deflate.h | 45 +- Modules/zlib/example.c | 92 +- Modules/zlib/gzclose.c | 25 + Modules/zlib/gzguts.h | 209 +++ Modules/zlib/gzio.c | 1026 ---------------- Modules/zlib/gzlib.c | 634 ++++++++++ Modules/zlib/gzread.c | 594 +++++++++ Modules/zlib/gzwrite.c | 577 +++++++++ Modules/zlib/infback.c | 105 +- Modules/zlib/inffast.c | 84 +- Modules/zlib/inffast.h | 4 +- Modules/zlib/inffixed.h | 6 +- Modules/zlib/inflate.c | 482 +++++-- Modules/zlib/inflate.h | 31 +- Modules/zlib/inftrees.c | 93 +- Modules/zlib/inftrees.h | 27 +- Modules/zlib/make_vms.com | 750 +++++++++-- Modules/zlib/minigzip.c | 345 +++++- Modules/zlib/trees.c | 145 +- Modules/zlib/trees.h | 4 +- Modules/zlib/uncompr.c | 6 +- Modules/zlib/zconf.h | 297 +++- Modules/zlib/zconf.h.cmakein | 513 ++++++++ Modules/zlib/zconf.h.in | 511 ++++++++ Modules/zlib/zconf.in.h | 332 ----- Modules/zlib/zlib.3 | 70 +- Modules/zlib/zlib.h | 1359 ++++++++++++++------- Modules/zlib/zlib.map | 83 + Modules/zlib/zlib.pc.in | 13 + Modules/zlib/zutil.c | 56 +- Modules/zlib/zutil.h | 142 +- 45 files changed, 8133 insertions(+), 3519 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -142,6 +142,8 @@ Extension Modules ----------------- +- Issue #22381: Update zlib to 1.2.8. + - Issue #22176: Update the ctypes module's libffi to v3.1. This release adds support for the Linux AArch64 and POWERPC ELF ABIv2 little endian architectures. diff --git a/Modules/zlib/ChangeLog b/Modules/zlib/ChangeLog --- a/Modules/zlib/ChangeLog +++ b/Modules/zlib/ChangeLog @@ -1,6 +1,623 @@ ChangeLog file for zlib +Changes in 1.2.8 (28 Apr 2013) +- Update contrib/minizip/iowin32.c for Windows RT [Vollant] +- Do not force Z_CONST for C++ +- Clean up contrib/vstudio [Ro?] +- Correct spelling error in zlib.h +- Fix mixed line endings in contrib/vstudio + +Changes in 1.2.7.3 (13 Apr 2013) +- Fix version numbers and DLL names in contrib/vstudio/*/zlib.rc + +Changes in 1.2.7.2 (13 Apr 2013) +- Change check for a four-byte type back to hexadecimal +- Fix typo in win32/Makefile.msc +- Add casts in gzwrite.c for pointer differences + +Changes in 1.2.7.1 (24 Mar 2013) +- Replace use of unsafe string functions with snprintf if available +- Avoid including stddef.h on Windows for Z_SOLO compile [Niessink] +- Fix gzgetc undefine when Z_PREFIX set [Turk] +- Eliminate use of mktemp in Makefile (not always available) +- Fix bug in 'F' mode for gzopen() +- Add inflateGetDictionary() function +- Correct comment in deflate.h +- Use _snprintf for snprintf in Microsoft C +- On Darwin, only use /usr/bin/libtool if libtool is not Apple +- Delete "--version" file if created by "ar --version" [Richard G.] +- Fix configure check for veracity of compiler error return codes +- Fix CMake compilation of static lib for MSVC2010 x64 +- Remove unused variable in infback9.c +- Fix argument checks in gzlog_compress() and gzlog_write() +- Clean up the usage of z_const and respect const usage within zlib +- Clean up examples/gzlog.[ch] comparisons of different types +- Avoid shift equal to bits in type (caused endless loop) +- Fix unintialized value bug in gzputc() introduced by const patches +- Fix memory allocation error in examples/zran.c [Nor] +- Fix bug where gzopen(), gzclose() would write an empty file +- Fix bug in gzclose() when gzwrite() runs out of memory +- Check for input buffer malloc failure in examples/gzappend.c +- Add note to contrib/blast to use binary mode in stdio +- Fix comparisons of differently signed integers in contrib/blast +- Check for invalid code length codes in contrib/puff +- Fix serious but very rare decompression bug in inftrees.c +- Update inflateBack() comments, since inflate() can be faster +- Use underscored I/O function names for WINAPI_FAMILY +- Add _tr_flush_bits to the external symbols prefixed by --zprefix +- Add contrib/vstudio/vc10 pre-build step for static only +- Quote --version-script argument in CMakeLists.txt +- Don't specify --version-script on Apple platforms in CMakeLists.txt +- Fix casting error in contrib/testzlib/testzlib.c +- Fix types in contrib/minizip to match result of get_crc_table() +- Simplify contrib/vstudio/vc10 with 'd' suffix +- Add TOP support to win32/Makefile.msc +- Suport i686 and amd64 assembler builds in CMakeLists.txt +- Fix typos in the use of _LARGEFILE64_SOURCE in zconf.h +- Add vc11 and vc12 build files to contrib/vstudio +- Add gzvprintf() as an undocumented function in zlib +- Fix configure for Sun shell +- Remove runtime check in configure for four-byte integer type +- Add casts and consts to ease user conversion to C++ +- Add man pages for minizip and miniunzip +- In Makefile uninstall, don't rm if preceding cd fails +- Do not return Z_BUF_ERROR if deflateParam() has nothing to write + +Changes in 1.2.7 (2 May 2012) +- Replace use of memmove() with a simple copy for portability +- Test for existence of strerror +- Restore gzgetc_ for backward compatibility with 1.2.6 +- Fix build with non-GNU make on Solaris +- Require gcc 4.0 or later on Mac OS X to use the hidden attribute +- Include unistd.h for Watcom C +- Use __WATCOMC__ instead of __WATCOM__ +- Do not use the visibility attribute if NO_VIZ defined +- Improve the detection of no hidden visibility attribute +- Avoid using __int64 for gcc or solo compilation +- Cast to char * in gzprintf to avoid warnings [Zinser] +- Fix make_vms.com for VAX [Zinser] +- Don't use library or built-in byte swaps +- Simplify test and use of gcc hidden attribute +- Fix bug in gzclose_w() when gzwrite() fails to allocate memory +- Add "x" (O_EXCL) and "e" (O_CLOEXEC) modes support to gzopen() +- Fix bug in test/minigzip.c for configure --solo +- Fix contrib/vstudio project link errors [Mohanathas] +- Add ability to choose the builder in make_vms.com [Schweda] +- Add DESTDIR support to mingw32 win32/Makefile.gcc +- Fix comments in win32/Makefile.gcc for proper usage +- Allow overriding the default install locations for cmake +- Generate and install the pkg-config file with cmake +- Build both a static and a shared version of zlib with cmake +- Include version symbols for cmake builds +- If using cmake with MSVC, add the source directory to the includes +- Remove unneeded EXTRA_CFLAGS from win32/Makefile.gcc [Truta] +- Move obsolete emx makefile to old [Truta] +- Allow the use of -Wundef when compiling or using zlib +- Avoid the use of the -u option with mktemp +- Improve inflate() documentation on the use of Z_FINISH +- Recognize clang as gcc +- Add gzopen_w() in Windows for wide character path names +- Rename zconf.h in CMakeLists.txt to move it out of the way +- Add source directory in CMakeLists.txt for building examples +- Look in build directory for zlib.pc in CMakeLists.txt +- Remove gzflags from zlibvc.def in vc9 and vc10 +- Fix contrib/minizip compilation in the MinGW environment +- Update ./configure for Solaris, support --64 [Mooney] +- Remove -R. from Solaris shared build (possible security issue) +- Avoid race condition for parallel make (-j) running example +- Fix type mismatch between get_crc_table() and crc_table +- Fix parsing of version with "-" in CMakeLists.txt [Snider, Ziegler] +- Fix the path to zlib.map in CMakeLists.txt +- Force the native libtool in Mac OS X to avoid GNU libtool [Beebe] +- Add instructions to win32/Makefile.gcc for shared install [Torri] + +Changes in 1.2.6.1 (12 Feb 2012) +- Avoid the use of the Objective-C reserved name "id" +- Include io.h in gzguts.h for Microsoft compilers +- Fix problem with ./configure --prefix and gzgetc macro +- Include gz_header definition when compiling zlib solo +- Put gzflags() functionality back in zutil.c +- Avoid library header include in crc32.c for Z_SOLO +- Use name in GCC_CLASSIC as C compiler for coverage testing, if set +- Minor cleanup in contrib/minizip/zip.c [Vollant] +- Update make_vms.com [Zinser] +- Remove unnecessary gzgetc_ function +- Use optimized byte swap operations for Microsoft and GNU [Snyder] +- Fix minor typo in zlib.h comments [Rzesniowiecki] + +Changes in 1.2.6 (29 Jan 2012) +- Update the Pascal interface in contrib/pascal +- Fix function numbers for gzgetc_ in zlibvc.def files +- Fix configure.ac for contrib/minizip [Schiffer] +- Fix large-entry detection in minizip on 64-bit systems [Schiffer] +- Have ./configure use the compiler return code for error indication +- Fix CMakeLists.txt for cross compilation [McClure] +- Fix contrib/minizip/zip.c for 64-bit architectures [Dalsnes] +- Fix compilation of contrib/minizip on FreeBSD [Marquez] +- Correct suggested usages in win32/Makefile.msc [Shachar, Horvath] +- Include io.h for Turbo C / Borland C on all platforms [Truta] +- Make version explicit in contrib/minizip/configure.ac [Bosmans] +- Avoid warning for no encryption in contrib/minizip/zip.c [Vollant] +- Minor cleanup up contrib/minizip/unzip.c [Vollant] +- Fix bug when compiling minizip with C++ [Vollant] +- Protect for long name and extra fields in contrib/minizip [Vollant] +- Avoid some warnings in contrib/minizip [Vollant] +- Add -I../.. -L../.. to CFLAGS for minizip and miniunzip +- Add missing libs to minizip linker command +- Add support for VPATH builds in contrib/minizip +- Add an --enable-demos option to contrib/minizip/configure +- Add the generation of configure.log by ./configure +- Exit when required parameters not provided to win32/Makefile.gcc +- Have gzputc return the character written instead of the argument +- Use the -m option on ldconfig for BSD systems [Tobias] +- Correct in zlib.map when deflateResetKeep was added + +Changes in 1.2.5.3 (15 Jan 2012) +- Restore gzgetc function for binary compatibility +- Do not use _lseeki64 under Borland C++ [Truta] +- Update win32/Makefile.msc to build test/*.c [Truta] +- Remove old/visualc6 given CMakefile and other alternatives +- Update AS400 build files and documentation [Monnerat] +- Update win32/Makefile.gcc to build test/*.c [Truta] +- Permit stronger flushes after Z_BLOCK flushes +- Avoid extraneous empty blocks when doing empty flushes +- Permit Z_NULL arguments to deflatePending +- Allow deflatePrime() to insert bits in the middle of a stream +- Remove second empty static block for Z_PARTIAL_FLUSH +- Write out all of the available bits when using Z_BLOCK +- Insert the first two strings in the hash table after a flush + +Changes in 1.2.5.2 (17 Dec 2011) +- fix ld error: unable to find version dependency 'ZLIB_1.2.5' +- use relative symlinks for shared libs +- Avoid searching past window for Z_RLE strategy +- Assure that high-water mark initialization is always applied in deflate +- Add assertions to fill_window() in deflate.c to match comments +- Update python link in README +- Correct spelling error in gzread.c +- Fix bug in gzgets() for a concatenated empty gzip stream +- Correct error in comment for gz_make() +- Change gzread() and related to ignore junk after gzip streams +- Allow gzread() and related to continue after gzclearerr() +- Allow gzrewind() and gzseek() after a premature end-of-file +- Simplify gzseek() now that raw after gzip is ignored +- Change gzgetc() to a macro for speed (~40% speedup in testing) +- Fix gzclose() to return the actual error last encountered +- Always add large file support for windows +- Include zconf.h for windows large file support +- Include zconf.h.cmakein for windows large file support +- Update zconf.h.cmakein on make distclean +- Merge vestigial vsnprintf determination from zutil.h to gzguts.h +- Clarify how gzopen() appends in zlib.h comments +- Correct documentation of gzdirect() since junk at end now ignored +- Add a transparent write mode to gzopen() when 'T' is in the mode +- Update python link in zlib man page +- Get inffixed.h and MAKEFIXED result to match +- Add a ./config --solo option to make zlib subset with no libary use +- Add undocumented inflateResetKeep() function for CAB file decoding +- Add --cover option to ./configure for gcc coverage testing +- Add #define ZLIB_CONST option to use const in the z_stream interface +- Add comment to gzdopen() in zlib.h to use dup() when using fileno() +- Note behavior of uncompress() to provide as much data as it can +- Add files in contrib/minizip to aid in building libminizip +- Split off AR options in Makefile.in and configure +- Change ON macro to Z_ARG to avoid application conflicts +- Facilitate compilation with Borland C++ for pragmas and vsnprintf +- Include io.h for Turbo C / Borland C++ +- Move example.c and minigzip.c to test/ +- Simplify incomplete code table filling in inflate_table() +- Remove code from inflate.c and infback.c that is impossible to execute +- Test the inflate code with full coverage +- Allow deflateSetDictionary, inflateSetDictionary at any time (in raw) +- Add deflateResetKeep and fix inflateResetKeep to retain dictionary +- Fix gzwrite.c to accommodate reduced memory zlib compilation +- Have inflate() with Z_FINISH avoid the allocation of a window +- Do not set strm->adler when doing raw inflate +- Fix gzeof() to behave just like feof() when read is not past end of file +- Fix bug in gzread.c when end-of-file is reached +- Avoid use of Z_BUF_ERROR in gz* functions except for premature EOF +- Document gzread() capability to read concurrently written files +- Remove hard-coding of resource compiler in CMakeLists.txt [Blammo] + +Changes in 1.2.5.1 (10 Sep 2011) +- Update FAQ entry on shared builds (#13) +- Avoid symbolic argument to chmod in Makefile.in +- Fix bug and add consts in contrib/puff [Oberhumer] +- Update contrib/puff/zeros.raw test file to have all block types +- Add full coverage test for puff in contrib/puff/Makefile +- Fix static-only-build install in Makefile.in +- Fix bug in unzGetCurrentFileInfo() in contrib/minizip [Kuno] +- Add libz.a dependency to shared in Makefile.in for parallel builds +- Spell out "number" (instead of "nb") in zlib.h for total_in, total_out +- Replace $(...) with `...` in configure for non-bash sh [Bowler] +- Add darwin* to Darwin* and solaris* to SunOS\ 5* in configure [Groffen] +- Add solaris* to Linux* in configure to allow gcc use [Groffen] +- Add *bsd* to Linux* case in configure [Bar-Lev] +- Add inffast.obj to dependencies in win32/Makefile.msc +- Correct spelling error in deflate.h [Kohler] +- Change libzdll.a again to libz.dll.a (!) in win32/Makefile.gcc +- Add test to configure for GNU C looking for gcc in output of $cc -v +- Add zlib.pc generation to win32/Makefile.gcc [Weigelt] +- Fix bug in zlib.h for _FILE_OFFSET_BITS set and _LARGEFILE64_SOURCE not +- Add comment in zlib.h that adler32_combine with len2 < 0 makes no sense +- Make NO_DIVIDE option in adler32.c much faster (thanks to John Reiser) +- Make stronger test in zconf.h to include unistd.h for LFS +- Apply Darwin patches for 64-bit file offsets to contrib/minizip [Slack] +- Fix zlib.h LFS support when Z_PREFIX used +- Add updated as400 support (removed from old) [Monnerat] +- Avoid deflate sensitivity to volatile input data +- Avoid division in adler32_combine for NO_DIVIDE +- Clarify the use of Z_FINISH with deflateBound() amount of space +- Set binary for output file in puff.c +- Use u4 type for crc_table to avoid conversion warnings +- Apply casts in zlib.h to avoid conversion warnings +- Add OF to prototypes for adler32_combine_ and crc32_combine_ [Miller] +- Improve inflateSync() documentation to note indeterminancy +- Add deflatePending() function to return the amount of pending output +- Correct the spelling of "specification" in FAQ [Randers-Pehrson] +- Add a check in configure for stdarg.h, use for gzprintf() +- Check that pointers fit in ints when gzprint() compiled old style +- Add dummy name before $(SHAREDLIBV) in Makefile [Bar-Lev, Bowler] +- Delete line in configure that adds -L. libz.a to LDFLAGS [Weigelt] +- Add debug records in assmebler code [Londer] +- Update RFC references to use http://tools.ietf.org/html/... [Li] +- Add --archs option, use of libtool to configure for Mac OS X [Borstel] + +Changes in 1.2.5 (19 Apr 2010) +- Disable visibility attribute in win32/Makefile.gcc [Bar-Lev] +- Default to libdir as sharedlibdir in configure [Nieder] +- Update copyright dates on modified source files +- Update trees.c to be able to generate modified trees.h +- Exit configure for MinGW, suggesting win32/Makefile.gcc +- Check for NULL path in gz_open [Homurlu] + +Changes in 1.2.4.5 (18 Apr 2010) +- Set sharedlibdir in configure [Torok] +- Set LDFLAGS in Makefile.in [Bar-Lev] +- Avoid mkdir objs race condition in Makefile.in [Bowler] +- Add ZLIB_INTERNAL in front of internal inter-module functions and arrays +- Define ZLIB_INTERNAL to hide internal functions and arrays for GNU C +- Don't use hidden attribute when it is a warning generator (e.g. Solaris) + +Changes in 1.2.4.4 (18 Apr 2010) +- Fix CROSS_PREFIX executable testing, CHOST extract, mingw* [Torok] +- Undefine _LARGEFILE64_SOURCE in zconf.h if it is zero, but not if empty +- Try to use bash or ksh regardless of functionality of /bin/sh +- Fix configure incompatibility with NetBSD sh +- Remove attempt to run under bash or ksh since have better NetBSD fix +- Fix win32/Makefile.gcc for MinGW [Bar-Lev] +- Add diagnostic messages when using CROSS_PREFIX in configure +- Added --sharedlibdir option to configure [Weigelt] +- Use hidden visibility attribute when available [Frysinger] + +Changes in 1.2.4.3 (10 Apr 2010) +- Only use CROSS_PREFIX in configure for ar and ranlib if they exist +- Use CROSS_PREFIX for nm [Bar-Lev] +- Assume _LARGEFILE64_SOURCE defined is equivalent to true +- Avoid use of undefined symbols in #if with && and || +- Make *64 prototypes in gzguts.h consistent with functions +- Add -shared load option for MinGW in configure [Bowler] +- Move z_off64_t to public interface, use instead of off64_t +- Remove ! from shell test in configure (not portable to Solaris) +- Change +0 macro tests to -0 for possibly increased portability + +Changes in 1.2.4.2 (9 Apr 2010) +- Add consistent carriage returns to readme.txt's in masmx86 and masmx64 +- Really provide prototypes for *64 functions when building without LFS +- Only define unlink() in minigzip.c if unistd.h not included +- Update README to point to contrib/vstudio project files +- Move projects/vc6 to old/ and remove projects/ +- Include stdlib.h in minigzip.c for setmode() definition under WinCE +- Clean up assembler builds in win32/Makefile.msc [Rowe] +- Include sys/types.h for Microsoft for off_t definition +- Fix memory leak on error in gz_open() +- Symbolize nm as $NM in configure [Weigelt] +- Use TEST_LDSHARED instead of LDSHARED to link test programs [Weigelt] +- Add +0 to _FILE_OFFSET_BITS and _LFS64_LARGEFILE in case not defined +- Fix bug in gzeof() to take into account unused input data +- Avoid initialization of structures with variables in puff.c +- Updated win32/README-WIN32.txt [Rowe] + +Changes in 1.2.4.1 (28 Mar 2010) +- Remove the use of [a-z] constructs for sed in configure [gentoo 310225] +- Remove $(SHAREDLIB) from LIBS in Makefile.in [Creech] +- Restore "for debugging" comment on sprintf() in gzlib.c +- Remove fdopen for MVS from gzguts.h +- Put new README-WIN32.txt in win32 [Rowe] +- Add check for shell to configure and invoke another shell if needed +- Fix big fat stinking bug in gzseek() on uncompressed files +- Remove vestigial F_OPEN64 define in zutil.h +- Set and check the value of _LARGEFILE_SOURCE and _LARGEFILE64_SOURCE +- Avoid errors on non-LFS systems when applications define LFS macros +- Set EXE to ".exe" in configure for MINGW [Kahle] +- Match crc32() in crc32.c exactly to the prototype in zlib.h [Sherrill] +- Add prefix for cross-compilation in win32/makefile.gcc [Bar-Lev] +- Add DLL install in win32/makefile.gcc [Bar-Lev] +- Allow Linux* or linux* from uname in configure [Bar-Lev] +- Allow ldconfig to be redefined in configure and Makefile.in [Bar-Lev] +- Add cross-compilation prefixes to configure [Bar-Lev] +- Match type exactly in gz_load() invocation in gzread.c +- Match type exactly of zcalloc() in zutil.c to zlib.h alloc_func +- Provide prototypes for *64 functions when building zlib without LFS +- Don't use -lc when linking shared library on MinGW +- Remove errno.h check in configure and vestigial errno code in zutil.h + +Changes in 1.2.4 (14 Mar 2010) +- Fix VER3 extraction in configure for no fourth subversion +- Update zlib.3, add docs to Makefile.in to make .pdf out of it +- Add zlib.3.pdf to distribution +- Don't set error code in gzerror() if passed pointer is NULL +- Apply destination directory fixes to CMakeLists.txt [Lowman] +- Move #cmakedefine's to a new zconf.in.cmakein +- Restore zconf.h for builds that don't use configure or cmake +- Add distclean to dummy Makefile for convenience +- Update and improve INDEX, README, and FAQ +- Update CMakeLists.txt for the return of zconf.h [Lowman] +- Update contrib/vstudio/vc9 and vc10 [Vollant] +- Change libz.dll.a back to libzdll.a in win32/Makefile.gcc +- Apply license and readme changes to contrib/asm686 [Raiter] +- Check file name lengths and add -c option in minigzip.c [Li] +- Update contrib/amd64 and contrib/masmx86/ [Vollant] +- Avoid use of "eof" parameter in trees.c to not shadow library variable +- Update make_vms.com for removal of zlibdefs.h [Zinser] +- Update assembler code and vstudio projects in contrib [Vollant] +- Remove outdated assembler code contrib/masm686 and contrib/asm586 +- Remove old vc7 and vc8 from contrib/vstudio +- Update win32/Makefile.msc, add ZLIB_VER_SUBREVISION [Rowe] +- Fix memory leaks in gzclose_r() and gzclose_w(), file leak in gz_open() +- Add contrib/gcc_gvmat64 for longest_match and inflate_fast [Vollant] +- Remove *64 functions from win32/zlib.def (they're not 64-bit yet) +- Fix bug in void-returning vsprintf() case in gzwrite.c +- Fix name change from inflate.h in contrib/inflate86/inffas86.c +- Check if temporary file exists before removing in make_vms.com [Zinser] +- Fix make install and uninstall for --static option +- Fix usage of _MSC_VER in gzguts.h and zutil.h [Truta] +- Update readme.txt in contrib/masmx64 and masmx86 to assemble + +Changes in 1.2.3.9 (21 Feb 2010) +- Expunge gzio.c +- Move as400 build information to old +- Fix updates in contrib/minizip and contrib/vstudio +- Add const to vsnprintf test in configure to avoid warnings [Weigelt] +- Delete zconf.h (made by configure) [Weigelt] +- Change zconf.in.h to zconf.h.in per convention [Weigelt] +- Check for NULL buf in gzgets() +- Return empty string for gzgets() with len == 1 (like fgets()) +- Fix description of gzgets() in zlib.h for end-of-file, NULL return +- Update minizip to 1.1 [Vollant] +- Avoid MSVC loss of data warnings in gzread.c, gzwrite.c +- Note in zlib.h that gzerror() should be used to distinguish from EOF +- Remove use of snprintf() from gzlib.c +- Fix bug in gzseek() +- Update contrib/vstudio, adding vc9 and vc10 [Kuno, Vollant] +- Fix zconf.h generation in CMakeLists.txt [Lowman] +- Improve comments in zconf.h where modified by configure + +Changes in 1.2.3.8 (13 Feb 2010) +- Clean up text files (tabs, trailing whitespace, etc.) [Oberhumer] +- Use z_off64_t in gz_zero() and gz_skip() to match state->skip +- Avoid comparison problem when sizeof(int) == sizeof(z_off64_t) +- Revert to Makefile.in from 1.2.3.6 (live with the clutter) +- Fix missing error return in gzflush(), add zlib.h note +- Add *64 functions to zlib.map [Levin] +- Fix signed/unsigned comparison in gz_comp() +- Use SFLAGS when testing shared linking in configure +- Add --64 option to ./configure to use -m64 with gcc +- Fix ./configure --help to correctly name options +- Have make fail if a test fails [Levin] +- Avoid buffer overrun in contrib/masmx64/gvmat64.asm [Simpson] +- Remove assembler object files from contrib + +Changes in 1.2.3.7 (24 Jan 2010) +- Always gzopen() with O_LARGEFILE if available +- Fix gzdirect() to work immediately after gzopen() or gzdopen() +- Make gzdirect() more precise when the state changes while reading +- Improve zlib.h documentation in many places +- Catch memory allocation failure in gz_open() +- Complete close operation if seek forward in gzclose_w() fails +- Return Z_ERRNO from gzclose_r() if close() fails +- Return Z_STREAM_ERROR instead of EOF for gzclose() being passed NULL +- Return zero for gzwrite() errors to match zlib.h description +- Return -1 on gzputs() error to match zlib.h description +- Add zconf.in.h to allow recovery from configure modification [Weigelt] +- Fix static library permissions in Makefile.in [Weigelt] +- Avoid warnings in configure tests that hide functionality [Weigelt] +- Add *BSD and DragonFly to Linux case in configure [gentoo 123571] +- Change libzdll.a to libz.dll.a in win32/Makefile.gcc [gentoo 288212] +- Avoid access of uninitialized data for first inflateReset2 call [Gomes] +- Keep object files in subdirectories to reduce the clutter somewhat +- Remove default Makefile and zlibdefs.h, add dummy Makefile +- Add new external functions to Z_PREFIX, remove duplicates, z_z_ -> z_ +- Remove zlibdefs.h completely -- modify zconf.h instead + +Changes in 1.2.3.6 (17 Jan 2010) +- Avoid void * arithmetic in gzread.c and gzwrite.c +- Make compilers happier with const char * for gz_error message +- Avoid unused parameter warning in inflate.c +- Avoid signed-unsigned comparison warning in inflate.c +- Indent #pragma's for traditional C +- Fix usage of strwinerror() in glib.c, change to gz_strwinerror() +- Correct email address in configure for system options +- Update make_vms.com and add make_vms.com to contrib/minizip [Zinser] +- Update zlib.map [Brown] +- Fix Makefile.in for Solaris 10 make of example64 and minizip64 [Torok] +- Apply various fixes to CMakeLists.txt [Lowman] +- Add checks on len in gzread() and gzwrite() +- Add error message for no more room for gzungetc() +- Remove zlib version check in gzwrite() +- Defer compression of gzprintf() result until need to +- Use snprintf() in gzdopen() if available +- Remove USE_MMAP configuration determination (only used by minigzip) +- Remove examples/pigz.c (available separately) +- Update examples/gun.c to 1.6 + +Changes in 1.2.3.5 (8 Jan 2010) +- Add space after #if in zutil.h for some compilers +- Fix relatively harmless bug in deflate_fast() [Exarevsky] +- Fix same problem in deflate_slow() +- Add $(SHAREDLIBV) to LIBS in Makefile.in [Brown] +- Add deflate_rle() for faster Z_RLE strategy run-length encoding +- Add deflate_huff() for faster Z_HUFFMAN_ONLY encoding +- Change name of "write" variable in inffast.c to avoid library collisions +- Fix premature EOF from gzread() in gzio.c [Brown] +- Use zlib header window size if windowBits is 0 in inflateInit2() +- Remove compressBound() call in deflate.c to avoid linking compress.o +- Replace use of errno in gz* with functions, support WinCE [Alves] +- Provide alternative to perror() in minigzip.c for WinCE [Alves] +- Don't use _vsnprintf on later versions of MSVC [Lowman] +- Add CMake build script and input file [Lowman] +- Update contrib/minizip to 1.1 [Svensson, Vollant] +- Moved nintendods directory from contrib to . +- Replace gzio.c with a new set of routines with the same functionality +- Add gzbuffer(), gzoffset(), gzclose_r(), gzclose_w() as part of above +- Update contrib/minizip to 1.1b +- Change gzeof() to return 0 on error instead of -1 to agree with zlib.h + +Changes in 1.2.3.4 (21 Dec 2009) +- Use old school .SUFFIXES in Makefile.in for FreeBSD compatibility +- Update comments in configure and Makefile.in for default --shared +- Fix test -z's in configure [Marquess] +- Build examplesh and minigzipsh when not testing +- Change NULL's to Z_NULL's in deflate.c and in comments in zlib.h +- Import LDFLAGS from the environment in configure +- Fix configure to populate SFLAGS with discovered CFLAGS options +- Adapt make_vms.com to the new Makefile.in [Zinser] +- Add zlib2ansi script for C++ compilation [Marquess] +- Add _FILE_OFFSET_BITS=64 test to make test (when applicable) +- Add AMD64 assembler code for longest match to contrib [Teterin] +- Include options from $SFLAGS when doing $LDSHARED +- Simplify 64-bit file support by introducing z_off64_t type +- Make shared object files in objs directory to work around old Sun cc +- Use only three-part version number for Darwin shared compiles +- Add rc option to ar in Makefile.in for when ./configure not run +- Add -WI,-rpath,. to LDFLAGS for OSF 1 V4* +- Set LD_LIBRARYN32_PATH for SGI IRIX shared compile +- Protect against _FILE_OFFSET_BITS being defined when compiling zlib +- Rename Makefile.in targets allstatic to static and allshared to shared +- Fix static and shared Makefile.in targets to be independent +- Correct error return bug in gz_open() by setting state [Brown] +- Put spaces before ;;'s in configure for better sh compatibility +- Add pigz.c (parallel implementation of gzip) to examples/ +- Correct constant in crc32.c to UL [Leventhal] +- Reject negative lengths in crc32_combine() +- Add inflateReset2() function to work like inflateEnd()/inflateInit2() +- Include sys/types.h for _LARGEFILE64_SOURCE [Brown] +- Correct typo in doc/algorithm.txt [Janik] +- Fix bug in adler32_combine() [Zhu] +- Catch missing-end-of-block-code error in all inflates and in puff + Assures that random input to inflate eventually results in an error +- Added enough.c (calculation of ENOUGH for inftrees.h) to examples/ +- Update ENOUGH and its usage to reflect discovered bounds +- Fix gzerror() error report on empty input file [Brown] +- Add ush casts in trees.c to avoid pedantic runtime errors +- Fix typo in zlib.h uncompress() description [Reiss] +- Correct inflate() comments with regard to automatic header detection +- Remove deprecation comment on Z_PARTIAL_FLUSH (it stays) +- Put new version of gzlog (2.0) in examples with interruption recovery +- Add puff compile option to permit invalid distance-too-far streams +- Add puff TEST command options, ability to read piped input +- Prototype the *64 functions in zlib.h when _FILE_OFFSET_BITS == 64, but + _LARGEFILE64_SOURCE not defined +- Fix Z_FULL_FLUSH to truly erase the past by resetting s->strstart +- Fix deflateSetDictionary() to use all 32K for output consistency +- Remove extraneous #define MIN_LOOKAHEAD in deflate.c (in deflate.h) +- Clear bytes after deflate lookahead to avoid use of uninitialized data +- Change a limit in inftrees.c to be more transparent to Coverity Prevent +- Update win32/zlib.def with exported symbols from zlib.h +- Correct spelling errors in zlib.h [Willem, Sobrado] +- Allow Z_BLOCK for deflate() to force a new block +- Allow negative bits in inflatePrime() to delete existing bit buffer +- Add Z_TREES flush option to inflate() to return at end of trees +- Add inflateMark() to return current state information for random access +- Add Makefile for NintendoDS to contrib [Costa] +- Add -w in configure compile tests to avoid spurious warnings [Beucler] +- Fix typos in zlib.h comments for deflateSetDictionary() +- Fix EOF detection in transparent gzread() [Maier] + +Changes in 1.2.3.3 (2 October 2006) +- Make --shared the default for configure, add a --static option +- Add compile option to permit invalid distance-too-far streams +- Add inflateUndermine() function which is required to enable above +- Remove use of "this" variable name for C++ compatibility [Marquess] +- Add testing of shared library in make test, if shared library built +- Use ftello() and fseeko() if available instead of ftell() and fseek() +- Provide two versions of all functions that use the z_off_t type for + binary compatibility -- a normal version and a 64-bit offset version, + per the Large File Support Extension when _LARGEFILE64_SOURCE is + defined; use the 64-bit versions by default when _FILE_OFFSET_BITS + is defined to be 64 +- Add a --uname= option to configure to perhaps help with cross-compiling + +Changes in 1.2.3.2 (3 September 2006) +- Turn off silly Borland warnings [Hay] +- Use off64_t and define _LARGEFILE64_SOURCE when present +- Fix missing dependency on inffixed.h in Makefile.in +- Rig configure --shared to build both shared and static [Teredesai, Truta] +- Remove zconf.in.h and instead create a new zlibdefs.h file +- Fix contrib/minizip/unzip.c non-encrypted after encrypted [Vollant] +- Add treebuild.xml (see http://treebuild.metux.de/) [Weigelt] + +Changes in 1.2.3.1 (16 August 2006) +- Add watcom directory with OpenWatcom make files [Daniel] +- Remove #undef of FAR in zconf.in.h for MVS [Fedtke] +- Update make_vms.com [Zinser] +- Use -fPIC for shared build in configure [Teredesai, Nicholson] +- Use only major version number for libz.so on IRIX and OSF1 [Reinholdtsen] +- Use fdopen() (not _fdopen()) for Interix in zutil.h [B?ck] +- Add some FAQ entries about the contrib directory +- Update the MVS question in the FAQ +- Avoid extraneous reads after EOF in gzio.c [Brown] +- Correct spelling of "successfully" in gzio.c [Randers-Pehrson] +- Add comments to zlib.h about gzerror() usage [Brown] +- Set extra flags in gzip header in gzopen() like deflate() does +- Make configure options more compatible with double-dash conventions + [Weigelt] +- Clean up compilation under Solaris SunStudio cc [Rowe, Reinholdtsen] +- Fix uninstall target in Makefile.in [Truta] +- Add pkgconfig support [Weigelt] +- Use $(DESTDIR) macro in Makefile.in [Reinholdtsen, Weigelt] +- Replace set_data_type() with a more accurate detect_data_type() in + trees.c, according to the txtvsbin.txt document [Truta] +- Swap the order of #include and #include "zlib.h" in + gzio.c, example.c and minigzip.c [Truta] +- Shut up annoying VS2005 warnings about standard C deprecation [Rowe, + Truta] (where?) +- Fix target "clean" from win32/Makefile.bor [Truta] +- Create .pdb and .manifest files in win32/makefile.msc [Ziegler, Rowe] +- Update zlib www home address in win32/DLL_FAQ.txt [Truta] +- Update contrib/masmx86/inffas32.asm for VS2005 [Vollant, Van Wassenhove] +- Enable browse info in the "Debug" and "ASM Debug" configurations in + the Visual C++ 6 project, and set (non-ASM) "Debug" as default [Truta] +- Add pkgconfig support [Weigelt] +- Add ZLIB_VER_MAJOR, ZLIB_VER_MINOR and ZLIB_VER_REVISION in zlib.h, + for use in win32/zlib1.rc [Polushin, Rowe, Truta] +- Add a document that explains the new text detection scheme to + doc/txtvsbin.txt [Truta] +- Add rfc1950.txt, rfc1951.txt and rfc1952.txt to doc/ [Truta] +- Move algorithm.txt into doc/ [Truta] +- Synchronize FAQ with website +- Fix compressBound(), was low for some pathological cases [Fearnley] +- Take into account wrapper variations in deflateBound() +- Set examples/zpipe.c input and output to binary mode for Windows +- Update examples/zlib_how.html with new zpipe.c (also web site) +- Fix some warnings in examples/gzlog.c and examples/zran.c (it seems + that gcc became pickier in 4.0) +- Add zlib.map for Linux: "All symbols from zlib-1.1.4 remain + un-versioned, the patch adds versioning only for symbols introduced in + zlib-1.2.0 or later. It also declares as local those symbols which are + not designed to be exported." [Levin] +- Update Z_PREFIX list in zconf.in.h, add --zprefix option to configure +- Do not initialize global static by default in trees.c, add a response + NO_INIT_GLOBAL_POINTERS to initialize them if needed [Marquess] +- Don't use strerror() in gzio.c under WinCE [Yakimov] +- Don't use errno.h in zutil.h under WinCE [Yakimov] +- Move arguments for AR to its usage to allow replacing ar [Marot] +- Add HAVE_VISIBILITY_PRAGMA in zconf.in.h for Mozilla [Randers-Pehrson] +- Improve inflateInit() and inflateInit2() documentation +- Fix structure size comment in inflate.h +- Change configure help option from --h* to --help [Santos] + Changes in 1.2.3 (18 July 2005) - Apply security vulnerability fixes to contrib/infback9 as well - Clean up some text files (carriage returns, trailing space) @@ -13,7 +630,7 @@ compile - Fix some spelling errors in comments [Betts] - Correct inflateInit2() error return documentation in zlib.h -- Added zran.c example of compressed data random access to examples +- Add zran.c example of compressed data random access to examples directory, shows use of inflatePrime() - Fix cast for assignments to strm->state in inflate.c and infback.c - Fix zlibCompileFlags() in zutil.c to use 1L for long shifts [Oberhumer] diff --git a/Modules/zlib/FAQ b/Modules/zlib/FAQ --- a/Modules/zlib/FAQ +++ b/Modules/zlib/FAQ @@ -3,8 +3,8 @@ If your question is not there, please check the zlib home page -http://www.zlib.org which may have more recent information. -The lastest zlib FAQ is at http://www.gzip.org/zlib/zlib_faq.html +http://zlib.net/ which may have more recent information. +The lastest zlib FAQ is at http://zlib.net/zlib_faq.html 1. Is zlib Y2K-compliant? @@ -13,54 +13,51 @@ 2. Where can I get a Windows DLL version? - The zlib sources can be compiled without change to produce a DLL. - See the file win32/DLL_FAQ.txt in the zlib distribution. - Pointers to the precompiled DLL are found in the zlib web site at - http://www.zlib.org. + The zlib sources can be compiled without change to produce a DLL. See the + file win32/DLL_FAQ.txt in the zlib distribution. Pointers to the + precompiled DLL are found in the zlib web site at http://zlib.net/ . 3. Where can I get a Visual Basic interface to zlib? See - * http://www.dogma.net/markn/articles/zlibtool/zlibtool.htm - * contrib/visual-basic.txt in the zlib distribution + * http://marknelson.us/1997/01/01/zlib-engine/ * win32/DLL_FAQ.txt in the zlib distribution 4. compress() returns Z_BUF_ERROR. - Make sure that before the call of compress, the length of the compressed - buffer is equal to the total size of the compressed buffer and not - zero. For Visual Basic, check that this parameter is passed by reference + Make sure that before the call of compress(), the length of the compressed + buffer is equal to the available size of the compressed buffer and not + zero. For Visual Basic, check that this parameter is passed by reference ("as any"), not by value ("as long"). 5. deflate() or inflate() returns Z_BUF_ERROR. - Before making the call, make sure that avail_in and avail_out are not - zero. When setting the parameter flush equal to Z_FINISH, also make sure - that avail_out is big enough to allow processing all pending input. - Note that a Z_BUF_ERROR is not fatal--another call to deflate() or - inflate() can be made with more input or output space. A Z_BUF_ERROR - may in fact be unavoidable depending on how the functions are used, since - it is not possible to tell whether or not there is more output pending - when strm.avail_out returns with zero. + Before making the call, make sure that avail_in and avail_out are not zero. + When setting the parameter flush equal to Z_FINISH, also make sure that + avail_out is big enough to allow processing all pending input. Note that a + Z_BUF_ERROR is not fatal--another call to deflate() or inflate() can be + made with more input or output space. A Z_BUF_ERROR may in fact be + unavoidable depending on how the functions are used, since it is not + possible to tell whether or not there is more output pending when + strm.avail_out returns with zero. See http://zlib.net/zlib_how.html for a + heavily annotated example. 6. Where's the zlib documentation (man pages, etc.)? - It's in zlib.h for the moment, and Francis S. Lin has converted it to a - web page zlib.html. Volunteers to transform this to Unix-style man pages, - please contact us (zlib at gzip.org). Examples of zlib usage are in the files - example.c and minigzip.c. + It's in zlib.h . Examples of zlib usage are in the files test/example.c + and test/minigzip.c, with more in examples/ . 7. Why don't you use GNU autoconf or libtool or ...? - Because we would like to keep zlib as a very small and simple - package. zlib is rather portable and doesn't need much configuration. + Because we would like to keep zlib as a very small and simple package. + zlib is rather portable and doesn't need much configuration. 8. I found a bug in zlib. - Most of the time, such problems are due to an incorrect usage of - zlib. Please try to reproduce the problem with a small program and send - the corresponding source to us at zlib at gzip.org . Do not send - multi-megabyte data files without prior agreement. + Most of the time, such problems are due to an incorrect usage of zlib. + Please try to reproduce the problem with a small program and send the + corresponding source to us at zlib at gzip.org . Do not send multi-megabyte + data files without prior agreement. 9. Why do I get "undefined reference to gzputc"? @@ -82,13 +79,15 @@ 12. Can zlib handle .Z files? - No, sorry. You have to spawn an uncompress or gunzip subprocess, or adapt + No, sorry. You have to spawn an uncompress or gunzip subprocess, or adapt the code of uncompress on your own. 13. How can I make a Unix shared library? - make clean - ./configure -s + By default a shared (and a static) library is built for Unix. So: + + make distclean + ./configure make 14. How do I install a shared zlib library on Unix? @@ -99,8 +98,10 @@ However, many flavors of Unix come with a shared zlib already installed. Before going to the trouble of compiling a shared version of zlib and - trying to install it, you may want to check if it's already there! If you - can #include , it's there. The -lz option will probably link to it. + trying to install it, you may want to check if it's already there! If you + can #include , it's there. The -lz option will probably link to + it. You can check the version at the top of zlib.h or with the + ZLIB_VERSION symbol defined in zlib.h . 15. I have a question about OttoPDF. @@ -109,8 +110,8 @@ 16. Can zlib decode Flate data in an Adobe PDF file? - Yes. See http://www.fastio.com/ (ClibPDF), or http://www.pdflib.com/ . - To modify PDF forms, see http://sourceforge.net/projects/acroformtool/ . + Yes. See http://www.pdflib.com/ . To modify PDF forms, see + http://sourceforge.net/projects/acroformtool/ . 17. Why am I getting this "register_frame_info not found" error on Solaris? @@ -121,67 +122,67 @@ symbol __register_frame_info: referenced symbol not found The symbol __register_frame_info is not part of zlib, it is generated by - the C compiler (cc or gcc). You must recompile applications using zlib - which have this problem. This problem is specific to Solaris. See + the C compiler (cc or gcc). You must recompile applications using zlib + which have this problem. This problem is specific to Solaris. See http://www.sunfreeware.com for Solaris versions of zlib and applications using zlib. 18. Why does gzip give an error on a file I make with compress/deflate? The compress and deflate functions produce data in the zlib format, which - is different and incompatible with the gzip format. The gz* functions in - zlib on the other hand use the gzip format. Both the zlib and gzip - formats use the same compressed data format internally, but have different - headers and trailers around the compressed data. + is different and incompatible with the gzip format. The gz* functions in + zlib on the other hand use the gzip format. Both the zlib and gzip formats + use the same compressed data format internally, but have different headers + and trailers around the compressed data. 19. Ok, so why are there two different formats? - The gzip format was designed to retain the directory information about - a single file, such as the name and last modification date. The zlib - format on the other hand was designed for in-memory and communication - channel applications, and has a much more compact header and trailer and - uses a faster integrity check than gzip. + The gzip format was designed to retain the directory information about a + single file, such as the name and last modification date. The zlib format + on the other hand was designed for in-memory and communication channel + applications, and has a much more compact header and trailer and uses a + faster integrity check than gzip. 20. Well that's nice, but how do I make a gzip file in memory? You can request that deflate write the gzip format instead of the zlib - format using deflateInit2(). You can also request that inflate decode - the gzip format using inflateInit2(). Read zlib.h for more details. + format using deflateInit2(). You can also request that inflate decode the + gzip format using inflateInit2(). Read zlib.h for more details. 21. Is zlib thread-safe? - Yes. However any library routines that zlib uses and any application- - provided memory allocation routines must also be thread-safe. zlib's gz* + Yes. However any library routines that zlib uses and any application- + provided memory allocation routines must also be thread-safe. zlib's gz* functions use stdio library routines, and most of zlib's functions use the - library memory allocation routines by default. zlib's Init functions allow - for the application to provide custom memory allocation routines. + library memory allocation routines by default. zlib's *Init* functions + allow for the application to provide custom memory allocation routines. Of course, you should only operate on any given zlib or gzip stream from a single thread at a time. 22. Can I use zlib in my commercial application? - Yes. Please read the license in zlib.h. + Yes. Please read the license in zlib.h. 23. Is zlib under the GNU license? - No. Please read the license in zlib.h. + No. Please read the license in zlib.h. 24. The license says that altered source versions must be "plainly marked". So what exactly do I need to do to meet that requirement? - You need to change the ZLIB_VERSION and ZLIB_VERNUM #defines in zlib.h. In + You need to change the ZLIB_VERSION and ZLIB_VERNUM #defines in zlib.h. In particular, the final version number needs to be changed to "f", and an - identification string should be appended to ZLIB_VERSION. Version numbers + identification string should be appended to ZLIB_VERSION. Version numbers x.x.x.f are reserved for modifications to zlib by others than the zlib - maintainers. For example, if the version of the base zlib you are altering + maintainers. For example, if the version of the base zlib you are altering is "1.2.3.4", then in zlib.h you should change ZLIB_VERNUM to 0x123f, and - ZLIB_VERSION to something like "1.2.3.f-zachary-mods-v3". You can also + ZLIB_VERSION to something like "1.2.3.f-zachary-mods-v3". You can also update the version strings in deflate.c and inftrees.c. For altered source distributions, you should also note the origin and nature of the changes in zlib.h, as well as in ChangeLog and README, along - with the dates of the alterations. The origin should include at least your + with the dates of the alterations. The origin should include at least your name (or your company's name), and an email address to contact for help or issues with the library. @@ -197,105 +198,112 @@ 26. Will zlib work on a 64-bit machine? - It should. It has been tested on 64-bit machines, and has no dependence - on any data types being limited to 32-bits in length. If you have any + Yes. It has been tested on 64-bit machines, and has no dependence on any + data types being limited to 32-bits in length. If you have any difficulties, please provide a complete problem report to zlib at gzip.org 27. Will zlib decompress data from the PKWare Data Compression Library? - No. The PKWare DCL uses a completely different compressed data format - than does PKZIP and zlib. However, you can look in zlib's contrib/blast + No. The PKWare DCL uses a completely different compressed data format than + does PKZIP and zlib. However, you can look in zlib's contrib/blast directory for a possible solution to your problem. 28. Can I access data randomly in a compressed stream? - No, not without some preparation. If when compressing you periodically - use Z_FULL_FLUSH, carefully write all the pending data at those points, - and keep an index of those locations, then you can start decompression - at those points. You have to be careful to not use Z_FULL_FLUSH too - often, since it can significantly degrade compression. + No, not without some preparation. If when compressing you periodically use + Z_FULL_FLUSH, carefully write all the pending data at those points, and + keep an index of those locations, then you can start decompression at those + points. You have to be careful to not use Z_FULL_FLUSH too often, since it + can significantly degrade compression. Alternatively, you can scan a + deflate stream once to generate an index, and then use that index for + random access. See examples/zran.c . 29. Does zlib work on MVS, OS/390, CICS, etc.? - We don't know for sure. We have heard occasional reports of success on - these systems. If you do use it on one of these, please provide us with - a report, instructions, and patches that we can reference when we get - these questions. Thanks. + It has in the past, but we have not heard of any recent evidence. There + were working ports of zlib 1.1.4 to MVS, but those links no longer work. + If you know of recent, successful applications of zlib on these operating + systems, please let us know. Thanks. -30. Is there some simpler, easier to read version of inflate I can look at - to understand the deflate format? +30. Is there some simpler, easier to read version of inflate I can look at to + understand the deflate format? - First off, you should read RFC 1951. Second, yes. Look in zlib's + First off, you should read RFC 1951. Second, yes. Look in zlib's contrib/puff directory. 31. Does zlib infringe on any patents? - As far as we know, no. In fact, that was originally the whole point behind - zlib. Look here for some more information: + As far as we know, no. In fact, that was originally the whole point behind + zlib. Look here for some more information: http://www.gzip.org/#faq11 32. Can zlib work with greater than 4 GB of data? - Yes. inflate() and deflate() will process any amount of data correctly. + Yes. inflate() and deflate() will process any amount of data correctly. Each call of inflate() or deflate() is limited to input and output chunks of the maximum value that can be stored in the compiler's "unsigned int" - type, but there is no limit to the number of chunks. Note however that the - strm.total_in and strm_total_out counters may be limited to 4 GB. These + type, but there is no limit to the number of chunks. Note however that the + strm.total_in and strm_total_out counters may be limited to 4 GB. These counters are provided as a convenience and are not used internally by - inflate() or deflate(). The application can easily set up its own counters + inflate() or deflate(). The application can easily set up its own counters updated after each call of inflate() or deflate() to count beyond 4 GB. compress() and uncompress() may be limited to 4 GB, since they operate in a - single call. gzseek() and gztell() may be limited to 4 GB depending on how - zlib is compiled. See the zlibCompileFlags() function in zlib.h. + single call. gzseek() and gztell() may be limited to 4 GB depending on how + zlib is compiled. See the zlibCompileFlags() function in zlib.h. - The word "may" appears several times above since there is a 4 GB limit - only if the compiler's "long" type is 32 bits. If the compiler's "long" - type is 64 bits, then the limit is 16 exabytes. + The word "may" appears several times above since there is a 4 GB limit only + if the compiler's "long" type is 32 bits. If the compiler's "long" type is + 64 bits, then the limit is 16 exabytes. 33. Does zlib have any security vulnerabilities? - The only one that we are aware of is potentially in gzprintf(). If zlib - is compiled to use sprintf() or vsprintf(), then there is no protection - against a buffer overflow of a 4K string space, other than the caller of - gzprintf() assuring that the output will not exceed 4K. On the other - hand, if zlib is compiled to use snprintf() or vsnprintf(), which should - normally be the case, then there is no vulnerability. The ./configure - script will display warnings if an insecure variation of sprintf() will - be used by gzprintf(). Also the zlibCompileFlags() function will return - information on what variant of sprintf() is used by gzprintf(). + The only one that we are aware of is potentially in gzprintf(). If zlib is + compiled to use sprintf() or vsprintf(), then there is no protection + against a buffer overflow of an 8K string space (or other value as set by + gzbuffer()), other than the caller of gzprintf() assuring that the output + will not exceed 8K. On the other hand, if zlib is compiled to use + snprintf() or vsnprintf(), which should normally be the case, then there is + no vulnerability. The ./configure script will display warnings if an + insecure variation of sprintf() will be used by gzprintf(). Also the + zlibCompileFlags() function will return information on what variant of + sprintf() is used by gzprintf(). If you don't have snprintf() or vsnprintf() and would like one, you can find a portable implementation here: http://www.ijs.si/software/snprintf/ - Note that you should be using the most recent version of zlib. Versions - 1.1.3 and before were subject to a double-free vulnerability. + Note that you should be using the most recent version of zlib. Versions + 1.1.3 and before were subject to a double-free vulnerability, and versions + 1.2.1 and 1.2.2 were subject to an access exception when decompressing + invalid compressed data. 34. Is there a Java version of zlib? Probably what you want is to use zlib in Java. zlib is already included as part of the Java SDK in the java.util.zip package. If you really want a version of zlib written in the Java language, look on the zlib home - page for links: http://www.zlib.org/ + page for links: http://zlib.net/ . 35. I get this or that compiler or source-code scanner warning when I crank it up to maximally-pedantic. Can't you guys write proper code? Many years ago, we gave up attempting to avoid warnings on every compiler - in the universe. It just got to be a waste of time, and some compilers - were downright silly. So now, we simply make sure that the code always - works. + in the universe. It just got to be a waste of time, and some compilers + were downright silly as well as contradicted each other. So now, we simply + make sure that the code always works. 36. Valgrind (or some similar memory access checker) says that deflate is performing a conditional jump that depends on an uninitialized value. Isn't that a bug? - No. That is intentional for performance reasons, and the output of - deflate is not affected. This only started showing up recently since - zlib 1.2.x uses malloc() by default for allocations, whereas earlier - versions used calloc(), which zeros out the allocated memory. + No. That is intentional for performance reasons, and the output of deflate + is not affected. This only started showing up recently since zlib 1.2.x + uses malloc() by default for allocations, whereas earlier versions used + calloc(), which zeros out the allocated memory. Even though the code was + correct, versions 1.2.4 and later was changed to not stimulate these + checkers. 37. Will zlib read the (insert any ancient or arcane format here) compressed data format? @@ -305,20 +313,21 @@ 38. How can I encrypt/decrypt zip files with zlib? - zlib doesn't support encryption. The original PKZIP encryption is very weak - and can be broken with freely available programs. To get strong encryption, - use GnuPG, http://www.gnupg.org/ , which already includes zlib compression. - For PKZIP compatible "encryption", look at http://www.info-zip.org/ + zlib doesn't support encryption. The original PKZIP encryption is very + weak and can be broken with freely available programs. To get strong + encryption, use GnuPG, http://www.gnupg.org/ , which already includes zlib + compression. For PKZIP compatible "encryption", look at + http://www.info-zip.org/ 39. What's the difference between the "gzip" and "deflate" HTTP 1.1 encodings? - "gzip" is the gzip format, and "deflate" is the zlib format. They should - probably have called the second one "zlib" instead to avoid confusion - with the raw deflate compressed data format. While the HTTP 1.1 RFC 2616 + "gzip" is the gzip format, and "deflate" is the zlib format. They should + probably have called the second one "zlib" instead to avoid confusion with + the raw deflate compressed data format. While the HTTP 1.1 RFC 2616 correctly points to the zlib specification in RFC 1950 for the "deflate" transfer encoding, there have been reports of servers and browsers that incorrectly produce or expect raw deflate data per the deflate - specficiation in RFC 1951, most notably Microsoft. So even though the + specification in RFC 1951, most notably Microsoft. So even though the "deflate" transfer encoding using the zlib format would be the more efficient approach (and in fact exactly what the zlib format was designed for), using the "gzip" transfer encoding is probably more reliable due to @@ -328,12 +337,32 @@ 40. Does zlib support the new "Deflate64" format introduced by PKWare? - No. PKWare has apparently decided to keep that format proprietary, since - they have not documented it as they have previous compression formats. - In any case, the compression improvements are so modest compared to other - more modern approaches, that it's not worth the effort to implement. + No. PKWare has apparently decided to keep that format proprietary, since + they have not documented it as they have previous compression formats. In + any case, the compression improvements are so modest compared to other more + modern approaches, that it's not worth the effort to implement. -41. Can you please sign these lengthy legal documents and fax them back to us +41. I'm having a problem with the zip functions in zlib, can you help? + + There are no zip functions in zlib. You are probably using minizip by + Giles Vollant, which is found in the contrib directory of zlib. It is not + part of zlib. In fact none of the stuff in contrib is part of zlib. The + files in there are not supported by the zlib authors. You need to contact + the authors of the respective contribution for help. + +42. The match.asm code in contrib is under the GNU General Public License. + Since it's part of zlib, doesn't that mean that all of zlib falls under the + GNU GPL? + + No. The files in contrib are not part of zlib. They were contributed by + other authors and are provided as a convenience to the user within the zlib + distribution. Each item in contrib has its own license. + +43. Is zlib subject to export controls? What is its ECCN? + + zlib is not subject to export controls, and so is classified as EAR99. + +44. Can you please sign these lengthy legal documents and fax them back to us so that we can use your software in our product? No. Go away. Shoo. diff --git a/Modules/zlib/INDEX b/Modules/zlib/INDEX --- a/Modules/zlib/INDEX +++ b/Modules/zlib/INDEX @@ -1,23 +1,37 @@ +CMakeLists.txt cmake build file ChangeLog history of changes FAQ Frequently Asked Questions about zlib INDEX this file -Makefile makefile for Unix (generated by configure) -Makefile.in makefile for Unix (template for configure) +Makefile dummy Makefile that tells you to ./configure +Makefile.in template for Unix Makefile README guess what -algorithm.txt description of the (de)compression algorithm configure configure script for Unix -zconf.in.h template for zconf.h (used by configure) +make_vms.com makefile for VMS +test/example.c zlib usages examples for build testing +test/minigzip.c minimal gzip-like functionality for build testing +test/infcover.c inf*.c code coverage for build coverage testing +treebuild.xml XML description of source file dependencies +zconf.h.cmakein zconf.h template for cmake +zconf.h.in zconf.h template for configure +zlib.3 Man page for zlib +zlib.3.pdf Man page in PDF format +zlib.map Linux symbol information +zlib.pc.in Template for pkg-config descriptor +zlib.pc.cmakein zlib.pc template for cmake +zlib2ansi perl script to convert source files for C++ compilation amiga/ makefiles for Amiga SAS C -as400/ makefiles for IBM AS/400 +as400/ makefiles for AS/400 +doc/ documentation for formats and algorithms msdos/ makefiles for MSDOS +nintendods/ makefile for Nintendo DS old/ makefiles for various architectures and zlib documentation files that have not yet been updated for zlib 1.2.x -projects/ projects for various Integrated Development Environments qnx/ makefiles for QNX +watcom/ makefiles for OpenWatcom win32/ makefiles for Windows - zlib public header files (must be kept): + zlib public header files (required for library use): zconf.h zlib.h @@ -28,7 +42,11 @@ crc32.h deflate.c deflate.h -gzio.c +gzclose.c +gzguts.h +gzlib.c +gzread.c +gzwrite.c infback.c inffast.c inffast.h @@ -43,9 +61,8 @@ zutil.c zutil.h - source files for sample programs: -example.c -minigzip.c + source files for sample programs +See examples/README.examples - unsupported contribution by third parties + unsupported contributions by third parties See contrib/README.contrib diff --git a/Modules/zlib/Makefile b/Modules/zlib/Makefile --- a/Modules/zlib/Makefile +++ b/Modules/zlib/Makefile @@ -1,154 +1,5 @@ -# Makefile for zlib -# Copyright (C) 1995-2005 Jean-loup Gailly. -# For conditions of distribution and use, see copyright notice in zlib.h +all: + - at echo "Please use ./configure first. Thank you." -# To compile and test, type: -# ./configure; make test -# The call of configure is optional if you don't have special requirements -# If you wish to build zlib as a shared library, use: ./configure -s - -# To use the asm code, type: -# cp contrib/asm?86/match.S ./match.S -# make LOC=-DASMV OBJA=match.o - -# To install /usr/local/lib/libz.* and /usr/local/include/zlib.h, type: -# make install -# To install in $HOME instead of /usr/local, use: -# make install prefix=$HOME - -CC=cc - -CFLAGS=-O -#CFLAGS=-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7 -#CFLAGS=-g -DDEBUG -#CFLAGS=-O3 -Wall -Wwrite-strings -Wpointer-arith -Wconversion \ -# -Wstrict-prototypes -Wmissing-prototypes - -LDFLAGS=libz.a -LDSHARED=$(CC) -CPP=$(CC) -E - -LIBS=libz.a -SHAREDLIB=libz.so -SHAREDLIBV=libz.so.1.2.3 -SHAREDLIBM=libz.so.1 - -AR=ar rc -RANLIB=ranlib -TAR=tar -SHELL=/bin/sh -EXE= - -prefix = /usr/local -exec_prefix = ${prefix} -libdir = ${exec_prefix}/lib -includedir = ${prefix}/include -mandir = ${prefix}/share/man -man3dir = ${mandir}/man3 - -OBJS = adler32.o compress.o crc32.o gzio.o uncompr.o deflate.o trees.o \ - zutil.o inflate.o infback.o inftrees.o inffast.o - -OBJA = -# to use the asm code: make OBJA=match.o - -TEST_OBJS = example.o minigzip.o - -all: example$(EXE) minigzip$(EXE) - -check: test -test: all - @LD_LIBRARY_PATH=.:$(LD_LIBRARY_PATH) ; export LD_LIBRARY_PATH; \ - echo hello world | ./minigzip | ./minigzip -d || \ - echo ' *** minigzip test FAILED ***' ; \ - if ./example; then \ - echo ' *** zlib test OK ***'; \ - else \ - echo ' *** zlib test FAILED ***'; \ - fi - -libz.a: $(OBJS) $(OBJA) - $(AR) $@ $(OBJS) $(OBJA) - -@ ($(RANLIB) $@ || true) >/dev/null 2>&1 - -match.o: match.S - $(CPP) match.S > _match.s - $(CC) -c _match.s - mv _match.o match.o - rm -f _match.s - -$(SHAREDLIBV): $(OBJS) - $(LDSHARED) -o $@ $(OBJS) - rm -f $(SHAREDLIB) $(SHAREDLIBM) - ln -s $@ $(SHAREDLIB) - ln -s $@ $(SHAREDLIBM) - -example$(EXE): example.o $(LIBS) - $(CC) $(CFLAGS) -o $@ example.o $(LDFLAGS) - -minigzip$(EXE): minigzip.o $(LIBS) - $(CC) $(CFLAGS) -o $@ minigzip.o $(LDFLAGS) - -install: $(LIBS) - - at if [ ! -d $(exec_prefix) ]; then mkdir -p $(exec_prefix); fi - - at if [ ! -d $(includedir) ]; then mkdir -p $(includedir); fi - - at if [ ! -d $(libdir) ]; then mkdir -p $(libdir); fi - - at if [ ! -d $(man3dir) ]; then mkdir -p $(man3dir); fi - cp zlib.h zconf.h $(includedir) - chmod 644 $(includedir)/zlib.h $(includedir)/zconf.h - cp $(LIBS) $(libdir) - cd $(libdir); chmod 755 $(LIBS) - -@(cd $(libdir); $(RANLIB) libz.a || true) >/dev/null 2>&1 - cd $(libdir); if test -f $(SHAREDLIBV); then \ - rm -f $(SHAREDLIB) $(SHAREDLIBM); \ - ln -s $(SHAREDLIBV) $(SHAREDLIB); \ - ln -s $(SHAREDLIBV) $(SHAREDLIBM); \ - (ldconfig || true) >/dev/null 2>&1; \ - fi - cp zlib.3 $(man3dir) - chmod 644 $(man3dir)/zlib.3 -# The ranlib in install is needed on NeXTSTEP which checks file times -# ldconfig is for Linux - -uninstall: - cd $(includedir); \ - cd $(libdir); rm -f libz.a; \ - if test -f $(SHAREDLIBV); then \ - rm -f $(SHAREDLIBV) $(SHAREDLIB) $(SHAREDLIBM); \ - fi - cd $(man3dir); rm -f zlib.3 - -mostlyclean: clean -clean: - rm -f *.o *~ example$(EXE) minigzip$(EXE) \ - libz.* foo.gz so_locations \ - _match.s maketree contrib/infback9/*.o - -maintainer-clean: distclean -distclean: clean - cp -p Makefile.in Makefile - cp -p zconf.in.h zconf.h - rm -f .DS_Store - -tags: - etags *.[ch] - -depend: - makedepend -- $(CFLAGS) -- *.[ch] - -# DO NOT DELETE THIS LINE -- make depend depends on it. - -adler32.o: zlib.h zconf.h -compress.o: zlib.h zconf.h -crc32.o: crc32.h zlib.h zconf.h -deflate.o: deflate.h zutil.h zlib.h zconf.h -example.o: zlib.h zconf.h -gzio.o: zutil.h zlib.h zconf.h -inffast.o: zutil.h zlib.h zconf.h inftrees.h inflate.h inffast.h -inflate.o: zutil.h zlib.h zconf.h inftrees.h inflate.h inffast.h -infback.o: zutil.h zlib.h zconf.h inftrees.h inflate.h inffast.h -inftrees.o: zutil.h zlib.h zconf.h inftrees.h -minigzip.o: zlib.h zconf.h -trees.o: deflate.h zutil.h zlib.h zconf.h trees.h -uncompr.o: zlib.h zconf.h -zutil.o: zutil.h zlib.h zconf.h +distclean: + make -f Makefile.in distclean diff --git a/Modules/zlib/Makefile.in b/Modules/zlib/Makefile.in --- a/Modules/zlib/Makefile.in +++ b/Modules/zlib/Makefile.in @@ -1,11 +1,11 @@ # Makefile for zlib -# Copyright (C) 1995-2005 Jean-loup Gailly. +# Copyright (C) 1995-2013 Jean-loup Gailly, Mark Adler # For conditions of distribution and use, see copyright notice in zlib.h # To compile and test, type: # ./configure; make test -# The call of configure is optional if you don't have special requirements -# If you wish to build zlib as a shared library, use: ./configure -s +# Normally configure builds both a static and a shared library. +# If you want to build just a static library, use: ./configure --static # To use the asm code, type: # cp contrib/asm?86/match.S ./match.S @@ -24,17 +24,23 @@ #CFLAGS=-O3 -Wall -Wwrite-strings -Wpointer-arith -Wconversion \ # -Wstrict-prototypes -Wmissing-prototypes -LDFLAGS=libz.a +SFLAGS=-O +LDFLAGS= +TEST_LDFLAGS=-L. libz.a LDSHARED=$(CC) CPP=$(CC) -E -LIBS=libz.a +STATICLIB=libz.a SHAREDLIB=libz.so -SHAREDLIBV=libz.so.1.2.3 +SHAREDLIBV=libz.so.1.2.8 SHAREDLIBM=libz.so.1 +LIBS=$(STATICLIB) $(SHAREDLIBV) -AR=ar rc +AR=ar +ARFLAGS=rc RANLIB=ranlib +LDCONFIG=ldconfig +LDSHAREDLIBC=-lc TAR=tar SHELL=/bin/sh EXE= @@ -42,33 +48,84 @@ prefix = /usr/local exec_prefix = ${prefix} libdir = ${exec_prefix}/lib +sharedlibdir = ${libdir} includedir = ${prefix}/include mandir = ${prefix}/share/man man3dir = ${mandir}/man3 +pkgconfigdir = ${libdir}/pkgconfig -OBJS = adler32.o compress.o crc32.o gzio.o uncompr.o deflate.o trees.o \ - zutil.o inflate.o infback.o inftrees.o inffast.o +OBJZ = adler32.o crc32.o deflate.o infback.o inffast.o inflate.o inftrees.o trees.o zutil.o +OBJG = compress.o uncompr.o gzclose.o gzlib.o gzread.o gzwrite.o +OBJC = $(OBJZ) $(OBJG) +PIC_OBJZ = adler32.lo crc32.lo deflate.lo infback.lo inffast.lo inflate.lo inftrees.lo trees.lo zutil.lo +PIC_OBJG = compress.lo uncompr.lo gzclose.lo gzlib.lo gzread.lo gzwrite.lo +PIC_OBJC = $(PIC_OBJZ) $(PIC_OBJG) + +# to use the asm code: make OBJA=match.o, PIC_OBJA=match.lo OBJA = -# to use the asm code: make OBJA=match.o +PIC_OBJA = -TEST_OBJS = example.o minigzip.o +OBJS = $(OBJC) $(OBJA) -all: example$(EXE) minigzip$(EXE) +PIC_OBJS = $(PIC_OBJC) $(PIC_OBJA) + +all: static shared + +static: example$(EXE) minigzip$(EXE) + +shared: examplesh$(EXE) minigzipsh$(EXE) + +all64: example64$(EXE) minigzip64$(EXE) check: test -test: all - @LD_LIBRARY_PATH=.:$(LD_LIBRARY_PATH) ; export LD_LIBRARY_PATH; \ - echo hello world | ./minigzip | ./minigzip -d || \ - echo ' *** minigzip test FAILED ***' ; \ - if ./example; then \ + +test: all teststatic testshared + +teststatic: static + @TMPST=tmpst_$$; \ + if echo hello world | ./minigzip | ./minigzip -d && ./example $$TMPST ; then \ echo ' *** zlib test OK ***'; \ else \ - echo ' *** zlib test FAILED ***'; \ - fi + echo ' *** zlib test FAILED ***'; false; \ + fi; \ + rm -f $$TMPST -libz.a: $(OBJS) $(OBJA) - $(AR) $@ $(OBJS) $(OBJA) +testshared: shared + @LD_LIBRARY_PATH=`pwd`:$(LD_LIBRARY_PATH) ; export LD_LIBRARY_PATH; \ + LD_LIBRARYN32_PATH=`pwd`:$(LD_LIBRARYN32_PATH) ; export LD_LIBRARYN32_PATH; \ + DYLD_LIBRARY_PATH=`pwd`:$(DYLD_LIBRARY_PATH) ; export DYLD_LIBRARY_PATH; \ + SHLIB_PATH=`pwd`:$(SHLIB_PATH) ; export SHLIB_PATH; \ + TMPSH=tmpsh_$$; \ + if echo hello world | ./minigzipsh | ./minigzipsh -d && ./examplesh $$TMPSH; then \ + echo ' *** zlib shared test OK ***'; \ + else \ + echo ' *** zlib shared test FAILED ***'; false; \ + fi; \ + rm -f $$TMPSH + +test64: all64 + @TMP64=tmp64_$$; \ + if echo hello world | ./minigzip64 | ./minigzip64 -d && ./example64 $$TMP64; then \ + echo ' *** zlib 64-bit test OK ***'; \ + else \ + echo ' *** zlib 64-bit test FAILED ***'; false; \ + fi; \ + rm -f $$TMP64 + +infcover.o: test/infcover.c zlib.h zconf.h + $(CC) $(CFLAGS) -I. -c -o $@ test/infcover.c + +infcover: infcover.o libz.a + $(CC) $(CFLAGS) -o $@ infcover.o libz.a + +cover: infcover + rm -f *.gcda + ./infcover + gcov inf*.c + +libz.a: $(OBJS) + $(AR) $(ARFLAGS) $@ $(OBJS) -@ ($(RANLIB) $@ || true) >/dev/null 2>&1 match.o: match.S @@ -77,58 +134,130 @@ mv _match.o match.o rm -f _match.s -$(SHAREDLIBV): $(OBJS) - $(LDSHARED) -o $@ $(OBJS) +match.lo: match.S + $(CPP) match.S > _match.s + $(CC) -c -fPIC _match.s + mv _match.o match.lo + rm -f _match.s + +example.o: test/example.c zlib.h zconf.h + $(CC) $(CFLAGS) -I. -c -o $@ test/example.c + +minigzip.o: test/minigzip.c zlib.h zconf.h + $(CC) $(CFLAGS) -I. -c -o $@ test/minigzip.c + +example64.o: test/example.c zlib.h zconf.h + $(CC) $(CFLAGS) -I. -D_FILE_OFFSET_BITS=64 -c -o $@ test/example.c + +minigzip64.o: test/minigzip.c zlib.h zconf.h + $(CC) $(CFLAGS) -I. -D_FILE_OFFSET_BITS=64 -c -o $@ test/minigzip.c + +.SUFFIXES: .lo + +.c.lo: + - at mkdir objs 2>/dev/null || test -d objs + $(CC) $(SFLAGS) -DPIC -c -o objs/$*.o $< + - at mv objs/$*.o $@ + +placebo $(SHAREDLIBV): $(PIC_OBJS) libz.a + $(LDSHARED) $(SFLAGS) -o $@ $(PIC_OBJS) $(LDSHAREDLIBC) $(LDFLAGS) rm -f $(SHAREDLIB) $(SHAREDLIBM) ln -s $@ $(SHAREDLIB) ln -s $@ $(SHAREDLIBM) + - at rmdir objs -example$(EXE): example.o $(LIBS) - $(CC) $(CFLAGS) -o $@ example.o $(LDFLAGS) +example$(EXE): example.o $(STATICLIB) + $(CC) $(CFLAGS) -o $@ example.o $(TEST_LDFLAGS) -minigzip$(EXE): minigzip.o $(LIBS) - $(CC) $(CFLAGS) -o $@ minigzip.o $(LDFLAGS) +minigzip$(EXE): minigzip.o $(STATICLIB) + $(CC) $(CFLAGS) -o $@ minigzip.o $(TEST_LDFLAGS) -install: $(LIBS) - - at if [ ! -d $(exec_prefix) ]; then mkdir -p $(exec_prefix); fi - - at if [ ! -d $(includedir) ]; then mkdir -p $(includedir); fi - - at if [ ! -d $(libdir) ]; then mkdir -p $(libdir); fi - - at if [ ! -d $(man3dir) ]; then mkdir -p $(man3dir); fi - cp zlib.h zconf.h $(includedir) - chmod 644 $(includedir)/zlib.h $(includedir)/zconf.h - cp $(LIBS) $(libdir) - cd $(libdir); chmod 755 $(LIBS) - -@(cd $(libdir); $(RANLIB) libz.a || true) >/dev/null 2>&1 - cd $(libdir); if test -f $(SHAREDLIBV); then \ - rm -f $(SHAREDLIB) $(SHAREDLIBM); \ - ln -s $(SHAREDLIBV) $(SHAREDLIB); \ - ln -s $(SHAREDLIBV) $(SHAREDLIBM); \ - (ldconfig || true) >/dev/null 2>&1; \ +examplesh$(EXE): example.o $(SHAREDLIBV) + $(CC) $(CFLAGS) -o $@ example.o -L. $(SHAREDLIBV) + +minigzipsh$(EXE): minigzip.o $(SHAREDLIBV) + $(CC) $(CFLAGS) -o $@ minigzip.o -L. $(SHAREDLIBV) + +example64$(EXE): example64.o $(STATICLIB) + $(CC) $(CFLAGS) -o $@ example64.o $(TEST_LDFLAGS) + +minigzip64$(EXE): minigzip64.o $(STATICLIB) + $(CC) $(CFLAGS) -o $@ minigzip64.o $(TEST_LDFLAGS) + +install-libs: $(LIBS) + - at if [ ! -d $(DESTDIR)$(exec_prefix) ]; then mkdir -p $(DESTDIR)$(exec_prefix); fi + - at if [ ! -d $(DESTDIR)$(libdir) ]; then mkdir -p $(DESTDIR)$(libdir); fi + - at if [ ! -d $(DESTDIR)$(sharedlibdir) ]; then mkdir -p $(DESTDIR)$(sharedlibdir); fi + - at if [ ! -d $(DESTDIR)$(man3dir) ]; then mkdir -p $(DESTDIR)$(man3dir); fi + - at if [ ! -d $(DESTDIR)$(pkgconfigdir) ]; then mkdir -p $(DESTDIR)$(pkgconfigdir); fi + cp $(STATICLIB) $(DESTDIR)$(libdir) + chmod 644 $(DESTDIR)$(libdir)/$(STATICLIB) + -@($(RANLIB) $(DESTDIR)$(libdir)/libz.a || true) >/dev/null 2>&1 + - at if test -n "$(SHAREDLIBV)"; then \ + cp $(SHAREDLIBV) $(DESTDIR)$(sharedlibdir); \ + echo "cp $(SHAREDLIBV) $(DESTDIR)$(sharedlibdir)"; \ + chmod 755 $(DESTDIR)$(sharedlibdir)/$(SHAREDLIBV); \ + echo "chmod 755 $(DESTDIR)$(sharedlibdir)/$(SHAREDLIBV)"; \ + rm -f $(DESTDIR)$(sharedlibdir)/$(SHAREDLIB) $(DESTDIR)$(sharedlibdir)/$(SHAREDLIBM); \ + ln -s $(SHAREDLIBV) $(DESTDIR)$(sharedlibdir)/$(SHAREDLIB); \ + ln -s $(SHAREDLIBV) $(DESTDIR)$(sharedlibdir)/$(SHAREDLIBM); \ + ($(LDCONFIG) || true) >/dev/null 2>&1; \ fi - cp zlib.3 $(man3dir) - chmod 644 $(man3dir)/zlib.3 + cp zlib.3 $(DESTDIR)$(man3dir) + chmod 644 $(DESTDIR)$(man3dir)/zlib.3 + cp zlib.pc $(DESTDIR)$(pkgconfigdir) + chmod 644 $(DESTDIR)$(pkgconfigdir)/zlib.pc # The ranlib in install is needed on NeXTSTEP which checks file times # ldconfig is for Linux +install: install-libs + - at if [ ! -d $(DESTDIR)$(includedir) ]; then mkdir -p $(DESTDIR)$(includedir); fi + cp zlib.h zconf.h $(DESTDIR)$(includedir) + chmod 644 $(DESTDIR)$(includedir)/zlib.h $(DESTDIR)$(includedir)/zconf.h + uninstall: - cd $(includedir); \ - cd $(libdir); rm -f libz.a; \ - if test -f $(SHAREDLIBV); then \ + cd $(DESTDIR)$(includedir) && rm -f zlib.h zconf.h + cd $(DESTDIR)$(libdir) && rm -f libz.a; \ + if test -n "$(SHAREDLIBV)" -a -f $(SHAREDLIBV); then \ rm -f $(SHAREDLIBV) $(SHAREDLIB) $(SHAREDLIBM); \ fi - cd $(man3dir); rm -f zlib.3 + cd $(DESTDIR)$(man3dir) && rm -f zlib.3 + cd $(DESTDIR)$(pkgconfigdir) && rm -f zlib.pc + +docs: zlib.3.pdf + +zlib.3.pdf: zlib.3 + groff -mandoc -f H -T ps zlib.3 | ps2pdf - zlib.3.pdf + +zconf.h.cmakein: zconf.h.in + -@ TEMPFILE=zconfh_$$; \ + echo "/#define ZCONF_H/ a\\\\\n#cmakedefine Z_PREFIX\\\\\n#cmakedefine Z_HAVE_UNISTD_H\n" >> $$TEMPFILE &&\ + sed -f $$TEMPFILE zconf.h.in > zconf.h.cmakein &&\ + touch -r zconf.h.in zconf.h.cmakein &&\ + rm $$TEMPFILE + +zconf: zconf.h.in + cp -p zconf.h.in zconf.h mostlyclean: clean clean: - rm -f *.o *~ example$(EXE) minigzip$(EXE) \ + rm -f *.o *.lo *~ \ + example$(EXE) minigzip$(EXE) examplesh$(EXE) minigzipsh$(EXE) \ + example64$(EXE) minigzip64$(EXE) \ + infcover \ libz.* foo.gz so_locations \ _match.s maketree contrib/infback9/*.o + rm -rf objs + rm -f *.gcda *.gcno *.gcov + rm -f contrib/infback9/*.gcda contrib/infback9/*.gcno contrib/infback9/*.gcov maintainer-clean: distclean -distclean: clean - cp -p Makefile.in Makefile - cp -p zconf.in.h zconf.h - rm -f .DS_Store +distclean: clean zconf zconf.h.cmakein docs + rm -f Makefile zlib.pc configure.log + - at rm -f .DS_Store + - at printf 'all:\n\t- at echo "Please use ./configure first. Thank you."\n' > Makefile + - at printf '\ndistclean:\n\tmake -f Makefile.in distclean\n' >> Makefile + - at touch -r Makefile.in Makefile tags: etags *.[ch] @@ -138,17 +267,22 @@ # DO NOT DELETE THIS LINE -- make depend depends on it. -adler32.o: zlib.h zconf.h -compress.o: zlib.h zconf.h -crc32.o: crc32.h zlib.h zconf.h +adler32.o zutil.o: zutil.h zlib.h zconf.h +gzclose.o gzlib.o gzread.o gzwrite.o: zlib.h zconf.h gzguts.h +compress.o example.o minigzip.o uncompr.o: zlib.h zconf.h +crc32.o: zutil.h zlib.h zconf.h crc32.h deflate.o: deflate.h zutil.h zlib.h zconf.h -example.o: zlib.h zconf.h -gzio.o: zutil.h zlib.h zconf.h +infback.o inflate.o: zutil.h zlib.h zconf.h inftrees.h inflate.h inffast.h inffixed.h inffast.o: zutil.h zlib.h zconf.h inftrees.h inflate.h inffast.h -inflate.o: zutil.h zlib.h zconf.h inftrees.h inflate.h inffast.h -infback.o: zutil.h zlib.h zconf.h inftrees.h inflate.h inffast.h inftrees.o: zutil.h zlib.h zconf.h inftrees.h -minigzip.o: zlib.h zconf.h trees.o: deflate.h zutil.h zlib.h zconf.h trees.h -uncompr.o: zlib.h zconf.h -zutil.o: zutil.h zlib.h zconf.h + +adler32.lo zutil.lo: zutil.h zlib.h zconf.h +gzclose.lo gzlib.lo gzread.lo gzwrite.lo: zlib.h zconf.h gzguts.h +compress.lo example.lo minigzip.lo uncompr.lo: zlib.h zconf.h +crc32.lo: zutil.h zlib.h zconf.h crc32.h +deflate.lo: deflate.h zutil.h zlib.h zconf.h +infback.lo inflate.lo: zutil.h zlib.h zconf.h inftrees.h inflate.h inffast.h inffixed.h +inffast.lo: zutil.h zlib.h zconf.h inftrees.h inflate.h inffast.h +inftrees.lo: zutil.h zlib.h zconf.h inftrees.h +trees.lo: deflate.h zutil.h zlib.h zconf.h trees.h diff --git a/Modules/zlib/README b/Modules/zlib/README --- a/Modules/zlib/README +++ b/Modules/zlib/README @@ -1,56 +1,52 @@ ZLIB DATA COMPRESSION LIBRARY -zlib 1.2.3 is a general purpose data compression library. All the code is +zlib 1.2.8 is a general purpose data compression library. All the code is thread safe. The data format used by the zlib library is described by RFCs (Request for Comments) 1950 to 1952 in the files -http://www.ietf.org/rfc/rfc1950.txt (zlib format), rfc1951.txt (deflate format) -and rfc1952.txt (gzip format). These documents are also available in other -formats from ftp://ftp.uu.net/graphics/png/documents/zlib/zdoc-index.html +http://tools.ietf.org/html/rfc1950 (zlib format), rfc1951 (deflate format) and +rfc1952 (gzip format). All functions of the compression library are documented in the file zlib.h -(volunteer to write man pages welcome, contact zlib at gzip.org). A usage example -of the library is given in the file example.c which also tests that the library -is working correctly. Another example is given in the file minigzip.c. The -compression library itself is composed of all source files except example.c and -minigzip.c. +(volunteer to write man pages welcome, contact zlib at gzip.org). A usage example +of the library is given in the file test/example.c which also tests that +the library is working correctly. Another example is given in the file +test/minigzip.c. The compression library itself is composed of all source +files in the root directory. To compile all files and run the test program, follow the instructions given at -the top of Makefile. In short "make test; make install" should work for most -machines. For Unix: "./configure; make test; make install". For MSDOS, use one -of the special makefiles such as Makefile.msc. For VMS, use make_vms.com. +the top of Makefile.in. In short "./configure; make test", and if that goes +well, "make install" should work for most flavors of Unix. For Windows, use +one of the special makefiles in win32/ or contrib/vstudio/ . For VMS, use +make_vms.com. Questions about zlib should be sent to , or to Gilles Vollant - for the Windows DLL version. The zlib home page is -http://www.zlib.org or http://www.gzip.org/zlib/ Before reporting a problem, -please check this site to verify that you have the latest version of zlib; -otherwise get the latest version and check whether the problem still exists or -not. + for the Windows DLL version. The zlib home page is +http://zlib.net/ . Before reporting a problem, please check this site to +verify that you have the latest version of zlib; otherwise get the latest +version and check whether the problem still exists or not. -PLEASE read the zlib FAQ http://www.gzip.org/zlib/zlib_faq.html before asking -for help. +PLEASE read the zlib FAQ http://zlib.net/zlib_faq.html before asking for help. -Mark Nelson wrote an article about zlib for the Jan. 1997 -issue of Dr. Dobb's Journal; a copy of the article is available in -http://dogma.net/markn/articles/zlibtool/zlibtool.htm +Mark Nelson wrote an article about zlib for the Jan. 1997 +issue of Dr. Dobb's Journal; a copy of the article is available at +http://marknelson.us/1997/01/01/zlib-engine/ . -The changes made in version 1.2.3 are documented in the file ChangeLog. +The changes made in version 1.2.8 are documented in the file ChangeLog. -Unsupported third party contributions are provided in directory "contrib". +Unsupported third party contributions are provided in directory contrib/ . -A Java implementation of zlib is available in the Java Development Kit -http://java.sun.com/j2se/1.4.2/docs/api/java/util/zip/package-summary.html -See the zlib home page http://www.zlib.org for details. +zlib is available in Java using the java.util.zip package, documented at +http://java.sun.com/developer/technicalArticles/Programming/compression/ . -A Perl interface to zlib written by Paul Marquess is in the -CPAN (Comprehensive Perl Archive Network) sites -http://www.cpan.org/modules/by-module/Compress/ +A Perl interface to zlib written by Paul Marquess is available +at CPAN (Comprehensive Perl Archive Network) sites, including +http://search.cpan.org/~pmqs/IO-Compress-Zlib/ . A Python interface to zlib written by A.M. Kuchling is available in Python 1.5 and later versions, see -http://www.python.org/doc/lib/module-zlib.html +http://docs.python.org/library/zlib.html . -A zlib binding for TCL written by Andreas Kupries is -availlable at http://www.oche.de/~akupries/soft/trf/trf_zip.html +zlib is built into tcl: http://wiki.tcl.tk/4610 . An experimental package to read and write files in .zip format, written on top of zlib by Gilles Vollant , is available in the @@ -74,25 +70,21 @@ - zlib doesn't work on HP-UX 9.05 with some versions of /bin/cc. It works with other compilers. Use "make test" to check your compiler. -- gzdopen is not supported on RISCOS, BEOS and by some Mac compilers. +- gzdopen is not supported on RISCOS or BEOS. - For PalmOs, see http://palmzlib.sourceforge.net/ -- When building a shared, i.e. dynamic library on Mac OS X, the library must be - installed before testing (do "make install" before "make test"), since the - library location is specified in the library. - Acknowledgments: - The deflate format used by zlib was defined by Phil Katz. The deflate - and zlib specifications were written by L. Peter Deutsch. Thanks to all the - people who reported problems and suggested various improvements in zlib; - they are too numerous to cite here. + The deflate format used by zlib was defined by Phil Katz. The deflate and + zlib specifications were written by L. Peter Deutsch. Thanks to all the + people who reported problems and suggested various improvements in zlib; they + are too numerous to cite here. Copyright notice: - (C) 1995-2004 Jean-loup Gailly and Mark Adler + (C) 1995-2013 Jean-loup Gailly and Mark Adler This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages @@ -113,13 +105,11 @@ Jean-loup Gailly Mark Adler jloup at gzip.org madler at alumni.caltech.edu -If you use the zlib library in a product, we would appreciate *not* -receiving lengthy legal documents to sign. The sources are provided -for free but without warranty of any kind. The library has been -entirely written by Jean-loup Gailly and Mark Adler; it does not -include third-party code. +If you use the zlib library in a product, we would appreciate *not* receiving +lengthy legal documents to sign. The sources are provided for free but without +warranty of any kind. The library has been entirely written by Jean-loup +Gailly and Mark Adler; it does not include third-party code. -If you redistribute modified sources, we would appreciate that you include -in the file ChangeLog history information documenting your changes. Please -read the FAQ for more information on the distribution of modified source -versions. +If you redistribute modified sources, we would appreciate that you include in +the file ChangeLog history information documenting your changes. Please read +the FAQ for more information on the distribution of modified source versions. diff --git a/Modules/zlib/adler32.c b/Modules/zlib/adler32.c --- a/Modules/zlib/adler32.c +++ b/Modules/zlib/adler32.c @@ -1,14 +1,17 @@ /* adler32.c -- compute the Adler-32 checksum of a data stream - * Copyright (C) 1995-2004 Mark Adler + * Copyright (C) 1995-2011 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ /* @(#) $Id$ */ -#define ZLIB_INTERNAL -#include "zlib.h" +#include "zutil.h" -#define BASE 65521UL /* largest prime smaller than 65536 */ +#define local static + +local uLong adler32_combine_ OF((uLong adler1, uLong adler2, z_off64_t len2)); + +#define BASE 65521 /* largest prime smaller than 65536 */ #define NMAX 5552 /* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */ @@ -18,39 +21,44 @@ #define DO8(buf,i) DO4(buf,i); DO4(buf,i+4); #define DO16(buf) DO8(buf,0); DO8(buf,8); -/* use NO_DIVIDE if your processor does not do division in hardware */ +/* use NO_DIVIDE if your processor does not do division in hardware -- + try it both ways to see which is faster */ #ifdef NO_DIVIDE +/* note that this assumes BASE is 65521, where 65536 % 65521 == 15 + (thank you to John Reiser for pointing this out) */ +# define CHOP(a) \ + do { \ + unsigned long tmp = a >> 16; \ + a &= 0xffffUL; \ + a += (tmp << 4) - tmp; \ + } while (0) +# define MOD28(a) \ + do { \ + CHOP(a); \ + if (a >= BASE) a -= BASE; \ + } while (0) # define MOD(a) \ do { \ - if (a >= (BASE << 16)) a -= (BASE << 16); \ - if (a >= (BASE << 15)) a -= (BASE << 15); \ - if (a >= (BASE << 14)) a -= (BASE << 14); \ - if (a >= (BASE << 13)) a -= (BASE << 13); \ - if (a >= (BASE << 12)) a -= (BASE << 12); \ - if (a >= (BASE << 11)) a -= (BASE << 11); \ - if (a >= (BASE << 10)) a -= (BASE << 10); \ - if (a >= (BASE << 9)) a -= (BASE << 9); \ - if (a >= (BASE << 8)) a -= (BASE << 8); \ - if (a >= (BASE << 7)) a -= (BASE << 7); \ - if (a >= (BASE << 6)) a -= (BASE << 6); \ - if (a >= (BASE << 5)) a -= (BASE << 5); \ - if (a >= (BASE << 4)) a -= (BASE << 4); \ - if (a >= (BASE << 3)) a -= (BASE << 3); \ - if (a >= (BASE << 2)) a -= (BASE << 2); \ - if (a >= (BASE << 1)) a -= (BASE << 1); \ - if (a >= BASE) a -= BASE; \ + CHOP(a); \ + MOD28(a); \ } while (0) -# define MOD4(a) \ - do { \ - if (a >= (BASE << 4)) a -= (BASE << 4); \ - if (a >= (BASE << 3)) a -= (BASE << 3); \ - if (a >= (BASE << 2)) a -= (BASE << 2); \ - if (a >= (BASE << 1)) a -= (BASE << 1); \ +# define MOD63(a) \ + do { /* this assumes a is not negative */ \ + z_off64_t tmp = a >> 32; \ + a &= 0xffffffffL; \ + a += (tmp << 8) - (tmp << 5) + tmp; \ + tmp = a >> 16; \ + a &= 0xffffL; \ + a += (tmp << 4) - tmp; \ + tmp = a >> 16; \ + a &= 0xffffL; \ + a += (tmp << 4) - tmp; \ if (a >= BASE) a -= BASE; \ } while (0) #else # define MOD(a) a %= BASE -# define MOD4(a) a %= BASE +# define MOD28(a) a %= BASE +# define MOD63(a) a %= BASE #endif /* ========================================================================= */ @@ -89,7 +97,7 @@ } if (adler >= BASE) adler -= BASE; - MOD4(sum2); /* only added so many BASE's */ + MOD28(sum2); /* only added so many BASE's */ return adler | (sum2 << 16); } @@ -125,25 +133,47 @@ } /* ========================================================================= */ +local uLong adler32_combine_(adler1, adler2, len2) + uLong adler1; + uLong adler2; + z_off64_t len2; +{ + unsigned long sum1; + unsigned long sum2; + unsigned rem; + + /* for negative len, return invalid adler32 as a clue for debugging */ + if (len2 < 0) + return 0xffffffffUL; + + /* the derivation of this formula is left as an exercise for the reader */ + MOD63(len2); /* assumes len2 >= 0 */ + rem = (unsigned)len2; + sum1 = adler1 & 0xffff; + sum2 = rem * sum1; + MOD(sum2); + sum1 += (adler2 & 0xffff) + BASE - 1; + sum2 += ((adler1 >> 16) & 0xffff) + ((adler2 >> 16) & 0xffff) + BASE - rem; + if (sum1 >= BASE) sum1 -= BASE; + if (sum1 >= BASE) sum1 -= BASE; + if (sum2 >= (BASE << 1)) sum2 -= (BASE << 1); + if (sum2 >= BASE) sum2 -= BASE; + return sum1 | (sum2 << 16); +} + +/* ========================================================================= */ uLong ZEXPORT adler32_combine(adler1, adler2, len2) uLong adler1; uLong adler2; z_off_t len2; { - unsigned long sum1; - unsigned long sum2; - unsigned rem; + return adler32_combine_(adler1, adler2, len2); +} - /* the derivation of this formula is left as an exercise for the reader */ - rem = (unsigned)(len2 % BASE); - sum1 = adler1 & 0xffff; - sum2 = rem * sum1; - MOD(sum2); - sum1 += (adler2 & 0xffff) + BASE - 1; - sum2 += ((adler1 >> 16) & 0xffff) + ((adler2 >> 16) & 0xffff) + BASE - rem; - if (sum1 > BASE) sum1 -= BASE; - if (sum1 > BASE) sum1 -= BASE; - if (sum2 > (BASE << 1)) sum2 -= (BASE << 1); - if (sum2 > BASE) sum2 -= BASE; - return sum1 | (sum2 << 16); +uLong ZEXPORT adler32_combine64(adler1, adler2, len2) + uLong adler1; + uLong adler2; + z_off64_t len2; +{ + return adler32_combine_(adler1, adler2, len2); } diff --git a/Modules/zlib/algorithm.txt b/Modules/zlib/algorithm.txt --- a/Modules/zlib/algorithm.txt +++ b/Modules/zlib/algorithm.txt @@ -121,7 +121,7 @@ kbytes. You can imagine that filling in a 2^15 entry table for a 15-bit code would take too long if you're only decoding several thousand symbols. At the other extreme, you could make a new table for every bit in the code. In fact, -that's essentially a Huffman tree. But then you spend two much time +that's essentially a Huffman tree. But then you spend too much time traversing the tree while decoding, even for short symbols. So the number of bits for the first lookup table is a trade of the time to @@ -206,4 +206,4 @@ pp. 337-343. ``DEFLATE Compressed Data Format Specification'' available in -http://www.ietf.org/rfc/rfc1951.txt +http://tools.ietf.org/html/rfc1951 diff --git a/Modules/zlib/compress.c b/Modules/zlib/compress.c --- a/Modules/zlib/compress.c +++ b/Modules/zlib/compress.c @@ -1,5 +1,5 @@ /* compress.c -- compress a memory buffer - * Copyright (C) 1995-2003 Jean-loup Gailly. + * Copyright (C) 1995-2005 Jean-loup Gailly. * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -29,7 +29,7 @@ z_stream stream; int err; - stream.next_in = (Bytef*)source; + stream.next_in = (z_const Bytef *)source; stream.avail_in = (uInt)sourceLen; #ifdef MAXSEG_64K /* Check for source > 64K on 16-bit machine: */ @@ -75,5 +75,6 @@ uLong ZEXPORT compressBound (sourceLen) uLong sourceLen; { - return sourceLen + (sourceLen >> 12) + (sourceLen >> 14) + 11; + return sourceLen + (sourceLen >> 12) + (sourceLen >> 14) + + (sourceLen >> 25) + 13; } diff --git a/Modules/zlib/configure b/Modules/zlib/configure --- a/Modules/zlib/configure +++ b/Modules/zlib/configure @@ -1,108 +1,239 @@ #!/bin/sh -# configure script for zlib. This script is needed only if -# you wish to build a shared library and your system supports them, -# of if you need special compiler, flags or install directory. -# Otherwise, you can just use directly "make test; make install" +# configure script for zlib. # -# To create a shared library, use "configure --shared"; by default a static -# library is created. If the primitive shared library support provided here -# does not work, use ftp://prep.ai.mit.edu/pub/gnu/libtool-*.tar.gz +# Normally configure builds both a static and a shared library. +# If you want to build just a static library, use: ./configure --static # # To impose specific compiler or flags or install directory, use for example: # prefix=$HOME CC=cc CFLAGS="-O4" ./configure # or for csh/tcsh users: # (setenv prefix $HOME; setenv CC cc; setenv CFLAGS "-O4"; ./configure) -# LDSHARED is the command to be used to create a shared library # Incorrect settings of CC or CFLAGS may prevent creating a shared library. # If you have problems, try without defining CC and CFLAGS before reporting # an error. -LIBS=libz.a -LDFLAGS="-L. ${LIBS}" +# start off configure.log +echo -------------------- >> configure.log +echo $0 $* >> configure.log +date >> configure.log + +# set command prefix for cross-compilation +if [ -n "${CHOST}" ]; then + uname="`echo "${CHOST}" | sed -e 's/^[^-]*-\([^-]*\)$/\1/' -e 's/^[^-]*-[^-]*-\([^-]*\)$/\1/' -e 's/^[^-]*-[^-]*-\([^-]*\)-.*$/\1/'`" + CROSS_PREFIX="${CHOST}-" +fi + +# destination name for static library +STATICLIB=libz.a + +# extract zlib version numbers from zlib.h VER=`sed -n -e '/VERSION "/s/.*"\(.*\)".*/\1/p' < zlib.h` +VER3=`sed -n -e '/VERSION "/s/.*"\([0-9]*\\.[0-9]*\\.[0-9]*\).*/\1/p' < zlib.h` VER2=`sed -n -e '/VERSION "/s/.*"\([0-9]*\\.[0-9]*\)\\..*/\1/p' < zlib.h` VER1=`sed -n -e '/VERSION "/s/.*"\([0-9]*\)\\..*/\1/p' < zlib.h` -AR=${AR-"ar rc"} -RANLIB=${RANLIB-"ranlib"} + +# establish commands for library building +if "${CROSS_PREFIX}ar" --version >/dev/null 2>/dev/null || test $? -lt 126; then + AR=${AR-"${CROSS_PREFIX}ar"} + test -n "${CROSS_PREFIX}" && echo Using ${AR} | tee -a configure.log +else + AR=${AR-"ar"} + test -n "${CROSS_PREFIX}" && echo Using ${AR} | tee -a configure.log +fi +ARFLAGS=${ARFLAGS-"rc"} +if "${CROSS_PREFIX}ranlib" --version >/dev/null 2>/dev/null || test $? -lt 126; then + RANLIB=${RANLIB-"${CROSS_PREFIX}ranlib"} + test -n "${CROSS_PREFIX}" && echo Using ${RANLIB} | tee -a configure.log +else + RANLIB=${RANLIB-"ranlib"} +fi +if "${CROSS_PREFIX}nm" --version >/dev/null 2>/dev/null || test $? -lt 126; then + NM=${NM-"${CROSS_PREFIX}nm"} + test -n "${CROSS_PREFIX}" && echo Using ${NM} | tee -a configure.log +else + NM=${NM-"nm"} +fi + +# set defaults before processing command line options +LDCONFIG=${LDCONFIG-"ldconfig"} +LDSHAREDLIBC="${LDSHAREDLIBC--lc}" +ARCHS= prefix=${prefix-/usr/local} exec_prefix=${exec_prefix-'${prefix}'} libdir=${libdir-'${exec_prefix}/lib'} +sharedlibdir=${sharedlibdir-'${libdir}'} includedir=${includedir-'${prefix}/include'} mandir=${mandir-'${prefix}/share/man'} shared_ext='.so' -shared=0 +shared=1 +solo=0 +cover=0 +zprefix=0 +zconst=0 +build64=0 gcc=0 old_cc="$CC" old_cflags="$CFLAGS" +OBJC='$(OBJZ) $(OBJG)' +PIC_OBJC='$(PIC_OBJZ) $(PIC_OBJG)' +# leave this script, optionally in a bad way +leave() +{ + if test "$*" != "0"; then + echo "** $0 aborting." | tee -a configure.log + fi + rm -f $test.[co] $test $test$shared_ext $test.gcno ./--version + echo -------------------- >> configure.log + echo >> configure.log + echo >> configure.log + exit $1 +} + +# process command line options while test $# -ge 1 do case "$1" in - -h* | --h*) - echo 'usage:' - echo ' configure [--shared] [--prefix=PREFIX] [--exec_prefix=EXPREFIX]' - echo ' [--libdir=LIBDIR] [--includedir=INCLUDEDIR]' - exit 0;; - -p*=* | --p*=*) prefix=`echo $1 | sed 's/[-a-z_]*=//'`; shift;; - -e*=* | --e*=*) exec_prefix=`echo $1 | sed 's/[-a-z_]*=//'`; shift;; - -l*=* | --libdir=*) libdir=`echo $1 | sed 's/[-a-z_]*=//'`; shift;; - -i*=* | --includedir=*) includedir=`echo $1 | sed 's/[-a-z_]*=//'`;shift;; - -p* | --p*) prefix="$2"; shift; shift;; - -e* | --e*) exec_prefix="$2"; shift; shift;; - -l* | --l*) libdir="$2"; shift; shift;; - -i* | --i*) includedir="$2"; shift; shift;; - -s* | --s*) shared=1; shift;; - *) echo "unknown option: $1"; echo "$0 --help for help"; exit 1;; + -h* | --help) + echo 'usage:' | tee -a configure.log + echo ' configure [--const] [--zprefix] [--prefix=PREFIX] [--eprefix=EXPREFIX]' | tee -a configure.log + echo ' [--static] [--64] [--libdir=LIBDIR] [--sharedlibdir=LIBDIR]' | tee -a configure.log + echo ' [--includedir=INCLUDEDIR] [--archs="-arch i386 -arch x86_64"]' | tee -a configure.log + exit 0 ;; + -p*=* | --prefix=*) prefix=`echo $1 | sed 's/.*=//'`; shift ;; + -e*=* | --eprefix=*) exec_prefix=`echo $1 | sed 's/.*=//'`; shift ;; + -l*=* | --libdir=*) libdir=`echo $1 | sed 's/.*=//'`; shift ;; + --sharedlibdir=*) sharedlibdir=`echo $1 | sed 's/.*=//'`; shift ;; + -i*=* | --includedir=*) includedir=`echo $1 | sed 's/.*=//'`;shift ;; + -u*=* | --uname=*) uname=`echo $1 | sed 's/.*=//'`;shift ;; + -p* | --prefix) prefix="$2"; shift; shift ;; + -e* | --eprefix) exec_prefix="$2"; shift; shift ;; + -l* | --libdir) libdir="$2"; shift; shift ;; + -i* | --includedir) includedir="$2"; shift; shift ;; + -s* | --shared | --enable-shared) shared=1; shift ;; + -t | --static) shared=0; shift ;; + --solo) solo=1; shift ;; + --cover) cover=1; shift ;; + -z* | --zprefix) zprefix=1; shift ;; + -6* | --64) build64=1; shift ;; + -a*=* | --archs=*) ARCHS=`echo $1 | sed 's/.*=//'`; shift ;; + --sysconfdir=*) echo "ignored option: --sysconfdir" | tee -a configure.log; shift ;; + --localstatedir=*) echo "ignored option: --localstatedir" | tee -a configure.log; shift ;; + -c* | --const) zconst=1; shift ;; + *) + echo "unknown option: $1" | tee -a configure.log + echo "$0 --help for help" | tee -a configure.log + leave 1;; esac done +# temporary file name test=ztest$$ + +# put arguments in log, also put test file in log if used in arguments +show() +{ + case "$*" in + *$test.c*) + echo === $test.c === >> configure.log + cat $test.c >> configure.log + echo === >> configure.log;; + esac + echo $* >> configure.log +} + +# check for gcc vs. cc and set compile and link flags based on the system identified by uname cat > $test.c <&1` in + *gcc*) gcc=1 ;; esac -if test "$gcc" -eq 1 && ($cc -c $cflags $test.c) 2>/dev/null; then +show $cc -c $test.c +if test "$gcc" -eq 1 && ($cc -c $test.c) >> configure.log 2>&1; then + echo ... using gcc >> configure.log CC="$cc" - SFLAGS=${CFLAGS-"-fPIC -O3"} - CFLAGS="$cflags" - case `(uname -s || echo unknown) 2>/dev/null` in - Linux | linux | GNU | GNU/*) LDSHARED=${LDSHARED-"$cc -shared -Wl,-soname,libz.so.1"};; - CYGWIN* | Cygwin* | cygwin* | OS/2* ) - EXE='.exe';; + CFLAGS="${CFLAGS--O3} ${ARCHS}" + SFLAGS="${CFLAGS--O3} -fPIC" + LDFLAGS="${LDFLAGS} ${ARCHS}" + if test $build64 -eq 1; then + CFLAGS="${CFLAGS} -m64" + SFLAGS="${SFLAGS} -m64" + fi + if test "${ZLIBGCCWARN}" = "YES"; then + if test "$zconst" -eq 1; then + CFLAGS="${CFLAGS} -Wall -Wextra -Wcast-qual -pedantic -DZLIB_CONST" + else + CFLAGS="${CFLAGS} -Wall -Wextra -pedantic" + fi + fi + if test -z "$uname"; then + uname=`(uname -s || echo unknown) 2>/dev/null` + fi + case "$uname" in + Linux* | linux* | GNU | GNU/* | solaris*) + LDSHARED=${LDSHARED-"$cc -shared -Wl,-soname,libz.so.1,--version-script,zlib.map"} ;; + *BSD | *bsd* | DragonFly) + LDSHARED=${LDSHARED-"$cc -shared -Wl,-soname,libz.so.1,--version-script,zlib.map"} + LDCONFIG="ldconfig -m" ;; + CYGWIN* | Cygwin* | cygwin* | OS/2*) + EXE='.exe' ;; + MINGW* | mingw*) +# temporary bypass + rm -f $test.[co] $test $test$shared_ext + echo "Please use win32/Makefile.gcc instead." | tee -a configure.log + leave 1 + LDSHARED=${LDSHARED-"$cc -shared"} + LDSHAREDLIBC="" + EXE='.exe' ;; QNX*) # This is for QNX6. I suppose that the QNX rule below is for QNX2,QNX4 # (alain.bonnefoy at icbt.com) - LDSHARED=${LDSHARED-"$cc -shared -Wl,-hlibz.so.1"};; + LDSHARED=${LDSHARED-"$cc -shared -Wl,-hlibz.so.1"} ;; HP-UX*) LDSHARED=${LDSHARED-"$cc -shared $SFLAGS"} case `(uname -m || echo unknown) 2>/dev/null` in ia64) shared_ext='.so' - SHAREDLIB='libz.so';; + SHAREDLIB='libz.so' ;; *) shared_ext='.sl' - SHAREDLIB='libz.sl';; - esac;; - Darwin*) shared_ext='.dylib' + SHAREDLIB='libz.sl' ;; + esac ;; + Darwin* | darwin*) + shared_ext='.dylib' SHAREDLIB=libz$shared_ext SHAREDLIBV=libz.$VER$shared_ext SHAREDLIBM=libz.$VER1$shared_ext - LDSHARED=${LDSHARED-"$cc -dynamiclib -install_name $libdir/$SHAREDLIBM -compatibility_version $VER1 -current_version $VER"};; - *) LDSHARED=${LDSHARED-"$cc -shared"};; + LDSHARED=${LDSHARED-"$cc -dynamiclib -install_name $libdir/$SHAREDLIBM -compatibility_version $VER1 -current_version $VER3"} + if libtool -V 2>&1 | grep Apple > /dev/null; then + AR="libtool" + else + AR="/usr/bin/libtool" + fi + ARFLAGS="-o" ;; + *) LDSHARED=${LDSHARED-"$cc -shared"} ;; esac else # find system name and corresponding cc options CC=${CC-cc} - case `(uname -sr || echo unknown) 2>/dev/null` in + gcc=0 + echo ... using $CC >> configure.log + if test -z "$uname"; then + uname=`(uname -sr || echo unknown) 2>/dev/null` + fi + case "$uname" in HP-UX*) SFLAGS=${CFLAGS-"-O +z"} CFLAGS=${CFLAGS-"-O"} # LDSHARED=${LDSHARED-"ld -b +vnocompatwarnings"} @@ -110,350 +241,591 @@ case `(uname -m || echo unknown) 2>/dev/null` in ia64) shared_ext='.so' - SHAREDLIB='libz.so';; + SHAREDLIB='libz.so' ;; *) shared_ext='.sl' - SHAREDLIB='libz.sl';; - esac;; + SHAREDLIB='libz.sl' ;; + esac ;; IRIX*) SFLAGS=${CFLAGS-"-ansi -O2 -rpath ."} CFLAGS=${CFLAGS-"-ansi -O2"} - LDSHARED=${LDSHARED-"cc -shared"};; + LDSHARED=${LDSHARED-"cc -shared -Wl,-soname,libz.so.1"} ;; OSF1\ V4*) SFLAGS=${CFLAGS-"-O -std1"} CFLAGS=${CFLAGS-"-O -std1"} - LDSHARED=${LDSHARED-"cc -shared -Wl,-soname,libz.so -Wl,-msym -Wl,-rpath,$(libdir) -Wl,-set_version,${VER}:1.0"};; + LDFLAGS="${LDFLAGS} -Wl,-rpath,." + LDSHARED=${LDSHARED-"cc -shared -Wl,-soname,libz.so -Wl,-msym -Wl,-rpath,$(libdir) -Wl,-set_version,${VER}:1.0"} ;; OSF1*) SFLAGS=${CFLAGS-"-O -std1"} CFLAGS=${CFLAGS-"-O -std1"} - LDSHARED=${LDSHARED-"cc -shared"};; + LDSHARED=${LDSHARED-"cc -shared -Wl,-soname,libz.so.1"} ;; QNX*) SFLAGS=${CFLAGS-"-4 -O"} CFLAGS=${CFLAGS-"-4 -O"} LDSHARED=${LDSHARED-"cc"} RANLIB=${RANLIB-"true"} - AR="cc -A";; + AR="cc" + ARFLAGS="-A" ;; SCO_SV\ 3.2*) SFLAGS=${CFLAGS-"-O3 -dy -KPIC "} CFLAGS=${CFLAGS-"-O3"} - LDSHARED=${LDSHARED-"cc -dy -KPIC -G"};; - SunOS\ 5*) SFLAGS=${CFLAGS-"-fast -xcg89 -KPIC -R."} - CFLAGS=${CFLAGS-"-fast -xcg89"} - LDSHARED=${LDSHARED-"cc -G"};; + LDSHARED=${LDSHARED-"cc -dy -KPIC -G"} ;; + SunOS\ 5* | solaris*) + LDSHARED=${LDSHARED-"cc -G -h libz$shared_ext.$VER1"} + SFLAGS=${CFLAGS-"-fast -KPIC"} + CFLAGS=${CFLAGS-"-fast"} + if test $build64 -eq 1; then + # old versions of SunPRO/Workshop/Studio don't support -m64, + # but newer ones do. Check for it. + flag64=`$CC -flags | egrep -- '^-m64'` + if test x"$flag64" != x"" ; then + CFLAGS="${CFLAGS} -m64" + SFLAGS="${SFLAGS} -m64" + else + case `(uname -m || echo unknown) 2>/dev/null` in + i86*) + SFLAGS="$SFLAGS -xarch=amd64" + CFLAGS="$CFLAGS -xarch=amd64" ;; + *) + SFLAGS="$SFLAGS -xarch=v9" + CFLAGS="$CFLAGS -xarch=v9" ;; + esac + fi + fi + ;; SunOS\ 4*) SFLAGS=${CFLAGS-"-O2 -PIC"} CFLAGS=${CFLAGS-"-O2"} - LDSHARED=${LDSHARED-"ld"};; - SunStudio\ 9*) SFLAGS=${CFLAGS-"-DUSE_MMAP -fast -xcode=pic32 -xtarget=ultra3 -xarch=v9b"} - CFLAGS=${CFLAGS-"-DUSE_MMAP -fast -xtarget=ultra3 -xarch=v9b"} - LDSHARED=${LDSHARED-"cc -xarch=v9b"};; + LDSHARED=${LDSHARED-"ld"} ;; + SunStudio\ 9*) SFLAGS=${CFLAGS-"-fast -xcode=pic32 -xtarget=ultra3 -xarch=v9b"} + CFLAGS=${CFLAGS-"-fast -xtarget=ultra3 -xarch=v9b"} + LDSHARED=${LDSHARED-"cc -xarch=v9b"} ;; UNIX_System_V\ 4.2.0) SFLAGS=${CFLAGS-"-KPIC -O"} CFLAGS=${CFLAGS-"-O"} - LDSHARED=${LDSHARED-"cc -G"};; + LDSHARED=${LDSHARED-"cc -G"} ;; UNIX_SV\ 4.2MP) SFLAGS=${CFLAGS-"-Kconform_pic -O"} CFLAGS=${CFLAGS-"-O"} - LDSHARED=${LDSHARED-"cc -G"};; + LDSHARED=${LDSHARED-"cc -G"} ;; OpenUNIX\ 5) SFLAGS=${CFLAGS-"-KPIC -O"} CFLAGS=${CFLAGS-"-O"} - LDSHARED=${LDSHARED-"cc -G"};; + LDSHARED=${LDSHARED-"cc -G"} ;; AIX*) # Courtesy of dbakker at arrayasolutions.com SFLAGS=${CFLAGS-"-O -qmaxmem=8192"} CFLAGS=${CFLAGS-"-O -qmaxmem=8192"} - LDSHARED=${LDSHARED-"xlc -G"};; - # send working options for other systems to support at gzip.org + LDSHARED=${LDSHARED-"xlc -G"} ;; + # send working options for other systems to zlib at gzip.org *) SFLAGS=${CFLAGS-"-O"} CFLAGS=${CFLAGS-"-O"} - LDSHARED=${LDSHARED-"cc -shared"};; + LDSHARED=${LDSHARED-"cc -shared"} ;; esac fi +# destination names for shared library if not defined above SHAREDLIB=${SHAREDLIB-"libz$shared_ext"} SHAREDLIBV=${SHAREDLIBV-"libz$shared_ext.$VER"} SHAREDLIBM=${SHAREDLIBM-"libz$shared_ext.$VER1"} +echo >> configure.log + +# define functions for testing compiler and library characteristics and logging the results + +cat > $test.c </dev/null; then + try() + { + show $* + test "`( $* ) 2>&1 | tee -a configure.log`" = "" + } + echo - using any output from compiler to indicate an error >> configure.log +else +try() +{ + show $* + ( $* ) >> configure.log 2>&1 + ret=$? + if test $ret -ne 0; then + echo "(exit code "$ret")" >> configure.log + fi + return $ret +} +fi + +tryboth() +{ + show $* + got=`( $* ) 2>&1` + ret=$? + printf %s "$got" >> configure.log + if test $ret -ne 0; then + return $ret + fi + test "$got" = "" +} + +cat > $test.c << EOF +int foo() { return 0; } +EOF +echo "Checking for obsessive-compulsive compiler options..." >> configure.log +if try $CC -c $CFLAGS $test.c; then + : +else + echo "Compiler error reporting is too harsh for $0 (perhaps remove -Werror)." | tee -a configure.log + leave 1 +fi + +echo >> configure.log + +# see if shared library build supported +cat > $test.c <&1`" = "" && - test "`($LDSHARED -o $test$shared_ext $test.o) 2>&1`" = ""; then - CFLAGS="$SFLAGS" - LIBS="$SHAREDLIBV" - echo Building shared library $SHAREDLIBV with $CC. + if try $CC -w -c $SFLAGS $test.c && + try $LDSHARED $SFLAGS -o $test$shared_ext $test.o; then + echo Building shared library $SHAREDLIBV with $CC. | tee -a configure.log elif test -z "$old_cc" -a -z "$old_cflags"; then - echo No shared library support. + echo No shared library support. | tee -a configure.log shared=0; else - echo 'No shared library support; try without defining CC and CFLAGS' + echo 'No shared library support; try without defining CC and CFLAGS' | tee -a configure.log shared=0; fi fi if test $shared -eq 0; then LDSHARED="$CC" - echo Building static library $LIBS version $VER with $CC. + ALL="static" + TEST="all teststatic" + SHAREDLIB="" + SHAREDLIBV="" + SHAREDLIBM="" + echo Building static library $STATICLIB version $VER with $CC. | tee -a configure.log else - LDFLAGS="-L. ${SHAREDLIBV}" + ALL="static shared" + TEST="all teststatic testshared" fi +# check for underscores in external names for use by assembler code +CPP=${CPP-"$CC -E"} +case $CFLAGS in + *ASMV*) + echo >> configure.log + show "$NM $test.o | grep _hello" + if test "`$NM $test.o | grep _hello | tee -a configure.log`" = ""; then + CPP="$CPP -DNO_UNDERLINE" + echo Checking for underline in external names... No. | tee -a configure.log + else + echo Checking for underline in external names... Yes. | tee -a configure.log + fi ;; +esac + +echo >> configure.log + +# check for large file support, and if none, check for fseeko() +cat > $test.c < +off64_t dummy = 0; +EOF +if try $CC -c $CFLAGS -D_LARGEFILE64_SOURCE=1 $test.c; then + CFLAGS="${CFLAGS} -D_LARGEFILE64_SOURCE=1" + SFLAGS="${SFLAGS} -D_LARGEFILE64_SOURCE=1" + ALL="${ALL} all64" + TEST="${TEST} test64" + echo "Checking for off64_t... Yes." | tee -a configure.log + echo "Checking for fseeko... Yes." | tee -a configure.log +else + echo "Checking for off64_t... No." | tee -a configure.log + echo >> configure.log + cat > $test.c < +int main(void) { + fseeko(NULL, 0, 0); + return 0; +} +EOF + if try $CC $CFLAGS -o $test $test.c; then + echo "Checking for fseeko... Yes." | tee -a configure.log + else + CFLAGS="${CFLAGS} -DNO_FSEEKO" + SFLAGS="${SFLAGS} -DNO_FSEEKO" + echo "Checking for fseeko... No." | tee -a configure.log + fi +fi + +echo >> configure.log + +# check for strerror() for use by gz* functions +cat > $test.c < +#include +int main() { return strlen(strerror(errno)); } +EOF +if try $CC $CFLAGS -o $test $test.c; then + echo "Checking for strerror... Yes." | tee -a configure.log +else + CFLAGS="${CFLAGS} -DNO_STRERROR" + SFLAGS="${SFLAGS} -DNO_STRERROR" + echo "Checking for strerror... No." | tee -a configure.log +fi + +# copy clean zconf.h for subsequent edits +cp -p zconf.h.in zconf.h + +echo >> configure.log + +# check for unistd.h and save result in zconf.h cat > $test.c < int main() { return 0; } EOF -if test "`($CC -c $CFLAGS $test.c) 2>&1`" = ""; then - sed < zconf.in.h "/HAVE_UNISTD_H/s%0%1%" > zconf.h - echo "Checking for unistd.h... Yes." +if try $CC -c $CFLAGS $test.c; then + sed < zconf.h "/^#ifdef HAVE_UNISTD_H.* may be/s/def HAVE_UNISTD_H\(.*\) may be/ 1\1 was/" > zconf.temp.h + mv zconf.temp.h zconf.h + echo "Checking for unistd.h... Yes." | tee -a configure.log else - cp -p zconf.in.h zconf.h - echo "Checking for unistd.h... No." + echo "Checking for unistd.h... No." | tee -a configure.log fi +echo >> configure.log + +# check for stdarg.h and save result in zconf.h +cat > $test.c < +int main() { return 0; } +EOF +if try $CC -c $CFLAGS $test.c; then + sed < zconf.h "/^#ifdef HAVE_STDARG_H.* may be/s/def HAVE_STDARG_H\(.*\) may be/ 1\1 was/" > zconf.temp.h + mv zconf.temp.h zconf.h + echo "Checking for stdarg.h... Yes." | tee -a configure.log +else + echo "Checking for stdarg.h... No." | tee -a configure.log +fi + +# if the z_ prefix was requested, save that in zconf.h +if test $zprefix -eq 1; then + sed < zconf.h "/#ifdef Z_PREFIX.* may be/s/def Z_PREFIX\(.*\) may be/ 1\1 was/" > zconf.temp.h + mv zconf.temp.h zconf.h + echo >> configure.log + echo "Using z_ prefix on all symbols." | tee -a configure.log +fi + +# if --solo compilation was requested, save that in zconf.h and remove gz stuff from object lists +if test $solo -eq 1; then + sed '/#define ZCONF_H/a\ +#define Z_SOLO + +' < zconf.h > zconf.temp.h + mv zconf.temp.h zconf.h +OBJC='$(OBJZ)' +PIC_OBJC='$(PIC_OBJZ)' +fi + +# if code coverage testing was requested, use older gcc if defined, e.g. "gcc-4.2" on Mac OS X +if test $cover -eq 1; then + CFLAGS="${CFLAGS} -fprofile-arcs -ftest-coverage" + if test -n "$GCC_CLASSIC"; then + CC=$GCC_CLASSIC + fi +fi + +echo >> configure.log + +# conduct a series of tests to resolve eight possible cases of using "vs" or "s" printf functions +# (using stdarg or not), with or without "n" (proving size of buffer), and with or without a +# return value. The most secure result is vsnprintf() with a return value. snprintf() with a +# return value is secure as well, but then gzprintf() will be limited to 20 arguments. cat > $test.c < #include #include "zconf.h" - int main() { #ifndef STDC choke me #endif - return 0; } EOF +if try $CC -c $CFLAGS $test.c; then + echo "Checking whether to use vs[n]printf() or s[n]printf()... using vs[n]printf()." | tee -a configure.log -if test "`($CC -c $CFLAGS $test.c) 2>&1`" = ""; then - echo "Checking whether to use vs[n]printf() or s[n]printf()... using vs[n]printf()" - + echo >> configure.log cat > $test.c < #include - -int mytest(char *fmt, ...) +int mytest(const char *fmt, ...) { char buf[20]; va_list ap; - va_start(ap, fmt); vsnprintf(buf, sizeof(buf), fmt, ap); va_end(ap); return 0; } +int main() +{ + return (mytest("Hello%d\n", 1)); +} +EOF + if try $CC $CFLAGS -o $test $test.c; then + echo "Checking for vsnprintf() in stdio.h... Yes." | tee -a configure.log + echo >> configure.log + cat >$test.c < +#include +int mytest(const char *fmt, ...) +{ + int n; + char buf[20]; + va_list ap; + va_start(ap, fmt); + n = vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + return n; +} int main() { return (mytest("Hello%d\n", 1)); } EOF - if test "`($CC $CFLAGS -o $test $test.c) 2>&1`" = ""; then - echo "Checking for vsnprintf() in stdio.h... Yes." + if try $CC -c $CFLAGS $test.c; then + echo "Checking for return value of vsnprintf()... Yes." | tee -a configure.log + else + CFLAGS="$CFLAGS -DHAS_vsnprintf_void" + SFLAGS="$SFLAGS -DHAS_vsnprintf_void" + echo "Checking for return value of vsnprintf()... No." | tee -a configure.log + echo " WARNING: apparently vsnprintf() does not return a value. zlib" | tee -a configure.log + echo " can build but will be open to possible string-format security" | tee -a configure.log + echo " vulnerabilities." | tee -a configure.log + fi + else + CFLAGS="$CFLAGS -DNO_vsnprintf" + SFLAGS="$SFLAGS -DNO_vsnprintf" + echo "Checking for vsnprintf() in stdio.h... No." | tee -a configure.log + echo " WARNING: vsnprintf() not found, falling back to vsprintf(). zlib" | tee -a configure.log + echo " can build but will be open to possible buffer-overflow security" | tee -a configure.log + echo " vulnerabilities." | tee -a configure.log + echo >> configure.log cat >$test.c < #include - -int mytest(char *fmt, ...) +int mytest(const char *fmt, ...) { int n; char buf[20]; va_list ap; - va_start(ap, fmt); - n = vsnprintf(buf, sizeof(buf), fmt, ap); + n = vsprintf(buf, fmt, ap); va_end(ap); return n; } - int main() { return (mytest("Hello%d\n", 1)); } EOF - if test "`($CC -c $CFLAGS $test.c) 2>&1`" = ""; then - echo "Checking for return value of vsnprintf()... Yes." - else - CFLAGS="$CFLAGS -DHAS_vsnprintf_void" - echo "Checking for return value of vsnprintf()... No." - echo " WARNING: apparently vsnprintf() does not return a value. zlib" - echo " can build but will be open to possible string-format security" - echo " vulnerabilities." - fi - else - CFLAGS="$CFLAGS -DNO_vsnprintf" - echo "Checking for vsnprintf() in stdio.h... No." - echo " WARNING: vsnprintf() not found, falling back to vsprintf(). zlib" - echo " can build but will be open to possible buffer-overflow security" - echo " vulnerabilities." - - cat >$test.c < -#include - -int mytest(char *fmt, ...) -{ - int n; - char buf[20]; - va_list ap; - - va_start(ap, fmt); - n = vsprintf(buf, fmt, ap); - va_end(ap); - return n; -} - -int main() -{ - return (mytest("Hello%d\n", 1)); -} -EOF - - if test "`($CC -c $CFLAGS $test.c) 2>&1`" = ""; then - echo "Checking for return value of vsprintf()... Yes." + if try $CC -c $CFLAGS $test.c; then + echo "Checking for return value of vsprintf()... Yes." | tee -a configure.log else CFLAGS="$CFLAGS -DHAS_vsprintf_void" - echo "Checking for return value of vsprintf()... No." - echo " WARNING: apparently vsprintf() does not return a value. zlib" - echo " can build but will be open to possible string-format security" - echo " vulnerabilities." + SFLAGS="$SFLAGS -DHAS_vsprintf_void" + echo "Checking for return value of vsprintf()... No." | tee -a configure.log + echo " WARNING: apparently vsprintf() does not return a value. zlib" | tee -a configure.log + echo " can build but will be open to possible string-format security" | tee -a configure.log + echo " vulnerabilities." | tee -a configure.log fi fi else - echo "Checking whether to use vs[n]printf() or s[n]printf()... using s[n]printf()" + echo "Checking whether to use vs[n]printf() or s[n]printf()... using s[n]printf()." | tee -a configure.log + echo >> configure.log cat >$test.c < - int mytest() { char buf[20]; - snprintf(buf, sizeof(buf), "%s", "foo"); return 0; } - int main() { return (mytest()); } EOF - if test "`($CC $CFLAGS -o $test $test.c) 2>&1`" = ""; then - echo "Checking for snprintf() in stdio.h... Yes." + if try $CC $CFLAGS -o $test $test.c; then + echo "Checking for snprintf() in stdio.h... Yes." | tee -a configure.log + echo >> configure.log cat >$test.c < - int mytest() { char buf[20]; - return snprintf(buf, sizeof(buf), "%s", "foo"); } - int main() { return (mytest()); } EOF - if test "`($CC -c $CFLAGS $test.c) 2>&1`" = ""; then - echo "Checking for return value of snprintf()... Yes." + if try $CC -c $CFLAGS $test.c; then + echo "Checking for return value of snprintf()... Yes." | tee -a configure.log else CFLAGS="$CFLAGS -DHAS_snprintf_void" - echo "Checking for return value of snprintf()... No." - echo " WARNING: apparently snprintf() does not return a value. zlib" - echo " can build but will be open to possible string-format security" - echo " vulnerabilities." + SFLAGS="$SFLAGS -DHAS_snprintf_void" + echo "Checking for return value of snprintf()... No." | tee -a configure.log + echo " WARNING: apparently snprintf() does not return a value. zlib" | tee -a configure.log + echo " can build but will be open to possible string-format security" | tee -a configure.log + echo " vulnerabilities." | tee -a configure.log fi else CFLAGS="$CFLAGS -DNO_snprintf" - echo "Checking for snprintf() in stdio.h... No." - echo " WARNING: snprintf() not found, falling back to sprintf(). zlib" - echo " can build but will be open to possible buffer-overflow security" - echo " vulnerabilities." + SFLAGS="$SFLAGS -DNO_snprintf" + echo "Checking for snprintf() in stdio.h... No." | tee -a configure.log + echo " WARNING: snprintf() not found, falling back to sprintf(). zlib" | tee -a configure.log + echo " can build but will be open to possible buffer-overflow security" | tee -a configure.log + echo " vulnerabilities." | tee -a configure.log + echo >> configure.log cat >$test.c < - int mytest() { char buf[20]; - return sprintf(buf, "%s", "foo"); } - int main() { return (mytest()); } EOF - if test "`($CC -c $CFLAGS $test.c) 2>&1`" = ""; then - echo "Checking for return value of sprintf()... Yes." + if try $CC -c $CFLAGS $test.c; then + echo "Checking for return value of sprintf()... Yes." | tee -a configure.log else CFLAGS="$CFLAGS -DHAS_sprintf_void" - echo "Checking for return value of sprintf()... No." - echo " WARNING: apparently sprintf() does not return a value. zlib" - echo " can build but will be open to possible string-format security" - echo " vulnerabilities." + SFLAGS="$SFLAGS -DHAS_sprintf_void" + echo "Checking for return value of sprintf()... No." | tee -a configure.log + echo " WARNING: apparently sprintf() does not return a value. zlib" | tee -a configure.log + echo " can build but will be open to possible string-format security" | tee -a configure.log + echo " vulnerabilities." | tee -a configure.log fi fi fi -cat >$test.c < -int main() { return 0; } +# see if we can hide zlib internal symbols that are linked between separate source files +if test "$gcc" -eq 1; then + echo >> configure.log + cat > $test.c <&1`" = ""; then - echo "Checking for errno.h... Yes." -else - echo "Checking for errno.h... No." - CFLAGS="$CFLAGS -DNO_ERRNO_H" + if tryboth $CC -c $CFLAGS $test.c; then + CFLAGS="$CFLAGS -DHAVE_HIDDEN" + SFLAGS="$SFLAGS -DHAVE_HIDDEN" + echo "Checking for attribute(visibility) support... Yes." | tee -a configure.log + else + echo "Checking for attribute(visibility) support... No." | tee -a configure.log + fi fi -cat > $test.c < -#include -#include -caddr_t hello() { - return mmap((caddr_t)0, (off_t)0, PROT_READ, MAP_SHARED, 0, (off_t)0); -} -EOF -if test "`($CC -c $CFLAGS $test.c) 2>&1`" = ""; then - CFLAGS="$CFLAGS -DUSE_MMAP" - echo Checking for mmap support... Yes. -else - echo Checking for mmap support... No. -fi +# show the results in the log +echo >> configure.log +echo ALL = $ALL >> configure.log +echo AR = $AR >> configure.log +echo ARFLAGS = $ARFLAGS >> configure.log +echo CC = $CC >> configure.log +echo CFLAGS = $CFLAGS >> configure.log +echo CPP = $CPP >> configure.log +echo EXE = $EXE >> configure.log +echo LDCONFIG = $LDCONFIG >> configure.log +echo LDFLAGS = $LDFLAGS >> configure.log +echo LDSHARED = $LDSHARED >> configure.log +echo LDSHAREDLIBC = $LDSHAREDLIBC >> configure.log +echo OBJC = $OBJC >> configure.log +echo PIC_OBJC = $PIC_OBJC >> configure.log +echo RANLIB = $RANLIB >> configure.log +echo SFLAGS = $SFLAGS >> configure.log +echo SHAREDLIB = $SHAREDLIB >> configure.log +echo SHAREDLIBM = $SHAREDLIBM >> configure.log +echo SHAREDLIBV = $SHAREDLIBV >> configure.log +echo STATICLIB = $STATICLIB >> configure.log +echo TEST = $TEST >> configure.log +echo VER = $VER >> configure.log +echo Z_U4 = $Z_U4 >> configure.log +echo exec_prefix = $exec_prefix >> configure.log +echo includedir = $includedir >> configure.log +echo libdir = $libdir >> configure.log +echo mandir = $mandir >> configure.log +echo prefix = $prefix >> configure.log +echo sharedlibdir = $sharedlibdir >> configure.log +echo uname = $uname >> configure.log -CPP=${CPP-"$CC -E"} -case $CFLAGS in - *ASMV*) - if test "`nm $test.o | grep _hello`" = ""; then - CPP="$CPP -DNO_UNDERLINE" - echo Checking for underline in external names... No. - else - echo Checking for underline in external names... Yes. - fi;; -esac +# udpate Makefile with the configure results +sed < Makefile.in " +/^CC *=/s#=.*#=$CC# +/^CFLAGS *=/s#=.*#=$CFLAGS# +/^SFLAGS *=/s#=.*#=$SFLAGS# +/^LDFLAGS *=/s#=.*#=$LDFLAGS# +/^LDSHARED *=/s#=.*#=$LDSHARED# +/^CPP *=/s#=.*#=$CPP# +/^STATICLIB *=/s#=.*#=$STATICLIB# +/^SHAREDLIB *=/s#=.*#=$SHAREDLIB# +/^SHAREDLIBV *=/s#=.*#=$SHAREDLIBV# +/^SHAREDLIBM *=/s#=.*#=$SHAREDLIBM# +/^AR *=/s#=.*#=$AR# +/^ARFLAGS *=/s#=.*#=$ARFLAGS# +/^RANLIB *=/s#=.*#=$RANLIB# +/^LDCONFIG *=/s#=.*#=$LDCONFIG# +/^LDSHAREDLIBC *=/s#=.*#=$LDSHAREDLIBC# +/^EXE *=/s#=.*#=$EXE# +/^prefix *=/s#=.*#=$prefix# +/^exec_prefix *=/s#=.*#=$exec_prefix# +/^libdir *=/s#=.*#=$libdir# +/^sharedlibdir *=/s#=.*#=$sharedlibdir# +/^includedir *=/s#=.*#=$includedir# +/^mandir *=/s#=.*#=$mandir# +/^OBJC *=/s#=.*#= $OBJC# +/^PIC_OBJC *=/s#=.*#= $PIC_OBJC# +/^all: */s#:.*#: $ALL# +/^test: */s#:.*#: $TEST# +" > Makefile -rm -f $test.[co] $test $test$shared_ext - -# udpate Makefile -sed < Makefile.in " +# create zlib.pc with the configure results +sed < zlib.pc.in " /^CC *=/s#=.*#=$CC# /^CFLAGS *=/s#=.*#=$CFLAGS# /^CPP *=/s#=.*#=$CPP# /^LDSHARED *=/s#=.*#=$LDSHARED# -/^LIBS *=/s#=.*#=$LIBS# +/^STATICLIB *=/s#=.*#=$STATICLIB# /^SHAREDLIB *=/s#=.*#=$SHAREDLIB# /^SHAREDLIBV *=/s#=.*#=$SHAREDLIBV# /^SHAREDLIBM *=/s#=.*#=$SHAREDLIBM# /^AR *=/s#=.*#=$AR# +/^ARFLAGS *=/s#=.*#=$ARFLAGS# /^RANLIB *=/s#=.*#=$RANLIB# /^EXE *=/s#=.*#=$EXE# /^prefix *=/s#=.*#=$prefix# /^exec_prefix *=/s#=.*#=$exec_prefix# /^libdir *=/s#=.*#=$libdir# +/^sharedlibdir *=/s#=.*#=$sharedlibdir# /^includedir *=/s#=.*#=$includedir# /^mandir *=/s#=.*#=$mandir# /^LDFLAGS *=/s#=.*#=$LDFLAGS# -" > Makefile +" | sed -e " +s/\@VERSION\@/$VER/g; +" > zlib.pc + +# done +leave 0 diff --git a/Modules/zlib/crc32.c b/Modules/zlib/crc32.c --- a/Modules/zlib/crc32.c +++ b/Modules/zlib/crc32.c @@ -1,5 +1,5 @@ /* crc32.c -- compute the CRC-32 of a data stream - * Copyright (C) 1995-2005 Mark Adler + * Copyright (C) 1995-2006, 2010, 2011, 2012 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h * * Thanks to Rodney Brown for his contribution of faster @@ -17,6 +17,8 @@ of the crc tables. Therefore, if you #define DYNAMIC_CRC_TABLE, you should first call get_crc_table() to initialize the tables before allowing more than one thread to use crc32(). + + DYNAMIC_CRC_TABLE and MAKECRCH can be #defined to write out crc32.h. */ #ifdef MAKECRCH @@ -30,31 +32,11 @@ #define local static -/* Find a four-byte integer type for crc32_little() and crc32_big(). */ -#ifndef NOBYFOUR -# ifdef STDC /* need ANSI C limits.h to determine sizes */ -# include -# define BYFOUR -# if (UINT_MAX == 0xffffffffUL) - typedef unsigned int u4; -# else -# if (ULONG_MAX == 0xffffffffUL) - typedef unsigned long u4; -# else -# if (USHRT_MAX == 0xffffffffUL) - typedef unsigned short u4; -# else -# undef BYFOUR /* can't find a four-byte integer type! */ -# endif -# endif -# endif -# endif /* STDC */ -#endif /* !NOBYFOUR */ - /* Definitions for doing the crc four data bytes at a time. */ +#if !defined(NOBYFOUR) && defined(Z_U4) +# define BYFOUR +#endif #ifdef BYFOUR -# define REV(w) (((w)>>24)+(((w)>>8)&0xff00)+ \ - (((w)&0xff00)<<8)+(((w)&0xff)<<24)) local unsigned long crc32_little OF((unsigned long, const unsigned char FAR *, unsigned)); local unsigned long crc32_big OF((unsigned long, @@ -68,14 +50,16 @@ local unsigned long gf2_matrix_times OF((unsigned long *mat, unsigned long vec)); local void gf2_matrix_square OF((unsigned long *square, unsigned long *mat)); +local uLong crc32_combine_ OF((uLong crc1, uLong crc2, z_off64_t len2)); + #ifdef DYNAMIC_CRC_TABLE local volatile int crc_table_empty = 1; -local unsigned long FAR crc_table[TBLS][256]; +local z_crc_t FAR crc_table[TBLS][256]; local void make_crc_table OF((void)); #ifdef MAKECRCH - local void write_table OF((FILE *, const unsigned long FAR *)); + local void write_table OF((FILE *, const z_crc_t FAR *)); #endif /* MAKECRCH */ /* Generate tables for a byte-wise 32-bit CRC calculation on the polynomial: @@ -105,9 +89,9 @@ */ local void make_crc_table() { - unsigned long c; + z_crc_t c; int n, k; - unsigned long poly; /* polynomial exclusive-or pattern */ + z_crc_t poly; /* polynomial exclusive-or pattern */ /* terms of polynomial defining this crc (except x^32): */ static volatile int first = 1; /* flag to limit concurrent making */ static const unsigned char p[] = {0,1,2,4,5,7,8,10,11,12,16,22,23,26}; @@ -119,13 +103,13 @@ first = 0; /* make exclusive-or pattern from polynomial (0xedb88320UL) */ - poly = 0UL; - for (n = 0; n < sizeof(p)/sizeof(unsigned char); n++) - poly |= 1UL << (31 - p[n]); + poly = 0; + for (n = 0; n < (int)(sizeof(p)/sizeof(unsigned char)); n++) + poly |= (z_crc_t)1 << (31 - p[n]); /* generate a crc for every 8-bit value */ for (n = 0; n < 256; n++) { - c = (unsigned long)n; + c = (z_crc_t)n; for (k = 0; k < 8; k++) c = c & 1 ? poly ^ (c >> 1) : c >> 1; crc_table[0][n] = c; @@ -136,11 +120,11 @@ and then the byte reversal of those as well as the first table */ for (n = 0; n < 256; n++) { c = crc_table[0][n]; - crc_table[4][n] = REV(c); + crc_table[4][n] = ZSWAP32(c); for (k = 1; k < 4; k++) { c = crc_table[0][c & 0xff] ^ (c >> 8); crc_table[k][n] = c; - crc_table[k + 4][n] = REV(c); + crc_table[k + 4][n] = ZSWAP32(c); } } #endif /* BYFOUR */ @@ -162,7 +146,7 @@ if (out == NULL) return; fprintf(out, "/* crc32.h -- tables for rapid CRC calculation\n"); fprintf(out, " * Generated automatically by crc32.c\n */\n\n"); - fprintf(out, "local const unsigned long FAR "); + fprintf(out, "local const z_crc_t FAR "); fprintf(out, "crc_table[TBLS][256] =\n{\n {\n"); write_table(out, crc_table[0]); # ifdef BYFOUR @@ -182,12 +166,13 @@ #ifdef MAKECRCH local void write_table(out, table) FILE *out; - const unsigned long FAR *table; + const z_crc_t FAR *table; { int n; for (n = 0; n < 256; n++) - fprintf(out, "%s0x%08lxUL%s", n % 5 ? "" : " ", table[n], + fprintf(out, "%s0x%08lxUL%s", n % 5 ? "" : " ", + (unsigned long)(table[n]), n == 255 ? "\n" : (n % 5 == 4 ? ",\n" : ", ")); } #endif /* MAKECRCH */ @@ -202,13 +187,13 @@ /* ========================================================================= * This function can be used by asm versions of crc32() */ -const unsigned long FAR * ZEXPORT get_crc_table() +const z_crc_t FAR * ZEXPORT get_crc_table() { #ifdef DYNAMIC_CRC_TABLE if (crc_table_empty) make_crc_table(); #endif /* DYNAMIC_CRC_TABLE */ - return (const unsigned long FAR *)crc_table; + return (const z_crc_t FAR *)crc_table; } /* ========================================================================= */ @@ -219,7 +204,7 @@ unsigned long ZEXPORT crc32(crc, buf, len) unsigned long crc; const unsigned char FAR *buf; - unsigned len; + uInt len; { if (buf == Z_NULL) return 0UL; @@ -230,7 +215,7 @@ #ifdef BYFOUR if (sizeof(void *) == sizeof(ptrdiff_t)) { - u4 endian; + z_crc_t endian; endian = 1; if (*((unsigned char *)(&endian))) @@ -264,17 +249,17 @@ const unsigned char FAR *buf; unsigned len; { - register u4 c; - register const u4 FAR *buf4; + register z_crc_t c; + register const z_crc_t FAR *buf4; - c = (u4)crc; + c = (z_crc_t)crc; c = ~c; while (len && ((ptrdiff_t)buf & 3)) { c = crc_table[0][(c ^ *buf++) & 0xff] ^ (c >> 8); len--; } - buf4 = (const u4 FAR *)(const void FAR *)buf; + buf4 = (const z_crc_t FAR *)(const void FAR *)buf; while (len >= 32) { DOLIT32; len -= 32; @@ -304,17 +289,17 @@ const unsigned char FAR *buf; unsigned len; { - register u4 c; - register const u4 FAR *buf4; + register z_crc_t c; + register const z_crc_t FAR *buf4; - c = REV((u4)crc); + c = ZSWAP32((z_crc_t)crc); c = ~c; while (len && ((ptrdiff_t)buf & 3)) { c = crc_table[4][(c >> 24) ^ *buf++] ^ (c << 8); len--; } - buf4 = (const u4 FAR *)(const void FAR *)buf; + buf4 = (const z_crc_t FAR *)(const void FAR *)buf; buf4--; while (len >= 32) { DOBIG32; @@ -331,7 +316,7 @@ c = crc_table[4][(c >> 24) ^ *buf++] ^ (c << 8); } while (--len); c = ~c; - return (unsigned long)(REV(c)); + return (unsigned long)(ZSWAP32(c)); } #endif /* BYFOUR */ @@ -367,22 +352,22 @@ } /* ========================================================================= */ -uLong ZEXPORT crc32_combine(crc1, crc2, len2) +local uLong crc32_combine_(crc1, crc2, len2) uLong crc1; uLong crc2; - z_off_t len2; + z_off64_t len2; { int n; unsigned long row; unsigned long even[GF2_DIM]; /* even-power-of-two zeros operator */ unsigned long odd[GF2_DIM]; /* odd-power-of-two zeros operator */ - /* degenerate case */ - if (len2 == 0) + /* degenerate case (also disallow negative lengths) */ + if (len2 <= 0) return crc1; /* put operator for one zero bit in odd */ - odd[0] = 0xedb88320L; /* CRC-32 polynomial */ + odd[0] = 0xedb88320UL; /* CRC-32 polynomial */ row = 1; for (n = 1; n < GF2_DIM; n++) { odd[n] = row; @@ -421,3 +406,20 @@ crc1 ^= crc2; return crc1; } + +/* ========================================================================= */ +uLong ZEXPORT crc32_combine(crc1, crc2, len2) + uLong crc1; + uLong crc2; + z_off_t len2; +{ + return crc32_combine_(crc1, crc2, len2); +} + +uLong ZEXPORT crc32_combine64(crc1, crc2, len2) + uLong crc1; + uLong crc2; + z_off64_t len2; +{ + return crc32_combine_(crc1, crc2, len2); +} diff --git a/Modules/zlib/crc32.h b/Modules/zlib/crc32.h --- a/Modules/zlib/crc32.h +++ b/Modules/zlib/crc32.h @@ -2,7 +2,7 @@ * Generated automatically by crc32.c */ -local const unsigned long FAR crc_table[TBLS][256] = +local const z_crc_t FAR crc_table[TBLS][256] = { { 0x00000000UL, 0x77073096UL, 0xee0e612cUL, 0x990951baUL, 0x076dc419UL, diff --git a/Modules/zlib/deflate.c b/Modules/zlib/deflate.c --- a/Modules/zlib/deflate.c +++ b/Modules/zlib/deflate.c @@ -1,5 +1,5 @@ /* deflate.c -- compress data using the deflation algorithm - * Copyright (C) 1995-2005 Jean-loup Gailly. + * Copyright (C) 1995-2013 Jean-loup Gailly and Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -37,7 +37,7 @@ * REFERENCES * * Deutsch, L.P.,"DEFLATE Compressed Data Format Specification". - * Available in http://www.ietf.org/rfc/rfc1951.txt + * Available in http://tools.ietf.org/html/rfc1951 * * A description of the Rabin and Karp algorithm is given in the book * "Algorithms" by R. Sedgewick, Addison-Wesley, p252. @@ -52,7 +52,7 @@ #include "deflate.h" const char deflate_copyright[] = - " deflate 1.2.3 Copyright 1995-2005 Jean-loup Gailly "; + " deflate 1.2.8 Copyright 1995-2013 Jean-loup Gailly and Mark Adler "; /* If you use the zlib library in a product, an acknowledgment is welcome in the documentation of your product. If for some reason you cannot @@ -79,19 +79,18 @@ #ifndef FASTEST local block_state deflate_slow OF((deflate_state *s, int flush)); #endif +local block_state deflate_rle OF((deflate_state *s, int flush)); +local block_state deflate_huff OF((deflate_state *s, int flush)); local void lm_init OF((deflate_state *s)); local void putShortMSB OF((deflate_state *s, uInt b)); local void flush_pending OF((z_streamp strm)); local int read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); -#ifndef FASTEST #ifdef ASMV void match_init OF((void)); /* asm code initialization */ uInt longest_match OF((deflate_state *s, IPos cur_match)); #else local uInt longest_match OF((deflate_state *s, IPos cur_match)); #endif -#endif -local uInt longest_match_fast OF((deflate_state *s, IPos cur_match)); #ifdef DEBUG local void check_match OF((deflate_state *s, IPos start, IPos match, @@ -110,11 +109,6 @@ #endif /* Matches of length 3 are discarded if their distance exceeds TOO_FAR */ -#define MIN_LOOKAHEAD (MAX_MATCH+MIN_MATCH+1) -/* Minimum amount of lookahead, except at the end of the input file. - * See deflate.c for comments about the MIN_MATCH+1. - */ - /* Values for max_lazy_match, good_match and max_chain_length, depending on * the desired pack level (0..9). The values given below have been tuned to * exclude worst case performance for pathological files. Better values may be @@ -161,9 +155,12 @@ struct static_tree_desc_s {int dummy;}; /* for buggy compilers */ #endif +/* rank Z_BLOCK between Z_NO_FLUSH and Z_PARTIAL_FLUSH */ +#define RANK(f) (((f) << 1) - ((f) > 4 ? 9 : 0)) + /* =========================================================================== * Update a hash value with the given input byte - * IN assertion: all calls to UPDATE_HASH are made with consecutive + * IN assertion: all calls to to UPDATE_HASH are made with consecutive * input characters, so that a running hash key can be computed from the * previous key instead of complete recalculation each time. */ @@ -176,7 +173,7 @@ * the previous length of the hash chain. * If this file is compiled with -DFASTEST, the compression level is forced * to 1, and no hash chains are maintained. - * IN assertion: all calls to INSERT_STRING are made with consecutive + * IN assertion: all calls to to INSERT_STRING are made with consecutive * input characters and the first MIN_MATCH bytes of str are valid * (except for the last MIN_MATCH-1 bytes of the input file). */ @@ -241,10 +238,19 @@ strm->msg = Z_NULL; if (strm->zalloc == (alloc_func)0) { +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else strm->zalloc = zcalloc; strm->opaque = (voidpf)0; +#endif } - if (strm->zfree == (free_func)0) strm->zfree = zcfree; + if (strm->zfree == (free_func)0) +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zfree = zcfree; +#endif #ifdef FASTEST if (level != 0) level = 1; @@ -288,6 +294,8 @@ s->prev = (Posf *) ZALLOC(strm, s->w_size, sizeof(Pos)); s->head = (Posf *) ZALLOC(strm, s->hash_size, sizeof(Pos)); + s->high_water = 0; /* nothing written to s->window yet */ + s->lit_bufsize = 1 << (memLevel + 6); /* 16K elements by default */ overlay = (ushf *) ZALLOC(strm, s->lit_bufsize, sizeof(ush)+2); @@ -297,7 +305,7 @@ if (s->window == Z_NULL || s->prev == Z_NULL || s->head == Z_NULL || s->pending_buf == Z_NULL) { s->status = FINISH_STATE; - strm->msg = (char*)ERR_MSG(Z_MEM_ERROR); + strm->msg = ERR_MSG(Z_MEM_ERROR); deflateEnd (strm); return Z_MEM_ERROR; } @@ -318,43 +326,70 @@ uInt dictLength; { deflate_state *s; - uInt length = dictLength; - uInt n; - IPos hash_head = 0; + uInt str, n; + int wrap; + unsigned avail; + z_const unsigned char *next; - if (strm == Z_NULL || strm->state == Z_NULL || dictionary == Z_NULL || - strm->state->wrap == 2 || - (strm->state->wrap == 1 && strm->state->status != INIT_STATE)) + if (strm == Z_NULL || strm->state == Z_NULL || dictionary == Z_NULL) + return Z_STREAM_ERROR; + s = strm->state; + wrap = s->wrap; + if (wrap == 2 || (wrap == 1 && s->status != INIT_STATE) || s->lookahead) return Z_STREAM_ERROR; - s = strm->state; - if (s->wrap) + /* when using zlib wrappers, compute Adler-32 for provided dictionary */ + if (wrap == 1) strm->adler = adler32(strm->adler, dictionary, dictLength); + s->wrap = 0; /* avoid computing Adler-32 in read_buf */ - if (length < MIN_MATCH) return Z_OK; - if (length > MAX_DIST(s)) { - length = MAX_DIST(s); - dictionary += dictLength - length; /* use the tail of the dictionary */ + /* if dictionary would fill window, just replace the history */ + if (dictLength >= s->w_size) { + if (wrap == 0) { /* already empty otherwise */ + CLEAR_HASH(s); + s->strstart = 0; + s->block_start = 0L; + s->insert = 0; + } + dictionary += dictLength - s->w_size; /* use the tail */ + dictLength = s->w_size; } - zmemcpy(s->window, dictionary, length); - s->strstart = length; - s->block_start = (long)length; - /* Insert all strings in the hash table (except for the last two bytes). - * s->lookahead stays null, so s->ins_h will be recomputed at the next - * call of fill_window. - */ - s->ins_h = s->window[0]; - UPDATE_HASH(s, s->ins_h, s->window[1]); - for (n = 0; n <= length - MIN_MATCH; n++) { - INSERT_STRING(s, n, hash_head); + /* insert dictionary into window and hash */ + avail = strm->avail_in; + next = strm->next_in; + strm->avail_in = dictLength; + strm->next_in = (z_const Bytef *)dictionary; + fill_window(s); + while (s->lookahead >= MIN_MATCH) { + str = s->strstart; + n = s->lookahead - (MIN_MATCH-1); + do { + UPDATE_HASH(s, s->ins_h, s->window[str + MIN_MATCH-1]); +#ifndef FASTEST + s->prev[str & s->w_mask] = s->head[s->ins_h]; +#endif + s->head[s->ins_h] = (Pos)str; + str++; + } while (--n); + s->strstart = str; + s->lookahead = MIN_MATCH-1; + fill_window(s); } - if (hash_head) hash_head = 0; /* to make compiler happy */ + s->strstart += s->lookahead; + s->block_start = (long)s->strstart; + s->insert = s->lookahead; + s->lookahead = 0; + s->match_length = s->prev_length = MIN_MATCH-1; + s->match_available = 0; + strm->next_in = next; + strm->avail_in = avail; + s->wrap = wrap; return Z_OK; } /* ========================================================================= */ -int ZEXPORT deflateReset (strm) +int ZEXPORT deflateResetKeep (strm) z_streamp strm; { deflate_state *s; @@ -384,12 +419,23 @@ s->last_flush = Z_NO_FLUSH; _tr_init(s); - lm_init(s); return Z_OK; } /* ========================================================================= */ +int ZEXPORT deflateReset (strm) + z_streamp strm; +{ + int ret; + + ret = deflateResetKeep(strm); + if (ret == Z_OK) + lm_init(strm->state); + return ret; +} + +/* ========================================================================= */ int ZEXPORT deflateSetHeader (strm, head) z_streamp strm; gz_headerp head; @@ -401,14 +447,42 @@ } /* ========================================================================= */ +int ZEXPORT deflatePending (strm, pending, bits) + unsigned *pending; + int *bits; + z_streamp strm; +{ + if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (pending != Z_NULL) + *pending = strm->state->pending; + if (bits != Z_NULL) + *bits = strm->state->bi_valid; + return Z_OK; +} + +/* ========================================================================= */ int ZEXPORT deflatePrime (strm, bits, value) z_streamp strm; int bits; int value; { + deflate_state *s; + int put; + if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; - strm->state->bi_valid = bits; - strm->state->bi_buf = (ush)(value & ((1 << bits) - 1)); + s = strm->state; + if ((Bytef *)(s->d_buf) < s->pending_out + ((Buf_size + 7) >> 3)) + return Z_BUF_ERROR; + do { + put = Buf_size - s->bi_valid; + if (put > bits) + put = bits; + s->bi_buf |= (ush)((value & ((1 << put) - 1)) << s->bi_valid); + s->bi_valid += put; + _tr_flush_bits(s); + value >>= put; + bits -= put; + } while (bits); return Z_OK; } @@ -435,9 +509,12 @@ } func = configuration_table[s->level].func; - if (func != configuration_table[level].func && strm->total_in != 0) { + if ((strategy != s->strategy || func != configuration_table[level].func) && + strm->total_in != 0) { /* Flush the last buffer: */ - err = deflate(strm, Z_PARTIAL_FLUSH); + err = deflate(strm, Z_BLOCK); + if (err == Z_BUF_ERROR && s->pending == 0) + err = Z_OK; } if (s->level != level) { s->level = level; @@ -481,33 +558,66 @@ * resulting from using fixed blocks instead of stored blocks, which deflate * can emit on compressed data for some combinations of the parameters. * - * This function could be more sophisticated to provide closer upper bounds - * for every combination of windowBits and memLevel, as well as wrap. - * But even the conservative upper bound of about 14% expansion does not - * seem onerous for output buffer allocation. + * This function could be more sophisticated to provide closer upper bounds for + * every combination of windowBits and memLevel. But even the conservative + * upper bound of about 14% expansion does not seem onerous for output buffer + * allocation. */ uLong ZEXPORT deflateBound(strm, sourceLen) z_streamp strm; uLong sourceLen; { deflate_state *s; - uLong destLen; + uLong complen, wraplen; + Bytef *str; - /* conservative upper bound */ - destLen = sourceLen + - ((sourceLen + 7) >> 3) + ((sourceLen + 63) >> 6) + 11; + /* conservative upper bound for compressed data */ + complen = sourceLen + + ((sourceLen + 7) >> 3) + ((sourceLen + 63) >> 6) + 5; - /* if can't get parameters, return conservative bound */ + /* if can't get parameters, return conservative bound plus zlib wrapper */ if (strm == Z_NULL || strm->state == Z_NULL) - return destLen; + return complen + 6; + + /* compute wrapper length */ + s = strm->state; + switch (s->wrap) { + case 0: /* raw deflate */ + wraplen = 0; + break; + case 1: /* zlib wrapper */ + wraplen = 6 + (s->strstart ? 4 : 0); + break; + case 2: /* gzip wrapper */ + wraplen = 18; + if (s->gzhead != Z_NULL) { /* user-supplied gzip header */ + if (s->gzhead->extra != Z_NULL) + wraplen += 2 + s->gzhead->extra_len; + str = s->gzhead->name; + if (str != Z_NULL) + do { + wraplen++; + } while (*str++); + str = s->gzhead->comment; + if (str != Z_NULL) + do { + wraplen++; + } while (*str++); + if (s->gzhead->hcrc) + wraplen += 2; + } + break; + default: /* for compiler happiness */ + wraplen = 6; + } /* if not default parameters, return conservative bound */ - s = strm->state; if (s->w_bits != 15 || s->hash_bits != 8 + 7) - return destLen; + return complen + wraplen; /* default settings: return tight bound for that case */ - return compressBound(sourceLen); + return sourceLen + (sourceLen >> 12) + (sourceLen >> 14) + + (sourceLen >> 25) + 13 - 6 + wraplen; } /* ========================================================================= @@ -532,19 +642,22 @@ local void flush_pending(strm) z_streamp strm; { - unsigned len = strm->state->pending; + unsigned len; + deflate_state *s = strm->state; + _tr_flush_bits(s); + len = s->pending; if (len > strm->avail_out) len = strm->avail_out; if (len == 0) return; - zmemcpy(strm->next_out, strm->state->pending_out, len); + zmemcpy(strm->next_out, s->pending_out, len); strm->next_out += len; - strm->state->pending_out += len; + s->pending_out += len; strm->total_out += len; strm->avail_out -= len; - strm->state->pending -= len; - if (strm->state->pending == 0) { - strm->state->pending_out = strm->state->pending_buf; + s->pending -= len; + if (s->pending == 0) { + s->pending_out = s->pending_buf; } } @@ -557,7 +670,7 @@ deflate_state *s; if (strm == Z_NULL || strm->state == Z_NULL || - flush > Z_FINISH || flush < 0) { + flush > Z_BLOCK || flush < 0) { return Z_STREAM_ERROR; } s = strm->state; @@ -581,7 +694,7 @@ put_byte(s, 31); put_byte(s, 139); put_byte(s, 8); - if (s->gzhead == NULL) { + if (s->gzhead == Z_NULL) { put_byte(s, 0); put_byte(s, 0); put_byte(s, 0); @@ -608,7 +721,7 @@ (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2 ? 4 : 0)); put_byte(s, s->gzhead->os & 0xff); - if (s->gzhead->extra != NULL) { + if (s->gzhead->extra != Z_NULL) { put_byte(s, s->gzhead->extra_len & 0xff); put_byte(s, (s->gzhead->extra_len >> 8) & 0xff); } @@ -650,7 +763,7 @@ } #ifdef GZIP if (s->status == EXTRA_STATE) { - if (s->gzhead->extra != NULL) { + if (s->gzhead->extra != Z_NULL) { uInt beg = s->pending; /* start of bytes to update crc */ while (s->gzindex < (s->gzhead->extra_len & 0xffff)) { @@ -678,7 +791,7 @@ s->status = NAME_STATE; } if (s->status == NAME_STATE) { - if (s->gzhead->name != NULL) { + if (s->gzhead->name != Z_NULL) { uInt beg = s->pending; /* start of bytes to update crc */ int val; @@ -709,7 +822,7 @@ s->status = COMMENT_STATE; } if (s->status == COMMENT_STATE) { - if (s->gzhead->comment != NULL) { + if (s->gzhead->comment != Z_NULL) { uInt beg = s->pending; /* start of bytes to update crc */ int val; @@ -771,7 +884,7 @@ * flushes. For repeated and useless calls with Z_FINISH, we keep * returning Z_STREAM_END instead of Z_BUF_ERROR. */ - } else if (strm->avail_in == 0 && flush <= old_flush && + } else if (strm->avail_in == 0 && RANK(flush) <= RANK(old_flush) && flush != Z_FINISH) { ERR_RETURN(strm, Z_BUF_ERROR); } @@ -787,7 +900,9 @@ (flush != Z_NO_FLUSH && s->status != FINISH_STATE)) { block_state bstate; - bstate = (*(configuration_table[s->level].func))(s, flush); + bstate = s->strategy == Z_HUFFMAN_ONLY ? deflate_huff(s, flush) : + (s->strategy == Z_RLE ? deflate_rle(s, flush) : + (*(configuration_table[s->level].func))(s, flush)); if (bstate == finish_started || bstate == finish_done) { s->status = FINISH_STATE; @@ -808,13 +923,18 @@ if (bstate == block_done) { if (flush == Z_PARTIAL_FLUSH) { _tr_align(s); - } else { /* FULL_FLUSH or SYNC_FLUSH */ + } else if (flush != Z_BLOCK) { /* FULL_FLUSH or SYNC_FLUSH */ _tr_stored_block(s, (char*)0, 0L, 0); /* For a full flush, this empty block will be recognized * as a special marker by inflate_sync(). */ if (flush == Z_FULL_FLUSH) { CLEAR_HASH(s); /* forget history */ + if (s->lookahead == 0) { + s->strstart = 0; + s->block_start = 0L; + s->insert = 0; + } } } flush_pending(strm); @@ -909,12 +1029,12 @@ ss = source->state; - zmemcpy(dest, source, sizeof(z_stream)); + zmemcpy((voidpf)dest, (voidpf)source, sizeof(z_stream)); ds = (deflate_state *) ZALLOC(dest, 1, sizeof(deflate_state)); if (ds == Z_NULL) return Z_MEM_ERROR; dest->state = (struct internal_state FAR *) ds; - zmemcpy(ds, ss, sizeof(deflate_state)); + zmemcpy((voidpf)ds, (voidpf)ss, sizeof(deflate_state)); ds->strm = dest; ds->window = (Bytef *) ZALLOC(dest, ds->w_size, 2*sizeof(Byte)); @@ -930,8 +1050,8 @@ } /* following zmemcpy do not work for 16-bit MSDOS */ zmemcpy(ds->window, ss->window, ds->w_size * 2 * sizeof(Byte)); - zmemcpy(ds->prev, ss->prev, ds->w_size * sizeof(Pos)); - zmemcpy(ds->head, ss->head, ds->hash_size * sizeof(Pos)); + zmemcpy((voidpf)ds->prev, (voidpf)ss->prev, ds->w_size * sizeof(Pos)); + zmemcpy((voidpf)ds->head, (voidpf)ss->head, ds->hash_size * sizeof(Pos)); zmemcpy(ds->pending_buf, ss->pending_buf, (uInt)ds->pending_buf_size); ds->pending_out = ds->pending_buf + (ss->pending_out - ss->pending_buf); @@ -965,15 +1085,15 @@ strm->avail_in -= len; + zmemcpy(buf, strm->next_in, len); if (strm->state->wrap == 1) { - strm->adler = adler32(strm->adler, strm->next_in, len); + strm->adler = adler32(strm->adler, buf, len); } #ifdef GZIP else if (strm->state->wrap == 2) { - strm->adler = crc32(strm->adler, strm->next_in, len); + strm->adler = crc32(strm->adler, buf, len); } #endif - zmemcpy(buf, strm->next_in, len); strm->next_in += len; strm->total_in += len; @@ -1000,6 +1120,7 @@ s->strstart = 0; s->block_start = 0L; s->lookahead = 0; + s->insert = 0; s->match_length = s->prev_length = MIN_MATCH-1; s->match_available = 0; s->ins_h = 0; @@ -1167,12 +1288,13 @@ return s->lookahead; } #endif /* ASMV */ -#endif /* FASTEST */ + +#else /* FASTEST */ /* --------------------------------------------------------------------------- - * Optimized version for level == 1 or strategy == Z_RLE only + * Optimized version for FASTEST only */ -local uInt longest_match_fast(s, cur_match) +local uInt longest_match(s, cur_match) deflate_state *s; IPos cur_match; /* current match */ { @@ -1225,6 +1347,8 @@ return (uInt)len <= s->lookahead ? (uInt)len : s->lookahead; } +#endif /* FASTEST */ + #ifdef DEBUG /* =========================================================================== * Check that the match at match_start is indeed a match. @@ -1271,6 +1395,8 @@ unsigned more; /* Amount of free space at the end of the window. */ uInt wsize = s->w_size; + Assert(s->lookahead < MIN_LOOKAHEAD, "already enough lookahead"); + do { more = (unsigned)(s->window_size -(ulg)s->lookahead -(ulg)s->strstart); @@ -1303,7 +1429,6 @@ later. (Using level 0 permanently is not an optimal usage of zlib, so we don't care about this pathological case.) */ - /* %%% avoid this when Z_RLE */ n = s->hash_size; p = &s->head[n]; do { @@ -1324,7 +1449,7 @@ #endif more += wsize; } - if (s->strm->avail_in == 0) return; + if (s->strm->avail_in == 0) break; /* If there was no sliding: * strstart <= WSIZE+MAX_DIST-1 && lookahead <= MIN_LOOKAHEAD - 1 && @@ -1343,39 +1468,88 @@ s->lookahead += n; /* Initialize the hash value now that we have some input: */ - if (s->lookahead >= MIN_MATCH) { - s->ins_h = s->window[s->strstart]; - UPDATE_HASH(s, s->ins_h, s->window[s->strstart+1]); + if (s->lookahead + s->insert >= MIN_MATCH) { + uInt str = s->strstart - s->insert; + s->ins_h = s->window[str]; + UPDATE_HASH(s, s->ins_h, s->window[str + 1]); #if MIN_MATCH != 3 Call UPDATE_HASH() MIN_MATCH-3 more times #endif + while (s->insert) { + UPDATE_HASH(s, s->ins_h, s->window[str + MIN_MATCH-1]); +#ifndef FASTEST + s->prev[str & s->w_mask] = s->head[s->ins_h]; +#endif + s->head[s->ins_h] = (Pos)str; + str++; + s->insert--; + if (s->lookahead + s->insert < MIN_MATCH) + break; + } } /* If the whole input has less than MIN_MATCH bytes, ins_h is garbage, * but this is not important since only literal bytes will be emitted. */ } while (s->lookahead < MIN_LOOKAHEAD && s->strm->avail_in != 0); + + /* If the WIN_INIT bytes after the end of the current data have never been + * written, then zero those bytes in order to avoid memory check reports of + * the use of uninitialized (or uninitialised as Julian writes) bytes by + * the longest match routines. Update the high water mark for the next + * time through here. WIN_INIT is set to MAX_MATCH since the longest match + * routines allow scanning to strstart + MAX_MATCH, ignoring lookahead. + */ + if (s->high_water < s->window_size) { + ulg curr = s->strstart + (ulg)(s->lookahead); + ulg init; + + if (s->high_water < curr) { + /* Previous high water mark below current data -- zero WIN_INIT + * bytes or up to end of window, whichever is less. + */ + init = s->window_size - curr; + if (init > WIN_INIT) + init = WIN_INIT; + zmemzero(s->window + curr, (unsigned)init); + s->high_water = curr + init; + } + else if (s->high_water < (ulg)curr + WIN_INIT) { + /* High water mark at or above current data, but below current data + * plus WIN_INIT -- zero out to current data plus WIN_INIT, or up + * to end of window, whichever is less. + */ + init = (ulg)curr + WIN_INIT - s->high_water; + if (init > s->window_size - s->high_water) + init = s->window_size - s->high_water; + zmemzero(s->window + s->high_water, (unsigned)init); + s->high_water += init; + } + } + + Assert((ulg)s->strstart <= s->window_size - MIN_LOOKAHEAD, + "not enough room for search"); } /* =========================================================================== * Flush the current block, with given end-of-file flag. * IN assertion: strstart is set to the end of the current match. */ -#define FLUSH_BLOCK_ONLY(s, eof) { \ +#define FLUSH_BLOCK_ONLY(s, last) { \ _tr_flush_block(s, (s->block_start >= 0L ? \ (charf *)&s->window[(unsigned)s->block_start] : \ (charf *)Z_NULL), \ (ulg)((long)s->strstart - s->block_start), \ - (eof)); \ + (last)); \ s->block_start = s->strstart; \ flush_pending(s->strm); \ Tracev((stderr,"[FLUSH]")); \ } /* Same but force premature exit if necessary. */ -#define FLUSH_BLOCK(s, eof) { \ - FLUSH_BLOCK_ONLY(s, eof); \ - if (s->strm->avail_out == 0) return (eof) ? finish_started : need_more; \ +#define FLUSH_BLOCK(s, last) { \ + FLUSH_BLOCK_ONLY(s, last); \ + if (s->strm->avail_out == 0) return (last) ? finish_started : need_more; \ } /* =========================================================================== @@ -1434,8 +1608,14 @@ FLUSH_BLOCK(s, 0); } } - FLUSH_BLOCK(s, flush == Z_FINISH); - return flush == Z_FINISH ? finish_done : block_done; + s->insert = 0; + if (flush == Z_FINISH) { + FLUSH_BLOCK(s, 1); + return finish_done; + } + if ((long)s->strstart > s->block_start) + FLUSH_BLOCK(s, 0); + return block_done; } /* =========================================================================== @@ -1449,7 +1629,7 @@ deflate_state *s; int flush; { - IPos hash_head = NIL; /* head of the hash chain */ + IPos hash_head; /* head of the hash chain */ int bflush; /* set if current block must be flushed */ for (;;) { @@ -1469,6 +1649,7 @@ /* Insert the string window[strstart .. strstart+2] in the * dictionary, and set hash_head to the head of the hash chain: */ + hash_head = NIL; if (s->lookahead >= MIN_MATCH) { INSERT_STRING(s, s->strstart, hash_head); } @@ -1481,19 +1662,8 @@ * of window index 0 (in particular we have to avoid a match * of the string with itself at the start of the input file). */ -#ifdef FASTEST - if ((s->strategy != Z_HUFFMAN_ONLY && s->strategy != Z_RLE) || - (s->strategy == Z_RLE && s->strstart - hash_head == 1)) { - s->match_length = longest_match_fast (s, hash_head); - } -#else - if (s->strategy != Z_HUFFMAN_ONLY && s->strategy != Z_RLE) { - s->match_length = longest_match (s, hash_head); - } else if (s->strategy == Z_RLE && s->strstart - hash_head == 1) { - s->match_length = longest_match_fast (s, hash_head); - } -#endif - /* longest_match() or longest_match_fast() sets match_start */ + s->match_length = longest_match (s, hash_head); + /* longest_match() sets match_start */ } if (s->match_length >= MIN_MATCH) { check_match(s, s->strstart, s->match_start, s->match_length); @@ -1541,8 +1711,14 @@ } if (bflush) FLUSH_BLOCK(s, 0); } - FLUSH_BLOCK(s, flush == Z_FINISH); - return flush == Z_FINISH ? finish_done : block_done; + s->insert = s->strstart < MIN_MATCH-1 ? s->strstart : MIN_MATCH-1; + if (flush == Z_FINISH) { + FLUSH_BLOCK(s, 1); + return finish_done; + } + if (s->last_lit) + FLUSH_BLOCK(s, 0); + return block_done; } #ifndef FASTEST @@ -1555,7 +1731,7 @@ deflate_state *s; int flush; { - IPos hash_head = NIL; /* head of hash chain */ + IPos hash_head; /* head of hash chain */ int bflush; /* set if current block must be flushed */ /* Process the input block. */ @@ -1576,6 +1752,7 @@ /* Insert the string window[strstart .. strstart+2] in the * dictionary, and set hash_head to the head of the hash chain: */ + hash_head = NIL; if (s->lookahead >= MIN_MATCH) { INSERT_STRING(s, s->strstart, hash_head); } @@ -1591,12 +1768,8 @@ * of window index 0 (in particular we have to avoid a match * of the string with itself at the start of the input file). */ - if (s->strategy != Z_HUFFMAN_ONLY && s->strategy != Z_RLE) { - s->match_length = longest_match (s, hash_head); - } else if (s->strategy == Z_RLE && s->strstart - hash_head == 1) { - s->match_length = longest_match_fast (s, hash_head); - } - /* longest_match() or longest_match_fast() sets match_start */ + s->match_length = longest_match (s, hash_head); + /* longest_match() sets match_start */ if (s->match_length <= 5 && (s->strategy == Z_FILTERED #if TOO_FAR <= 32767 @@ -1669,12 +1842,17 @@ _tr_tally_lit(s, s->window[s->strstart-1], bflush); s->match_available = 0; } - FLUSH_BLOCK(s, flush == Z_FINISH); - return flush == Z_FINISH ? finish_done : block_done; + s->insert = s->strstart < MIN_MATCH-1 ? s->strstart : MIN_MATCH-1; + if (flush == Z_FINISH) { + FLUSH_BLOCK(s, 1); + return finish_done; + } + if (s->last_lit) + FLUSH_BLOCK(s, 0); + return block_done; } #endif /* FASTEST */ -#if 0 /* =========================================================================== * For Z_RLE, simply look for runs of bytes, generate matches only of distance * one. Do not maintain a hash table. (It will be regenerated if this run of @@ -1684,43 +1862,52 @@ deflate_state *s; int flush; { - int bflush; /* set if current block must be flushed */ - uInt run; /* length of run */ - uInt max; /* maximum length of run */ - uInt prev; /* byte at distance one to match */ - Bytef *scan; /* scan for end of run */ + int bflush; /* set if current block must be flushed */ + uInt prev; /* byte at distance one to match */ + Bytef *scan, *strend; /* scan goes up to strend for length of run */ for (;;) { /* Make sure that we always have enough lookahead, except * at the end of the input file. We need MAX_MATCH bytes - * for the longest encodable run. + * for the longest run, plus one for the unrolled loop. */ - if (s->lookahead < MAX_MATCH) { + if (s->lookahead <= MAX_MATCH) { fill_window(s); - if (s->lookahead < MAX_MATCH && flush == Z_NO_FLUSH) { + if (s->lookahead <= MAX_MATCH && flush == Z_NO_FLUSH) { return need_more; } if (s->lookahead == 0) break; /* flush the current block */ } /* See how many times the previous byte repeats */ - run = 0; - if (s->strstart > 0) { /* if there is a previous byte, that is */ - max = s->lookahead < MAX_MATCH ? s->lookahead : MAX_MATCH; + s->match_length = 0; + if (s->lookahead >= MIN_MATCH && s->strstart > 0) { scan = s->window + s->strstart - 1; - prev = *scan++; - do { - if (*scan++ != prev) - break; - } while (++run < max); + prev = *scan; + if (prev == *++scan && prev == *++scan && prev == *++scan) { + strend = s->window + s->strstart + MAX_MATCH; + do { + } while (prev == *++scan && prev == *++scan && + prev == *++scan && prev == *++scan && + prev == *++scan && prev == *++scan && + prev == *++scan && prev == *++scan && + scan < strend); + s->match_length = MAX_MATCH - (int)(strend - scan); + if (s->match_length > s->lookahead) + s->match_length = s->lookahead; + } + Assert(scan <= s->window+(uInt)(s->window_size-1), "wild scan"); } /* Emit match if have run of MIN_MATCH or longer, else emit literal */ - if (run >= MIN_MATCH) { - check_match(s, s->strstart, s->strstart - 1, run); - _tr_tally_dist(s, 1, run - MIN_MATCH, bflush); - s->lookahead -= run; - s->strstart += run; + if (s->match_length >= MIN_MATCH) { + check_match(s, s->strstart, s->strstart - 1, s->match_length); + + _tr_tally_dist(s, 1, s->match_length - MIN_MATCH, bflush); + + s->lookahead -= s->match_length; + s->strstart += s->match_length; + s->match_length = 0; } else { /* No match, output a literal byte */ Tracevv((stderr,"%c", s->window[s->strstart])); @@ -1730,7 +1917,51 @@ } if (bflush) FLUSH_BLOCK(s, 0); } - FLUSH_BLOCK(s, flush == Z_FINISH); - return flush == Z_FINISH ? finish_done : block_done; + s->insert = 0; + if (flush == Z_FINISH) { + FLUSH_BLOCK(s, 1); + return finish_done; + } + if (s->last_lit) + FLUSH_BLOCK(s, 0); + return block_done; } -#endif + +/* =========================================================================== + * For Z_HUFFMAN_ONLY, do not look for matches. Do not maintain a hash table. + * (It will be regenerated if this run of deflate switches away from Huffman.) + */ +local block_state deflate_huff(s, flush) + deflate_state *s; + int flush; +{ + int bflush; /* set if current block must be flushed */ + + for (;;) { + /* Make sure that we have a literal to write. */ + if (s->lookahead == 0) { + fill_window(s); + if (s->lookahead == 0) { + if (flush == Z_NO_FLUSH) + return need_more; + break; /* flush the current block */ + } + } + + /* Output a literal byte */ + s->match_length = 0; + Tracevv((stderr,"%c", s->window[s->strstart])); + _tr_tally_lit (s, s->window[s->strstart], bflush); + s->lookahead--; + s->strstart++; + if (bflush) FLUSH_BLOCK(s, 0); + } + s->insert = 0; + if (flush == Z_FINISH) { + FLUSH_BLOCK(s, 1); + return finish_done; + } + if (s->last_lit) + FLUSH_BLOCK(s, 0); + return block_done; +} diff --git a/Modules/zlib/deflate.h b/Modules/zlib/deflate.h --- a/Modules/zlib/deflate.h +++ b/Modules/zlib/deflate.h @@ -1,5 +1,5 @@ /* deflate.h -- internal compression state - * Copyright (C) 1995-2004 Jean-loup Gailly + * Copyright (C) 1995-2012 Jean-loup Gailly * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -48,6 +48,9 @@ #define MAX_BITS 15 /* All codes must not exceed MAX_BITS bits */ +#define Buf_size 16 +/* size of bit buffer in bi_buf */ + #define INIT_STATE 42 #define EXTRA_STATE 69 #define NAME_STATE 73 @@ -101,7 +104,7 @@ int wrap; /* bit 0 true for zlib, bit 1 true for gzip */ gz_headerp gzhead; /* gzip header information to write */ uInt gzindex; /* where in extra, name, or comment */ - Byte method; /* STORED (for zip only) or DEFLATED */ + Byte method; /* can only be DEFLATED */ int last_flush; /* value of flush param for previous deflate call */ /* used by deflate.c: */ @@ -188,7 +191,7 @@ int nice_match; /* Stop searching when current match exceeds this */ /* used by trees.c: */ - /* Didn't use ct_data typedef below to supress compiler warning */ + /* Didn't use ct_data typedef below to suppress compiler warning */ struct ct_data_s dyn_ltree[HEAP_SIZE]; /* literal and length tree */ struct ct_data_s dyn_dtree[2*D_CODES+1]; /* distance tree */ struct ct_data_s bl_tree[2*BL_CODES+1]; /* Huffman tree for bit lengths */ @@ -244,7 +247,7 @@ ulg opt_len; /* bit length of current block with optimal trees */ ulg static_len; /* bit length of current block with static trees */ uInt matches; /* number of string matches in current block */ - int last_eob_len; /* bit length of EOB code for last block */ + uInt insert; /* bytes at end of window left to insert */ #ifdef DEBUG ulg compressed_len; /* total bit length of compressed file mod 2^32 */ @@ -260,6 +263,13 @@ * are always zero. */ + ulg high_water; + /* High water mark offset in window for initialized bytes -- bytes above + * this are set to zero in order to avoid memory check warnings when + * longest match routines access bytes past the input. This is then + * updated to the new high water mark. + */ + } FAR deflate_state; /* Output a byte on the stream. @@ -278,14 +288,19 @@ * distances are limited to MAX_DIST instead of WSIZE. */ +#define WIN_INIT MAX_MATCH +/* Number of bytes after end of data in window to initialize in order to avoid + memory checker errors from longest match routines */ + /* in trees.c */ -void _tr_init OF((deflate_state *s)); -int _tr_tally OF((deflate_state *s, unsigned dist, unsigned lc)); -void _tr_flush_block OF((deflate_state *s, charf *buf, ulg stored_len, - int eof)); -void _tr_align OF((deflate_state *s)); -void _tr_stored_block OF((deflate_state *s, charf *buf, ulg stored_len, - int eof)); +void ZLIB_INTERNAL _tr_init OF((deflate_state *s)); +int ZLIB_INTERNAL _tr_tally OF((deflate_state *s, unsigned dist, unsigned lc)); +void ZLIB_INTERNAL _tr_flush_block OF((deflate_state *s, charf *buf, + ulg stored_len, int last)); +void ZLIB_INTERNAL _tr_flush_bits OF((deflate_state *s)); +void ZLIB_INTERNAL _tr_align OF((deflate_state *s)); +void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf, + ulg stored_len, int last)); #define d_code(dist) \ ((dist) < 256 ? _dist_code[dist] : _dist_code[256+((dist)>>7)]) @@ -298,11 +313,11 @@ /* Inline versions of _tr_tally for speed: */ #if defined(GEN_TREES_H) || !defined(STDC) - extern uch _length_code[]; - extern uch _dist_code[]; + extern uch ZLIB_INTERNAL _length_code[]; + extern uch ZLIB_INTERNAL _dist_code[]; #else - extern const uch _length_code[]; - extern const uch _dist_code[]; + extern const uch ZLIB_INTERNAL _length_code[]; + extern const uch ZLIB_INTERNAL _dist_code[]; #endif # define _tr_tally_lit(s, c, flush) \ diff --git a/Modules/zlib/example.c b/Modules/zlib/example.c --- a/Modules/zlib/example.c +++ b/Modules/zlib/example.c @@ -1,12 +1,12 @@ /* example.c -- usage example of the zlib compression library - * Copyright (C) 1995-2004 Jean-loup Gailly. + * Copyright (C) 1995-2006, 2011 Jean-loup Gailly. * For conditions of distribution and use, see copyright notice in zlib.h */ /* @(#) $Id$ */ +#include "zlib.h" #include -#include "zlib.h" #ifdef STDC # include @@ -26,7 +26,7 @@ } \ } -const char hello[] = "hello, hello!"; +z_const char hello[] = "hello, hello!"; /* "hello world" would be more standard, but the repeated "hello" * stresses the compression code better, sorry... */ @@ -34,10 +34,6 @@ const char dictionary[] = "hello"; uLong dictId; /* Adler32 value of the dictionary */ -void test_compress OF((Byte *compr, uLong comprLen, - Byte *uncompr, uLong uncomprLen)); -void test_gzio OF((const char *fname, - Byte *uncompr, uLong uncomprLen)); void test_deflate OF((Byte *compr, uLong comprLen)); void test_inflate OF((Byte *compr, uLong comprLen, Byte *uncompr, uLong uncomprLen)); @@ -53,6 +49,39 @@ Byte *uncompr, uLong uncomprLen)); int main OF((int argc, char *argv[])); + +#ifdef Z_SOLO + +void *myalloc OF((void *, unsigned, unsigned)); +void myfree OF((void *, void *)); + +void *myalloc(q, n, m) + void *q; + unsigned n, m; +{ + q = Z_NULL; + return calloc(n, m); +} + +void myfree(void *q, void *p) +{ + q = Z_NULL; + free(p); +} + +static alloc_func zalloc = myalloc; +static free_func zfree = myfree; + +#else /* !Z_SOLO */ + +static alloc_func zalloc = (alloc_func)0; +static free_func zfree = (free_func)0; + +void test_compress OF((Byte *compr, uLong comprLen, + Byte *uncompr, uLong uncomprLen)); +void test_gzio OF((const char *fname, + Byte *uncompr, uLong uncomprLen)); + /* =========================================================================== * Test compress() and uncompress() */ @@ -163,6 +192,8 @@ #endif } +#endif /* Z_SOLO */ + /* =========================================================================== * Test deflate() with small buffers */ @@ -174,14 +205,14 @@ int err; uLong len = (uLong)strlen(hello)+1; - c_stream.zalloc = (alloc_func)0; - c_stream.zfree = (free_func)0; + c_stream.zalloc = zalloc; + c_stream.zfree = zfree; c_stream.opaque = (voidpf)0; err = deflateInit(&c_stream, Z_DEFAULT_COMPRESSION); CHECK_ERR(err, "deflateInit"); - c_stream.next_in = (Bytef*)hello; + c_stream.next_in = (z_const unsigned char *)hello; c_stream.next_out = compr; while (c_stream.total_in != len && c_stream.total_out < comprLen) { @@ -213,8 +244,8 @@ strcpy((char*)uncompr, "garbage"); - d_stream.zalloc = (alloc_func)0; - d_stream.zfree = (free_func)0; + d_stream.zalloc = zalloc; + d_stream.zfree = zfree; d_stream.opaque = (voidpf)0; d_stream.next_in = compr; @@ -252,8 +283,8 @@ z_stream c_stream; /* compression stream */ int err; - c_stream.zalloc = (alloc_func)0; - c_stream.zfree = (free_func)0; + c_stream.zalloc = zalloc; + c_stream.zfree = zfree; c_stream.opaque = (voidpf)0; err = deflateInit(&c_stream, Z_BEST_SPEED); @@ -309,8 +340,8 @@ strcpy((char*)uncompr, "garbage"); - d_stream.zalloc = (alloc_func)0; - d_stream.zfree = (free_func)0; + d_stream.zalloc = zalloc; + d_stream.zfree = zfree; d_stream.opaque = (voidpf)0; d_stream.next_in = compr; @@ -349,14 +380,14 @@ int err; uInt len = (uInt)strlen(hello)+1; - c_stream.zalloc = (alloc_func)0; - c_stream.zfree = (free_func)0; + c_stream.zalloc = zalloc; + c_stream.zfree = zfree; c_stream.opaque = (voidpf)0; err = deflateInit(&c_stream, Z_DEFAULT_COMPRESSION); CHECK_ERR(err, "deflateInit"); - c_stream.next_in = (Bytef*)hello; + c_stream.next_in = (z_const unsigned char *)hello; c_stream.next_out = compr; c_stream.avail_in = 3; c_stream.avail_out = (uInt)*comprLen; @@ -388,8 +419,8 @@ strcpy((char*)uncompr, "garbage"); - d_stream.zalloc = (alloc_func)0; - d_stream.zfree = (free_func)0; + d_stream.zalloc = zalloc; + d_stream.zfree = zfree; d_stream.opaque = (voidpf)0; d_stream.next_in = compr; @@ -430,22 +461,22 @@ z_stream c_stream; /* compression stream */ int err; - c_stream.zalloc = (alloc_func)0; - c_stream.zfree = (free_func)0; + c_stream.zalloc = zalloc; + c_stream.zfree = zfree; c_stream.opaque = (voidpf)0; err = deflateInit(&c_stream, Z_BEST_COMPRESSION); CHECK_ERR(err, "deflateInit"); err = deflateSetDictionary(&c_stream, - (const Bytef*)dictionary, sizeof(dictionary)); + (const Bytef*)dictionary, (int)sizeof(dictionary)); CHECK_ERR(err, "deflateSetDictionary"); dictId = c_stream.adler; c_stream.next_out = compr; c_stream.avail_out = (uInt)comprLen; - c_stream.next_in = (Bytef*)hello; + c_stream.next_in = (z_const unsigned char *)hello; c_stream.avail_in = (uInt)strlen(hello)+1; err = deflate(&c_stream, Z_FINISH); @@ -469,8 +500,8 @@ strcpy((char*)uncompr, "garbage"); - d_stream.zalloc = (alloc_func)0; - d_stream.zfree = (free_func)0; + d_stream.zalloc = zalloc; + d_stream.zfree = zfree; d_stream.opaque = (voidpf)0; d_stream.next_in = compr; @@ -491,7 +522,7 @@ exit(1); } err = inflateSetDictionary(&d_stream, (const Bytef*)dictionary, - sizeof(dictionary)); + (int)sizeof(dictionary)); } CHECK_ERR(err, "inflate with dict"); } @@ -540,10 +571,15 @@ printf("out of memory\n"); exit(1); } + +#ifdef Z_SOLO + argc = strlen(argv[0]); +#else test_compress(compr, comprLen, uncompr, uncomprLen); test_gzio((argc > 1 ? argv[1] : TESTFILE), uncompr, uncomprLen); +#endif test_deflate(compr, comprLen); test_inflate(compr, comprLen, uncompr, uncomprLen); diff --git a/Modules/zlib/gzclose.c b/Modules/zlib/gzclose.c new file mode 100644 --- /dev/null +++ b/Modules/zlib/gzclose.c @@ -0,0 +1,25 @@ +/* gzclose.c -- zlib gzclose() function + * Copyright (C) 2004, 2010 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include "gzguts.h" + +/* gzclose() is in a separate file so that it is linked in only if it is used. + That way the other gzclose functions can be used instead to avoid linking in + unneeded compression or decompression routines. */ +int ZEXPORT gzclose(file) + gzFile file; +{ +#ifndef NO_GZCOMPRESS + gz_statep state; + + if (file == NULL) + return Z_STREAM_ERROR; + state = (gz_statep)file; + + return state->mode == GZ_READ ? gzclose_r(file) : gzclose_w(file); +#else + return gzclose_r(file); +#endif +} diff --git a/Modules/zlib/gzguts.h b/Modules/zlib/gzguts.h new file mode 100644 --- /dev/null +++ b/Modules/zlib/gzguts.h @@ -0,0 +1,209 @@ +/* gzguts.h -- zlib internal header definitions for gz* operations + * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#ifdef _LARGEFILE64_SOURCE +# ifndef _LARGEFILE_SOURCE +# define _LARGEFILE_SOURCE 1 +# endif +# ifdef _FILE_OFFSET_BITS +# undef _FILE_OFFSET_BITS +# endif +#endif + +#ifdef HAVE_HIDDEN +# define ZLIB_INTERNAL __attribute__((visibility ("hidden"))) +#else +# define ZLIB_INTERNAL +#endif + +#include +#include "zlib.h" +#ifdef STDC +# include +# include +# include +#endif +#include + +#ifdef _WIN32 +# include +#endif + +#if defined(__TURBOC__) || defined(_MSC_VER) || defined(_WIN32) +# include +#endif + +#ifdef WINAPI_FAMILY +# define open _open +# define read _read +# define write _write +# define close _close +#endif + +#ifdef NO_DEFLATE /* for compatibility with old definition */ +# define NO_GZCOMPRESS +#endif + +#if defined(STDC99) || (defined(__TURBOC__) && __TURBOC__ >= 0x550) +# ifndef HAVE_VSNPRINTF +# define HAVE_VSNPRINTF +# endif +#endif + +#if defined(__CYGWIN__) +# ifndef HAVE_VSNPRINTF +# define HAVE_VSNPRINTF +# endif +#endif + +#if defined(MSDOS) && defined(__BORLANDC__) && (BORLANDC > 0x410) +# ifndef HAVE_VSNPRINTF +# define HAVE_VSNPRINTF +# endif +#endif + +#ifndef HAVE_VSNPRINTF +# ifdef MSDOS +/* vsnprintf may exist on some MS-DOS compilers (DJGPP?), + but for now we just assume it doesn't. */ +# define NO_vsnprintf +# endif +# ifdef __TURBOC__ +# define NO_vsnprintf +# endif +# ifdef WIN32 +/* In Win32, vsnprintf is available as the "non-ANSI" _vsnprintf. */ +# if !defined(vsnprintf) && !defined(NO_vsnprintf) +# if !defined(_MSC_VER) || ( defined(_MSC_VER) && _MSC_VER < 1500 ) +# define vsnprintf _vsnprintf +# endif +# endif +# endif +# ifdef __SASC +# define NO_vsnprintf +# endif +# ifdef VMS +# define NO_vsnprintf +# endif +# ifdef __OS400__ +# define NO_vsnprintf +# endif +# ifdef __MVS__ +# define NO_vsnprintf +# endif +#endif + +/* unlike snprintf (which is required in C99, yet still not supported by + Microsoft more than a decade later!), _snprintf does not guarantee null + termination of the result -- however this is only used in gzlib.c where + the result is assured to fit in the space provided */ +#ifdef _MSC_VER +# define snprintf _snprintf +#endif + +#ifndef local +# define local static +#endif +/* compile with -Dlocal if your debugger can't find static symbols */ + +/* gz* functions always use library allocation functions */ +#ifndef STDC + extern voidp malloc OF((uInt size)); + extern void free OF((voidpf ptr)); +#endif + +/* get errno and strerror definition */ +#if defined UNDER_CE +# include +# define zstrerror() gz_strwinerror((DWORD)GetLastError()) +#else +# ifndef NO_STRERROR +# include +# define zstrerror() strerror(errno) +# else +# define zstrerror() "stdio error (consult errno)" +# endif +#endif + +/* provide prototypes for these when building zlib without LFS */ +#if !defined(_LARGEFILE64_SOURCE) || _LFS64_LARGEFILE-0 == 0 + ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *)); + ZEXTERN z_off64_t ZEXPORT gzseek64 OF((gzFile, z_off64_t, int)); + ZEXTERN z_off64_t ZEXPORT gztell64 OF((gzFile)); + ZEXTERN z_off64_t ZEXPORT gzoffset64 OF((gzFile)); +#endif + +/* default memLevel */ +#if MAX_MEM_LEVEL >= 8 +# define DEF_MEM_LEVEL 8 +#else +# define DEF_MEM_LEVEL MAX_MEM_LEVEL +#endif + +/* default i/o buffer size -- double this for output when reading (this and + twice this must be able to fit in an unsigned type) */ +#define GZBUFSIZE 8192 + +/* gzip modes, also provide a little integrity check on the passed structure */ +#define GZ_NONE 0 +#define GZ_READ 7247 +#define GZ_WRITE 31153 +#define GZ_APPEND 1 /* mode set to GZ_WRITE after the file is opened */ + +/* values for gz_state how */ +#define LOOK 0 /* look for a gzip header */ +#define COPY 1 /* copy input directly */ +#define GZIP 2 /* decompress a gzip stream */ + +/* internal gzip file state data structure */ +typedef struct { + /* exposed contents for gzgetc() macro */ + struct gzFile_s x; /* "x" for exposed */ + /* x.have: number of bytes available at x.next */ + /* x.next: next output data to deliver or write */ + /* x.pos: current position in uncompressed data */ + /* used for both reading and writing */ + int mode; /* see gzip modes above */ + int fd; /* file descriptor */ + char *path; /* path or fd for error messages */ + unsigned size; /* buffer size, zero if not allocated yet */ + unsigned want; /* requested buffer size, default is GZBUFSIZE */ + unsigned char *in; /* input buffer */ + unsigned char *out; /* output buffer (double-sized when reading) */ + int direct; /* 0 if processing gzip, 1 if transparent */ + /* just for reading */ + int how; /* 0: get header, 1: copy, 2: decompress */ + z_off64_t start; /* where the gzip data started, for rewinding */ + int eof; /* true if end of input file reached */ + int past; /* true if read requested past end */ + /* just for writing */ + int level; /* compression level */ + int strategy; /* compression strategy */ + /* seek request */ + z_off64_t skip; /* amount to skip (already rewound if backwards) */ + int seek; /* true if seek request pending */ + /* error information */ + int err; /* error code */ + char *msg; /* error message */ + /* zlib inflate or deflate stream */ + z_stream strm; /* stream structure in-place (not a pointer) */ +} gz_state; +typedef gz_state FAR *gz_statep; + +/* shared functions */ +void ZLIB_INTERNAL gz_error OF((gz_statep, int, const char *)); +#if defined UNDER_CE +char ZLIB_INTERNAL *gz_strwinerror OF((DWORD error)); +#endif + +/* GT_OFF(x), where x is an unsigned value, is true if x > maximum z_off64_t + value -- needed when comparing unsigned to z_off64_t, which is signed + (possible z_off64_t types off_t, off64_t, and long are all signed) */ +#ifdef INT_MAX +# define GT_OFF(x) (sizeof(int) == sizeof(z_off64_t) && (x) > INT_MAX) +#else +unsigned ZLIB_INTERNAL gz_intmax OF((void)); +# define GT_OFF(x) (sizeof(int) == sizeof(z_off64_t) && (x) > gz_intmax()) +#endif diff --git a/Modules/zlib/gzio.c b/Modules/zlib/gzio.c deleted file mode 100644 --- a/Modules/zlib/gzio.c +++ /dev/null @@ -1,1026 +0,0 @@ -/* gzio.c -- IO on .gz files - * Copyright (C) 1995-2005 Jean-loup Gailly. - * For conditions of distribution and use, see copyright notice in zlib.h - * - * Compile this file with -DNO_GZCOMPRESS to avoid the compression code. - */ - -/* @(#) $Id$ */ - -#include - -#include "zutil.h" - -#ifdef NO_DEFLATE /* for compatibility with old definition */ -# define NO_GZCOMPRESS -#endif - -#ifndef NO_DUMMY_DECL -struct internal_state {int dummy;}; /* for buggy compilers */ -#endif - -#ifndef Z_BUFSIZE -# ifdef MAXSEG_64K -# define Z_BUFSIZE 4096 /* minimize memory usage for 16-bit DOS */ -# else -# define Z_BUFSIZE 16384 -# endif -#endif -#ifndef Z_PRINTF_BUFSIZE -# define Z_PRINTF_BUFSIZE 4096 -#endif - -#ifdef __MVS__ -# pragma map (fdopen , "\174\174FDOPEN") - FILE *fdopen(int, const char *); -#endif - -#ifndef STDC -extern voidp malloc OF((uInt size)); -extern void free OF((voidpf ptr)); -#endif - -#define ALLOC(size) malloc(size) -#define TRYFREE(p) {if (p) free(p);} - -static int const gz_magic[2] = {0x1f, 0x8b}; /* gzip magic header */ - -/* gzip flag byte */ -#define ASCII_FLAG 0x01 /* bit 0 set: file probably ascii text */ -#define HEAD_CRC 0x02 /* bit 1 set: header CRC present */ -#define EXTRA_FIELD 0x04 /* bit 2 set: extra field present */ -#define ORIG_NAME 0x08 /* bit 3 set: original file name present */ -#define COMMENT 0x10 /* bit 4 set: file comment present */ -#define RESERVED 0xE0 /* bits 5..7: reserved */ - -typedef struct gz_stream { - z_stream stream; - int z_err; /* error code for last stream operation */ - int z_eof; /* set if end of input file */ - FILE *file; /* .gz file */ - Byte *inbuf; /* input buffer */ - Byte *outbuf; /* output buffer */ - uLong crc; /* crc32 of uncompressed data */ - char *msg; /* error message */ - char *path; /* path name for debugging only */ - int transparent; /* 1 if input file is not a .gz file */ - char mode; /* 'w' or 'r' */ - z_off_t start; /* start of compressed data in file (header skipped) */ - z_off_t in; /* bytes into deflate or inflate */ - z_off_t out; /* bytes out of deflate or inflate */ - int back; /* one character push-back */ - int last; /* true if push-back is last character */ -} gz_stream; - - -local gzFile gz_open OF((const char *path, const char *mode, int fd)); -local int do_flush OF((gzFile file, int flush)); -local int get_byte OF((gz_stream *s)); -local void check_header OF((gz_stream *s)); -local int destroy OF((gz_stream *s)); -local void putLong OF((FILE *file, uLong x)); -local uLong getLong OF((gz_stream *s)); - -/* =========================================================================== - Opens a gzip (.gz) file for reading or writing. The mode parameter - is as in fopen ("rb" or "wb"). The file is given either by file descriptor - or path name (if fd == -1). - gz_open returns NULL if the file could not be opened or if there was - insufficient memory to allocate the (de)compression state; errno - can be checked to distinguish the two cases (if errno is zero, the - zlib error is Z_MEM_ERROR). -*/ -local gzFile gz_open (path, mode, fd) - const char *path; - const char *mode; - int fd; -{ - int err; - int level = Z_DEFAULT_COMPRESSION; /* compression level */ - int strategy = Z_DEFAULT_STRATEGY; /* compression strategy */ - char *p = (char*)mode; - gz_stream *s; - char fmode[80]; /* copy of mode, without the compression level */ - char *m = fmode; - - if (!path || !mode) return Z_NULL; - - s = (gz_stream *)ALLOC(sizeof(gz_stream)); - if (!s) return Z_NULL; - - s->stream.zalloc = (alloc_func)0; - s->stream.zfree = (free_func)0; - s->stream.opaque = (voidpf)0; - s->stream.next_in = s->inbuf = Z_NULL; - s->stream.next_out = s->outbuf = Z_NULL; - s->stream.avail_in = s->stream.avail_out = 0; - s->file = NULL; - s->z_err = Z_OK; - s->z_eof = 0; - s->in = 0; - s->out = 0; - s->back = EOF; - s->crc = crc32(0L, Z_NULL, 0); - s->msg = NULL; - s->transparent = 0; - - s->path = (char*)ALLOC(strlen(path)+1); - if (s->path == NULL) { - return destroy(s), (gzFile)Z_NULL; - } - strcpy(s->path, path); /* do this early for debugging */ - - s->mode = '\0'; - do { - if (*p == 'r') s->mode = 'r'; - if (*p == 'w' || *p == 'a') s->mode = 'w'; - if (*p >= '0' && *p <= '9') { - level = *p - '0'; - } else if (*p == 'f') { - strategy = Z_FILTERED; - } else if (*p == 'h') { - strategy = Z_HUFFMAN_ONLY; - } else if (*p == 'R') { - strategy = Z_RLE; - } else { - *m++ = *p; /* copy the mode */ - } - } while (*p++ && m != fmode + sizeof(fmode)); - if (s->mode == '\0') return destroy(s), (gzFile)Z_NULL; - - if (s->mode == 'w') { -#ifdef NO_GZCOMPRESS - err = Z_STREAM_ERROR; -#else - err = deflateInit2(&(s->stream), level, - Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL, strategy); - /* windowBits is passed < 0 to suppress zlib header */ - - s->stream.next_out = s->outbuf = (Byte*)ALLOC(Z_BUFSIZE); -#endif - if (err != Z_OK || s->outbuf == Z_NULL) { - return destroy(s), (gzFile)Z_NULL; - } - } else { - s->stream.next_in = s->inbuf = (Byte*)ALLOC(Z_BUFSIZE); - - err = inflateInit2(&(s->stream), -MAX_WBITS); - /* windowBits is passed < 0 to tell that there is no zlib header. - * Note that in this case inflate *requires* an extra "dummy" byte - * after the compressed stream in order to complete decompression and - * return Z_STREAM_END. Here the gzip CRC32 ensures that 4 bytes are - * present after the compressed stream. - */ - if (err != Z_OK || s->inbuf == Z_NULL) { - return destroy(s), (gzFile)Z_NULL; - } - } - s->stream.avail_out = Z_BUFSIZE; - - errno = 0; - s->file = fd < 0 ? F_OPEN(path, fmode) : (FILE*)fdopen(fd, fmode); - - if (s->file == NULL) { - return destroy(s), (gzFile)Z_NULL; - } - if (s->mode == 'w') { - /* Write a very simple .gz header: - */ - fprintf(s->file, "%c%c%c%c%c%c%c%c%c%c", gz_magic[0], gz_magic[1], - Z_DEFLATED, 0 /*flags*/, 0,0,0,0 /*time*/, 0 /*xflags*/, OS_CODE); - s->start = 10L; - /* We use 10L instead of ftell(s->file) to because ftell causes an - * fflush on some systems. This version of the library doesn't use - * start anyway in write mode, so this initialization is not - * necessary. - */ - } else { - check_header(s); /* skip the .gz header */ - s->start = ftell(s->file) - s->stream.avail_in; - } - - return (gzFile)s; -} - -/* =========================================================================== - Opens a gzip (.gz) file for reading or writing. -*/ -gzFile ZEXPORT gzopen (path, mode) - const char *path; - const char *mode; -{ - return gz_open (path, mode, -1); -} - -/* =========================================================================== - Associate a gzFile with the file descriptor fd. fd is not dup'ed here - to mimic the behavio(u)r of fdopen. -*/ -gzFile ZEXPORT gzdopen (fd, mode) - int fd; - const char *mode; -{ - char name[46]; /* allow for up to 128-bit integers */ - - if (fd < 0) return (gzFile)Z_NULL; - sprintf(name, "", fd); /* for debugging */ - - return gz_open (name, mode, fd); -} - -/* =========================================================================== - * Update the compression level and strategy - */ -int ZEXPORT gzsetparams (file, level, strategy) - gzFile file; - int level; - int strategy; -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL || s->mode != 'w') return Z_STREAM_ERROR; - - /* Make room to allow flushing */ - if (s->stream.avail_out == 0) { - - s->stream.next_out = s->outbuf; - if (fwrite(s->outbuf, 1, Z_BUFSIZE, s->file) != Z_BUFSIZE) { - s->z_err = Z_ERRNO; - } - s->stream.avail_out = Z_BUFSIZE; - } - - return deflateParams (&(s->stream), level, strategy); -} - -/* =========================================================================== - Read a byte from a gz_stream; update next_in and avail_in. Return EOF - for end of file. - IN assertion: the stream s has been sucessfully opened for reading. -*/ -local int get_byte(s) - gz_stream *s; -{ - if (s->z_eof) return EOF; - if (s->stream.avail_in == 0) { - errno = 0; - s->stream.avail_in = (uInt)fread(s->inbuf, 1, Z_BUFSIZE, s->file); - if (s->stream.avail_in == 0) { - s->z_eof = 1; - if (ferror(s->file)) s->z_err = Z_ERRNO; - return EOF; - } - s->stream.next_in = s->inbuf; - } - s->stream.avail_in--; - return *(s->stream.next_in)++; -} - -/* =========================================================================== - Check the gzip header of a gz_stream opened for reading. Set the stream - mode to transparent if the gzip magic header is not present; set s->err - to Z_DATA_ERROR if the magic header is present but the rest of the header - is incorrect. - IN assertion: the stream s has already been created sucessfully; - s->stream.avail_in is zero for the first time, but may be non-zero - for concatenated .gz files. -*/ -local void check_header(s) - gz_stream *s; -{ - int method; /* method byte */ - int flags; /* flags byte */ - uInt len; - int c; - - /* Assure two bytes in the buffer so we can peek ahead -- handle case - where first byte of header is at the end of the buffer after the last - gzip segment */ - len = s->stream.avail_in; - if (len < 2) { - if (len) s->inbuf[0] = s->stream.next_in[0]; - errno = 0; - len = (uInt)fread(s->inbuf + len, 1, Z_BUFSIZE >> len, s->file); - if (len == 0 && ferror(s->file)) s->z_err = Z_ERRNO; - s->stream.avail_in += len; - s->stream.next_in = s->inbuf; - if (s->stream.avail_in < 2) { - s->transparent = s->stream.avail_in; - return; - } - } - - /* Peek ahead to check the gzip magic header */ - if (s->stream.next_in[0] != gz_magic[0] || - s->stream.next_in[1] != gz_magic[1]) { - s->transparent = 1; - return; - } - s->stream.avail_in -= 2; - s->stream.next_in += 2; - - /* Check the rest of the gzip header */ - method = get_byte(s); - flags = get_byte(s); - if (method != Z_DEFLATED || (flags & RESERVED) != 0) { - s->z_err = Z_DATA_ERROR; - return; - } - - /* Discard time, xflags and OS code: */ - for (len = 0; len < 6; len++) (void)get_byte(s); - - if ((flags & EXTRA_FIELD) != 0) { /* skip the extra field */ - len = (uInt)get_byte(s); - len += ((uInt)get_byte(s))<<8; - /* len is garbage if EOF but the loop below will quit anyway */ - while (len-- != 0 && get_byte(s) != EOF) ; - } - if ((flags & ORIG_NAME) != 0) { /* skip the original file name */ - while ((c = get_byte(s)) != 0 && c != EOF) ; - } - if ((flags & COMMENT) != 0) { /* skip the .gz file comment */ - while ((c = get_byte(s)) != 0 && c != EOF) ; - } - if ((flags & HEAD_CRC) != 0) { /* skip the header crc */ - for (len = 0; len < 2; len++) (void)get_byte(s); - } - s->z_err = s->z_eof ? Z_DATA_ERROR : Z_OK; -} - - /* =========================================================================== - * Cleanup then free the given gz_stream. Return a zlib error code. - Try freeing in the reverse order of allocations. - */ -local int destroy (s) - gz_stream *s; -{ - int err = Z_OK; - - if (!s) return Z_STREAM_ERROR; - - TRYFREE(s->msg); - - if (s->stream.state != NULL) { - if (s->mode == 'w') { -#ifdef NO_GZCOMPRESS - err = Z_STREAM_ERROR; -#else - err = deflateEnd(&(s->stream)); -#endif - } else if (s->mode == 'r') { - err = inflateEnd(&(s->stream)); - } - } - if (s->file != NULL && fclose(s->file)) { -#ifdef ESPIPE - if (errno != ESPIPE) /* fclose is broken for pipes in HP/UX */ -#endif - err = Z_ERRNO; - } - if (s->z_err < 0) err = s->z_err; - - TRYFREE(s->inbuf); - TRYFREE(s->outbuf); - TRYFREE(s->path); - TRYFREE(s); - return err; -} - -/* =========================================================================== - Reads the given number of uncompressed bytes from the compressed file. - gzread returns the number of bytes actually read (0 for end of file). -*/ -int ZEXPORT gzread (file, buf, len) - gzFile file; - voidp buf; - unsigned len; -{ - gz_stream *s = (gz_stream*)file; - Bytef *start = (Bytef*)buf; /* starting point for crc computation */ - Byte *next_out; /* == stream.next_out but not forced far (for MSDOS) */ - - if (s == NULL || s->mode != 'r') return Z_STREAM_ERROR; - - if (s->z_err == Z_DATA_ERROR || s->z_err == Z_ERRNO) return -1; - if (s->z_err == Z_STREAM_END) return 0; /* EOF */ - - next_out = (Byte*)buf; - s->stream.next_out = (Bytef*)buf; - s->stream.avail_out = len; - - if (s->stream.avail_out && s->back != EOF) { - *next_out++ = s->back; - s->stream.next_out++; - s->stream.avail_out--; - s->back = EOF; - s->out++; - start++; - if (s->last) { - s->z_err = Z_STREAM_END; - return 1; - } - } - - while (s->stream.avail_out != 0) { - - if (s->transparent) { - /* Copy first the lookahead bytes: */ - uInt n = s->stream.avail_in; - if (n > s->stream.avail_out) n = s->stream.avail_out; - if (n > 0) { - zmemcpy(s->stream.next_out, s->stream.next_in, n); - next_out += n; - s->stream.next_out = next_out; - s->stream.next_in += n; - s->stream.avail_out -= n; - s->stream.avail_in -= n; - } - if (s->stream.avail_out > 0) { - s->stream.avail_out -= - (uInt)fread(next_out, 1, s->stream.avail_out, s->file); - } - len -= s->stream.avail_out; - s->in += len; - s->out += len; - if (len == 0) s->z_eof = 1; - return (int)len; - } - if (s->stream.avail_in == 0 && !s->z_eof) { - - errno = 0; - s->stream.avail_in = (uInt)fread(s->inbuf, 1, Z_BUFSIZE, s->file); - if (s->stream.avail_in == 0) { - s->z_eof = 1; - if (ferror(s->file)) { - s->z_err = Z_ERRNO; - break; - } - } - s->stream.next_in = s->inbuf; - } - s->in += s->stream.avail_in; - s->out += s->stream.avail_out; - s->z_err = inflate(&(s->stream), Z_NO_FLUSH); - s->in -= s->stream.avail_in; - s->out -= s->stream.avail_out; - - if (s->z_err == Z_STREAM_END) { - /* Check CRC and original size */ - s->crc = crc32(s->crc, start, (uInt)(s->stream.next_out - start)); - start = s->stream.next_out; - - if (getLong(s) != s->crc) { - s->z_err = Z_DATA_ERROR; - } else { - (void)getLong(s); - /* The uncompressed length returned by above getlong() may be - * different from s->out in case of concatenated .gz files. - * Check for such files: - */ - check_header(s); - if (s->z_err == Z_OK) { - inflateReset(&(s->stream)); - s->crc = crc32(0L, Z_NULL, 0); - } - } - } - if (s->z_err != Z_OK || s->z_eof) break; - } - s->crc = crc32(s->crc, start, (uInt)(s->stream.next_out - start)); - - if (len == s->stream.avail_out && - (s->z_err == Z_DATA_ERROR || s->z_err == Z_ERRNO)) - return -1; - return (int)(len - s->stream.avail_out); -} - - -/* =========================================================================== - Reads one byte from the compressed file. gzgetc returns this byte - or -1 in case of end of file or error. -*/ -int ZEXPORT gzgetc(file) - gzFile file; -{ - unsigned char c; - - return gzread(file, &c, 1) == 1 ? c : -1; -} - - -/* =========================================================================== - Push one byte back onto the stream. -*/ -int ZEXPORT gzungetc(c, file) - int c; - gzFile file; -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL || s->mode != 'r' || c == EOF || s->back != EOF) return EOF; - s->back = c; - s->out--; - s->last = (s->z_err == Z_STREAM_END); - if (s->last) s->z_err = Z_OK; - s->z_eof = 0; - return c; -} - - -/* =========================================================================== - Reads bytes from the compressed file until len-1 characters are - read, or a newline character is read and transferred to buf, or an - end-of-file condition is encountered. The string is then terminated - with a null character. - gzgets returns buf, or Z_NULL in case of error. - - The current implementation is not optimized at all. -*/ -char * ZEXPORT gzgets(file, buf, len) - gzFile file; - char *buf; - int len; -{ - char *b = buf; - if (buf == Z_NULL || len <= 0) return Z_NULL; - - while (--len > 0 && gzread(file, buf, 1) == 1 && *buf++ != '\n') ; - *buf = '\0'; - return b == buf && len > 0 ? Z_NULL : b; -} - - -#ifndef NO_GZCOMPRESS -/* =========================================================================== - Writes the given number of uncompressed bytes into the compressed file. - gzwrite returns the number of bytes actually written (0 in case of error). -*/ -int ZEXPORT gzwrite (file, buf, len) - gzFile file; - voidpc buf; - unsigned len; -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL || s->mode != 'w') return Z_STREAM_ERROR; - - s->stream.next_in = (Bytef*)buf; - s->stream.avail_in = len; - - while (s->stream.avail_in != 0) { - - if (s->stream.avail_out == 0) { - - s->stream.next_out = s->outbuf; - if (fwrite(s->outbuf, 1, Z_BUFSIZE, s->file) != Z_BUFSIZE) { - s->z_err = Z_ERRNO; - break; - } - s->stream.avail_out = Z_BUFSIZE; - } - s->in += s->stream.avail_in; - s->out += s->stream.avail_out; - s->z_err = deflate(&(s->stream), Z_NO_FLUSH); - s->in -= s->stream.avail_in; - s->out -= s->stream.avail_out; - if (s->z_err != Z_OK) break; - } - s->crc = crc32(s->crc, (const Bytef *)buf, len); - - return (int)(len - s->stream.avail_in); -} - - -/* =========================================================================== - Converts, formats, and writes the args to the compressed file under - control of the format string, as in fprintf. gzprintf returns the number of - uncompressed bytes actually written (0 in case of error). -*/ -#ifdef STDC -#include - -int ZEXPORTVA gzprintf (gzFile file, const char *format, /* args */ ...) -{ - char buf[Z_PRINTF_BUFSIZE]; - va_list va; - int len; - - buf[sizeof(buf) - 1] = 0; - va_start(va, format); -#ifdef NO_vsnprintf -# ifdef HAS_vsprintf_void - (void)vsprintf(buf, format, va); - va_end(va); - for (len = 0; len < sizeof(buf); len++) - if (buf[len] == 0) break; -# else - len = vsprintf(buf, format, va); - va_end(va); -# endif -#else -# ifdef HAS_vsnprintf_void - (void)vsnprintf(buf, sizeof(buf), format, va); - va_end(va); - len = strlen(buf); -# else - len = vsnprintf(buf, sizeof(buf), format, va); - va_end(va); -# endif -#endif - if (len <= 0 || len >= (int)sizeof(buf) || buf[sizeof(buf) - 1] != 0) - return 0; - return gzwrite(file, buf, (unsigned)len); -} -#else /* not ANSI C */ - -int ZEXPORTVA gzprintf (file, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, - a11, a12, a13, a14, a15, a16, a17, a18, a19, a20) - gzFile file; - const char *format; - int a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, - a11, a12, a13, a14, a15, a16, a17, a18, a19, a20; -{ - char buf[Z_PRINTF_BUFSIZE]; - int len; - - buf[sizeof(buf) - 1] = 0; -#ifdef NO_snprintf -# ifdef HAS_sprintf_void - sprintf(buf, format, a1, a2, a3, a4, a5, a6, a7, a8, - a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); - for (len = 0; len < sizeof(buf); len++) - if (buf[len] == 0) break; -# else - len = sprintf(buf, format, a1, a2, a3, a4, a5, a6, a7, a8, - a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); -# endif -#else -# ifdef HAS_snprintf_void - snprintf(buf, sizeof(buf), format, a1, a2, a3, a4, a5, a6, a7, a8, - a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); - len = strlen(buf); -# else - len = snprintf(buf, sizeof(buf), format, a1, a2, a3, a4, a5, a6, a7, a8, - a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); -# endif -#endif - if (len <= 0 || len >= sizeof(buf) || buf[sizeof(buf) - 1] != 0) - return 0; - return gzwrite(file, buf, len); -} -#endif - -/* =========================================================================== - Writes c, converted to an unsigned char, into the compressed file. - gzputc returns the value that was written, or -1 in case of error. -*/ -int ZEXPORT gzputc(file, c) - gzFile file; - int c; -{ - unsigned char cc = (unsigned char) c; /* required for big endian systems */ - - return gzwrite(file, &cc, 1) == 1 ? (int)cc : -1; -} - - -/* =========================================================================== - Writes the given null-terminated string to the compressed file, excluding - the terminating null character. - gzputs returns the number of characters written, or -1 in case of error. -*/ -int ZEXPORT gzputs(file, s) - gzFile file; - const char *s; -{ - return gzwrite(file, (char*)s, (unsigned)strlen(s)); -} - - -/* =========================================================================== - Flushes all pending output into the compressed file. The parameter - flush is as in the deflate() function. -*/ -local int do_flush (file, flush) - gzFile file; - int flush; -{ - uInt len; - int done = 0; - gz_stream *s = (gz_stream*)file; - - if (s == NULL || s->mode != 'w') return Z_STREAM_ERROR; - - s->stream.avail_in = 0; /* should be zero already anyway */ - - for (;;) { - len = Z_BUFSIZE - s->stream.avail_out; - - if (len != 0) { - if ((uInt)fwrite(s->outbuf, 1, len, s->file) != len) { - s->z_err = Z_ERRNO; - return Z_ERRNO; - } - s->stream.next_out = s->outbuf; - s->stream.avail_out = Z_BUFSIZE; - } - if (done) break; - s->out += s->stream.avail_out; - s->z_err = deflate(&(s->stream), flush); - s->out -= s->stream.avail_out; - - /* Ignore the second of two consecutive flushes: */ - if (len == 0 && s->z_err == Z_BUF_ERROR) s->z_err = Z_OK; - - /* deflate has finished flushing only when it hasn't used up - * all the available space in the output buffer: - */ - done = (s->stream.avail_out != 0 || s->z_err == Z_STREAM_END); - - if (s->z_err != Z_OK && s->z_err != Z_STREAM_END) break; - } - return s->z_err == Z_STREAM_END ? Z_OK : s->z_err; -} - -int ZEXPORT gzflush (file, flush) - gzFile file; - int flush; -{ - gz_stream *s = (gz_stream*)file; - int err = do_flush (file, flush); - - if (err) return err; - fflush(s->file); - return s->z_err == Z_STREAM_END ? Z_OK : s->z_err; -} -#endif /* NO_GZCOMPRESS */ - -/* =========================================================================== - Sets the starting position for the next gzread or gzwrite on the given - compressed file. The offset represents a number of bytes in the - gzseek returns the resulting offset location as measured in bytes from - the beginning of the uncompressed stream, or -1 in case of error. - SEEK_END is not implemented, returns error. - In this version of the library, gzseek can be extremely slow. -*/ -z_off_t ZEXPORT gzseek (file, offset, whence) - gzFile file; - z_off_t offset; - int whence; -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL || whence == SEEK_END || - s->z_err == Z_ERRNO || s->z_err == Z_DATA_ERROR) { - return -1L; - } - - if (s->mode == 'w') { -#ifdef NO_GZCOMPRESS - return -1L; -#else - if (whence == SEEK_SET) { - offset -= s->in; - } - if (offset < 0) return -1L; - - /* At this point, offset is the number of zero bytes to write. */ - if (s->inbuf == Z_NULL) { - s->inbuf = (Byte*)ALLOC(Z_BUFSIZE); /* for seeking */ - if (s->inbuf == Z_NULL) return -1L; - zmemzero(s->inbuf, Z_BUFSIZE); - } - while (offset > 0) { - uInt size = Z_BUFSIZE; - if (offset < Z_BUFSIZE) size = (uInt)offset; - - size = gzwrite(file, s->inbuf, size); - if (size == 0) return -1L; - - offset -= size; - } - return s->in; -#endif - } - /* Rest of function is for reading only */ - - /* compute absolute position */ - if (whence == SEEK_CUR) { - offset += s->out; - } - if (offset < 0) return -1L; - - if (s->transparent) { - /* map to fseek */ - s->back = EOF; - s->stream.avail_in = 0; - s->stream.next_in = s->inbuf; - if (fseek(s->file, offset, SEEK_SET) < 0) return -1L; - - s->in = s->out = offset; - return offset; - } - - /* For a negative seek, rewind and use positive seek */ - if (offset >= s->out) { - offset -= s->out; - } else if (gzrewind(file) < 0) { - return -1L; - } - /* offset is now the number of bytes to skip. */ - - if (offset != 0 && s->outbuf == Z_NULL) { - s->outbuf = (Byte*)ALLOC(Z_BUFSIZE); - if (s->outbuf == Z_NULL) return -1L; - } - if (offset && s->back != EOF) { - s->back = EOF; - s->out++; - offset--; - if (s->last) s->z_err = Z_STREAM_END; - } - while (offset > 0) { - int size = Z_BUFSIZE; - if (offset < Z_BUFSIZE) size = (int)offset; - - size = gzread(file, s->outbuf, (uInt)size); - if (size <= 0) return -1L; - offset -= size; - } - return s->out; -} - -/* =========================================================================== - Rewinds input file. -*/ -int ZEXPORT gzrewind (file) - gzFile file; -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL || s->mode != 'r') return -1; - - s->z_err = Z_OK; - s->z_eof = 0; - s->back = EOF; - s->stream.avail_in = 0; - s->stream.next_in = s->inbuf; - s->crc = crc32(0L, Z_NULL, 0); - if (!s->transparent) (void)inflateReset(&s->stream); - s->in = 0; - s->out = 0; - return fseek(s->file, s->start, SEEK_SET); -} - -/* =========================================================================== - Returns the starting position for the next gzread or gzwrite on the - given compressed file. This position represents a number of bytes in the - uncompressed data stream. -*/ -z_off_t ZEXPORT gztell (file) - gzFile file; -{ - return gzseek(file, 0L, SEEK_CUR); -} - -/* =========================================================================== - Returns 1 when EOF has previously been detected reading the given - input stream, otherwise zero. -*/ -int ZEXPORT gzeof (file) - gzFile file; -{ - gz_stream *s = (gz_stream*)file; - - /* With concatenated compressed files that can have embedded - * crc trailers, z_eof is no longer the only/best indicator of EOF - * on a gz_stream. Handle end-of-stream error explicitly here. - */ - if (s == NULL || s->mode != 'r') return 0; - if (s->z_eof) return 1; - return s->z_err == Z_STREAM_END; -} - -/* =========================================================================== - Returns 1 if reading and doing so transparently, otherwise zero. -*/ -int ZEXPORT gzdirect (file) - gzFile file; -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL || s->mode != 'r') return 0; - return s->transparent; -} - -/* =========================================================================== - Outputs a long in LSB order to the given file -*/ -local void putLong (file, x) - FILE *file; - uLong x; -{ - int n; - for (n = 0; n < 4; n++) { - fputc((int)(x & 0xff), file); - x >>= 8; - } -} - -/* =========================================================================== - Reads a long in LSB order from the given gz_stream. Sets z_err in case - of error. -*/ -local uLong getLong (s) - gz_stream *s; -{ - uLong x = (uLong)get_byte(s); - int c; - - x += ((uLong)get_byte(s))<<8; - x += ((uLong)get_byte(s))<<16; - c = get_byte(s); - if (c == EOF) s->z_err = Z_DATA_ERROR; - x += ((uLong)c)<<24; - return x; -} - -/* =========================================================================== - Flushes all pending output if necessary, closes the compressed file - and deallocates all the (de)compression state. -*/ -int ZEXPORT gzclose (file) - gzFile file; -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL) return Z_STREAM_ERROR; - - if (s->mode == 'w') { -#ifdef NO_GZCOMPRESS - return Z_STREAM_ERROR; -#else - if (do_flush (file, Z_FINISH) != Z_OK) - return destroy((gz_stream*)file); - - putLong (s->file, s->crc); - putLong (s->file, (uLong)(s->in & 0xffffffff)); -#endif - } - return destroy((gz_stream*)file); -} - -#ifdef STDC -# define zstrerror(errnum) strerror(errnum) -#else -# define zstrerror(errnum) "" -#endif - -/* =========================================================================== - Returns the error message for the last error which occurred on the - given compressed file. errnum is set to zlib error number. If an - error occurred in the file system and not in the compression library, - errnum is set to Z_ERRNO and the application may consult errno - to get the exact error code. -*/ -const char * ZEXPORT gzerror (file, errnum) - gzFile file; - int *errnum; -{ - char *m; - gz_stream *s = (gz_stream*)file; - - if (s == NULL) { - *errnum = Z_STREAM_ERROR; - return (const char*)ERR_MSG(Z_STREAM_ERROR); - } - *errnum = s->z_err; - if (*errnum == Z_OK) return (const char*)""; - - m = (char*)(*errnum == Z_ERRNO ? zstrerror(errno) : s->stream.msg); - - if (m == NULL || *m == '\0') m = (char*)ERR_MSG(s->z_err); - - TRYFREE(s->msg); - s->msg = (char*)ALLOC(strlen(s->path) + strlen(m) + 3); - if (s->msg == Z_NULL) return (const char*)ERR_MSG(Z_MEM_ERROR); - strcpy(s->msg, s->path); - strcat(s->msg, ": "); - strcat(s->msg, m); - return (const char*)s->msg; -} - -/* =========================================================================== - Clear the error and end-of-file flags, and do the same for the real file. -*/ -void ZEXPORT gzclearerr (file) - gzFile file; -{ - gz_stream *s = (gz_stream*)file; - - if (s == NULL) return; - if (s->z_err != Z_STREAM_END) s->z_err = Z_OK; - s->z_eof = 0; - clearerr(s->file); -} diff --git a/Modules/zlib/gzlib.c b/Modules/zlib/gzlib.c new file mode 100644 --- /dev/null +++ b/Modules/zlib/gzlib.c @@ -0,0 +1,634 @@ +/* gzlib.c -- zlib functions common to reading and writing gzip files + * Copyright (C) 2004, 2010, 2011, 2012, 2013 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include "gzguts.h" + +#if defined(_WIN32) && !defined(__BORLANDC__) +# define LSEEK _lseeki64 +#else +#if defined(_LARGEFILE64_SOURCE) && _LFS64_LARGEFILE-0 +# define LSEEK lseek64 +#else +# define LSEEK lseek +#endif +#endif + +/* Local functions */ +local void gz_reset OF((gz_statep)); +local gzFile gz_open OF((const void *, int, const char *)); + +#if defined UNDER_CE + +/* Map the Windows error number in ERROR to a locale-dependent error message + string and return a pointer to it. Typically, the values for ERROR come + from GetLastError. + + The string pointed to shall not be modified by the application, but may be + overwritten by a subsequent call to gz_strwinerror + + The gz_strwinerror function does not change the current setting of + GetLastError. */ +char ZLIB_INTERNAL *gz_strwinerror (error) + DWORD error; +{ + static char buf[1024]; + + wchar_t *msgbuf; + DWORD lasterr = GetLastError(); + DWORD chars = FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM + | FORMAT_MESSAGE_ALLOCATE_BUFFER, + NULL, + error, + 0, /* Default language */ + (LPVOID)&msgbuf, + 0, + NULL); + if (chars != 0) { + /* If there is an \r\n appended, zap it. */ + if (chars >= 2 + && msgbuf[chars - 2] == '\r' && msgbuf[chars - 1] == '\n') { + chars -= 2; + msgbuf[chars] = 0; + } + + if (chars > sizeof (buf) - 1) { + chars = sizeof (buf) - 1; + msgbuf[chars] = 0; + } + + wcstombs(buf, msgbuf, chars + 1); + LocalFree(msgbuf); + } + else { + sprintf(buf, "unknown win32 error (%ld)", error); + } + + SetLastError(lasterr); + return buf; +} + +#endif /* UNDER_CE */ + +/* Reset gzip file state */ +local void gz_reset(state) + gz_statep state; +{ + state->x.have = 0; /* no output data available */ + if (state->mode == GZ_READ) { /* for reading ... */ + state->eof = 0; /* not at end of file */ + state->past = 0; /* have not read past end yet */ + state->how = LOOK; /* look for gzip header */ + } + state->seek = 0; /* no seek request pending */ + gz_error(state, Z_OK, NULL); /* clear error */ + state->x.pos = 0; /* no uncompressed data yet */ + state->strm.avail_in = 0; /* no input data yet */ +} + +/* Open a gzip file either by name or file descriptor. */ +local gzFile gz_open(path, fd, mode) + const void *path; + int fd; + const char *mode; +{ + gz_statep state; + size_t len; + int oflag; +#ifdef O_CLOEXEC + int cloexec = 0; +#endif +#ifdef O_EXCL + int exclusive = 0; +#endif + + /* check input */ + if (path == NULL) + return NULL; + + /* allocate gzFile structure to return */ + state = (gz_statep)malloc(sizeof(gz_state)); + if (state == NULL) + return NULL; + state->size = 0; /* no buffers allocated yet */ + state->want = GZBUFSIZE; /* requested buffer size */ + state->msg = NULL; /* no error message yet */ + + /* interpret mode */ + state->mode = GZ_NONE; + state->level = Z_DEFAULT_COMPRESSION; + state->strategy = Z_DEFAULT_STRATEGY; + state->direct = 0; + while (*mode) { + if (*mode >= '0' && *mode <= '9') + state->level = *mode - '0'; + else + switch (*mode) { + case 'r': + state->mode = GZ_READ; + break; +#ifndef NO_GZCOMPRESS + case 'w': + state->mode = GZ_WRITE; + break; + case 'a': + state->mode = GZ_APPEND; + break; +#endif + case '+': /* can't read and write at the same time */ + free(state); + return NULL; + case 'b': /* ignore -- will request binary anyway */ + break; +#ifdef O_CLOEXEC + case 'e': + cloexec = 1; + break; +#endif +#ifdef O_EXCL + case 'x': + exclusive = 1; + break; +#endif + case 'f': + state->strategy = Z_FILTERED; + break; + case 'h': + state->strategy = Z_HUFFMAN_ONLY; + break; + case 'R': + state->strategy = Z_RLE; + break; + case 'F': + state->strategy = Z_FIXED; + break; + case 'T': + state->direct = 1; + break; + default: /* could consider as an error, but just ignore */ + ; + } + mode++; + } + + /* must provide an "r", "w", or "a" */ + if (state->mode == GZ_NONE) { + free(state); + return NULL; + } + + /* can't force transparent read */ + if (state->mode == GZ_READ) { + if (state->direct) { + free(state); + return NULL; + } + state->direct = 1; /* for empty file */ + } + + /* save the path name for error messages */ +#ifdef _WIN32 + if (fd == -2) { + len = wcstombs(NULL, path, 0); + if (len == (size_t)-1) + len = 0; + } + else +#endif + len = strlen((const char *)path); + state->path = (char *)malloc(len + 1); + if (state->path == NULL) { + free(state); + return NULL; + } +#ifdef _WIN32 + if (fd == -2) + if (len) + wcstombs(state->path, path, len + 1); + else + *(state->path) = 0; + else +#endif +#if !defined(NO_snprintf) && !defined(NO_vsnprintf) + snprintf(state->path, len + 1, "%s", (const char *)path); +#else + strcpy(state->path, path); +#endif + + /* compute the flags for open() */ + oflag = +#ifdef O_LARGEFILE + O_LARGEFILE | +#endif +#ifdef O_BINARY + O_BINARY | +#endif +#ifdef O_CLOEXEC + (cloexec ? O_CLOEXEC : 0) | +#endif + (state->mode == GZ_READ ? + O_RDONLY : + (O_WRONLY | O_CREAT | +#ifdef O_EXCL + (exclusive ? O_EXCL : 0) | +#endif + (state->mode == GZ_WRITE ? + O_TRUNC : + O_APPEND))); + + /* open the file with the appropriate flags (or just use fd) */ + state->fd = fd > -1 ? fd : ( +#ifdef _WIN32 + fd == -2 ? _wopen(path, oflag, 0666) : +#endif + open((const char *)path, oflag, 0666)); + if (state->fd == -1) { + free(state->path); + free(state); + return NULL; + } + if (state->mode == GZ_APPEND) + state->mode = GZ_WRITE; /* simplify later checks */ + + /* save the current position for rewinding (only if reading) */ + if (state->mode == GZ_READ) { + state->start = LSEEK(state->fd, 0, SEEK_CUR); + if (state->start == -1) state->start = 0; + } + + /* initialize stream */ + gz_reset(state); + + /* return stream */ + return (gzFile)state; +} + +/* -- see zlib.h -- */ +gzFile ZEXPORT gzopen(path, mode) + const char *path; + const char *mode; +{ + return gz_open(path, -1, mode); +} + +/* -- see zlib.h -- */ +gzFile ZEXPORT gzopen64(path, mode) + const char *path; + const char *mode; +{ + return gz_open(path, -1, mode); +} + +/* -- see zlib.h -- */ +gzFile ZEXPORT gzdopen(fd, mode) + int fd; + const char *mode; +{ + char *path; /* identifier for error messages */ + gzFile gz; + + if (fd == -1 || (path = (char *)malloc(7 + 3 * sizeof(int))) == NULL) + return NULL; +#if !defined(NO_snprintf) && !defined(NO_vsnprintf) + snprintf(path, 7 + 3 * sizeof(int), "", fd); /* for debugging */ +#else + sprintf(path, "", fd); /* for debugging */ +#endif + gz = gz_open(path, fd, mode); + free(path); + return gz; +} + +/* -- see zlib.h -- */ +#ifdef _WIN32 +gzFile ZEXPORT gzopen_w(path, mode) + const wchar_t *path; + const char *mode; +{ + return gz_open(path, -2, mode); +} +#endif + +/* -- see zlib.h -- */ +int ZEXPORT gzbuffer(file, size) + gzFile file; + unsigned size; +{ + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return -1; + state = (gz_statep)file; + if (state->mode != GZ_READ && state->mode != GZ_WRITE) + return -1; + + /* make sure we haven't already allocated memory */ + if (state->size != 0) + return -1; + + /* check and set requested size */ + if (size < 2) + size = 2; /* need two bytes to check magic header */ + state->want = size; + return 0; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzrewind(file) + gzFile file; +{ + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return -1; + state = (gz_statep)file; + + /* check that we're reading and that there's no error */ + if (state->mode != GZ_READ || + (state->err != Z_OK && state->err != Z_BUF_ERROR)) + return -1; + + /* back up and start over */ + if (LSEEK(state->fd, state->start, SEEK_SET) == -1) + return -1; + gz_reset(state); + return 0; +} + +/* -- see zlib.h -- */ +z_off64_t ZEXPORT gzseek64(file, offset, whence) + gzFile file; + z_off64_t offset; + int whence; +{ + unsigned n; + z_off64_t ret; + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return -1; + state = (gz_statep)file; + if (state->mode != GZ_READ && state->mode != GZ_WRITE) + return -1; + + /* check that there's no error */ + if (state->err != Z_OK && state->err != Z_BUF_ERROR) + return -1; + + /* can only seek from start or relative to current position */ + if (whence != SEEK_SET && whence != SEEK_CUR) + return -1; + + /* normalize offset to a SEEK_CUR specification */ + if (whence == SEEK_SET) + offset -= state->x.pos; + else if (state->seek) + offset += state->skip; + state->seek = 0; + + /* if within raw area while reading, just go there */ + if (state->mode == GZ_READ && state->how == COPY && + state->x.pos + offset >= 0) { + ret = LSEEK(state->fd, offset - state->x.have, SEEK_CUR); + if (ret == -1) + return -1; + state->x.have = 0; + state->eof = 0; + state->past = 0; + state->seek = 0; + gz_error(state, Z_OK, NULL); + state->strm.avail_in = 0; + state->x.pos += offset; + return state->x.pos; + } + + /* calculate skip amount, rewinding if needed for back seek when reading */ + if (offset < 0) { + if (state->mode != GZ_READ) /* writing -- can't go backwards */ + return -1; + offset += state->x.pos; + if (offset < 0) /* before start of file! */ + return -1; + if (gzrewind(file) == -1) /* rewind, then skip to offset */ + return -1; + } + + /* if reading, skip what's in output buffer (one less gzgetc() check) */ + if (state->mode == GZ_READ) { + n = GT_OFF(state->x.have) || (z_off64_t)state->x.have > offset ? + (unsigned)offset : state->x.have; + state->x.have -= n; + state->x.next += n; + state->x.pos += n; + offset -= n; + } + + /* request skip (if not zero) */ + if (offset) { + state->seek = 1; + state->skip = offset; + } + return state->x.pos + offset; +} + +/* -- see zlib.h -- */ +z_off_t ZEXPORT gzseek(file, offset, whence) + gzFile file; + z_off_t offset; + int whence; +{ + z_off64_t ret; + + ret = gzseek64(file, (z_off64_t)offset, whence); + return ret == (z_off_t)ret ? (z_off_t)ret : -1; +} + +/* -- see zlib.h -- */ +z_off64_t ZEXPORT gztell64(file) + gzFile file; +{ + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return -1; + state = (gz_statep)file; + if (state->mode != GZ_READ && state->mode != GZ_WRITE) + return -1; + + /* return position */ + return state->x.pos + (state->seek ? state->skip : 0); +} + +/* -- see zlib.h -- */ +z_off_t ZEXPORT gztell(file) + gzFile file; +{ + z_off64_t ret; + + ret = gztell64(file); + return ret == (z_off_t)ret ? (z_off_t)ret : -1; +} + +/* -- see zlib.h -- */ +z_off64_t ZEXPORT gzoffset64(file) + gzFile file; +{ + z_off64_t offset; + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return -1; + state = (gz_statep)file; + if (state->mode != GZ_READ && state->mode != GZ_WRITE) + return -1; + + /* compute and return effective offset in file */ + offset = LSEEK(state->fd, 0, SEEK_CUR); + if (offset == -1) + return -1; + if (state->mode == GZ_READ) /* reading */ + offset -= state->strm.avail_in; /* don't count buffered input */ + return offset; +} + +/* -- see zlib.h -- */ +z_off_t ZEXPORT gzoffset(file) + gzFile file; +{ + z_off64_t ret; + + ret = gzoffset64(file); + return ret == (z_off_t)ret ? (z_off_t)ret : -1; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzeof(file) + gzFile file; +{ + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return 0; + state = (gz_statep)file; + if (state->mode != GZ_READ && state->mode != GZ_WRITE) + return 0; + + /* return end-of-file state */ + return state->mode == GZ_READ ? state->past : 0; +} + +/* -- see zlib.h -- */ +const char * ZEXPORT gzerror(file, errnum) + gzFile file; + int *errnum; +{ + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return NULL; + state = (gz_statep)file; + if (state->mode != GZ_READ && state->mode != GZ_WRITE) + return NULL; + + /* return error information */ + if (errnum != NULL) + *errnum = state->err; + return state->err == Z_MEM_ERROR ? "out of memory" : + (state->msg == NULL ? "" : state->msg); +} + +/* -- see zlib.h -- */ +void ZEXPORT gzclearerr(file) + gzFile file; +{ + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return; + state = (gz_statep)file; + if (state->mode != GZ_READ && state->mode != GZ_WRITE) + return; + + /* clear error and end-of-file */ + if (state->mode == GZ_READ) { + state->eof = 0; + state->past = 0; + } + gz_error(state, Z_OK, NULL); +} + +/* Create an error message in allocated memory and set state->err and + state->msg accordingly. Free any previous error message already there. Do + not try to free or allocate space if the error is Z_MEM_ERROR (out of + memory). Simply save the error message as a static string. If there is an + allocation failure constructing the error message, then convert the error to + out of memory. */ +void ZLIB_INTERNAL gz_error(state, err, msg) + gz_statep state; + int err; + const char *msg; +{ + /* free previously allocated message and clear */ + if (state->msg != NULL) { + if (state->err != Z_MEM_ERROR) + free(state->msg); + state->msg = NULL; + } + + /* if fatal, set state->x.have to 0 so that the gzgetc() macro fails */ + if (err != Z_OK && err != Z_BUF_ERROR) + state->x.have = 0; + + /* set error code, and if no message, then done */ + state->err = err; + if (msg == NULL) + return; + + /* for an out of memory error, return literal string when requested */ + if (err == Z_MEM_ERROR) + return; + + /* construct error message with path */ + if ((state->msg = (char *)malloc(strlen(state->path) + strlen(msg) + 3)) == + NULL) { + state->err = Z_MEM_ERROR; + return; + } +#if !defined(NO_snprintf) && !defined(NO_vsnprintf) + snprintf(state->msg, strlen(state->path) + strlen(msg) + 3, + "%s%s%s", state->path, ": ", msg); +#else + strcpy(state->msg, state->path); + strcat(state->msg, ": "); + strcat(state->msg, msg); +#endif + return; +} + +#ifndef INT_MAX +/* portably return maximum value for an int (when limits.h presumed not + available) -- we need to do this to cover cases where 2's complement not + used, since C standard permits 1's complement and sign-bit representations, + otherwise we could just use ((unsigned)-1) >> 1 */ +unsigned ZLIB_INTERNAL gz_intmax() +{ + unsigned p, q; + + p = 1; + do { + q = p; + p <<= 1; + p++; + } while (p > q); + return q >> 1; +} +#endif diff --git a/Modules/zlib/gzread.c b/Modules/zlib/gzread.c new file mode 100644 --- /dev/null +++ b/Modules/zlib/gzread.c @@ -0,0 +1,594 @@ +/* gzread.c -- zlib functions for reading gzip files + * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include "gzguts.h" + +/* Local functions */ +local int gz_load OF((gz_statep, unsigned char *, unsigned, unsigned *)); +local int gz_avail OF((gz_statep)); +local int gz_look OF((gz_statep)); +local int gz_decomp OF((gz_statep)); +local int gz_fetch OF((gz_statep)); +local int gz_skip OF((gz_statep, z_off64_t)); + +/* Use read() to load a buffer -- return -1 on error, otherwise 0. Read from + state->fd, and update state->eof, state->err, and state->msg as appropriate. + This function needs to loop on read(), since read() is not guaranteed to + read the number of bytes requested, depending on the type of descriptor. */ +local int gz_load(state, buf, len, have) + gz_statep state; + unsigned char *buf; + unsigned len; + unsigned *have; +{ + int ret; + + *have = 0; + do { + ret = read(state->fd, buf + *have, len - *have); + if (ret <= 0) + break; + *have += ret; + } while (*have < len); + if (ret < 0) { + gz_error(state, Z_ERRNO, zstrerror()); + return -1; + } + if (ret == 0) + state->eof = 1; + return 0; +} + +/* Load up input buffer and set eof flag if last data loaded -- return -1 on + error, 0 otherwise. Note that the eof flag is set when the end of the input + file is reached, even though there may be unused data in the buffer. Once + that data has been used, no more attempts will be made to read the file. + If strm->avail_in != 0, then the current data is moved to the beginning of + the input buffer, and then the remainder of the buffer is loaded with the + available data from the input file. */ +local int gz_avail(state) + gz_statep state; +{ + unsigned got; + z_streamp strm = &(state->strm); + + if (state->err != Z_OK && state->err != Z_BUF_ERROR) + return -1; + if (state->eof == 0) { + if (strm->avail_in) { /* copy what's there to the start */ + unsigned char *p = state->in; + unsigned const char *q = strm->next_in; + unsigned n = strm->avail_in; + do { + *p++ = *q++; + } while (--n); + } + if (gz_load(state, state->in + strm->avail_in, + state->size - strm->avail_in, &got) == -1) + return -1; + strm->avail_in += got; + strm->next_in = state->in; + } + return 0; +} + +/* Look for gzip header, set up for inflate or copy. state->x.have must be 0. + If this is the first time in, allocate required memory. state->how will be + left unchanged if there is no more input data available, will be set to COPY + if there is no gzip header and direct copying will be performed, or it will + be set to GZIP for decompression. If direct copying, then leftover input + data from the input buffer will be copied to the output buffer. In that + case, all further file reads will be directly to either the output buffer or + a user buffer. If decompressing, the inflate state will be initialized. + gz_look() will return 0 on success or -1 on failure. */ +local int gz_look(state) + gz_statep state; +{ + z_streamp strm = &(state->strm); + + /* allocate read buffers and inflate memory */ + if (state->size == 0) { + /* allocate buffers */ + state->in = (unsigned char *)malloc(state->want); + state->out = (unsigned char *)malloc(state->want << 1); + if (state->in == NULL || state->out == NULL) { + if (state->out != NULL) + free(state->out); + if (state->in != NULL) + free(state->in); + gz_error(state, Z_MEM_ERROR, "out of memory"); + return -1; + } + state->size = state->want; + + /* allocate inflate memory */ + state->strm.zalloc = Z_NULL; + state->strm.zfree = Z_NULL; + state->strm.opaque = Z_NULL; + state->strm.avail_in = 0; + state->strm.next_in = Z_NULL; + if (inflateInit2(&(state->strm), 15 + 16) != Z_OK) { /* gunzip */ + free(state->out); + free(state->in); + state->size = 0; + gz_error(state, Z_MEM_ERROR, "out of memory"); + return -1; + } + } + + /* get at least the magic bytes in the input buffer */ + if (strm->avail_in < 2) { + if (gz_avail(state) == -1) + return -1; + if (strm->avail_in == 0) + return 0; + } + + /* look for gzip magic bytes -- if there, do gzip decoding (note: there is + a logical dilemma here when considering the case of a partially written + gzip file, to wit, if a single 31 byte is written, then we cannot tell + whether this is a single-byte file, or just a partially written gzip + file -- for here we assume that if a gzip file is being written, then + the header will be written in a single operation, so that reading a + single byte is sufficient indication that it is not a gzip file) */ + if (strm->avail_in > 1 && + strm->next_in[0] == 31 && strm->next_in[1] == 139) { + inflateReset(strm); + state->how = GZIP; + state->direct = 0; + return 0; + } + + /* no gzip header -- if we were decoding gzip before, then this is trailing + garbage. Ignore the trailing garbage and finish. */ + if (state->direct == 0) { + strm->avail_in = 0; + state->eof = 1; + state->x.have = 0; + return 0; + } + + /* doing raw i/o, copy any leftover input to output -- this assumes that + the output buffer is larger than the input buffer, which also assures + space for gzungetc() */ + state->x.next = state->out; + if (strm->avail_in) { + memcpy(state->x.next, strm->next_in, strm->avail_in); + state->x.have = strm->avail_in; + strm->avail_in = 0; + } + state->how = COPY; + state->direct = 1; + return 0; +} + +/* Decompress from input to the provided next_out and avail_out in the state. + On return, state->x.have and state->x.next point to the just decompressed + data. If the gzip stream completes, state->how is reset to LOOK to look for + the next gzip stream or raw data, once state->x.have is depleted. Returns 0 + on success, -1 on failure. */ +local int gz_decomp(state) + gz_statep state; +{ + int ret = Z_OK; + unsigned had; + z_streamp strm = &(state->strm); + + /* fill output buffer up to end of deflate stream */ + had = strm->avail_out; + do { + /* get more input for inflate() */ + if (strm->avail_in == 0 && gz_avail(state) == -1) + return -1; + if (strm->avail_in == 0) { + gz_error(state, Z_BUF_ERROR, "unexpected end of file"); + break; + } + + /* decompress and handle errors */ + ret = inflate(strm, Z_NO_FLUSH); + if (ret == Z_STREAM_ERROR || ret == Z_NEED_DICT) { + gz_error(state, Z_STREAM_ERROR, + "internal error: inflate stream corrupt"); + return -1; + } + if (ret == Z_MEM_ERROR) { + gz_error(state, Z_MEM_ERROR, "out of memory"); + return -1; + } + if (ret == Z_DATA_ERROR) { /* deflate stream invalid */ + gz_error(state, Z_DATA_ERROR, + strm->msg == NULL ? "compressed data error" : strm->msg); + return -1; + } + } while (strm->avail_out && ret != Z_STREAM_END); + + /* update available output */ + state->x.have = had - strm->avail_out; + state->x.next = strm->next_out - state->x.have; + + /* if the gzip stream completed successfully, look for another */ + if (ret == Z_STREAM_END) + state->how = LOOK; + + /* good decompression */ + return 0; +} + +/* Fetch data and put it in the output buffer. Assumes state->x.have is 0. + Data is either copied from the input file or decompressed from the input + file depending on state->how. If state->how is LOOK, then a gzip header is + looked for to determine whether to copy or decompress. Returns -1 on error, + otherwise 0. gz_fetch() will leave state->how as COPY or GZIP unless the + end of the input file has been reached and all data has been processed. */ +local int gz_fetch(state) + gz_statep state; +{ + z_streamp strm = &(state->strm); + + do { + switch(state->how) { + case LOOK: /* -> LOOK, COPY (only if never GZIP), or GZIP */ + if (gz_look(state) == -1) + return -1; + if (state->how == LOOK) + return 0; + break; + case COPY: /* -> COPY */ + if (gz_load(state, state->out, state->size << 1, &(state->x.have)) + == -1) + return -1; + state->x.next = state->out; + return 0; + case GZIP: /* -> GZIP or LOOK (if end of gzip stream) */ + strm->avail_out = state->size << 1; + strm->next_out = state->out; + if (gz_decomp(state) == -1) + return -1; + } + } while (state->x.have == 0 && (!state->eof || strm->avail_in)); + return 0; +} + +/* Skip len uncompressed bytes of output. Return -1 on error, 0 on success. */ +local int gz_skip(state, len) + gz_statep state; + z_off64_t len; +{ + unsigned n; + + /* skip over len bytes or reach end-of-file, whichever comes first */ + while (len) + /* skip over whatever is in output buffer */ + if (state->x.have) { + n = GT_OFF(state->x.have) || (z_off64_t)state->x.have > len ? + (unsigned)len : state->x.have; + state->x.have -= n; + state->x.next += n; + state->x.pos += n; + len -= n; + } + + /* output buffer empty -- return if we're at the end of the input */ + else if (state->eof && state->strm.avail_in == 0) + break; + + /* need more data to skip -- load up output buffer */ + else { + /* get more output, looking for header if required */ + if (gz_fetch(state) == -1) + return -1; + } + return 0; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzread(file, buf, len) + gzFile file; + voidp buf; + unsigned len; +{ + unsigned got, n; + gz_statep state; + z_streamp strm; + + /* get internal structure */ + if (file == NULL) + return -1; + state = (gz_statep)file; + strm = &(state->strm); + + /* check that we're reading and that there's no (serious) error */ + if (state->mode != GZ_READ || + (state->err != Z_OK && state->err != Z_BUF_ERROR)) + return -1; + + /* since an int is returned, make sure len fits in one, otherwise return + with an error (this avoids the flaw in the interface) */ + if ((int)len < 0) { + gz_error(state, Z_DATA_ERROR, "requested length does not fit in int"); + return -1; + } + + /* if len is zero, avoid unnecessary operations */ + if (len == 0) + return 0; + + /* process a skip request */ + if (state->seek) { + state->seek = 0; + if (gz_skip(state, state->skip) == -1) + return -1; + } + + /* get len bytes to buf, or less than len if at the end */ + got = 0; + do { + /* first just try copying data from the output buffer */ + if (state->x.have) { + n = state->x.have > len ? len : state->x.have; + memcpy(buf, state->x.next, n); + state->x.next += n; + state->x.have -= n; + } + + /* output buffer empty -- return if we're at the end of the input */ + else if (state->eof && strm->avail_in == 0) { + state->past = 1; /* tried to read past end */ + break; + } + + /* need output data -- for small len or new stream load up our output + buffer */ + else if (state->how == LOOK || len < (state->size << 1)) { + /* get more output, looking for header if required */ + if (gz_fetch(state) == -1) + return -1; + continue; /* no progress yet -- go back to copy above */ + /* the copy above assures that we will leave with space in the + output buffer, allowing at least one gzungetc() to succeed */ + } + + /* large len -- read directly into user buffer */ + else if (state->how == COPY) { /* read directly */ + if (gz_load(state, (unsigned char *)buf, len, &n) == -1) + return -1; + } + + /* large len -- decompress directly into user buffer */ + else { /* state->how == GZIP */ + strm->avail_out = len; + strm->next_out = (unsigned char *)buf; + if (gz_decomp(state) == -1) + return -1; + n = state->x.have; + state->x.have = 0; + } + + /* update progress */ + len -= n; + buf = (char *)buf + n; + got += n; + state->x.pos += n; + } while (len); + + /* return number of bytes read into user buffer (will fit in int) */ + return (int)got; +} + +/* -- see zlib.h -- */ +#ifdef Z_PREFIX_SET +# undef z_gzgetc +#else +# undef gzgetc +#endif +int ZEXPORT gzgetc(file) + gzFile file; +{ + int ret; + unsigned char buf[1]; + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return -1; + state = (gz_statep)file; + + /* check that we're reading and that there's no (serious) error */ + if (state->mode != GZ_READ || + (state->err != Z_OK && state->err != Z_BUF_ERROR)) + return -1; + + /* try output buffer (no need to check for skip request) */ + if (state->x.have) { + state->x.have--; + state->x.pos++; + return *(state->x.next)++; + } + + /* nothing there -- try gzread() */ + ret = gzread(file, buf, 1); + return ret < 1 ? -1 : buf[0]; +} + +int ZEXPORT gzgetc_(file) +gzFile file; +{ + return gzgetc(file); +} + +/* -- see zlib.h -- */ +int ZEXPORT gzungetc(c, file) + int c; + gzFile file; +{ + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return -1; + state = (gz_statep)file; + + /* check that we're reading and that there's no (serious) error */ + if (state->mode != GZ_READ || + (state->err != Z_OK && state->err != Z_BUF_ERROR)) + return -1; + + /* process a skip request */ + if (state->seek) { + state->seek = 0; + if (gz_skip(state, state->skip) == -1) + return -1; + } + + /* can't push EOF */ + if (c < 0) + return -1; + + /* if output buffer empty, put byte at end (allows more pushing) */ + if (state->x.have == 0) { + state->x.have = 1; + state->x.next = state->out + (state->size << 1) - 1; + state->x.next[0] = c; + state->x.pos--; + state->past = 0; + return c; + } + + /* if no room, give up (must have already done a gzungetc()) */ + if (state->x.have == (state->size << 1)) { + gz_error(state, Z_DATA_ERROR, "out of room to push characters"); + return -1; + } + + /* slide output data if needed and insert byte before existing data */ + if (state->x.next == state->out) { + unsigned char *src = state->out + state->x.have; + unsigned char *dest = state->out + (state->size << 1); + while (src > state->out) + *--dest = *--src; + state->x.next = dest; + } + state->x.have++; + state->x.next--; + state->x.next[0] = c; + state->x.pos--; + state->past = 0; + return c; +} + +/* -- see zlib.h -- */ +char * ZEXPORT gzgets(file, buf, len) + gzFile file; + char *buf; + int len; +{ + unsigned left, n; + char *str; + unsigned char *eol; + gz_statep state; + + /* check parameters and get internal structure */ + if (file == NULL || buf == NULL || len < 1) + return NULL; + state = (gz_statep)file; + + /* check that we're reading and that there's no (serious) error */ + if (state->mode != GZ_READ || + (state->err != Z_OK && state->err != Z_BUF_ERROR)) + return NULL; + + /* process a skip request */ + if (state->seek) { + state->seek = 0; + if (gz_skip(state, state->skip) == -1) + return NULL; + } + + /* copy output bytes up to new line or len - 1, whichever comes first -- + append a terminating zero to the string (we don't check for a zero in + the contents, let the user worry about that) */ + str = buf; + left = (unsigned)len - 1; + if (left) do { + /* assure that something is in the output buffer */ + if (state->x.have == 0 && gz_fetch(state) == -1) + return NULL; /* error */ + if (state->x.have == 0) { /* end of file */ + state->past = 1; /* read past end */ + break; /* return what we have */ + } + + /* look for end-of-line in current output buffer */ + n = state->x.have > left ? left : state->x.have; + eol = (unsigned char *)memchr(state->x.next, '\n', n); + if (eol != NULL) + n = (unsigned)(eol - state->x.next) + 1; + + /* copy through end-of-line, or remainder if not found */ + memcpy(buf, state->x.next, n); + state->x.have -= n; + state->x.next += n; + state->x.pos += n; + left -= n; + buf += n; + } while (left && eol == NULL); + + /* return terminated string, or if nothing, end of file */ + if (buf == str) + return NULL; + buf[0] = 0; + return str; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzdirect(file) + gzFile file; +{ + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return 0; + state = (gz_statep)file; + + /* if the state is not known, but we can find out, then do so (this is + mainly for right after a gzopen() or gzdopen()) */ + if (state->mode == GZ_READ && state->how == LOOK && state->x.have == 0) + (void)gz_look(state); + + /* return 1 if transparent, 0 if processing a gzip stream */ + return state->direct; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzclose_r(file) + gzFile file; +{ + int ret, err; + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return Z_STREAM_ERROR; + state = (gz_statep)file; + + /* check that we're reading */ + if (state->mode != GZ_READ) + return Z_STREAM_ERROR; + + /* free memory and close file */ + if (state->size) { + inflateEnd(&(state->strm)); + free(state->out); + free(state->in); + } + err = state->err == Z_BUF_ERROR ? Z_BUF_ERROR : Z_OK; + gz_error(state, Z_OK, NULL); + free(state->path); + ret = close(state->fd); + free(state); + return ret ? Z_ERRNO : err; +} diff --git a/Modules/zlib/gzwrite.c b/Modules/zlib/gzwrite.c new file mode 100644 --- /dev/null +++ b/Modules/zlib/gzwrite.c @@ -0,0 +1,577 @@ +/* gzwrite.c -- zlib functions for writing gzip files + * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include "gzguts.h" + +/* Local functions */ +local int gz_init OF((gz_statep)); +local int gz_comp OF((gz_statep, int)); +local int gz_zero OF((gz_statep, z_off64_t)); + +/* Initialize state for writing a gzip file. Mark initialization by setting + state->size to non-zero. Return -1 on failure or 0 on success. */ +local int gz_init(state) + gz_statep state; +{ + int ret; + z_streamp strm = &(state->strm); + + /* allocate input buffer */ + state->in = (unsigned char *)malloc(state->want); + if (state->in == NULL) { + gz_error(state, Z_MEM_ERROR, "out of memory"); + return -1; + } + + /* only need output buffer and deflate state if compressing */ + if (!state->direct) { + /* allocate output buffer */ + state->out = (unsigned char *)malloc(state->want); + if (state->out == NULL) { + free(state->in); + gz_error(state, Z_MEM_ERROR, "out of memory"); + return -1; + } + + /* allocate deflate memory, set up for gzip compression */ + strm->zalloc = Z_NULL; + strm->zfree = Z_NULL; + strm->opaque = Z_NULL; + ret = deflateInit2(strm, state->level, Z_DEFLATED, + MAX_WBITS + 16, DEF_MEM_LEVEL, state->strategy); + if (ret != Z_OK) { + free(state->out); + free(state->in); + gz_error(state, Z_MEM_ERROR, "out of memory"); + return -1; + } + } + + /* mark state as initialized */ + state->size = state->want; + + /* initialize write buffer if compressing */ + if (!state->direct) { + strm->avail_out = state->size; + strm->next_out = state->out; + state->x.next = strm->next_out; + } + return 0; +} + +/* Compress whatever is at avail_in and next_in and write to the output file. + Return -1 if there is an error writing to the output file, otherwise 0. + flush is assumed to be a valid deflate() flush value. If flush is Z_FINISH, + then the deflate() state is reset to start a new gzip stream. If gz->direct + is true, then simply write to the output file without compressing, and + ignore flush. */ +local int gz_comp(state, flush) + gz_statep state; + int flush; +{ + int ret, got; + unsigned have; + z_streamp strm = &(state->strm); + + /* allocate memory if this is the first time through */ + if (state->size == 0 && gz_init(state) == -1) + return -1; + + /* write directly if requested */ + if (state->direct) { + got = write(state->fd, strm->next_in, strm->avail_in); + if (got < 0 || (unsigned)got != strm->avail_in) { + gz_error(state, Z_ERRNO, zstrerror()); + return -1; + } + strm->avail_in = 0; + return 0; + } + + /* run deflate() on provided input until it produces no more output */ + ret = Z_OK; + do { + /* write out current buffer contents if full, or if flushing, but if + doing Z_FINISH then don't write until we get to Z_STREAM_END */ + if (strm->avail_out == 0 || (flush != Z_NO_FLUSH && + (flush != Z_FINISH || ret == Z_STREAM_END))) { + have = (unsigned)(strm->next_out - state->x.next); + if (have && ((got = write(state->fd, state->x.next, have)) < 0 || + (unsigned)got != have)) { + gz_error(state, Z_ERRNO, zstrerror()); + return -1; + } + if (strm->avail_out == 0) { + strm->avail_out = state->size; + strm->next_out = state->out; + } + state->x.next = strm->next_out; + } + + /* compress */ + have = strm->avail_out; + ret = deflate(strm, flush); + if (ret == Z_STREAM_ERROR) { + gz_error(state, Z_STREAM_ERROR, + "internal error: deflate stream corrupt"); + return -1; + } + have -= strm->avail_out; + } while (have); + + /* if that completed a deflate stream, allow another to start */ + if (flush == Z_FINISH) + deflateReset(strm); + + /* all done, no errors */ + return 0; +} + +/* Compress len zeros to output. Return -1 on error, 0 on success. */ +local int gz_zero(state, len) + gz_statep state; + z_off64_t len; +{ + int first; + unsigned n; + z_streamp strm = &(state->strm); + + /* consume whatever's left in the input buffer */ + if (strm->avail_in && gz_comp(state, Z_NO_FLUSH) == -1) + return -1; + + /* compress len zeros (len guaranteed > 0) */ + first = 1; + while (len) { + n = GT_OFF(state->size) || (z_off64_t)state->size > len ? + (unsigned)len : state->size; + if (first) { + memset(state->in, 0, n); + first = 0; + } + strm->avail_in = n; + strm->next_in = state->in; + state->x.pos += n; + if (gz_comp(state, Z_NO_FLUSH) == -1) + return -1; + len -= n; + } + return 0; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzwrite(file, buf, len) + gzFile file; + voidpc buf; + unsigned len; +{ + unsigned put = len; + gz_statep state; + z_streamp strm; + + /* get internal structure */ + if (file == NULL) + return 0; + state = (gz_statep)file; + strm = &(state->strm); + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return 0; + + /* since an int is returned, make sure len fits in one, otherwise return + with an error (this avoids the flaw in the interface) */ + if ((int)len < 0) { + gz_error(state, Z_DATA_ERROR, "requested length does not fit in int"); + return 0; + } + + /* if len is zero, avoid unnecessary operations */ + if (len == 0) + return 0; + + /* allocate memory if this is the first time through */ + if (state->size == 0 && gz_init(state) == -1) + return 0; + + /* check for seek request */ + if (state->seek) { + state->seek = 0; + if (gz_zero(state, state->skip) == -1) + return 0; + } + + /* for small len, copy to input buffer, otherwise compress directly */ + if (len < state->size) { + /* copy to input buffer, compress when full */ + do { + unsigned have, copy; + + if (strm->avail_in == 0) + strm->next_in = state->in; + have = (unsigned)((strm->next_in + strm->avail_in) - state->in); + copy = state->size - have; + if (copy > len) + copy = len; + memcpy(state->in + have, buf, copy); + strm->avail_in += copy; + state->x.pos += copy; + buf = (const char *)buf + copy; + len -= copy; + if (len && gz_comp(state, Z_NO_FLUSH) == -1) + return 0; + } while (len); + } + else { + /* consume whatever's left in the input buffer */ + if (strm->avail_in && gz_comp(state, Z_NO_FLUSH) == -1) + return 0; + + /* directly compress user buffer to file */ + strm->avail_in = len; + strm->next_in = (z_const Bytef *)buf; + state->x.pos += len; + if (gz_comp(state, Z_NO_FLUSH) == -1) + return 0; + } + + /* input was all buffered or compressed (put will fit in int) */ + return (int)put; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzputc(file, c) + gzFile file; + int c; +{ + unsigned have; + unsigned char buf[1]; + gz_statep state; + z_streamp strm; + + /* get internal structure */ + if (file == NULL) + return -1; + state = (gz_statep)file; + strm = &(state->strm); + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return -1; + + /* check for seek request */ + if (state->seek) { + state->seek = 0; + if (gz_zero(state, state->skip) == -1) + return -1; + } + + /* try writing to input buffer for speed (state->size == 0 if buffer not + initialized) */ + if (state->size) { + if (strm->avail_in == 0) + strm->next_in = state->in; + have = (unsigned)((strm->next_in + strm->avail_in) - state->in); + if (have < state->size) { + state->in[have] = c; + strm->avail_in++; + state->x.pos++; + return c & 0xff; + } + } + + /* no room in buffer or not initialized, use gz_write() */ + buf[0] = c; + if (gzwrite(file, buf, 1) != 1) + return -1; + return c & 0xff; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzputs(file, str) + gzFile file; + const char *str; +{ + int ret; + unsigned len; + + /* write string */ + len = (unsigned)strlen(str); + ret = gzwrite(file, str, len); + return ret == 0 && len != 0 ? -1 : ret; +} + +#if defined(STDC) || defined(Z_HAVE_STDARG_H) +#include + +/* -- see zlib.h -- */ +int ZEXPORTVA gzvprintf(gzFile file, const char *format, va_list va) +{ + int size, len; + gz_statep state; + z_streamp strm; + + /* get internal structure */ + if (file == NULL) + return -1; + state = (gz_statep)file; + strm = &(state->strm); + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return 0; + + /* make sure we have some buffer space */ + if (state->size == 0 && gz_init(state) == -1) + return 0; + + /* check for seek request */ + if (state->seek) { + state->seek = 0; + if (gz_zero(state, state->skip) == -1) + return 0; + } + + /* consume whatever's left in the input buffer */ + if (strm->avail_in && gz_comp(state, Z_NO_FLUSH) == -1) + return 0; + + /* do the printf() into the input buffer, put length in len */ + size = (int)(state->size); + state->in[size - 1] = 0; +#ifdef NO_vsnprintf +# ifdef HAS_vsprintf_void + (void)vsprintf((char *)(state->in), format, va); + for (len = 0; len < size; len++) + if (state->in[len] == 0) break; +# else + len = vsprintf((char *)(state->in), format, va); +# endif +#else +# ifdef HAS_vsnprintf_void + (void)vsnprintf((char *)(state->in), size, format, va); + len = strlen((char *)(state->in)); +# else + len = vsnprintf((char *)(state->in), size, format, va); +# endif +#endif + + /* check that printf() results fit in buffer */ + if (len <= 0 || len >= (int)size || state->in[size - 1] != 0) + return 0; + + /* update buffer and position, defer compression until needed */ + strm->avail_in = (unsigned)len; + strm->next_in = state->in; + state->x.pos += len; + return len; +} + +int ZEXPORTVA gzprintf(gzFile file, const char *format, ...) +{ + va_list va; + int ret; + + va_start(va, format); + ret = gzvprintf(file, format, va); + va_end(va); + return ret; +} + +#else /* !STDC && !Z_HAVE_STDARG_H */ + +/* -- see zlib.h -- */ +int ZEXPORTVA gzprintf (file, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, + a11, a12, a13, a14, a15, a16, a17, a18, a19, a20) + gzFile file; + const char *format; + int a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, + a11, a12, a13, a14, a15, a16, a17, a18, a19, a20; +{ + int size, len; + gz_statep state; + z_streamp strm; + + /* get internal structure */ + if (file == NULL) + return -1; + state = (gz_statep)file; + strm = &(state->strm); + + /* check that can really pass pointer in ints */ + if (sizeof(int) != sizeof(void *)) + return 0; + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return 0; + + /* make sure we have some buffer space */ + if (state->size == 0 && gz_init(state) == -1) + return 0; + + /* check for seek request */ + if (state->seek) { + state->seek = 0; + if (gz_zero(state, state->skip) == -1) + return 0; + } + + /* consume whatever's left in the input buffer */ + if (strm->avail_in && gz_comp(state, Z_NO_FLUSH) == -1) + return 0; + + /* do the printf() into the input buffer, put length in len */ + size = (int)(state->size); + state->in[size - 1] = 0; +#ifdef NO_snprintf +# ifdef HAS_sprintf_void + sprintf((char *)(state->in), format, a1, a2, a3, a4, a5, a6, a7, a8, + a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); + for (len = 0; len < size; len++) + if (state->in[len] == 0) break; +# else + len = sprintf((char *)(state->in), format, a1, a2, a3, a4, a5, a6, a7, a8, + a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); +# endif +#else +# ifdef HAS_snprintf_void + snprintf((char *)(state->in), size, format, a1, a2, a3, a4, a5, a6, a7, a8, + a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); + len = strlen((char *)(state->in)); +# else + len = snprintf((char *)(state->in), size, format, a1, a2, a3, a4, a5, a6, + a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, + a19, a20); +# endif +#endif + + /* check that printf() results fit in buffer */ + if (len <= 0 || len >= (int)size || state->in[size - 1] != 0) + return 0; + + /* update buffer and position, defer compression until needed */ + strm->avail_in = (unsigned)len; + strm->next_in = state->in; + state->x.pos += len; + return len; +} + +#endif + +/* -- see zlib.h -- */ +int ZEXPORT gzflush(file, flush) + gzFile file; + int flush; +{ + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return -1; + state = (gz_statep)file; + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return Z_STREAM_ERROR; + + /* check flush parameter */ + if (flush < 0 || flush > Z_FINISH) + return Z_STREAM_ERROR; + + /* check for seek request */ + if (state->seek) { + state->seek = 0; + if (gz_zero(state, state->skip) == -1) + return -1; + } + + /* compress remaining data with requested flush */ + gz_comp(state, flush); + return state->err; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzsetparams(file, level, strategy) + gzFile file; + int level; + int strategy; +{ + gz_statep state; + z_streamp strm; + + /* get internal structure */ + if (file == NULL) + return Z_STREAM_ERROR; + state = (gz_statep)file; + strm = &(state->strm); + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return Z_STREAM_ERROR; + + /* if no change is requested, then do nothing */ + if (level == state->level && strategy == state->strategy) + return Z_OK; + + /* check for seek request */ + if (state->seek) { + state->seek = 0; + if (gz_zero(state, state->skip) == -1) + return -1; + } + + /* change compression parameters for subsequent input */ + if (state->size) { + /* flush previous input with previous parameters before changing */ + if (strm->avail_in && gz_comp(state, Z_PARTIAL_FLUSH) == -1) + return state->err; + deflateParams(strm, level, strategy); + } + state->level = level; + state->strategy = strategy; + return Z_OK; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzclose_w(file) + gzFile file; +{ + int ret = Z_OK; + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return Z_STREAM_ERROR; + state = (gz_statep)file; + + /* check that we're writing */ + if (state->mode != GZ_WRITE) + return Z_STREAM_ERROR; + + /* check for seek request */ + if (state->seek) { + state->seek = 0; + if (gz_zero(state, state->skip) == -1) + ret = state->err; + } + + /* flush, free memory, and close file */ + if (gz_comp(state, Z_FINISH) == -1) + ret = state->err; + if (state->size) { + if (!state->direct) { + (void)deflateEnd(&(state->strm)); + free(state->out); + } + free(state->in); + } + gz_error(state, Z_OK, NULL); + free(state->path); + if (close(state->fd) == -1) + ret = Z_ERRNO; + free(state); + return ret; +} diff --git a/Modules/zlib/infback.c b/Modules/zlib/infback.c --- a/Modules/zlib/infback.c +++ b/Modules/zlib/infback.c @@ -1,5 +1,5 @@ /* infback.c -- inflate using a call-back interface - * Copyright (C) 1995-2005 Mark Adler + * Copyright (C) 1995-2011 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -42,10 +42,19 @@ return Z_STREAM_ERROR; strm->msg = Z_NULL; /* in case we return an error */ if (strm->zalloc == (alloc_func)0) { +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else strm->zalloc = zcalloc; strm->opaque = (voidpf)0; +#endif } - if (strm->zfree == (free_func)0) strm->zfree = zcfree; + if (strm->zfree == (free_func)0) +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zfree = zcfree; +#endif state = (struct inflate_state FAR *)ZALLOC(strm, 1, sizeof(struct inflate_state)); if (state == Z_NULL) return Z_MEM_ERROR; @@ -55,7 +64,7 @@ state->wbits = windowBits; state->wsize = 1U << windowBits; state->window = window; - state->write = 0; + state->wnext = 0; state->whave = 0; return Z_OK; } @@ -246,14 +255,14 @@ void FAR *out_desc; { struct inflate_state FAR *state; - unsigned char FAR *next; /* next input */ + z_const unsigned char FAR *next; /* next input */ unsigned char FAR *put; /* next output */ unsigned have, left; /* available input and output */ unsigned long hold; /* bit buffer */ unsigned bits; /* bits in bit buffer */ unsigned copy; /* number of stored or match bytes to copy */ unsigned char FAR *from; /* where to copy match bytes from */ - code this; /* current decoding table entry */ + code here; /* current decoding table entry */ code last; /* parent table entry */ unsigned len; /* length to copy for repeats, bits to drop */ int ret; /* return code */ @@ -389,19 +398,18 @@ state->have = 0; while (state->have < state->nlen + state->ndist) { for (;;) { - this = state->lencode[BITS(state->lenbits)]; - if ((unsigned)(this.bits) <= bits) break; + here = state->lencode[BITS(state->lenbits)]; + if ((unsigned)(here.bits) <= bits) break; PULLBYTE(); } - if (this.val < 16) { - NEEDBITS(this.bits); - DROPBITS(this.bits); - state->lens[state->have++] = this.val; + if (here.val < 16) { + DROPBITS(here.bits); + state->lens[state->have++] = here.val; } else { - if (this.val == 16) { - NEEDBITS(this.bits + 2); - DROPBITS(this.bits); + if (here.val == 16) { + NEEDBITS(here.bits + 2); + DROPBITS(here.bits); if (state->have == 0) { strm->msg = (char *)"invalid bit length repeat"; state->mode = BAD; @@ -411,16 +419,16 @@ copy = 3 + BITS(2); DROPBITS(2); } - else if (this.val == 17) { - NEEDBITS(this.bits + 3); - DROPBITS(this.bits); + else if (here.val == 17) { + NEEDBITS(here.bits + 3); + DROPBITS(here.bits); len = 0; copy = 3 + BITS(3); DROPBITS(3); } else { - NEEDBITS(this.bits + 7); - DROPBITS(this.bits); + NEEDBITS(here.bits + 7); + DROPBITS(here.bits); len = 0; copy = 11 + BITS(7); DROPBITS(7); @@ -438,7 +446,16 @@ /* handle error breaks in while */ if (state->mode == BAD) break; - /* build code tables */ + /* check for end-of-block code (better have one) */ + if (state->lens[256] == 0) { + strm->msg = (char *)"invalid code -- missing end-of-block"; + state->mode = BAD; + break; + } + + /* build code tables -- note: do not change the lenbits or distbits + values here (9 and 6) without reading the comments in inftrees.h + concerning the ENOUGH constants, which depend on those values */ state->next = state->codes; state->lencode = (code const FAR *)(state->next); state->lenbits = 9; @@ -474,28 +491,28 @@ /* get a literal, length, or end-of-block code */ for (;;) { - this = state->lencode[BITS(state->lenbits)]; - if ((unsigned)(this.bits) <= bits) break; + here = state->lencode[BITS(state->lenbits)]; + if ((unsigned)(here.bits) <= bits) break; PULLBYTE(); } - if (this.op && (this.op & 0xf0) == 0) { - last = this; + if (here.op && (here.op & 0xf0) == 0) { + last = here; for (;;) { - this = state->lencode[last.val + + here = state->lencode[last.val + (BITS(last.bits + last.op) >> last.bits)]; - if ((unsigned)(last.bits + this.bits) <= bits) break; + if ((unsigned)(last.bits + here.bits) <= bits) break; PULLBYTE(); } DROPBITS(last.bits); } - DROPBITS(this.bits); - state->length = (unsigned)this.val; + DROPBITS(here.bits); + state->length = (unsigned)here.val; /* process literal */ - if (this.op == 0) { - Tracevv((stderr, this.val >= 0x20 && this.val < 0x7f ? + if (here.op == 0) { + Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ? "inflate: literal '%c'\n" : - "inflate: literal 0x%02x\n", this.val)); + "inflate: literal 0x%02x\n", here.val)); ROOM(); *put++ = (unsigned char)(state->length); left--; @@ -504,21 +521,21 @@ } /* process end of block */ - if (this.op & 32) { + if (here.op & 32) { Tracevv((stderr, "inflate: end of block\n")); state->mode = TYPE; break; } /* invalid code */ - if (this.op & 64) { + if (here.op & 64) { strm->msg = (char *)"invalid literal/length code"; state->mode = BAD; break; } /* length code -- get extra bits, if any */ - state->extra = (unsigned)(this.op) & 15; + state->extra = (unsigned)(here.op) & 15; if (state->extra != 0) { NEEDBITS(state->extra); state->length += BITS(state->extra); @@ -528,30 +545,30 @@ /* get distance code */ for (;;) { - this = state->distcode[BITS(state->distbits)]; - if ((unsigned)(this.bits) <= bits) break; + here = state->distcode[BITS(state->distbits)]; + if ((unsigned)(here.bits) <= bits) break; PULLBYTE(); } - if ((this.op & 0xf0) == 0) { - last = this; + if ((here.op & 0xf0) == 0) { + last = here; for (;;) { - this = state->distcode[last.val + + here = state->distcode[last.val + (BITS(last.bits + last.op) >> last.bits)]; - if ((unsigned)(last.bits + this.bits) <= bits) break; + if ((unsigned)(last.bits + here.bits) <= bits) break; PULLBYTE(); } DROPBITS(last.bits); } - DROPBITS(this.bits); - if (this.op & 64) { + DROPBITS(here.bits); + if (here.op & 64) { strm->msg = (char *)"invalid distance code"; state->mode = BAD; break; } - state->offset = (unsigned)this.val; + state->offset = (unsigned)here.val; /* get distance extra bits, if any */ - state->extra = (unsigned)(this.op) & 15; + state->extra = (unsigned)(here.op) & 15; if (state->extra != 0) { NEEDBITS(state->extra); state->offset += BITS(state->extra); diff --git a/Modules/zlib/inffast.c b/Modules/zlib/inffast.c --- a/Modules/zlib/inffast.c +++ b/Modules/zlib/inffast.c @@ -1,5 +1,5 @@ /* inffast.c -- fast decoding - * Copyright (C) 1995-2004 Mark Adler + * Copyright (C) 1995-2008, 2010, 2013 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -64,13 +64,13 @@ requires strm->avail_out >= 258 for each loop to avoid checking for output space. */ -void inflate_fast(strm, start) +void ZLIB_INTERNAL inflate_fast(strm, start) z_streamp strm; unsigned start; /* inflate()'s starting value for strm->avail_out */ { struct inflate_state FAR *state; - unsigned char FAR *in; /* local strm->next_in */ - unsigned char FAR *last; /* while in < last, enough input available */ + z_const unsigned char FAR *in; /* local strm->next_in */ + z_const unsigned char FAR *last; /* have enough input while in < last */ unsigned char FAR *out; /* local strm->next_out */ unsigned char FAR *beg; /* inflate()'s initial strm->next_out */ unsigned char FAR *end; /* while out < end, enough space available */ @@ -79,7 +79,7 @@ #endif unsigned wsize; /* window size or zero if not using window */ unsigned whave; /* valid bytes in the window */ - unsigned write; /* window write index */ + unsigned wnext; /* window write index */ unsigned char FAR *window; /* allocated sliding window, if wsize != 0 */ unsigned long hold; /* local strm->hold */ unsigned bits; /* local strm->bits */ @@ -87,7 +87,7 @@ code const FAR *dcode; /* local strm->distcode */ unsigned lmask; /* mask for first level of length codes */ unsigned dmask; /* mask for first level of distance codes */ - code this; /* retrieved table entry */ + code here; /* retrieved table entry */ unsigned op; /* code bits, operation, extra bits, or */ /* window position, window bytes to copy */ unsigned len; /* match length, unused bytes */ @@ -106,7 +106,7 @@ #endif wsize = state->wsize; whave = state->whave; - write = state->write; + wnext = state->wnext; window = state->window; hold = state->hold; bits = state->bits; @@ -124,20 +124,20 @@ hold += (unsigned long)(PUP(in)) << bits; bits += 8; } - this = lcode[hold & lmask]; + here = lcode[hold & lmask]; dolen: - op = (unsigned)(this.bits); + op = (unsigned)(here.bits); hold >>= op; bits -= op; - op = (unsigned)(this.op); + op = (unsigned)(here.op); if (op == 0) { /* literal */ - Tracevv((stderr, this.val >= 0x20 && this.val < 0x7f ? + Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ? "inflate: literal '%c'\n" : - "inflate: literal 0x%02x\n", this.val)); - PUP(out) = (unsigned char)(this.val); + "inflate: literal 0x%02x\n", here.val)); + PUP(out) = (unsigned char)(here.val); } else if (op & 16) { /* length base */ - len = (unsigned)(this.val); + len = (unsigned)(here.val); op &= 15; /* number of extra bits */ if (op) { if (bits < op) { @@ -155,14 +155,14 @@ hold += (unsigned long)(PUP(in)) << bits; bits += 8; } - this = dcode[hold & dmask]; + here = dcode[hold & dmask]; dodist: - op = (unsigned)(this.bits); + op = (unsigned)(here.bits); hold >>= op; bits -= op; - op = (unsigned)(this.op); + op = (unsigned)(here.op); if (op & 16) { /* distance base */ - dist = (unsigned)(this.val); + dist = (unsigned)(here.val); op &= 15; /* number of extra bits */ if (bits < op) { hold += (unsigned long)(PUP(in)) << bits; @@ -187,12 +187,34 @@ if (dist > op) { /* see if copy from window */ op = dist - op; /* distance back in window */ if (op > whave) { - strm->msg = (char *)"invalid distance too far back"; - state->mode = BAD; - break; + if (state->sane) { + strm->msg = + (char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + if (len <= op - whave) { + do { + PUP(out) = 0; + } while (--len); + continue; + } + len -= op - whave; + do { + PUP(out) = 0; + } while (--op > whave); + if (op == 0) { + from = out - dist; + do { + PUP(out) = PUP(from); + } while (--len); + continue; + } +#endif } from = window - OFF; - if (write == 0) { /* very common case */ + if (wnext == 0) { /* very common case */ from += wsize - op; if (op < len) { /* some from window */ len -= op; @@ -202,17 +224,17 @@ from = out - dist; /* rest from output */ } } - else if (write < op) { /* wrap around window */ - from += wsize + write - op; - op -= write; + else if (wnext < op) { /* wrap around window */ + from += wsize + wnext - op; + op -= wnext; if (op < len) { /* some from end of window */ len -= op; do { PUP(out) = PUP(from); } while (--op); from = window - OFF; - if (write < len) { /* some from start of window */ - op = write; + if (wnext < len) { /* some from start of window */ + op = wnext; len -= op; do { PUP(out) = PUP(from); @@ -222,7 +244,7 @@ } } else { /* contiguous in window */ - from += write - op; + from += wnext - op; if (op < len) { /* some from window */ len -= op; do { @@ -259,7 +281,7 @@ } } else if ((op & 64) == 0) { /* 2nd level distance code */ - this = dcode[this.val + (hold & ((1U << op) - 1))]; + here = dcode[here.val + (hold & ((1U << op) - 1))]; goto dodist; } else { @@ -269,7 +291,7 @@ } } else if ((op & 64) == 0) { /* 2nd level length code */ - this = lcode[this.val + (hold & ((1U << op) - 1))]; + here = lcode[here.val + (hold & ((1U << op) - 1))]; goto dolen; } else if (op & 32) { /* end-of-block */ @@ -305,7 +327,7 @@ inflate_fast() speedups that turned out slower (on a PowerPC G3 750CXe): - Using bit fields for code structure - Different op definition to avoid & for extra bits (do & for table bits) - - Three separate decoding do-loops for direct, window, and write == 0 + - Three separate decoding do-loops for direct, window, and wnext == 0 - Special case for distance > 1 copies to do overlapped load and store copy - Explicit branch predictions (based on measured branch probabilities) - Deferring match copy and interspersed it with decoding subsequent codes diff --git a/Modules/zlib/inffast.h b/Modules/zlib/inffast.h --- a/Modules/zlib/inffast.h +++ b/Modules/zlib/inffast.h @@ -1,5 +1,5 @@ /* inffast.h -- header to use inffast.c - * Copyright (C) 1995-2003 Mark Adler + * Copyright (C) 1995-2003, 2010 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -8,4 +8,4 @@ subject to change. Applications should only use zlib.h. */ -void inflate_fast OF((z_streamp strm, unsigned start)); +void ZLIB_INTERNAL inflate_fast OF((z_streamp strm, unsigned start)); diff --git a/Modules/zlib/inffixed.h b/Modules/zlib/inffixed.h --- a/Modules/zlib/inffixed.h +++ b/Modules/zlib/inffixed.h @@ -2,9 +2,9 @@ * Generated automatically by makefixed(). */ - /* WARNING: this file should *not* be used by applications. It - is part of the implementation of the compression library and - is subject to change. Applications should only use zlib.h. + /* WARNING: this file should *not* be used by applications. + It is part of the implementation of this library and is + subject to change. Applications should only use zlib.h. */ static const code lenfix[512] = { diff --git a/Modules/zlib/inflate.c b/Modules/zlib/inflate.c --- a/Modules/zlib/inflate.c +++ b/Modules/zlib/inflate.c @@ -1,5 +1,5 @@ /* inflate.c -- zlib decompression - * Copyright (C) 1995-2005 Mark Adler + * Copyright (C) 1995-2012 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -45,7 +45,7 @@ * - Rearrange window copies in inflate_fast() for speed and simplification * - Unroll last copy for window match in inflate_fast() * - Use local copies of window variables in inflate_fast() for speed - * - Pull out common write == 0 case for speed in inflate_fast() + * - Pull out common wnext == 0 case for speed in inflate_fast() * - Make op and len in inflate_fast() unsigned for consistency * - Add FAR to lcode and dcode declarations in inflate_fast() * - Simplified bad distance check in inflate_fast() @@ -93,13 +93,39 @@ /* function prototypes */ local void fixedtables OF((struct inflate_state FAR *state)); -local int updatewindow OF((z_streamp strm, unsigned out)); +local int updatewindow OF((z_streamp strm, const unsigned char FAR *end, + unsigned copy)); #ifdef BUILDFIXED void makefixed OF((void)); #endif -local unsigned syncsearch OF((unsigned FAR *have, unsigned char FAR *buf, +local unsigned syncsearch OF((unsigned FAR *have, const unsigned char FAR *buf, unsigned len)); +int ZEXPORT inflateResetKeep(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + + if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + strm->total_in = strm->total_out = state->total = 0; + strm->msg = Z_NULL; + if (state->wrap) /* to support ill-conceived Java test suite */ + strm->adler = state->wrap & 1; + state->mode = HEAD; + state->last = 0; + state->havedict = 0; + state->dmax = 32768U; + state->head = Z_NULL; + state->hold = 0; + state->bits = 0; + state->lencode = state->distcode = state->next = state->codes; + state->sane = 1; + state->back = -1; + Tracev((stderr, "inflate: reset\n")); + return Z_OK; +} + int ZEXPORT inflateReset(strm) z_streamp strm; { @@ -107,22 +133,98 @@ if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; - strm->total_in = strm->total_out = state->total = 0; - strm->msg = Z_NULL; - strm->adler = 1; /* to support ill-conceived Java test suite */ - state->mode = HEAD; - state->last = 0; - state->havedict = 0; - state->dmax = 32768U; - state->head = Z_NULL; state->wsize = 0; state->whave = 0; - state->write = 0; - state->hold = 0; - state->bits = 0; - state->lencode = state->distcode = state->next = state->codes; - Tracev((stderr, "inflate: reset\n")); - return Z_OK; + state->wnext = 0; + return inflateResetKeep(strm); +} + +int ZEXPORT inflateReset2(strm, windowBits) +z_streamp strm; +int windowBits; +{ + int wrap; + struct inflate_state FAR *state; + + /* get the state */ + if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + + /* extract wrap request from windowBits parameter */ + if (windowBits < 0) { + wrap = 0; + windowBits = -windowBits; + } + else { + wrap = (windowBits >> 4) + 1; +#ifdef GUNZIP + if (windowBits < 48) + windowBits &= 15; +#endif + } + + /* set number of window bits, free window if different */ + if (windowBits && (windowBits < 8 || windowBits > 15)) + return Z_STREAM_ERROR; + if (state->window != Z_NULL && state->wbits != (unsigned)windowBits) { + ZFREE(strm, state->window); + state->window = Z_NULL; + } + + /* update state and reset the rest of it */ + state->wrap = wrap; + state->wbits = (unsigned)windowBits; + return inflateReset(strm); +} + +int ZEXPORT inflateInit2_(strm, windowBits, version, stream_size) +z_streamp strm; +int windowBits; +const char *version; +int stream_size; +{ + int ret; + struct inflate_state FAR *state; + + if (version == Z_NULL || version[0] != ZLIB_VERSION[0] || + stream_size != (int)(sizeof(z_stream))) + return Z_VERSION_ERROR; + if (strm == Z_NULL) return Z_STREAM_ERROR; + strm->msg = Z_NULL; /* in case we return an error */ + if (strm->zalloc == (alloc_func)0) { +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zalloc = zcalloc; + strm->opaque = (voidpf)0; +#endif + } + if (strm->zfree == (free_func)0) +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zfree = zcfree; +#endif + state = (struct inflate_state FAR *) + ZALLOC(strm, 1, sizeof(struct inflate_state)); + if (state == Z_NULL) return Z_MEM_ERROR; + Tracev((stderr, "inflate: allocated\n")); + strm->state = (struct internal_state FAR *)state; + state->window = Z_NULL; + ret = inflateReset2(strm, windowBits); + if (ret != Z_OK) { + ZFREE(strm, state); + strm->state = Z_NULL; + } + return ret; +} + +int ZEXPORT inflateInit_(strm, version, stream_size) +z_streamp strm; +const char *version; +int stream_size; +{ + return inflateInit2_(strm, DEF_WBITS, version, stream_size); } int ZEXPORT inflatePrime(strm, bits, value) @@ -134,6 +236,11 @@ if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; + if (bits < 0) { + state->hold = 0; + state->bits = 0; + return Z_OK; + } if (bits > 16 || state->bits + bits > 32) return Z_STREAM_ERROR; value &= (1L << bits) - 1; state->hold += value << state->bits; @@ -141,57 +248,6 @@ return Z_OK; } -int ZEXPORT inflateInit2_(strm, windowBits, version, stream_size) -z_streamp strm; -int windowBits; -const char *version; -int stream_size; -{ - struct inflate_state FAR *state; - - if (version == Z_NULL || version[0] != ZLIB_VERSION[0] || - stream_size != (int)(sizeof(z_stream))) - return Z_VERSION_ERROR; - if (strm == Z_NULL) return Z_STREAM_ERROR; - strm->msg = Z_NULL; /* in case we return an error */ - if (strm->zalloc == (alloc_func)0) { - strm->zalloc = zcalloc; - strm->opaque = (voidpf)0; - } - if (strm->zfree == (free_func)0) strm->zfree = zcfree; - state = (struct inflate_state FAR *) - ZALLOC(strm, 1, sizeof(struct inflate_state)); - if (state == Z_NULL) return Z_MEM_ERROR; - Tracev((stderr, "inflate: allocated\n")); - strm->state = (struct internal_state FAR *)state; - if (windowBits < 0) { - state->wrap = 0; - windowBits = -windowBits; - } - else { - state->wrap = (windowBits >> 4) + 1; -#ifdef GUNZIP - if (windowBits < 48) windowBits &= 15; -#endif - } - if (windowBits < 8 || windowBits > 15) { - ZFREE(strm, state); - strm->state = Z_NULL; - return Z_STREAM_ERROR; - } - state->wbits = (unsigned)windowBits; - state->window = Z_NULL; - return inflateReset(strm); -} - -int ZEXPORT inflateInit_(strm, version, stream_size) -z_streamp strm; -const char *version; -int stream_size; -{ - return inflateInit2_(strm, DEF_WBITS, version, stream_size); -} - /* Return state with length and distance decoding tables and index sizes set to fixed code decoding. Normally this returns fixed tables from inffixed.h. @@ -286,8 +342,8 @@ low = 0; for (;;) { if ((low % 7) == 0) printf("\n "); - printf("{%u,%u,%d}", state.lencode[low].op, state.lencode[low].bits, - state.lencode[low].val); + printf("{%u,%u,%d}", (low & 127) == 99 ? 64 : state.lencode[low].op, + state.lencode[low].bits, state.lencode[low].val); if (++low == size) break; putchar(','); } @@ -320,12 +376,13 @@ output will fall in the output data, making match copies simpler and faster. The advantage may be dependent on the size of the processor's data caches. */ -local int updatewindow(strm, out) +local int updatewindow(strm, end, copy) z_streamp strm; -unsigned out; +const Bytef *end; +unsigned copy; { struct inflate_state FAR *state; - unsigned copy, dist; + unsigned dist; state = (struct inflate_state FAR *)strm->state; @@ -340,30 +397,29 @@ /* if window not in use yet, initialize */ if (state->wsize == 0) { state->wsize = 1U << state->wbits; - state->write = 0; + state->wnext = 0; state->whave = 0; } /* copy state->wsize or less output bytes into the circular window */ - copy = out - strm->avail_out; if (copy >= state->wsize) { - zmemcpy(state->window, strm->next_out - state->wsize, state->wsize); - state->write = 0; + zmemcpy(state->window, end - state->wsize, state->wsize); + state->wnext = 0; state->whave = state->wsize; } else { - dist = state->wsize - state->write; + dist = state->wsize - state->wnext; if (dist > copy) dist = copy; - zmemcpy(state->window + state->write, strm->next_out - copy, dist); + zmemcpy(state->window + state->wnext, end - copy, dist); copy -= dist; if (copy) { - zmemcpy(state->window, strm->next_out - copy, copy); - state->write = copy; + zmemcpy(state->window, end - copy, copy); + state->wnext = copy; state->whave = state->wsize; } else { - state->write += dist; - if (state->write == state->wsize) state->write = 0; + state->wnext += dist; + if (state->wnext == state->wsize) state->wnext = 0; if (state->whave < state->wsize) state->whave += dist; } } @@ -464,11 +520,6 @@ bits -= bits & 7; \ } while (0) -/* Reverse the bytes in a 32-bit value */ -#define REVERSE(q) \ - ((((q) >> 24) & 0xff) + (((q) >> 8) & 0xff00) + \ - (((q) & 0xff00) << 8) + (((q) & 0xff) << 24)) - /* inflate() uses a state machine to process as much input data and generate as much output data as possible before returning. The state machine is @@ -556,7 +607,7 @@ int flush; { struct inflate_state FAR *state; - unsigned char FAR *next; /* next input */ + z_const unsigned char FAR *next; /* next input */ unsigned char FAR *put; /* next output */ unsigned have, left; /* available input and output */ unsigned long hold; /* bit buffer */ @@ -564,7 +615,7 @@ unsigned in, out; /* save starting available input and output */ unsigned copy; /* number of stored or match bytes to copy */ unsigned char FAR *from; /* where to copy match bytes from */ - code this; /* current decoding table entry */ + code here; /* current decoding table entry */ code last; /* parent table entry */ unsigned len; /* length to copy for repeats, bits to drop */ int ret; /* return code */ @@ -619,7 +670,9 @@ } DROPBITS(4); len = BITS(4) + 8; - if (len > state->wbits) { + if (state->wbits == 0) + state->wbits = len; + else if (len > state->wbits) { strm->msg = (char *)"invalid window size"; state->mode = BAD; break; @@ -760,7 +813,7 @@ #endif case DICTID: NEEDBITS(32); - strm->adler = state->check = REVERSE(hold); + strm->adler = state->check = ZSWAP32(hold); INITBITS(); state->mode = DICT; case DICT: @@ -771,7 +824,7 @@ strm->adler = state->check = adler32(0L, Z_NULL, 0); state->mode = TYPE; case TYPE: - if (flush == Z_BLOCK) goto inf_leave; + if (flush == Z_BLOCK || flush == Z_TREES) goto inf_leave; case TYPEDO: if (state->last) { BYTEBITS(); @@ -791,7 +844,11 @@ fixedtables(state); Tracev((stderr, "inflate: fixed codes block%s\n", state->last ? " (last)" : "")); - state->mode = LEN; /* decode codes */ + state->mode = LEN_; /* decode codes */ + if (flush == Z_TREES) { + DROPBITS(2); + goto inf_leave; + } break; case 2: /* dynamic block */ Tracev((stderr, "inflate: dynamic codes block%s\n", @@ -816,6 +873,9 @@ Tracev((stderr, "infla