Mailman 3 December 2015 - Python-checkins

cpython (merge 3.5 -> default): Issue #6478: _strptime's regexp cache now is reset after changing timezone
by serhiy.storchaka Dec. 3, 2015

Dec. 3, 2015

https://hg.python.org/cpython/rev/ea576db13827 changeset: 99427:ea576db13827 parent: 99422:01397c11ebb8 parent: 99426:5fa855d20624 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Thu Dec 03 22:27:31 2015 +0200 summary: Issue #6478: _strptime's regexp cache now is reset after changing timezone with time.tzset(). files: Lib/_strptime.py | 22 +++++++++----- Lib/test/test_strptime.py | 40 ++++++++++++++++++++++---- Misc/NEWS | 3 ++ 3 files changed, 50 insertions(+), 15 deletions(-) diff --git a/Lib/_strptime.py b/Lib/_strptime.py --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -77,6 +77,8 @@ self.__calc_date_time() if _getlang() != self.lang: raise ValueError("locale changed during initialization") + if time.tzname != self.tzname or time.daylight != self.daylight: + raise ValueError("timezone changed during initialization") def __pad(self, seq, front): # Add '' to seq to either the front (is True), else the back. @@ -161,15 +163,17 @@ def __calc_timezone(self): # Set self.timezone by using time.tzname. - # Do not worry about possibility of time.tzname[0] == timetzname[1] - # and time.daylight; handle that in strptime . + # Do not worry about possibility of time.tzname[0] == time.tzname[1] + # and time.daylight; handle that in strptime. try: time.tzset() except AttributeError: pass - no_saving = frozenset({"utc", "gmt", time.tzname[0].lower()}) - if time.daylight: - has_saving = frozenset({time.tzname[1].lower()}) + self.tzname = time.tzname + self.daylight = time.daylight + no_saving = frozenset({"utc", "gmt", self.tzname[0].lower()}) + if self.daylight: + has_saving = frozenset({self.tzname[1].lower()}) else: has_saving = frozenset() self.timezone = (no_saving, has_saving) @@ -326,13 +330,15 @@ global _TimeRE_cache, _regex_cache with _cache_lock: - - if _getlang() != _TimeRE_cache.locale_time.lang: + locale_time = _TimeRE_cache.locale_time + if (_getlang() != locale_time.lang or + time.tzname != locale_time.tzname or + time.daylight != locale_time.daylight): _TimeRE_cache = TimeRE() _regex_cache.clear() + locale_time = _TimeRE_cache.locale_time if len(_regex_cache) > _CACHE_MAX_SIZE: _regex_cache.clear() - locale_time = _TimeRE_cache.locale_time format_regex = _regex_cache.get(format) if not format_regex: try: diff --git a/Lib/test/test_strptime.py b/Lib/test/test_strptime.py --- a/Lib/test/test_strptime.py +++ b/Lib/test/test_strptime.py @@ -4,6 +4,7 @@ import time import locale import re +import os import sys from test import support from datetime import date as datetime_date @@ -344,9 +345,10 @@ tz_name = time.tzname[0] if tz_name.upper() in ("UTC", "GMT"): self.skipTest('need non-UTC/GMT timezone') - try: - original_tzname = time.tzname - original_daylight = time.daylight + + with support.swap_attr(time, 'tzname', (tz_name, tz_name)), \ + support.swap_attr(time, 'daylight', 1), \ + support.swap_attr(time, 'tzset', lambda: None): time.tzname = (tz_name, tz_name) time.daylight = 1 tz_value = _strptime._strptime_time(tz_name, "%Z")[8] @@ -354,9 +356,6 @@ "%s lead to a timezone value of %s instead of -1 when " "time.daylight set to %s and passing in %s" % (time.tzname, tz_value, time.daylight, tz_name)) - finally: - time.tzname = original_tzname - time.daylight = original_daylight def test_date_time(self): # Test %c directive @@ -579,7 +578,7 @@ _strptime._strptime_time("10", "%d") self.assertIsNot(locale_time_id, _strptime._TimeRE_cache.locale_time) - def test_TimeRE_recreation(self): + def test_TimeRE_recreation_locale(self): # The TimeRE instance should be recreated upon changing the locale. locale_info = locale.getlocale(locale.LC_TIME) try: @@ -608,6 +607,33 @@ finally: locale.setlocale(locale.LC_TIME, locale_info) + @support.run_with_tz('STD-1DST') + def test_TimeRE_recreation_timezone(self): + # The TimeRE instance should be recreated upon changing the timezone. + oldtzname = time.tzname + tm = _strptime._strptime_time(time.tzname[0], '%Z') + self.assertEqual(tm.tm_isdst, 0) + tm = _strptime._strptime_time(time.tzname[1], '%Z') + self.assertEqual(tm.tm_isdst, 1) + # Get id of current cache object. + first_time_re = _strptime._TimeRE_cache + # Change the timezone and force a recreation of the cache. + os.environ['TZ'] = 'EST+05EDT,M3.2.0,M11.1.0' + time.tzset() + tm = _strptime._strptime_time(time.tzname[0], '%Z') + self.assertEqual(tm.tm_isdst, 0) + tm = _strptime._strptime_time(time.tzname[1], '%Z') + self.assertEqual(tm.tm_isdst, 1) + # Get the new cache object's id. + second_time_re = _strptime._TimeRE_cache + # They should not be equal. + self.assertIsNot(first_time_re, second_time_re) + # Make sure old names no longer accepted. + with self.assertRaises(ValueError): + _strptime._strptime_time(oldtzname[0], '%Z') + with self.assertRaises(ValueError): + _strptime._strptime_time(oldtzname[1], '%Z') + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -109,6 +109,9 @@ Library ------- +- Issue #6478: _strptime's regexp cache now is reset after changing timezone + with time.tzset(). + - Issue #14285: When executing a package with the "python -m package" option, and package initialization fails, a proper traceback is now reported. The "runpy" module now lets exceptions from package initialization pass back to -- Repository URL: https://hg.python.org/cpython

1 0

cpython (3.4): Issue #6478: _strptime's regexp cache now is reset after changing timezone
by serhiy.storchaka Dec. 3, 2015

Dec. 3, 2015

https://hg.python.org/cpython/rev/4b0a4da1aa27 changeset: 99425:4b0a4da1aa27 branch: 3.4 parent: 99418:67718032badb user: Serhiy Storchaka <storchaka(a)gmail.com> date: Thu Dec 03 22:21:07 2015 +0200 summary: Issue #6478: _strptime's regexp cache now is reset after changing timezone with time.tzset(). files: Lib/_strptime.py | 21 +++++++++---- Lib/test/test_strptime.py | 40 ++++++++++++++++++++++---- Misc/NEWS | 3 ++ 3 files changed, 50 insertions(+), 14 deletions(-) diff --git a/Lib/_strptime.py b/Lib/_strptime.py --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -77,6 +77,8 @@ self.__calc_date_time() if _getlang() != self.lang: raise ValueError("locale changed during initialization") + if time.tzname != self.tzname or time.daylight != self.daylight: + raise ValueError("timezone changed during initialization") def __pad(self, seq, front): # Add '' to seq to either the front (is True), else the back. @@ -161,15 +163,17 @@ def __calc_timezone(self): # Set self.timezone by using time.tzname. - # Do not worry about possibility of time.tzname[0] == timetzname[1] - # and time.daylight; handle that in strptime . + # Do not worry about possibility of time.tzname[0] == time.tzname[1] + # and time.daylight; handle that in strptime. try: time.tzset() except AttributeError: pass - no_saving = frozenset(["utc", "gmt", time.tzname[0].lower()]) - if time.daylight: - has_saving = frozenset([time.tzname[1].lower()]) + self.tzname = time.tzname + self.daylight = time.daylight + no_saving = frozenset(["utc", "gmt", self.tzname[0].lower()]) + if self.daylight: + has_saving = frozenset([self.tzname[1].lower()]) else: has_saving = frozenset() self.timezone = (no_saving, has_saving) @@ -308,12 +312,15 @@ global _TimeRE_cache, _regex_cache with _cache_lock: - if _getlang() != _TimeRE_cache.locale_time.lang: + locale_time = _TimeRE_cache.locale_time + if (_getlang() != locale_time.lang or + time.tzname != locale_time.tzname or + time.daylight != locale_time.daylight): _TimeRE_cache = TimeRE() _regex_cache.clear() + locale_time = _TimeRE_cache.locale_time if len(_regex_cache) > _CACHE_MAX_SIZE: _regex_cache.clear() - locale_time = _TimeRE_cache.locale_time format_regex = _regex_cache.get(format) if not format_regex: try: diff --git a/Lib/test/test_strptime.py b/Lib/test/test_strptime.py --- a/Lib/test/test_strptime.py +++ b/Lib/test/test_strptime.py @@ -4,6 +4,7 @@ import time import locale import re +import os import sys from test import support from datetime import date as datetime_date @@ -324,9 +325,10 @@ tz_name = time.tzname[0] if tz_name.upper() in ("UTC", "GMT"): self.skipTest('need non-UTC/GMT timezone') - try: - original_tzname = time.tzname - original_daylight = time.daylight + + with support.swap_attr(time, 'tzname', (tz_name, tz_name)), \ + support.swap_attr(time, 'daylight', 1), \ + support.swap_attr(time, 'tzset', lambda: None): time.tzname = (tz_name, tz_name) time.daylight = 1 tz_value = _strptime._strptime_time(tz_name, "%Z")[8] @@ -334,9 +336,6 @@ "%s lead to a timezone value of %s instead of -1 when " "time.daylight set to %s and passing in %s" % (time.tzname, tz_value, time.daylight, tz_name)) - finally: - time.tzname = original_tzname - time.daylight = original_daylight def test_date_time(self): # Test %c directive @@ -548,7 +547,7 @@ _strptime._strptime_time("10", "%d") self.assertIsNot(locale_time_id, _strptime._TimeRE_cache.locale_time) - def test_TimeRE_recreation(self): + def test_TimeRE_recreation_locale(self): # The TimeRE instance should be recreated upon changing the locale. locale_info = locale.getlocale(locale.LC_TIME) try: @@ -577,6 +576,33 @@ finally: locale.setlocale(locale.LC_TIME, locale_info) + @support.run_with_tz('STD-1DST') + def test_TimeRE_recreation_timezone(self): + # The TimeRE instance should be recreated upon changing the timezone. + oldtzname = time.tzname + tm = _strptime._strptime_time(time.tzname[0], '%Z') + self.assertEqual(tm.tm_isdst, 0) + tm = _strptime._strptime_time(time.tzname[1], '%Z') + self.assertEqual(tm.tm_isdst, 1) + # Get id of current cache object. + first_time_re = _strptime._TimeRE_cache + # Change the timezone and force a recreation of the cache. + os.environ['TZ'] = 'EST+05EDT,M3.2.0,M11.1.0' + time.tzset() + tm = _strptime._strptime_time(time.tzname[0], '%Z') + self.assertEqual(tm.tm_isdst, 0) + tm = _strptime._strptime_time(time.tzname[1], '%Z') + self.assertEqual(tm.tm_isdst, 1) + # Get the new cache object's id. + second_time_re = _strptime._TimeRE_cache + # They should not be equal. + self.assertIsNot(first_time_re, second_time_re) + # Make sure old names no longer accepted. + with self.assertRaises(ValueError): + _strptime._strptime_time(oldtzname[0], '%Z') + with self.assertRaises(ValueError): + _strptime._strptime_time(oldtzname[1], '%Z') + def test_main(): support.run_unittest( diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -115,6 +115,9 @@ Library ------- +- Issue #6478: _strptime's regexp cache now is reset after changing timezone + with time.tzset(). + - Issue #25177: Fixed problem with the mean of very small and very large numbers. As a side effect, statistics.mean and statistics.variance should be significantly faster. -- Repository URL: https://hg.python.org/cpython

1 0

cpython (2.7): Issue #19543: Added Py3k warning for decoding unicode.
by serhiy.storchaka Dec. 3, 2015

Dec. 3, 2015

https://hg.python.org/cpython/rev/c89a0f24d5f6 changeset: 99423:c89a0f24d5f6 branch: 2.7 parent: 99413:ff351607a90d user: Serhiy Storchaka <storchaka(a)gmail.com> date: Thu Dec 03 20:47:48 2015 +0200 summary: Issue #19543: Added Py3k warning for decoding unicode. files: Lib/test/test_unicode.py | 10 ++++++---- Misc/NEWS | 2 ++ Objects/unicodeobject.c | 3 +++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -1037,10 +1037,12 @@ self.assertEqual(unicode('Andr\202 x','ascii','ignore'), u"Andr x") self.assertEqual(unicode('Andr\202 x','ascii','replace'), u'Andr\uFFFD x') self.assertEqual(unicode('\202 x', 'ascii', 'replace'), u'\uFFFD x') - self.assertEqual(u'abcde'.decode('ascii', 'ignore'), - u'abcde'.decode('ascii', errors='ignore')) - self.assertEqual(u'abcde'.decode('ascii', 'replace'), - u'abcde'.decode(encoding='ascii', errors='replace')) + with test_support.check_py3k_warnings(): + self.assertEqual(u'abcde'.decode('ascii', 'ignore'), + u'abcde'.decode('ascii', errors='ignore')) + with test_support.check_py3k_warnings(): + self.assertEqual(u'abcde'.decode('ascii', 'replace'), + u'abcde'.decode(encoding='ascii', errors='replace')) # Error handling (unknown character names) self.assertEqual("\\N{foo}xx".decode("unicode-escape", "ignore"), u"xx") diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,8 @@ Core and Builtins ----------------- +- Issue #19543: Added Py3k warning for decoding unicode. + - Issue #24097: Fixed crash in object.__reduce__() if slot name is freed inside __getattr__. diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -1288,6 +1288,9 @@ goto onError; } + if (PyErr_WarnPy3k("decoding Unicode is not supported in 3.x", 1) < 0) + goto onError; + if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); -- Repository URL: https://hg.python.org/cpython

1 0

Benchmark Results for Python 2.7 2015-12-03
by lp_benchmark_robot＠intel.com Dec. 3, 2015

Dec. 3, 2015

Results for project Python 2.7, build date 2015-12-03 04:50:31 +0000 commit: ff351607a90d345d1335eef3518ab5f43f1ee118 revision date: 2015-12-02 13:39:37 +0000 environment: Haswell-EP cpu: Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz 2x18 cores, stepping 2, LLC 45 MB mem: 128 GB os: CentOS 7.1 kernel: Linux 3.10.0-229.4.2.el7.x86_64 Baseline results were generated using release v2.7.10, with hash 15c95b7d81dcf821daade360741e00714667653f from 2015-05-23 16:02:14+00:00 ---------------------------------------------------------------------------------- benchmark relative change since change since current rev run std_dev* last run baseline with PGO ---------------------------------------------------------------------------------- :-| django_v2 0.18% -0.59% 1.62% 8.58% :-) pybench 0.19% -0.12% 6.09% 7.21% :-( regex_v8 1.40% -0.20% -3.33% 9.06% :-) nbody 0.13% -0.03% 9.51% 2.45% :-) json_dump_v2 0.32% 0.07% 2.74% 13.46% :-| normal_startup 1.94% 0.40% -1.47% 2.15% :-| ssbench 0.35% 0.01% 0.35% 2.39% ---------------------------------------------------------------------------------- Note: Benchmark results for ssbench are measured in requests/second while all other are measured in seconds. * Relative Standard Deviation (Standard Deviation/Average) Our lab does a nightly source pull and build of the Python project and measures performance changes against the previous stable version and the previous nightly measurement. This is provided as a service to the community so that quality issues with current hardware can be identified quickly. Intel technologies' features and benefits depend on system configuration and may require enabled hardware, software or service activation. Performance varies depending on system configuration.

1 0

Benchmark Results for Python Default 2015-12-03
by lp_benchmark_robot＠intel.com Dec. 3, 2015

Dec. 3, 2015

Results for project Python default, build date 2015-12-03 04:02:59 +0000 commit: 01397c11ebb809a7ccc0a1b4c873183e318dc9dc revision date: 2015-12-03 01:36:03 +0000 environment: Haswell-EP cpu: Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz 2x18 cores, stepping 2, LLC 45 MB mem: 128 GB os: CentOS 7.1 kernel: Linux 3.10.0-229.4.2.el7.x86_64 Baseline results were generated using release v3.4.3, with hash b4cbecbc0781e89a309d03b60a1f75f8499250e6 from 2015-02-25 12:15:33+00:00 ---------------------------------------------------------------------------------- benchmark relative change since change since current rev run std_dev* last run baseline with PGO ---------------------------------------------------------------------------------- :-) django_v2 0.20% 2.73% 10.00% 15.45% :-( pybench 0.20% -0.14% -2.06% 9.28% :-( regex_v8 2.73% 1.82% -3.69% 4.90% :-( nbody 0.06% -2.57% -3.09% 12.18% :-| json_dump_v2 0.23% -0.21% -1.09% 11.34% :-| normal_startup 0.87% 0.09% 0.41% 4.48% ---------------------------------------------------------------------------------- Note: Benchmark results are measured in seconds. * Relative Standard Deviation (Standard Deviation/Average) Our lab does a nightly source pull and build of the Python project and measures performance changes against the previous stable version and the previous nightly measurement. This is provided as a service to the community so that quality issues with current hardware can be identified quickly. Intel technologies' features and benefits depend on system configuration and may require enabled hardware, software or service activation. Performance varies depending on system configuration.

1 0

Daily reference leaks (01397c11ebb8): sum=4
by solipsis＠pitrou.net Dec. 3, 2015

Dec. 3, 2015

results for 01397c11ebb8 on branch "default" -------------------------------------------- test_functools leaked [0, 2, 2] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflog0MKx4_', '--timeout', '7200']

1 0

cpython (merge 3.5 -> default): Issue #14285: Merge runpy exception fix from 3.5
by martin.panter Dec. 3, 2015

Dec. 3, 2015

https://hg.python.org/cpython/rev/01397c11ebb8 changeset: 99422:01397c11ebb8 parent: 99420:9e800b2aeeac parent: 99421:784a64a21fd0 user: Martin Panter <vadmium+py(a)gmail.com> date: Thu Dec 03 01:36:03 2015 +0000 summary: Issue #14285: Merge runpy exception fix from 3.5 files: Lib/runpy.py | 44 +++++------ Lib/test/test_cmd_line_script.py | 71 +++++++++++++++++-- Lib/test/test_runpy.py | 22 ++++++ Misc/NEWS | 5 + 4 files changed, 110 insertions(+), 32 deletions(-) diff --git a/Lib/runpy.py b/Lib/runpy.py --- a/Lib/runpy.py +++ b/Lib/runpy.py @@ -99,7 +99,7 @@ return mod_globals.copy() # Helper to get the loader, code and filename for a module -def _get_module_details(mod_name): +def _get_module_details(mod_name, error=ImportError): try: spec = importlib.util.find_spec(mod_name) except (ImportError, AttributeError, TypeError, ValueError) as ex: @@ -107,27 +107,34 @@ # importlib, where the latter raises other errors for cases where # pkgutil previously raised ImportError msg = "Error while finding spec for {!r} ({}: {})" - raise ImportError(msg.format(mod_name, type(ex), ex)) from ex + raise error(msg.format(mod_name, type(ex), ex)) from ex if spec is None: - raise ImportError("No module named %s" % mod_name) + raise error("No module named %s" % mod_name) if spec.submodule_search_locations is not None: if mod_name == "__main__" or mod_name.endswith(".__main__"): - raise ImportError("Cannot use package as __main__ module") + raise error("Cannot use package as __main__ module") + __import__(mod_name) # Do not catch exceptions initializing package try: pkg_main_name = mod_name + ".__main__" return _get_module_details(pkg_main_name) except ImportError as e: - raise ImportError(("%s; %r is a package and cannot " + + raise error(("%s; %r is a package and cannot " + "be directly executed") %(e, mod_name)) loader = spec.loader if loader is None: - raise ImportError("%r is a namespace package and cannot be executed" + raise error("%r is a namespace package and cannot be executed" % mod_name) - code = loader.get_code(mod_name) + try: + code = loader.get_code(mod_name) + except ImportError as e: + raise error(format(e)) from e if code is None: - raise ImportError("No code object available for %s" % mod_name) + raise error("No code object available for %s" % mod_name) return mod_name, spec, code +class _Error(Exception): + """Error that _run_module_as_main() should report without a traceback""" + # XXX ncoghlan: Should this be documented and made public? # (Current thoughts: don't repeat the mistake that lead to its # creation when run_module() no longer met the needs of @@ -148,20 +155,11 @@ """ try: if alter_argv or mod_name != "__main__": # i.e. -m switch - mod_name, mod_spec, code = _get_module_details(mod_name) + mod_name, mod_spec, code = _get_module_details(mod_name, _Error) else: # i.e. directory or zipfile execution - mod_name, mod_spec, code = _get_main_module_details() - except ImportError as exc: - # Try to provide a good error message - # for directories, zip files and the -m switch - if alter_argv: - # For -m switch, just display the exception - info = str(exc) - else: - # For directories/zipfiles, let the user - # know what the code was looking for - info = "can't find '__main__' module in %r" % sys.argv[0] - msg = "%s: %s" % (sys.executable, info) + mod_name, mod_spec, code = _get_main_module_details(_Error) + except _Error as exc: + msg = "%s: %s" % (sys.executable, exc) sys.exit(msg) main_globals = sys.modules["__main__"].__dict__ if alter_argv: @@ -184,7 +182,7 @@ # Leave the sys module alone return _run_code(code, {}, init_globals, run_name, mod_spec) -def _get_main_module_details(): +def _get_main_module_details(error=ImportError): # Helper that gives a nicer error message when attempting to # execute a zipfile or directory by invoking __main__.py # Also moves the standard __main__ out of the way so that the @@ -196,7 +194,7 @@ return _get_module_details(main_name) except ImportError as exc: if main_name in str(exc): - raise ImportError("can't find %r module in %r" % + raise error("can't find %r module in %r" % (main_name, sys.path[0])) from exc raise finally: diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -397,20 +397,73 @@ script_name, script_name, '', '', importlib.machinery.SourceFileLoader) + @contextlib.contextmanager + def setup_test_pkg(self, *args): + with support.temp_dir() as script_dir, \ + support.change_cwd(path=script_dir): + pkg_dir = os.path.join(script_dir, 'test_pkg') + make_pkg(pkg_dir, *args) + yield pkg_dir + + def check_dash_m_failure(self, *args): + rc, out, err = assert_python_failure('-m', *args, __isolated=False) + if verbose > 1: + print(repr(out)) + self.assertEqual(rc, 1) + return err + def test_dash_m_error_code_is_one(self): # If a module is invoked with the -m command line flag # and results in an error that the return code to the # shell is '1' - with support.temp_dir() as script_dir: - with support.change_cwd(path=script_dir): - pkg_dir = os.path.join(script_dir, 'test_pkg') - make_pkg(pkg_dir) - script_name = _make_test_script(pkg_dir, 'other', - "if __name__ == '__main__': raise ValueError") - rc, out, err = assert_python_failure('-m', 'test_pkg.other', *example_args) - if verbose > 1: - print(repr(out)) + with self.setup_test_pkg() as pkg_dir: + script_name = _make_test_script(pkg_dir, 'other', + "if __name__ == '__main__': raise ValueError") + err = self.check_dash_m_failure('test_pkg.other', *example_args) + self.assertIn(b'ValueError', err) + + def test_dash_m_errors(self): + # Exercise error reporting for various invalid package executions + tests = ( + ('builtins', br'No code object available'), + ('builtins.x', br'Error while finding spec.*AttributeError'), + ('builtins.x.y', br'Error while finding spec.*' + br'ImportError.*No module named.*not a package'), + ('os.path', br'loader.*cannot handle'), + ('importlib', br'No module named.*' + br'is a package and cannot be directly executed'), + ('importlib.nonexistant', br'No module named'), + ) + for name, regex in tests: + with self.subTest(name): + rc, _, err = assert_python_failure('-m', name) self.assertEqual(rc, 1) + self.assertRegex(err, regex) + self.assertNotIn(b'Traceback', err) + + def test_dash_m_init_traceback(self): + # These were wrapped in an ImportError and tracebacks were + # suppressed; see Issue 14285 + exceptions = (ImportError, AttributeError, TypeError, ValueError) + for exception in exceptions: + exception = exception.__name__ + init = "raise {0}('Exception in __init__.py')".format(exception) + with self.subTest(exception), \ + self.setup_test_pkg(init) as pkg_dir: + err = self.check_dash_m_failure('test_pkg') + self.assertIn(exception.encode('ascii'), err) + self.assertIn(b'Exception in __init__.py', err) + self.assertIn(b'Traceback', err) + + def test_dash_m_main_traceback(self): + # Ensure that an ImportError's traceback is reported + with self.setup_test_pkg() as pkg_dir: + main = "raise ImportError('Exception in __main__ module')" + _make_test_script(pkg_dir, '__main__', main) + err = self.check_dash_m_failure('test_pkg') + self.assertIn(b'ImportError', err) + self.assertIn(b'Exception in __main__ module', err) + self.assertIn(b'Traceback', err) def test_pep_409_verbiage(self): # Make sure PEP 409 syntax properly suppresses diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -439,6 +439,28 @@ if verbose > 1: print("Testing package depth:", depth) self._check_package(depth) + def test_run_package_init_exceptions(self): + # These were previously wrapped in an ImportError; see Issue 14285 + result = self._make_pkg("", 1, "__main__") + pkg_dir, _, mod_name, _ = result + mod_name = mod_name.replace(".__main__", "") + self.addCleanup(self._del_pkg, pkg_dir, 1, mod_name) + init = os.path.join(pkg_dir, "__runpy_pkg__", "__init__.py") + + exceptions = (ImportError, AttributeError, TypeError, ValueError) + for exception in exceptions: + name = exception.__name__ + with self.subTest(name): + source = "raise {0}('{0} in __init__.py.')".format(name) + with open(init, "wt", encoding="ascii") as mod_file: + mod_file.write(source) + try: + run_module(mod_name) + except exception as err: + self.assertNotIn("finding spec", format(err)) + else: + self.fail("Nothing raised; expected {}".format(name)) + def test_run_package_in_namespace_package(self): for depth in range(1, 4): if verbose > 1: print("Testing package depth:", depth) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -109,6 +109,11 @@ Library ------- +- Issue #14285: When executing a package with the "python -m package" option, + and package initialization fails, a proper traceback is now reported. The + "runpy" module now lets exceptions from package initialization pass back to + the caller, rather than raising ImportError. + - Issue #25177: Fixed problem with the mean of very small and very large numbers. As a side effect, statistics.mean and statistics.variance should be significantly faster. -- Repository URL: https://hg.python.org/cpython

1 0

cpython (3.5): Issue #14285: Do not catch __init__.py exceptions in runpy
by martin.panter Dec. 3, 2015

Dec. 3, 2015

https://hg.python.org/cpython/rev/784a64a21fd0 changeset: 99421:784a64a21fd0 branch: 3.5 parent: 99419:a0e2376768dc user: Martin Panter <vadmium+py(a)gmail.com> date: Thu Dec 03 01:23:10 2015 +0000 summary: Issue #14285: Do not catch __init__.py exceptions in runpy Initialize package before calling find_spec() for __main__, so that we do not incorrectly handle exceptions from __init__.py. When runpy is used from the Python CLI, use an internal exception rather than ImportError, to avoid catching unexpected exceptions. Also remove exception message rewriting in _run_module_as_main(), because it seems to be redundant with the _get_main_module_details() function. files: Lib/runpy.py | 44 +++++------ Lib/test/test_cmd_line_script.py | 71 +++++++++++++++++-- Lib/test/test_runpy.py | 22 ++++++ Misc/NEWS | 5 + 4 files changed, 110 insertions(+), 32 deletions(-) diff --git a/Lib/runpy.py b/Lib/runpy.py --- a/Lib/runpy.py +++ b/Lib/runpy.py @@ -99,7 +99,7 @@ return mod_globals.copy() # Helper to get the loader, code and filename for a module -def _get_module_details(mod_name): +def _get_module_details(mod_name, error=ImportError): try: spec = importlib.util.find_spec(mod_name) except (ImportError, AttributeError, TypeError, ValueError) as ex: @@ -107,27 +107,34 @@ # importlib, where the latter raises other errors for cases where # pkgutil previously raised ImportError msg = "Error while finding spec for {!r} ({}: {})" - raise ImportError(msg.format(mod_name, type(ex), ex)) from ex + raise error(msg.format(mod_name, type(ex), ex)) from ex if spec is None: - raise ImportError("No module named %s" % mod_name) + raise error("No module named %s" % mod_name) if spec.submodule_search_locations is not None: if mod_name == "__main__" or mod_name.endswith(".__main__"): - raise ImportError("Cannot use package as __main__ module") + raise error("Cannot use package as __main__ module") + __import__(mod_name) # Do not catch exceptions initializing package try: pkg_main_name = mod_name + ".__main__" return _get_module_details(pkg_main_name) except ImportError as e: - raise ImportError(("%s; %r is a package and cannot " + + raise error(("%s; %r is a package and cannot " + "be directly executed") %(e, mod_name)) loader = spec.loader if loader is None: - raise ImportError("%r is a namespace package and cannot be executed" + raise error("%r is a namespace package and cannot be executed" % mod_name) - code = loader.get_code(mod_name) + try: + code = loader.get_code(mod_name) + except ImportError as e: + raise error(format(e)) from e if code is None: - raise ImportError("No code object available for %s" % mod_name) + raise error("No code object available for %s" % mod_name) return mod_name, spec, code +class _Error(Exception): + """Error that _run_module_as_main() should report without a traceback""" + # XXX ncoghlan: Should this be documented and made public? # (Current thoughts: don't repeat the mistake that lead to its # creation when run_module() no longer met the needs of @@ -148,20 +155,11 @@ """ try: if alter_argv or mod_name != "__main__": # i.e. -m switch - mod_name, mod_spec, code = _get_module_details(mod_name) + mod_name, mod_spec, code = _get_module_details(mod_name, _Error) else: # i.e. directory or zipfile execution - mod_name, mod_spec, code = _get_main_module_details() - except ImportError as exc: - # Try to provide a good error message - # for directories, zip files and the -m switch - if alter_argv: - # For -m switch, just display the exception - info = str(exc) - else: - # For directories/zipfiles, let the user - # know what the code was looking for - info = "can't find '__main__' module in %r" % sys.argv[0] - msg = "%s: %s" % (sys.executable, info) + mod_name, mod_spec, code = _get_main_module_details(_Error) + except _Error as exc: + msg = "%s: %s" % (sys.executable, exc) sys.exit(msg) main_globals = sys.modules["__main__"].__dict__ if alter_argv: @@ -184,7 +182,7 @@ # Leave the sys module alone return _run_code(code, {}, init_globals, run_name, mod_spec) -def _get_main_module_details(): +def _get_main_module_details(error=ImportError): # Helper that gives a nicer error message when attempting to # execute a zipfile or directory by invoking __main__.py # Also moves the standard __main__ out of the way so that the @@ -196,7 +194,7 @@ return _get_module_details(main_name) except ImportError as exc: if main_name in str(exc): - raise ImportError("can't find %r module in %r" % + raise error("can't find %r module in %r" % (main_name, sys.path[0])) from exc raise finally: diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -397,20 +397,73 @@ script_name, script_name, '', '', importlib.machinery.SourceFileLoader) + @contextlib.contextmanager + def setup_test_pkg(self, *args): + with support.temp_dir() as script_dir, \ + support.change_cwd(path=script_dir): + pkg_dir = os.path.join(script_dir, 'test_pkg') + make_pkg(pkg_dir, *args) + yield pkg_dir + + def check_dash_m_failure(self, *args): + rc, out, err = assert_python_failure('-m', *args, __isolated=False) + if verbose > 1: + print(repr(out)) + self.assertEqual(rc, 1) + return err + def test_dash_m_error_code_is_one(self): # If a module is invoked with the -m command line flag # and results in an error that the return code to the # shell is '1' - with support.temp_dir() as script_dir: - with support.change_cwd(path=script_dir): - pkg_dir = os.path.join(script_dir, 'test_pkg') - make_pkg(pkg_dir) - script_name = _make_test_script(pkg_dir, 'other', - "if __name__ == '__main__': raise ValueError") - rc, out, err = assert_python_failure('-m', 'test_pkg.other', *example_args) - if verbose > 1: - print(repr(out)) + with self.setup_test_pkg() as pkg_dir: + script_name = _make_test_script(pkg_dir, 'other', + "if __name__ == '__main__': raise ValueError") + err = self.check_dash_m_failure('test_pkg.other', *example_args) + self.assertIn(b'ValueError', err) + + def test_dash_m_errors(self): + # Exercise error reporting for various invalid package executions + tests = ( + ('builtins', br'No code object available'), + ('builtins.x', br'Error while finding spec.*AttributeError'), + ('builtins.x.y', br'Error while finding spec.*' + br'ImportError.*No module named.*not a package'), + ('os.path', br'loader.*cannot handle'), + ('importlib', br'No module named.*' + br'is a package and cannot be directly executed'), + ('importlib.nonexistant', br'No module named'), + ) + for name, regex in tests: + with self.subTest(name): + rc, _, err = assert_python_failure('-m', name) self.assertEqual(rc, 1) + self.assertRegex(err, regex) + self.assertNotIn(b'Traceback', err) + + def test_dash_m_init_traceback(self): + # These were wrapped in an ImportError and tracebacks were + # suppressed; see Issue 14285 + exceptions = (ImportError, AttributeError, TypeError, ValueError) + for exception in exceptions: + exception = exception.__name__ + init = "raise {0}('Exception in __init__.py')".format(exception) + with self.subTest(exception), \ + self.setup_test_pkg(init) as pkg_dir: + err = self.check_dash_m_failure('test_pkg') + self.assertIn(exception.encode('ascii'), err) + self.assertIn(b'Exception in __init__.py', err) + self.assertIn(b'Traceback', err) + + def test_dash_m_main_traceback(self): + # Ensure that an ImportError's traceback is reported + with self.setup_test_pkg() as pkg_dir: + main = "raise ImportError('Exception in __main__ module')" + _make_test_script(pkg_dir, '__main__', main) + err = self.check_dash_m_failure('test_pkg') + self.assertIn(b'ImportError', err) + self.assertIn(b'Exception in __main__ module', err) + self.assertIn(b'Traceback', err) def test_pep_409_verbiage(self): # Make sure PEP 409 syntax properly suppresses diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -439,6 +439,28 @@ if verbose > 1: print("Testing package depth:", depth) self._check_package(depth) + def test_run_package_init_exceptions(self): + # These were previously wrapped in an ImportError; see Issue 14285 + result = self._make_pkg("", 1, "__main__") + pkg_dir, _, mod_name, _ = result + mod_name = mod_name.replace(".__main__", "") + self.addCleanup(self._del_pkg, pkg_dir, 1, mod_name) + init = os.path.join(pkg_dir, "__runpy_pkg__", "__init__.py") + + exceptions = (ImportError, AttributeError, TypeError, ValueError) + for exception in exceptions: + name = exception.__name__ + with self.subTest(name): + source = "raise {0}('{0} in __init__.py.')".format(name) + with open(init, "wt", encoding="ascii") as mod_file: + mod_file.write(source) + try: + run_module(mod_name) + except exception as err: + self.assertNotIn("finding spec", format(err)) + else: + self.fail("Nothing raised; expected {}".format(name)) + def test_run_package_in_namespace_package(self): for depth in range(1, 4): if verbose > 1: print("Testing package depth:", depth) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -22,6 +22,11 @@ Library ------- +- Issue #14285: When executing a package with the "python -m package" option, + and package initialization fails, a proper traceback is now reported. The + "runpy" module now lets exceptions from package initialization pass back to + the caller, rather than raising ImportError. + - Issue #25177: Fixed problem with the mean of very small and very large numbers. As a side effect, statistics.mean and statistics.variance should be significantly faster. -- Repository URL: https://hg.python.org/cpython

1 0

cpython (merge 3.5 -> default): Null merge
by serhiy.storchaka Dec. 2, 2015

Dec. 2, 2015

https://hg.python.org/cpython/rev/a89f92ac7d19 changeset: 99417:a89f92ac7d19 parent: 99415:7dcc03b344b5 parent: 99416:c0522985bd7c user: Serhiy Storchaka <storchaka(a)gmail.com> date: Thu Dec 03 00:58:13 2015 +0200 summary: Null merge files: -- Repository URL: https://hg.python.org/cpython

1 0

cpython (merge 3.4 -> 3.5): Issue #25709: Fixed problem with in-place string concatenation and utf-8 cache.
by serhiy.storchaka Dec. 2, 2015

Dec. 2, 2015

https://hg.python.org/cpython/rev/a0e2376768dc changeset: 99419:a0e2376768dc branch: 3.5 parent: 99416:c0522985bd7c parent: 99418:67718032badb user: Serhiy Storchaka <storchaka(a)gmail.com> date: Thu Dec 03 01:04:37 2015 +0200 summary: Issue #25709: Fixed problem with in-place string concatenation and utf-8 cache. files: Lib/test/test_unicode.py | 17 +++++++++++++++++ Misc/NEWS | 2 ++ Objects/unicodeobject.c | 5 +++++ 3 files changed, 24 insertions(+), 0 deletions(-) diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -2702,6 +2702,23 @@ self.assertTrue(astral >= bmp2) self.assertFalse(astral >= astral2) + @support.cpython_only + def test_pep393_utf8_caching_bug(self): + # Issue #25709: Problem with string concatenation and utf-8 cache + from _testcapi import getargs_s_hash + for k in 0x24, 0xa4, 0x20ac, 0x1f40d: + s = '' + for i in range(5): + # Due to CPython specific optimization the 's' string can be + # resized in-place. + s += chr(k) + # Parsing with the "s#" format code calls indirectly + # PyUnicode_AsUTF8AndSize() which creates the UTF-8 + # encoded string cached in the Unicode object. + self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) + # Check that the second call returns the same result + self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) + class StringModuleTest(unittest.TestCase): def test_formatter_parser(self): diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,8 @@ Core and Builtins ----------------- +- Issue #25709: Fixed problem with in-place string concatenation and utf-8 cache. + - Issue #24097: Fixed crash in object.__reduce__() if slot name is freed inside __getattr__. diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -722,6 +722,11 @@ } new_size = (struct_size + (length + 1) * char_size); + if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) { + PyObject_DEL(_PyUnicode_UTF8(unicode)); + _PyUnicode_UTF8(unicode) = NULL; + _PyUnicode_UTF8_LENGTH(unicode) = 0; + } _Py_DEC_REFTOTAL; _Py_ForgetReference(unicode); -- Repository URL: https://hg.python.org/cpython

1 0