cpython (merge default -> default): merged
http://hg.python.org/cpython/rev/d406e1392bbc changeset: 78353:d406e1392bbc parent: 78352:a391185ab5b2 parent: 78319:1d811e1097ed user: Barry Warsaw <barry@python.org> date: Sun Jul 29 16:40:04 2012 -0400 summary: merged files: Lib/importlib/_bootstrap.py | 89 +- Lib/test/support.py | 28 + Lib/test/test_import.py | 68 + Lib/test/test_io.py | 20 +- Lib/test/test_struct.py | 33 +- Lib/test/test_sys.py | 214 +- Lib/test/test_xml_etree_c.py | 24 +- Misc/NEWS | 11 + Modules/_freeze_importlib.c | 11 +- Modules/_io/bufferedio.c | 14 + Modules/_struct.c | 8 +- PCbuild/pcbuild.sln | 6 - Python/import.c | 30 +- Python/importlib.h | 8060 +++++++++++---------- 14 files changed, 4511 insertions(+), 4105 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -722,6 +722,56 @@ return _imp.init_frozen(fullname) +class WindowsRegistryImporter: + + """Meta path import for modules declared in the Windows registry. + """ + + REGISTRY_KEY = ( + "Software\\Python\\PythonCore\\{sys_version}" + "\\Modules\\{fullname}") + REGISTRY_KEY_DEBUG = ( + "Software\\Python\\PythonCore\\{sys_version}" + "\\Modules\\{fullname}\\Debug") + DEBUG_BUILD = False # Changed in _setup() + + @classmethod + def _open_registry(cls, key): + try: + return _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, key) + except WindowsError: + return _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, key) + + @classmethod + def _search_registry(cls, fullname): + if cls.DEBUG_BUILD: + registry_key = cls.REGISTRY_KEY_DEBUG + else: + registry_key = cls.REGISTRY_KEY + key = registry_key.format(fullname=fullname, + sys_version=sys.version[:3]) + try: + with cls._open_registry(key) as hkey: + filepath = _winreg.QueryValue(hkey, "") + except WindowsError: + return None + return filepath + + @classmethod + def find_module(cls, fullname, path=None): + """Find module named in the registry.""" + filepath = cls._search_registry(fullname) + if filepath is None: + return None + try: + _os.stat(filepath) + except OSError: + return None + for loader, suffixes, _ in _get_supported_file_loaders(): + if filepath.endswith(tuple(suffixes)): + return loader(fullname, filepath) + + class _LoaderBasics: """Base class of common code needed by both SourceLoader and @@ -1422,7 +1472,7 @@ parent = name.rpartition('.')[0] if parent: if parent not in sys.modules: - import_(parent) + _recursive_import(import_, parent) # Crazy side-effects! if name in sys.modules: return sys.modules[name] @@ -1500,6 +1550,12 @@ _lock_unlock_module(name) return module +def _recursive_import(import_, name): + """Common exit point for recursive calls to the import machinery + + This simplifies the process of stripping importlib from tracebacks + """ + return import_(name) def _handle_fromlist(module, fromlist, import_): """Figure out what __import__ should return. @@ -1519,7 +1575,8 @@ fromlist.extend(module.__all__) for x in fromlist: if not hasattr(module, x): - import_('{}.{}'.format(module.__name__, x)) + _recursive_import(import_, + '{}.{}'.format(module.__name__, x)) return module @@ -1538,6 +1595,17 @@ return package +def _get_supported_file_loaders(): + """Returns a list of file-based module loaders. + + Each item is a tuple (loader, suffixes, allow_packages). + """ + extensions = ExtensionFileLoader, _imp.extension_suffixes(), False + source = SourceFileLoader, SOURCE_SUFFIXES, True + bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES, True + return [extensions, source, bytecode] + + def __import__(name, globals={}, locals={}, fromlist=[], level=0): """Import a module. @@ -1620,6 +1688,10 @@ thread_module = None weakref_module = BuiltinImporter.load_module('_weakref') + if builtin_os == 'nt': + winreg_module = BuiltinImporter.load_module('winreg') + setattr(self_module, '_winreg', winreg_module) + setattr(self_module, '_os', os_module) setattr(self_module, '_thread', thread_module) setattr(self_module, '_weakref', weakref_module) @@ -1629,14 +1701,17 @@ setattr(self_module, '_relax_case', _make_relax_case()) if builtin_os == 'nt': SOURCE_SUFFIXES.append('.pyw') + if '_d.pyd' in _imp.extension_suffixes(): + WindowsRegistryImporter.DEBUG_BUILD = True def _install(sys_module, _imp_module): """Install importlib as the implementation of import.""" _setup(sys_module, _imp_module) - extensions = ExtensionFileLoader, _imp_module.extension_suffixes(), False - source = SourceFileLoader, SOURCE_SUFFIXES, True - bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES, True - supported_loaders = [extensions, source, bytecode] + supported_loaders = _get_supported_file_loaders() sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)]) - sys.meta_path.extend([BuiltinImporter, FrozenImporter, PathFinder]) + sys.meta_path.append(BuiltinImporter) + sys.meta_path.append(FrozenImporter) + if _os.__name__ == 'nt': + sys.meta_path.append(WindowsRegistryImporter) + sys.meta_path.append(PathFinder) diff --git a/Lib/test/support.py b/Lib/test/support.py --- a/Lib/test/support.py +++ b/Lib/test/support.py @@ -25,6 +25,7 @@ import logging.handlers import struct import tempfile +import _testcapi try: import _thread, threading @@ -1082,6 +1083,33 @@ return final_opt != '' and final_opt != '-O0' +_header = 'nP' +_align = '0n' +if hasattr(sys, "gettotalrefcount"): + _header = '2P' + _header + _align = '0P' +_vheader = _header + 'n' + +def calcobjsize(fmt): + return struct.calcsize(_header + fmt + _align) + +def calcvobjsize(fmt): + return struct.calcsize(_vheader + fmt + _align) + + +_TPFLAGS_HAVE_GC = 1<<14 +_TPFLAGS_HEAPTYPE = 1<<9 + +def check_sizeof(test, o, size): + result = sys.getsizeof(o) + # add GC header size + if ((type(o) == type) and (o.__flags__ & _TPFLAGS_HEAPTYPE) or\ + ((type(o) != type) and (type(o).__flags__ & _TPFLAGS_HAVE_GC))): + size += _testcapi.SIZEOF_PYGC_HEAD + msg = 'wrong size for %s: got %d, expected %d' \ + % (type(o), result, size) + test.assertEqual(result, size, msg) + #======================================================================= # Decorator for running a function in a different locale, correctly resetting # it afterwards. diff --git a/Lib/test/test_import.py b/Lib/test/test_import.py --- a/Lib/test/test_import.py +++ b/Lib/test/test_import.py @@ -844,6 +844,74 @@ self.fail("ZeroDivisionError should have been raised") self.assert_traceback(tb, [__file__, 'foo.py', 'bar.py']) + # A few more examples from issue #15425 + def test_syntax_error(self): + self.create_module("foo", "invalid syntax is invalid") + try: + import foo + except SyntaxError as e: + tb = e.__traceback__ + else: + self.fail("SyntaxError should have been raised") + self.assert_traceback(tb, [__file__]) + + def _setup_broken_package(self, parent, child): + pkg_name = "_parent_foo" + def cleanup(): + rmtree(pkg_name) + unload(pkg_name) + os.mkdir(pkg_name) + self.addCleanup(cleanup) + # Touch the __init__.py + init_path = os.path.join(pkg_name, '__init__.py') + with open(init_path, 'w') as f: + f.write(parent) + bar_path = os.path.join(pkg_name, 'bar.py') + with open(bar_path, 'w') as f: + f.write(child) + importlib.invalidate_caches() + return init_path, bar_path + + def test_broken_submodule(self): + init_path, bar_path = self._setup_broken_package("", "1/0") + try: + import _parent_foo.bar + except ZeroDivisionError as e: + tb = e.__traceback__ + else: + self.fail("ZeroDivisionError should have been raised") + self.assert_traceback(tb, [__file__, bar_path]) + + def test_broken_from(self): + init_path, bar_path = self._setup_broken_package("", "1/0") + try: + from _parent_foo import bar + except ZeroDivisionError as e: + tb = e.__traceback__ + else: + self.fail("ImportError should have been raised") + self.assert_traceback(tb, [__file__, bar_path]) + + def test_broken_parent(self): + init_path, bar_path = self._setup_broken_package("1/0", "") + try: + import _parent_foo.bar + except ZeroDivisionError as e: + tb = e.__traceback__ + else: + self.fail("ZeroDivisionError should have been raised") + self.assert_traceback(tb, [__file__, init_path]) + + def test_broken_parent_from(self): + init_path, bar_path = self._setup_broken_package("1/0", "") + try: + from _parent_foo import bar + except ZeroDivisionError as e: + tb = e.__traceback__ + else: + self.fail("ZeroDivisionError should have been raised") + self.assert_traceback(tb, [__file__, init_path]) + @cpython_only def test_import_bug(self): # We simulate a bug in importlib and check that it's not stripped diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -802,6 +802,20 @@ buf.raw = x +class SizeofTest: + + @support.cpython_only + def test_sizeof(self): + bufsize1 = 4096 + bufsize2 = 8192 + rawio = self.MockRawIO() + bufio = self.tp(rawio, buffer_size=bufsize1) + size = sys.getsizeof(bufio) - bufsize1 + rawio = self.MockRawIO() + bufio = self.tp(rawio, buffer_size=bufsize2) + self.assertEqual(sys.getsizeof(bufio), size + bufsize2) + + class BufferedReaderTest(unittest.TestCase, CommonBufferedTests): read_mode = "rb" @@ -999,7 +1013,7 @@ "failed for {}: {} != 0".format(n, rawio._extraneous_reads)) -class CBufferedReaderTest(BufferedReaderTest): +class CBufferedReaderTest(BufferedReaderTest, SizeofTest): tp = io.BufferedReader def test_constructor(self): @@ -1260,7 +1274,7 @@ self.tp(self.MockRawIO(), 8, 12) -class CBufferedWriterTest(BufferedWriterTest): +class CBufferedWriterTest(BufferedWriterTest, SizeofTest): tp = io.BufferedWriter def test_constructor(self): @@ -1650,7 +1664,7 @@ # You can't construct a BufferedRandom over a non-seekable stream. test_unseekable = None -class CBufferedRandomTest(BufferedRandomTest): +class CBufferedRandomTest(BufferedRandomTest, SizeofTest): tp = io.BufferedRandom def test_constructor(self): diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -3,7 +3,7 @@ import struct import sys -from test.support import run_unittest +from test import support ISBIGENDIAN = sys.byteorder == "big" IS32BIT = sys.maxsize == 0x7fffffff @@ -572,18 +572,29 @@ s = struct.Struct('i') s.__init__('ii') - def test_sizeof(self): - self.assertGreater(sys.getsizeof(struct.Struct('BHILfdspP')), - sys.getsizeof(struct.Struct('B'))) - self.assertGreater(sys.getsizeof(struct.Struct('123B')), - sys.getsizeof(struct.Struct('B'))) - self.assertGreater(sys.getsizeof(struct.Struct('B' * 1234)), - sys.getsizeof(struct.Struct('123B'))) - self.assertGreater(sys.getsizeof(struct.Struct('1234B')), - sys.getsizeof(struct.Struct('123B'))) + def check_sizeof(self, format_str, number_of_codes): + # The size of 'PyStructObject' + totalsize = support.calcobjsize('2n3P') + # The size taken up by the 'formatcode' dynamic array + totalsize += struct.calcsize('P2n0P') * (number_of_codes + 1) + support.check_sizeof(self, struct.Struct(format_str), totalsize) + + @support.cpython_only + def test__sizeof__(self): + for code in integer_codes: + self.check_sizeof(code, 1) + self.check_sizeof('BHILfdspP', 9) + self.check_sizeof('B' * 1234, 1234) + self.check_sizeof('fd', 2) + self.check_sizeof('xxxxxxxxxxxxxx', 0) + self.check_sizeof('100H', 100) + self.check_sizeof('187s', 1) + self.check_sizeof('20p', 1) + self.check_sizeof('0s', 1) + self.check_sizeof('0c', 0) def test_main(): - run_unittest(StructTest) + support.run_unittest(StructTest) if __name__ == '__main__': test_main() diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -612,22 +612,8 @@ class SizeofTest(unittest.TestCase): - TPFLAGS_HAVE_GC = 1<<14 - TPFLAGS_HEAPTYPE = 1<<9 - def setUp(self): - self.c = len(struct.pack('c', b' ')) - self.H = len(struct.pack('H', 0)) - self.i = len(struct.pack('i', 0)) - self.l = len(struct.pack('l', 0)) - self.P = len(struct.pack('P', 0)) - # due to missing size_t information from struct, it is assumed that - # sizeof(Py_ssize_t) = sizeof(void*) - self.header = 'PP' - self.vheader = self.header + 'P' - if hasattr(sys, "gettotalrefcount"): - self.header += '2P' - self.vheader += '2P' + self.P = struct.calcsize('P') self.longdigit = sys.int_info.sizeof_digit import _testcapi self.gc_headsize = _testcapi.SIZEOF_PYGC_HEAD @@ -637,129 +623,108 @@ self.file.close() test.support.unlink(test.support.TESTFN) - def check_sizeof(self, o, size): - result = sys.getsizeof(o) - # add GC header size - if ((type(o) == type) and (o.__flags__ & self.TPFLAGS_HEAPTYPE) or\ - ((type(o) != type) and (type(o).__flags__ & self.TPFLAGS_HAVE_GC))): - size += self.gc_headsize - msg = 'wrong size for %s: got %d, expected %d' \ - % (type(o), result, size) - self.assertEqual(result, size, msg) - - def calcsize(self, fmt): - """Wrapper around struct.calcsize which enforces the alignment of the - end of a structure to the alignment requirement of pointer. - - Note: This wrapper should only be used if a pointer member is included - and no member with a size larger than a pointer exists. - """ - return struct.calcsize(fmt + '0P') + check_sizeof = test.support.check_sizeof def test_gc_head_size(self): # Check that the gc header size is added to objects tracked by the gc. - h = self.header - vh = self.vheader - size = self.calcsize + vsize = test.support.calcvobjsize gc_header_size = self.gc_headsize # bool objects are not gc tracked - self.assertEqual(sys.getsizeof(True), size(vh) + self.longdigit) + self.assertEqual(sys.getsizeof(True), vsize('') + self.longdigit) # but lists are - self.assertEqual(sys.getsizeof([]), size(vh + 'PP') + gc_header_size) + self.assertEqual(sys.getsizeof([]), vsize('Pn') + gc_header_size) def test_default(self): - h = self.header - vh = self.vheader - size = self.calcsize - self.assertEqual(sys.getsizeof(True), size(vh) + self.longdigit) - self.assertEqual(sys.getsizeof(True, -1), size(vh) + self.longdigit) + size = test.support.calcvobjsize + self.assertEqual(sys.getsizeof(True), size('') + self.longdigit) + self.assertEqual(sys.getsizeof(True, -1), size('') + self.longdigit) def test_objecttypes(self): # check all types defined in Objects/ - h = self.header - vh = self.vheader - size = self.calcsize + size = test.support.calcobjsize + vsize = test.support.calcvobjsize check = self.check_sizeof # bool - check(True, size(vh) + self.longdigit) + check(True, vsize('') + self.longdigit) # buffer # XXX # builtin_function_or_method - check(len, size(h + '3P')) + check(len, size('3P')) # XXX check layout # bytearray samples = [b'', b'u'*100000] for sample in samples: x = bytearray(sample) - check(x, size(vh + 'iPP') + x.__alloc__() * self.c) + check(x, vsize('inP') + x.__alloc__()) # bytearray_iterator - check(iter(bytearray()), size(h + 'PP')) + check(iter(bytearray()), size('nP')) # cell def get_cell(): x = 42 def inner(): return x return inner - check(get_cell().__closure__[0], size(h + 'P')) + check(get_cell().__closure__[0], size('P')) # code - check(get_cell().__code__, size(h + '5i9Pi3P')) - check(get_cell.__code__, size(h + '5i9Pi3P')) + check(get_cell().__code__, size('5i9Pi3P')) + check(get_cell.__code__, size('5i9Pi3P')) def get_cell2(x): def inner(): return x return inner - check(get_cell2.__code__, size(h + '5i9Pi3P') + 1) + check(get_cell2.__code__, size('5i9Pi3P') + 1) # complex - check(complex(0,1), size(h + '2d')) + check(complex(0,1), size('2d')) # method_descriptor (descriptor object) - check(str.lower, size(h + '3PP')) + check(str.lower, size('3PP')) # classmethod_descriptor (descriptor object) # XXX # member_descriptor (descriptor object) import datetime - check(datetime.timedelta.days, size(h + '3PP')) + check(datetime.timedelta.days, size('3PP')) # getset_descriptor (descriptor object) import collections - check(collections.defaultdict.default_factory, size(h + '3PP')) + check(collections.defaultdict.default_factory, size('3PP')) # wrapper_descriptor (descriptor object) - check(int.__add__, size(h + '3P2P')) + check(int.__add__, size('3P2P')) # method-wrapper (descriptor object) - check({}.__iter__, size(h + '2P')) + check({}.__iter__, size('2P')) # dict - check({}, size(h + '3P' + '4P' + 8*'P2P')) + check({}, size('n2P' + '2nPn' + 8*'n2P')) longdict = {1:1, 2:2, 3:3, 4:4, 5:5, 6:6, 7:7, 8:8} - check(longdict, size(h + '3P' + '4P') + 16*size('P2P')) + check(longdict, size('n2P' + '2nPn') + 16*struct.calcsize('n2P')) # dictionary-keyiterator - check({}.keys(), size(h + 'P')) + check({}.keys(), size('P')) # dictionary-valueiterator - check({}.values(), size(h + 'P')) + check({}.values(), size('P')) # dictionary-itemiterator - check({}.items(), size(h + 'P')) + check({}.items(), size('P')) + # dictionary iterator + check(iter({}), size('P2nPn')) # dictproxy class C(object): pass - check(C.__dict__, size(h + 'P')) + check(C.__dict__, size('P')) # BaseException - check(BaseException(), size(h + '5Pi')) + check(BaseException(), size('5Pi')) # UnicodeEncodeError - check(UnicodeEncodeError("", "", 0, 0, ""), size(h + '5Pi 2P2PP')) + check(UnicodeEncodeError("", "", 0, 0, ""), size('5Pi 2P2nP')) # UnicodeDecodeError - # XXX -# check(UnicodeDecodeError("", "", 0, 0, ""), size(h + '5P2PP')) + check(UnicodeDecodeError("", b"", 0, 0, ""), size('5Pi 2P2nP')) # UnicodeTranslateError - check(UnicodeTranslateError("", 0, 1, ""), size(h + '5Pi 2P2PP')) + check(UnicodeTranslateError("", 0, 1, ""), size('5Pi 2P2nP')) # ellipses - check(Ellipsis, size(h + '')) + check(Ellipsis, size('')) # EncodingMap import codecs, encodings.iso8859_3 x = codecs.charmap_build(encodings.iso8859_3.decoding_table) - check(x, size(h + '32B2iB')) + check(x, size('32B2iB')) # enumerate - check(enumerate([]), size(h + 'l3P')) + check(enumerate([]), size('n3P')) # reverse - check(reversed(''), size(h + 'PP')) + check(reversed(''), size('nP')) # float - check(float(0), size(h + 'd')) + check(float(0), size('d')) # sys.floatinfo - check(sys.float_info, size(vh) + self.P * len(sys.float_info)) + check(sys.float_info, vsize('') + self.P * len(sys.float_info)) # frame import inspect CO_MAXBLOCKS = 20 @@ -768,10 +733,10 @@ nfrees = len(x.f_code.co_freevars) extras = x.f_code.co_stacksize + x.f_code.co_nlocals +\ ncells + nfrees - 1 - check(x, size(vh + '12P3i' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P')) + check(x, vsize('12P3i' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P')) # function def func(): pass - check(func, size(h + '12P')) + check(func, size('12P')) class c(): @staticmethod def foo(): @@ -780,68 +745,68 @@ def bar(cls): pass # staticmethod - check(foo, size(h + 'PP')) + check(foo, size('PP')) # classmethod - check(bar, size(h + 'PP')) + check(bar, size('PP')) # generator def get_gen(): yield 1 - check(get_gen(), size(h + 'Pi2P')) + check(get_gen(), size('Pb2P')) # iterator - check(iter('abc'), size(h + 'lP')) + check(iter('abc'), size('lP')) # callable-iterator import re - check(re.finditer('',''), size(h + '2P')) + check(re.finditer('',''), size('2P')) # list samples = [[], [1,2,3], ['1', '2', '3']] for sample in samples: - check(sample, size(vh + 'PP') + len(sample)*self.P) + check(sample, vsize('Pn') + len(sample)*self.P) # sortwrapper (list) # XXX # cmpwrapper (list) # XXX # listiterator (list) - check(iter([]), size(h + 'lP')) + check(iter([]), size('lP')) # listreverseiterator (list) - check(reversed([]), size(h + 'lP')) + check(reversed([]), size('nP')) # long - check(0, size(vh)) - check(1, size(vh) + self.longdigit) - check(-1, size(vh) + self.longdigit) + check(0, vsize('')) + check(1, vsize('') + self.longdigit) + check(-1, vsize('') + self.longdigit) PyLong_BASE = 2**sys.int_info.bits_per_digit - check(int(PyLong_BASE), size(vh) + 2*self.longdigit) - check(int(PyLong_BASE**2-1), size(vh) + 2*self.longdigit) - check(int(PyLong_BASE**2), size(vh) + 3*self.longdigit) + check(int(PyLong_BASE), vsize('') + 2*self.longdigit) + check(int(PyLong_BASE**2-1), vsize('') + 2*self.longdigit) + check(int(PyLong_BASE**2), vsize('') + 3*self.longdigit) # memoryview - check(memoryview(b''), size(h + 'PPiP4P2i5P3c2P')) + check(memoryview(b''), size('Pnin 2P2n2i5P 3cPn')) # module - check(unittest, size(h + '3P')) + check(unittest, size('PnP')) # None - check(None, size(h + '')) + check(None, size('')) # NotImplementedType - check(NotImplemented, size(h)) + check(NotImplemented, size('')) # object - check(object(), size(h + '')) + check(object(), size('')) # property (descriptor object) class C(object): def getx(self): return self.__x def setx(self, value): self.__x = value def delx(self): del self.__x x = property(getx, setx, delx, "") - check(x, size(h + '4Pi')) + check(x, size('4Pi')) # PyCapsule # XXX # rangeiterator - check(iter(range(1)), size(h + '4l')) + check(iter(range(1)), size('4l')) # reverse - check(reversed(''), size(h + 'PP')) + check(reversed(''), size('nP')) # range - check(range(1), size(h + '4P')) - check(range(66000), size(h + '4P')) + check(range(1), size('4P')) + check(range(66000), size('4P')) # set # frozenset PySet_MINSIZE = 8 samples = [[], range(10), range(50)] - s = size(h + '3P2P' + PySet_MINSIZE*'lP' + 'lP') + s = size('3n2P' + PySet_MINSIZE*'nP' + 'nP') for sample in samples: minused = len(sample) if minused == 0: tmp = 1 @@ -855,31 +820,31 @@ check(set(sample), s) check(frozenset(sample), s) else: - check(set(sample), s + newsize*struct.calcsize('lP')) - check(frozenset(sample), s + newsize*struct.calcsize('lP')) + check(set(sample), s + newsize*struct.calcsize('nP')) + check(frozenset(sample), s + newsize*struct.calcsize('nP')) # setiterator - check(iter(set()), size(h + 'P3P')) + check(iter(set()), size('P3n')) # slice - check(slice(0), size(h + '3P')) + check(slice(0), size('3P')) # super - check(super(int), size(h + '3P')) + check(super(int), size('3P')) # tuple - check((), size(vh)) - check((1,2,3), size(vh) + 3*self.P) + check((), vsize('')) + check((1,2,3), vsize('') + 3*self.P) # type # static type: PyTypeObject - s = size(vh + 'P2P15Pl4PP9PP11PI') + s = vsize('P2n15Pl4Pn9Pn11PI') check(int, s) # (PyTypeObject + PyNumberMethods + PyMappingMethods + # PySequenceMethods + PyBufferProcs + 4P) - s = size(vh + 'P2P15Pl4PP9PP11PI') + size('34P 3P 10P 2P 4P') + s = vsize('P2n15Pl4Pn9Pn11PI') + struct.calcsize('34P 3P 10P 2P 4P') # Separate block for PyDictKeysObject with 4 entries - s += size("PPPP") + 4*size("PPP") + s += struct.calcsize("2nPn") + 4*struct.calcsize("n2P") # class class newstyleclass(object): pass check(newstyleclass, s) # dict with shared keys - check(newstyleclass().__dict__, size(h+"PPP4P")) + check(newstyleclass().__dict__, size('n2P' + '2nPn')) # unicode # each tuple contains a string and its expected character size # don't put any static strings here, as they may contain @@ -887,8 +852,8 @@ samples = ['1'*100, '\xff'*50, '\u0100'*40, '\uffff'*100, '\U00010000'*30, '\U0010ffff'*100] - asciifields = h + "PPiP" - compactfields = asciifields + "PPP" + asciifields = "nniP" + compactfields = asciifields + "nPn" unicodefields = compactfields + "P" for s in samples: maxchar = ord(max(s)) @@ -912,32 +877,31 @@ # TODO: add check that forces layout of unicodefields # weakref import weakref - check(weakref.ref(int), size(h + '2Pl2P')) + check(weakref.ref(int), size('2Pn2P')) # weakproxy # XXX # weakcallableproxy - check(weakref.proxy(int), size(h + '2Pl2P')) + check(weakref.proxy(int), size('2Pn2P')) def test_pythontypes(self): # check all types defined in Python/ - h = self.header - vh = self.vheader - size = self.calcsize + size = test.support.calcobjsize + vsize = test.support.calcvobjsize check = self.check_sizeof # _ast.AST import _ast - check(_ast.AST(), size(h + 'P')) + check(_ast.AST(), size('P')) try: raise TypeError except TypeError: tb = sys.exc_info()[2] # traceback if tb != None: - check(tb, size(h + '2P2i')) + check(tb, size('2P2i')) # symtable entry # XXX # sys.flags - check(sys.flags, size(vh) + self.P * len(sys.flags)) + check(sys.flags, vsize('') + self.P * len(sys.flags)) def test_main(): diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py --- a/Lib/test/test_xml_etree_c.py +++ b/Lib/test/test_xml_etree_c.py @@ -41,38 +41,30 @@ @unittest.skipUnless(cET, 'requires _elementtree') +@support.cpython_only class SizeofTest(unittest.TestCase): def setUp(self): - import _testcapi - gc_headsize = _testcapi.SIZEOF_PYGC_HEAD - # object header - header = 'PP' - if hasattr(sys, "gettotalrefcount"): - # debug header - header = 'PP' + header - # fields - element = header + '5P' - self.elementsize = gc_headsize + struct.calcsize(element) + self.elementsize = support.calcobjsize('5P') # extra self.extra = struct.calcsize('PiiP4P') + check_sizeof = support.check_sizeof + def test_element(self): e = cET.Element('a') - self.assertEqual(sys.getsizeof(e), self.elementsize) + self.check_sizeof(e, self.elementsize) def test_element_with_attrib(self): e = cET.Element('a', href='about:') - self.assertEqual(sys.getsizeof(e), - self.elementsize + self.extra) + self.check_sizeof(e, self.elementsize + self.extra) def test_element_with_children(self): e = cET.Element('a') for i in range(5): cET.SubElement(e, 'span') # should have space for 8 children now - self.assertEqual(sys.getsizeof(e), - self.elementsize + self.extra + - struct.calcsize('8P')) + self.check_sizeof(e, self.elementsize + self.extra + + struct.calcsize('8P')) def test_main(): from test import test_xml_etree, test_xml_etree_c diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,11 @@ Core and Builtins ----------------- +- Issue #15425: Eliminated traceback noise from more situations involving + importlib + +- Issue #14578: Support modules registered in the Windows registry again. + - Issue #15466: Stop using TYPE_INT64 in marshal, to make importlib.h (and other byte code files) equal between 32-bit and 64-bit systems. @@ -236,6 +241,9 @@ Tests ----- +- Issue #15467: Move helpers for __sizeof__ tests into test_support. + Patch by Serhiy Storchaka. + - Issue #15320: Make iterating the list of tests thread-safe when running tests in multiprocess mode. Patch by Chris Jerdonek. @@ -329,6 +337,9 @@ Library ------- +- Issue #15487: Add a __sizeof__ implementation for buffered I/O objects. + Patch by Serhiy Storchaka. + - Issue #15187: Bugfix: remove temporary directories test_shutil was leaving behind. diff --git a/Modules/_freeze_importlib.c b/Modules/_freeze_importlib.c --- a/Modules/_freeze_importlib.c +++ b/Modules/_freeze_importlib.c @@ -21,6 +21,13 @@ {0, 0, 0} /* sentinel */ }; +#ifndef MS_WINDOWS +/* On Windows, this links with the regular pythonXY.dll, so this variable comes + from frozen.obj. In the Makefile, frozen.o is not linked into this executable, + so we define the variable here. */ +struct _frozen *PyImport_FrozenModules; +#endif + const char header[] = "/* Auto-generated by Modules/_freeze_importlib.c */"; int @@ -91,8 +98,8 @@ data_size = PyBytes_GET_SIZE(marshalled); /* Open the file in text mode. The hg checkout should be using the eol extension, - which in turn should cause the existing file to use CRLF */ - outfile = fopen(outpath, "wt"); + which in turn should cause the EOL style match the C library's text mode */ + outfile = fopen(outpath, "w"); if (outfile == NULL) { fprintf(stderr, "cannot open '%s' for writing\n", outpath); return 1; diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -398,6 +398,17 @@ Py_TYPE(self)->tp_free((PyObject *)self); } +static PyObject * +buffered_sizeof(buffered *self, void *unused) +{ + Py_ssize_t res; + + res = sizeof(buffered); + if (self->buffer) + res += self->buffer_size; + return PyLong_FromSsize_t(res); +} + static int buffered_traverse(buffered *self, visitproc visit, void *arg) { @@ -1699,6 +1710,7 @@ {"seek", (PyCFunction)buffered_seek, METH_VARARGS}, {"tell", (PyCFunction)buffered_tell, METH_NOARGS}, {"truncate", (PyCFunction)buffered_truncate, METH_VARARGS}, + {"__sizeof__", (PyCFunction)buffered_sizeof, METH_NOARGS}, {NULL, NULL} }; @@ -2079,6 +2091,7 @@ {"flush", (PyCFunction)buffered_flush, METH_NOARGS}, {"seek", (PyCFunction)buffered_seek, METH_VARARGS}, {"tell", (PyCFunction)buffered_tell, METH_NOARGS}, + {"__sizeof__", (PyCFunction)buffered_sizeof, METH_NOARGS}, {NULL, NULL} }; @@ -2470,6 +2483,7 @@ {"readline", (PyCFunction)buffered_readline, METH_VARARGS}, {"peek", (PyCFunction)buffered_peek, METH_VARARGS}, {"write", (PyCFunction)bufferedwriter_write, METH_VARARGS}, + {"__sizeof__", (PyCFunction)buffered_sizeof, METH_NOARGS}, {NULL, NULL} }; diff --git a/Modules/_struct.c b/Modules/_struct.c --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -1756,15 +1756,11 @@ "S.__sizeof__() -> size of S in memory, in bytes"); static PyObject * -s_sizeof(PyStructObject *self) +s_sizeof(PyStructObject *self, void *unused) { Py_ssize_t size; - formatcode *code; - size = sizeof(PyStructObject) + sizeof(formatcode); - for (code = self->s_codes; code->fmtdef != NULL; code++) { - size += sizeof(formatcode); - } + size = sizeof(PyStructObject) + sizeof(formatcode) * (self->s_len + 1); return PyLong_FromSsize_t(size); } diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -603,19 +603,13 @@ {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|x64.ActiveCfg = Release|Win32 {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|x64.Build.0 = Release|Win32 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|Win32.ActiveCfg = Debug|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|Win32.Build.0 = Debug|Win32 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|x64.ActiveCfg = Debug|x64 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|x64.Build.0 = Debug|x64 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|Win32.Build.0 = Release|Win32 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|x64.ActiveCfg = Release|Win32 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGUpdate|Win32.Build.0 = Release|Win32 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGUpdate|x64.ActiveCfg = Release|Win32 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|Win32.ActiveCfg = Release|Win32 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|Win32.Build.0 = Release|Win32 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|x64.ActiveCfg = Release|x64 - {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Python/import.c b/Python/import.c --- a/Python/import.c +++ b/Python/import.c @@ -1154,14 +1154,27 @@ { const char *importlib_filename = "<frozen importlib._bootstrap>"; const char *exec_funcname = "_exec_module"; + const char *get_code_funcname = "get_code"; + const char *recursive_import = "_recursive_import"; int always_trim = 0; + int trim_get_code = 0; int in_importlib = 0; PyObject *exception, *value, *base_tb, *tb; PyObject **prev_link, **outer_link = NULL; /* Synopsis: if it's an ImportError, we trim all importlib chunks - from the traceback. Otherwise, we trim only those chunks which - end with a call to "_exec_module". */ + from the traceback. If it's a SyntaxError, we trim any chunks that + end with a call to "get_code", We always trim chunks + which end with a call to "_exec_module". */ + + /* Thanks to issue 15425, we also strip any chunk ending with + * _recursive_import. This is used when making a recursive call to the + * full import machinery which means the inner stack gets stripped early + * and the normal heuristics won't fire properly for outer frames. A + * more elegant mechanism would be nice, as this one can misfire if + * builtins.__import__ has been replaced with a custom implementation. + * However, the current approach at least gets the job done. + */ PyErr_Fetch(&exception, &value, &base_tb); if (!exception || Py_VerboseFlag) @@ -1169,6 +1182,9 @@ if (PyType_IsSubtype((PyTypeObject *) exception, (PyTypeObject *) PyExc_ImportError)) always_trim = 1; + if (PyType_IsSubtype((PyTypeObject *) exception, + (PyTypeObject *) PyExc_SyntaxError)) + trim_get_code = 1; prev_link = &base_tb; tb = base_tb; @@ -1191,8 +1207,14 @@ if (in_importlib && (always_trim || - PyUnicode_CompareWithASCIIString(code->co_name, - exec_funcname) == 0)) { + (PyUnicode_CompareWithASCIIString(code->co_name, + exec_funcname) == 0) || + (PyUnicode_CompareWithASCIIString(code->co_name, + recursive_import) == 0) || + (trim_get_code && + PyUnicode_CompareWithASCIIString(code->co_name, + get_code_funcname) == 0) + )) { PyObject *tmp = *outer_link; *outer_link = next; Py_XINCREF(next); diff --git a/Python/importlib.h b/Python/importlib.h --- a/Python/importlib.h +++ b/Python/importlib.h [stripped] -- Repository URL: http://hg.python.org/cpython
participants (1)
-
barry.warsaw