[Python-checkins] r80137 - in python/branches/py3k: .bzrignore .hgignore Doc/c-api/import.rst Doc/library/compileall.rst Doc/library/imp.rst Doc/library/py_compile.rst Doc/library/runpy.rst Include/import.h Lib/compileall.py Lib/importlib/_bootstrap.py Lib/importlib/test/__main__.py Lib/importlib/test/source/test_file_loader.py Lib/importlib/test/source/test_finder.py Lib/importlib/test/source/test_source_encoding.py Lib/importlib/test/source/util.py Lib/importlib/util.py Lib/inspect.py Lib/py_compile.py Lib/pydoc.py Lib/runpy.py Lib/site.py Lib/test/script_helper.py Lib/test/support.py Lib/test/test_cmd_line_script.py Lib/test/test_compileall.py Lib/test/test_frozen.py Lib/test/test_imp.py Lib/test/test_import.py Lib/test/test_pkg.py Lib/test/test_pkgimport.py Lib/test/test_pydoc.py Lib/test/test_runpy.py Lib/test/test_site.py Lib/test/test_zipfile.py Lib/test/test_zipimport.py Lib/zipfile.py Makefile.pre.in Python/import.c Python/pythonrun.c
barry.warsaw
python-checkins at python.org
Sat Apr 17 02:19:56 CEST 2010
Author: barry.warsaw
Date: Sat Apr 17 02:19:56 2010
New Revision: 80137
Log:
PEP 3147
Modified:
python/branches/py3k/.bzrignore
python/branches/py3k/.hgignore
python/branches/py3k/Doc/c-api/import.rst
python/branches/py3k/Doc/library/compileall.rst
python/branches/py3k/Doc/library/imp.rst
python/branches/py3k/Doc/library/py_compile.rst
python/branches/py3k/Doc/library/runpy.rst
python/branches/py3k/Include/import.h
python/branches/py3k/Lib/compileall.py
python/branches/py3k/Lib/importlib/_bootstrap.py
python/branches/py3k/Lib/importlib/test/__main__.py
python/branches/py3k/Lib/importlib/test/source/test_file_loader.py
python/branches/py3k/Lib/importlib/test/source/test_finder.py
python/branches/py3k/Lib/importlib/test/source/test_source_encoding.py
python/branches/py3k/Lib/importlib/test/source/util.py
python/branches/py3k/Lib/importlib/util.py
python/branches/py3k/Lib/inspect.py
python/branches/py3k/Lib/py_compile.py
python/branches/py3k/Lib/pydoc.py
python/branches/py3k/Lib/runpy.py
python/branches/py3k/Lib/site.py
python/branches/py3k/Lib/test/script_helper.py
python/branches/py3k/Lib/test/support.py
python/branches/py3k/Lib/test/test_cmd_line_script.py
python/branches/py3k/Lib/test/test_compileall.py
python/branches/py3k/Lib/test/test_frozen.py
python/branches/py3k/Lib/test/test_imp.py
python/branches/py3k/Lib/test/test_import.py
python/branches/py3k/Lib/test/test_pkg.py
python/branches/py3k/Lib/test/test_pkgimport.py
python/branches/py3k/Lib/test/test_pydoc.py
python/branches/py3k/Lib/test/test_runpy.py
python/branches/py3k/Lib/test/test_site.py
python/branches/py3k/Lib/test/test_zipfile.py
python/branches/py3k/Lib/test/test_zipimport.py
python/branches/py3k/Lib/zipfile.py
python/branches/py3k/Makefile.pre.in
python/branches/py3k/Python/import.c
python/branches/py3k/Python/pythonrun.c
Modified: python/branches/py3k/.bzrignore
==============================================================================
--- python/branches/py3k/.bzrignore (original)
+++ python/branches/py3k/.bzrignore Sat Apr 17 02:19:56 2010
@@ -33,3 +33,4 @@
Lib/test/data/*
Lib/lib2to3/Grammar*.pickle
Lib/lib2to3/PatternGrammar*.pickle
+__pycache__
Modified: python/branches/py3k/.hgignore
==============================================================================
--- python/branches/py3k/.hgignore (original)
+++ python/branches/py3k/.hgignore Sat Apr 17 02:19:56 2010
@@ -54,3 +54,4 @@
PCbuild/*.ncb
PCbuild/*.bsc
PCbuild/Win32-temp-*
+__pycache__
Modified: python/branches/py3k/Doc/c-api/import.rst
==============================================================================
--- python/branches/py3k/Doc/c-api/import.rst (original)
+++ python/branches/py3k/Doc/c-api/import.rst Sat Apr 17 02:19:56 2010
@@ -124,12 +124,24 @@
If *name* points to a dotted name of the form ``package.module``, any package
structures not already created will still not be created.
+ See also :func:`PyImport_ExecCodeModuleEx` and
+ :func:`PyImport_ExecCodeModuleWithPathnames`.
+
.. cfunction:: PyObject* PyImport_ExecCodeModuleEx(char *name, PyObject *co, char *pathname)
Like :cfunc:`PyImport_ExecCodeModule`, but the :attr:`__file__` attribute of
the module object is set to *pathname* if it is non-``NULL``.
+ See also :func:`PyImport_ExecCodeModuleWithPathnames`.
+
+
+.. cfunction:: PyObject* PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, char *cpathname)
+
+ Like :cfunc:`PyImport_ExecCodeModuleEx`, but the :attr:`__cached__`
+ attribute of the module object is set to *cpathname* if it is
+ non-``NULL``. Of the three functions, this is the preferred one to use.
+
.. cfunction:: long PyImport_GetMagicNumber()
@@ -138,6 +150,11 @@
of the bytecode file, in little-endian byte order.
+.. cfunction:: const char * PyImport_GetMagicTag()
+
+ Return the magic tag string for :pep:`3147` format Python bytecode file
+ names.
+
.. cfunction:: PyObject* PyImport_GetModuleDict()
Return the dictionary used for the module administration (a.k.a.
Modified: python/branches/py3k/Doc/library/compileall.rst
==============================================================================
--- python/branches/py3k/Doc/library/compileall.rst (original)
+++ python/branches/py3k/Doc/library/compileall.rst Sat Apr 17 02:19:56 2010
@@ -17,9 +17,11 @@
sys.path``. Printing lists of the files compiled can be disabled with the
:option:`-q` flag. In addition, the :option:`-x` option takes a regular
expression argument. All files that match the expression will be skipped.
+The :option:`-b` flag may be given to write legacy ``.pyc`` file path names,
+otherwise :pep:`3147` style byte-compiled path names are written.
-.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=False)
+.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=False, legacy=False)
Recursively descend the directory tree named by *dir*, compiling all :file:`.py`
files along the way. The *maxlevels* parameter is used to limit the depth of
@@ -34,12 +36,16 @@
If *quiet* is true, nothing is printed to the standard output in normal
operation.
+ If *legacy* is true, old-style ``.pyc`` file path names are written,
+ otherwise (the default), :pep:`3147` style path names are written.
-.. function:: compile_path(skip_curdir=True, maxlevels=0, force=False)
+
+.. function:: compile_path(skip_curdir=True, maxlevels=0, force=False, legacy=False)
Byte-compile all the :file:`.py` files found along ``sys.path``. If
*skip_curdir* is true (the default), the current directory is not included in
- the search. The *maxlevels* and *force* parameters default to ``0`` and are
+ the search. The *maxlevels* parameter defaults to ``0``, and the *force*
+ and *legacy* parameters default to ``False``. All are
passed to the :func:`compile_dir` function.
To force a recompile of all the :file:`.py` files in the :file:`Lib/`
Modified: python/branches/py3k/Doc/library/imp.rst
==============================================================================
--- python/branches/py3k/Doc/library/imp.rst (original)
+++ python/branches/py3k/Doc/library/imp.rst Sat Apr 17 02:19:56 2010
@@ -204,8 +204,41 @@
function does nothing.
-The following constants with integer values, defined in this module, are used to
-indicate the search result of :func:`find_module`.
+The following functions and data provide conveniences for handling :pep:`3147`
+byte-compiled file paths.
+
+.. versionadded:: 3.2
+
+.. function:: cache_from_source(path, debug_override=None)
+
+ Return the PEP 3147 path to the byte-compiled file associated with the
+ source *path*. For example, if *path* is ``/foo/bar/baz.py`` the return
+ value would be ``/foo/bar/__pycache__/baz.cpython-32.pyc`` for Python 3.2.
+ The ``cpython-32`` string comes from the current magic tag (see
+ :func:`get_tag`). The returned path will end in ``.pyc`` when
+ ``__debug__`` is True or ``.pyo`` for an optimized Python
+ (i.e. ``__debug__`` is False). By passing in True or False for
+ *debug_override* you can override the system's value for ``__debug__`` for
+ extension selection.
+
+ *path* need not exist.
+
+.. function:: source_from_cache(path)
+
+ Given the *path* to a PEP 3147 file name, return the associated source code
+ file path. For example, if *path* is
+ ``/foo/bar/__pycache__/baz.cpython-32.pyc`` the returned path would be
+ ``/foo/bar/baz.py``. *path* need not exist, however if it does not conform
+ to PEP 3147 format, a ``ValueError`` is raised.
+
+.. function:: get_tag()
+
+ Return the PEP 3147 magic tag string matching this version of Python's
+ magic number, as returned by :func:`get_magic`.
+
+
+The following constants with integer values, defined in this module, are used
+to indicate the search result of :func:`find_module`.
.. data:: PY_SOURCE
Modified: python/branches/py3k/Doc/library/py_compile.rst
==============================================================================
--- python/branches/py3k/Doc/library/py_compile.rst (original)
+++ python/branches/py3k/Doc/library/py_compile.rst Sat Apr 17 02:19:56 2010
@@ -26,12 +26,16 @@
Compile a source file to byte-code and write out the byte-code cache file. The
source code is loaded from the file name *file*. The byte-code is written to
- *cfile*, which defaults to *file* ``+`` ``'c'`` (``'o'`` if optimization is
- enabled in the current interpreter). If *dfile* is specified, it is used as the
+ *cfile*, which defaults to the :PEP:`3147` path, ending in ``.pyc``
+ (``'.pyo`` if optimization is enabled in the current interpreter). For
+ example, if *file* is ``/foo/bar/baz.py`` *cfile* will default to
+ ``/foo/bar/__pycache__/baz.cpython-32.pyc`` for Python 3.2. If *dfile* is specified, it is used as the
name of the source file in error messages instead of *file*. If *doraise* is
true, a :exc:`PyCompileError` is raised when an error is encountered while
compiling *file*. If *doraise* is false (the default), an error string is
- written to ``sys.stderr``, but no exception is raised.
+ written to ``sys.stderr``, but no exception is raised. This function
+ returns the path to byte-compiled file, i.e. whatever *cfile* value was
+ used.
.. function:: main(args=None)
Modified: python/branches/py3k/Doc/library/runpy.rst
==============================================================================
--- python/branches/py3k/Doc/library/runpy.rst (original)
+++ python/branches/py3k/Doc/library/runpy.rst Sat Apr 17 02:19:56 2010
@@ -32,7 +32,8 @@
below are defined in the supplied dictionary, those definitions are
overridden by :func:`run_module`.
- The special global variables ``__name__``, ``__file__``, ``__loader__``
+ The special global variables ``__name__``, ``__file__``, ``__cached__``,
+ ``__loader__``
and ``__package__`` are set in the globals dictionary before the module
code is executed (Note that this is a minimal set of variables - other
variables may be set implicitly as an interpreter implementation detail).
@@ -45,6 +46,8 @@
loader does not make filename information available, this variable is set
to :const:`None`.
+ ``__cached__`` will be set to ``None``.
+
``__loader__`` is set to the PEP 302 module loader used to retrieve the
code for the module (This loader may be a wrapper around the standard
import mechanism).
Modified: python/branches/py3k/Include/import.h
==============================================================================
--- python/branches/py3k/Include/import.h (original)
+++ python/branches/py3k/Include/import.h Sat Apr 17 02:19:56 2010
@@ -8,9 +8,12 @@
#endif
PyAPI_FUNC(long) PyImport_GetMagicNumber(void);
+PyAPI_FUNC(const char *) PyImport_GetMagicTag(void);
PyAPI_FUNC(PyObject *) PyImport_ExecCodeModule(char *name, PyObject *co);
PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleEx(
char *name, PyObject *co, char *pathname);
+PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleWithPathnames(
+ char *name, PyObject *co, char *pathname, char *cpathname);
PyAPI_FUNC(PyObject *) PyImport_GetModuleDict(void);
PyAPI_FUNC(PyObject *) PyImport_AddModule(const char *name);
PyAPI_FUNC(PyObject *) PyImport_ImportModule(const char *name);
Modified: python/branches/py3k/Lib/compileall.py
==============================================================================
--- python/branches/py3k/Lib/compileall.py (original)
+++ python/branches/py3k/Lib/compileall.py Sat Apr 17 02:19:56 2010
@@ -12,6 +12,7 @@
"""
import os
+import errno
import sys
import py_compile
import struct
@@ -20,7 +21,7 @@
__all__ = ["compile_dir","compile_file","compile_path"]
def compile_dir(dir, maxlevels=10, ddir=None,
- force=0, rx=None, quiet=0):
+ force=False, rx=None, quiet=False, legacy=False):
"""Byte-compile all modules in the given directory tree.
Arguments (only dir is required):
@@ -29,8 +30,9 @@
maxlevels: maximum recursion level (default 10)
ddir: if given, purported directory name (this is the
directory name that will show up in error messages)
- force: if 1, force compilation, even if timestamps are up-to-date
- quiet: if 1, be quiet during compilation
+ force: if True, force compilation, even if timestamps are up-to-date
+ quiet: if True, be quiet during compilation
+ legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
"""
if not quiet:
@@ -49,24 +51,26 @@
else:
dfile = None
if not os.path.isdir(fullname):
- if not compile_file(fullname, ddir, force, rx, quiet):
+ if not compile_file(fullname, ddir, force, rx, quiet, legacy):
success = 0
elif maxlevels > 0 and \
name != os.curdir and name != os.pardir and \
os.path.isdir(fullname) and \
not os.path.islink(fullname):
if not compile_dir(fullname, maxlevels - 1, dfile, force, rx,
- quiet):
+ quiet, legacy):
success = 0
return success
-def compile_file(fullname, ddir=None, force=0, rx=None, quiet=0):
+def compile_file(fullname, ddir=None, force=0, rx=None, quiet=False,
+ legacy=False):
"""Byte-compile file.
- file: the file to byte-compile
+ fullname: the file to byte-compile
ddir: if given, purported directory name (this is the
directory name that will show up in error messages)
- force: if 1, force compilation, even if timestamps are up-to-date
- quiet: if 1, be quiet during compilation
+ force: if True, force compilation, even if timestamps are up-to-date
+ quiet: if True, be quiet during compilation
+ legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
"""
success = 1
@@ -80,13 +84,22 @@
if mo:
return success
if os.path.isfile(fullname):
+ if legacy:
+ cfile = fullname + ('c' if __debug__ else 'o')
+ else:
+ cfile = imp.cache_from_source(fullname)
+ cache_dir = os.path.dirname(cfile)
+ try:
+ os.mkdir(cache_dir)
+ except OSError as error:
+ if error.errno != errno.EEXIST:
+ raise
head, tail = name[:-3], name[-3:]
if tail == '.py':
if not force:
try:
mtime = int(os.stat(fullname).st_mtime)
expect = struct.pack('<4sl', imp.get_magic(), mtime)
- cfile = fullname + (__debug__ and 'c' or 'o')
with open(cfile, 'rb') as chandle:
actual = chandle.read(8)
if expect == actual:
@@ -96,14 +109,15 @@
if not quiet:
print('Compiling', fullname, '...')
try:
- ok = py_compile.compile(fullname, None, dfile, True)
+ ok = py_compile.compile(fullname, cfile, dfile, True)
except py_compile.PyCompileError as err:
if quiet:
print('*** Error compiling', fullname, '...')
else:
print('*** ', end='')
# escape non-printable characters in msg
- msg = err.msg.encode(sys.stdout.encoding, errors='backslashreplace')
+ msg = err.msg.encode(sys.stdout.encoding,
+ errors='backslashreplace')
msg = msg.decode(sys.stdout.encoding)
print(msg)
success = 0
@@ -119,15 +133,17 @@
success = 0
return success
-def compile_path(skip_curdir=1, maxlevels=0, force=0, quiet=0):
+def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=False,
+ legacy=False):
"""Byte-compile all module on sys.path.
Arguments (all optional):
skip_curdir: if true, skip current directory (default true)
maxlevels: max recursion level (default 0)
- force: as for compile_dir() (default 0)
- quiet: as for compile_dir() (default 0)
+ force: as for compile_dir() (default False)
+ quiet: as for compile_dir() (default False)
+ legacy: as for compile_dir() (default False)
"""
success = 1
@@ -136,7 +152,8 @@
print('Skipping current directory')
else:
success = success and compile_dir(dir, maxlevels, None,
- force, quiet=quiet)
+ force, quiet=quiet,
+ legacy=legacy)
return success
def expand_args(args, flist):
@@ -162,10 +179,10 @@
"""Script main program."""
import getopt
try:
- opts, args = getopt.getopt(sys.argv[1:], 'lfqd:x:i:')
+ opts, args = getopt.getopt(sys.argv[1:], 'lfqd:x:i:b')
except getopt.error as msg:
print(msg)
- print("usage: python compileall.py [-l] [-f] [-q] [-d destdir] " \
+ print("usage: python compileall.py [-l] [-f] [-q] [-d destdir] "
"[-x regexp] [-i list] [directory|file ...]")
print("-l: don't recurse down")
print("-f: force rebuild even if timestamps are up-to-date")
@@ -174,23 +191,27 @@
print(" if no directory arguments, -l sys.path is assumed")
print("-x regexp: skip files matching the regular expression regexp")
print(" the regexp is searched for in the full path of the file")
- print("-i list: expand list with its content (file and directory names)")
+ print("-i list: expand list with its content "
+ "(file and directory names)")
+ print("-b: Produce legacy byte-compile file paths")
sys.exit(2)
maxlevels = 10
ddir = None
- force = 0
- quiet = 0
+ force = False
+ quiet = False
rx = None
flist = None
+ legacy = False
for o, a in opts:
if o == '-l': maxlevels = 0
if o == '-d': ddir = a
- if o == '-f': force = 1
- if o == '-q': quiet = 1
+ if o == '-f': force = True
+ if o == '-q': quiet = True
if o == '-x':
import re
rx = re.compile(a)
if o == '-i': flist = a
+ if o == '-b': legacy = True
if ddir:
if len(args) != 1 and not os.path.isdir(args[0]):
print("-d destdir require exactly one directory argument")
@@ -207,13 +228,14 @@
for arg in args:
if os.path.isdir(arg):
if not compile_dir(arg, maxlevels, ddir,
- force, rx, quiet):
+ force, rx, quiet, legacy):
success = 0
else:
- if not compile_file(arg, ddir, force, rx, quiet):
+ if not compile_file(arg, ddir, force, rx,
+ quiet, legacy):
success = 0
else:
- success = compile_path()
+ success = compile_path(legacy=legacy)
except KeyboardInterrupt:
print("\n[interrupt]")
success = 0
Modified: python/branches/py3k/Lib/importlib/_bootstrap.py
==============================================================================
--- python/branches/py3k/Lib/importlib/_bootstrap.py (original)
+++ python/branches/py3k/Lib/importlib/_bootstrap.py Sat Apr 17 02:19:56 2010
@@ -488,6 +488,16 @@
"""Load a module from a source or bytecode file."""
+ def _find_path(self, ext_type):
+ """Return PEP 3147 path if ext_type is PY_COMPILED, otherwise
+ super()._find_path() is called."""
+ if ext_type == imp.PY_COMPILED:
+ # We don't really care what the extension on self._base_path is,
+ # as long as it has exactly one dot.
+ bytecode_path = imp.cache_from_source(self._base_path + '.py')
+ return (bytecode_path if _path_exists(bytecode_path) else None)
+ return super()._find_path(ext_type)
+
@_check_name
def source_mtime(self, name):
"""Return the modification time of the source for the specified
@@ -515,7 +525,16 @@
"""
bytecode_path = self.bytecode_path(name)
if not bytecode_path:
- bytecode_path = self._base_path + _suffix_list(imp.PY_COMPILED)[0]
+ source_path = self.source_path(name)
+ bytecode_path = imp.cache_from_source(source_path)
+ # Ensure that the __pycache__ directory exists. We can't use
+ # os.path.dirname() here.
+ dirname, sep, basename = bytecode_path.rpartition(path_sep)
+ try:
+ _os.mkdir(dirname)
+ except OSError as error:
+ if error.errno != errno.EEXIST:
+ raise
try:
# Assuming bytes.
with _closing(_io.FileIO(bytecode_path, 'w')) as bytecode_file:
Modified: python/branches/py3k/Lib/importlib/test/__main__.py
==============================================================================
--- python/branches/py3k/Lib/importlib/test/__main__.py (original)
+++ python/branches/py3k/Lib/importlib/test/__main__.py Sat Apr 17 02:19:56 2010
@@ -13,7 +13,12 @@
def test_main():
- start_dir = os.path.dirname(__file__)
+ if '__pycache__' in __file__:
+ parts = __file__.split(os.path.sep)
+ start_dir = sep.join(parts[:-2])
+ else:
+ start_dir = os.path.dirname(__file__)
+ # XXX 2010-03-18 barry: Fix __file__
top_dir = os.path.dirname(os.path.dirname(start_dir))
test_loader = unittest.TestLoader()
if '--builtin' in sys.argv:
Modified: python/branches/py3k/Lib/importlib/test/source/test_file_loader.py
==============================================================================
--- python/branches/py3k/Lib/importlib/test/source/test_file_loader.py (original)
+++ python/branches/py3k/Lib/importlib/test/source/test_file_loader.py Sat Apr 17 02:19:56 2010
@@ -127,7 +127,7 @@
except KeyError:
pass
py_compile.compile(mapping[name])
- bytecode_path = source_util.bytecode_path(mapping[name])
+ bytecode_path = imp.cache_from_source(mapping[name])
with open(bytecode_path, 'rb') as file:
bc = file.read()
new_bc = manipulator(bc)
@@ -226,7 +226,7 @@
zeros = b'\x00\x00\x00\x00'
with source_util.create_modules('_temp') as mapping:
py_compile.compile(mapping['_temp'])
- bytecode_path = source_util.bytecode_path(mapping['_temp'])
+ bytecode_path = imp.cache_from_source(mapping['_temp'])
with open(bytecode_path, 'r+b') as bytecode_file:
bytecode_file.seek(4)
bytecode_file.write(zeros)
@@ -242,9 +242,10 @@
def test_bad_marshal(self):
# Bad marshal data should raise a ValueError.
with source_util.create_modules('_temp') as mapping:
- bytecode_path = source_util.bytecode_path(mapping['_temp'])
+ bytecode_path = imp.cache_from_source(mapping['_temp'])
source_mtime = os.path.getmtime(mapping['_temp'])
source_timestamp = importlib._w_long(source_mtime)
+ source_util.ensure_bytecode_path(bytecode_path)
with open(bytecode_path, 'wb') as bytecode_file:
bytecode_file.write(imp.get_magic())
bytecode_file.write(source_timestamp)
@@ -260,7 +261,7 @@
with source_util.create_modules('_temp') as mapping:
# Create bytecode that will need to be re-created.
py_compile.compile(mapping['_temp'])
- bytecode_path = source_util.bytecode_path(mapping['_temp'])
+ bytecode_path = imp.cache_from_source(mapping['_temp'])
with open(bytecode_path, 'r+b') as bytecode_file:
bytecode_file.seek(0)
bytecode_file.write(b'\x00\x00\x00\x00')
Modified: python/branches/py3k/Lib/importlib/test/source/test_finder.py
==============================================================================
--- python/branches/py3k/Lib/importlib/test/source/test_finder.py (original)
+++ python/branches/py3k/Lib/importlib/test/source/test_finder.py Sat Apr 17 02:19:56 2010
@@ -1,7 +1,9 @@
from importlib import _bootstrap
from .. import abc
from . import util as source_util
+from test.support import make_legacy_pyc
import os
+import errno
import py_compile
import unittest
import warnings
@@ -52,6 +54,14 @@
if unlink:
for name in unlink:
os.unlink(mapping[name])
+ try:
+ make_legacy_pyc(mapping[name])
+ except OSError as error:
+ # Some tests do not set compile_=True so the source
+ # module will not get compiled and there will be no
+ # PEP 3147 pyc file to rename.
+ if error.errno != errno.ENOENT:
+ raise
loader = self.import_(mapping['.root'], test)
self.assertTrue(hasattr(loader, 'load_module'))
return loader
@@ -60,7 +70,8 @@
# [top-level source]
self.run_test('top_level')
# [top-level bc]
- self.run_test('top_level', compile_={'top_level'}, unlink={'top_level'})
+ self.run_test('top_level', compile_={'top_level'},
+ unlink={'top_level'})
# [top-level both]
self.run_test('top_level', compile_={'top_level'})
Modified: python/branches/py3k/Lib/importlib/test/source/test_source_encoding.py
==============================================================================
--- python/branches/py3k/Lib/importlib/test/source/test_source_encoding.py (original)
+++ python/branches/py3k/Lib/importlib/test/source/test_source_encoding.py Sat Apr 17 02:19:56 2010
@@ -33,7 +33,7 @@
def run_test(self, source):
with source_util.create_modules(self.module_name) as mapping:
- with open(mapping[self.module_name], 'wb')as file:
+ with open(mapping[self.module_name], 'wb') as file:
file.write(source)
loader = _bootstrap._PyPycFileLoader(self.module_name,
mapping[self.module_name], False)
Modified: python/branches/py3k/Lib/importlib/test/source/util.py
==============================================================================
--- python/branches/py3k/Lib/importlib/test/source/util.py (original)
+++ python/branches/py3k/Lib/importlib/test/source/util.py Sat Apr 17 02:19:56 2010
@@ -1,5 +1,6 @@
from .. import util
import contextlib
+import errno
import functools
import imp
import os
@@ -26,14 +27,16 @@
return wrapper
-def bytecode_path(source_path):
- for suffix, _, type_ in imp.get_suffixes():
- if type_ == imp.PY_COMPILED:
- bc_suffix = suffix
- break
- else:
- raise ValueError("no bytecode suffix is defined")
- return os.path.splitext(source_path)[0] + bc_suffix
+def ensure_bytecode_path(bytecode_path):
+ """Ensure that the __pycache__ directory for PEP 3147 pyc file exists.
+
+ :param bytecode_path: File system path to PEP 3147 pyc file.
+ """
+ try:
+ os.mkdir(os.path.dirname(bytecode_path))
+ except OSError as error:
+ if error.errno != errno.EEXIST:
+ raise
@contextlib.contextmanager
Modified: python/branches/py3k/Lib/importlib/util.py
==============================================================================
--- python/branches/py3k/Lib/importlib/util.py (original)
+++ python/branches/py3k/Lib/importlib/util.py Sat Apr 17 02:19:56 2010
@@ -1,4 +1,5 @@
"""Utility code for constructing importers, etc."""
+
from ._bootstrap import module_for_loader
from ._bootstrap import set_loader
from ._bootstrap import set_package
Modified: python/branches/py3k/Lib/inspect.py
==============================================================================
--- python/branches/py3k/Lib/inspect.py (original)
+++ python/branches/py3k/Lib/inspect.py Sat Apr 17 02:19:56 2010
@@ -54,6 +54,7 @@
"""Return true if the object is a module.
Module objects provide these attributes:
+ __cached__ pathname to byte compiled file
__doc__ documentation string
__file__ filename (missing for built-in modules)"""
return isinstance(object, types.ModuleType)
Modified: python/branches/py3k/Lib/py_compile.py
==============================================================================
--- python/branches/py3k/Lib/py_compile.py (original)
+++ python/branches/py3k/Lib/py_compile.py Sat Apr 17 02:19:56 2010
@@ -4,6 +4,7 @@
"""
import builtins
+import errno
import imp
import marshal
import os
@@ -37,16 +38,18 @@
can be accesses as class variable 'file'
msg: string message to be written as error message
- If no value is given, a default exception message will be given,
- consistent with 'standard' py_compile output.
- message (or default) can be accesses as class variable 'msg'
+ If no value is given, a default exception message will be
+ given, consistent with 'standard' py_compile output.
+ message (or default) can be accesses as class variable
+ 'msg'
"""
def __init__(self, exc_type, exc_value, file, msg=''):
exc_type_name = exc_type.__name__
if exc_type is SyntaxError:
- tbtext = ''.join(traceback.format_exception_only(exc_type, exc_value))
+ tbtext = ''.join(traceback.format_exception_only(
+ exc_type, exc_value))
errmsg = tbtext.replace('File "<string>"', 'File "%s"' % file)
else:
errmsg = "Sorry: %s: %s" % (exc_type_name,exc_value)
@@ -64,7 +67,7 @@
def wr_long(f, x):
"""Internal; write a 32-bit int to a file in little-endian order."""
- f.write(bytes([x & 0xff,
+ f.write(bytes([x & 0xff,
(x >> 8) & 0xff,
(x >> 16) & 0xff,
(x >> 24) & 0xff]))
@@ -72,20 +75,18 @@
def compile(file, cfile=None, dfile=None, doraise=False):
"""Byte-compile one Python source file to Python bytecode.
- Arguments:
-
- file: source filename
- cfile: target filename; defaults to source with 'c' or 'o' appended
- ('c' normally, 'o' in optimizing mode, giving .pyc or .pyo)
- dfile: purported filename; defaults to source (this is the filename
- that will show up in error messages)
- doraise: flag indicating whether or not an exception should be
- raised when a compile error is found. If an exception
- occurs and this flag is set to False, a string
- indicating the nature of the exception will be printed,
- and the function will return to the caller. If an
- exception occurs and this flag is set to True, a
- PyCompileError exception will be raised.
+ :param file: The source file name.
+ :param cfile: The target byte compiled file name. When not given, this
+ defaults to the PEP 3147 location.
+ :param dfile: Purported file name, i.e. the file name that shows up in
+ error messages. Defaults to the source file name.
+ :param doraise: Flag indicating whether or not an exception should be
+ raised when a compile error is found. If an exception occurs and this
+ flag is set to False, a string indicating the nature of the exception
+ will be printed, and the function will return to the caller. If an
+ exception occurs and this flag is set to True, a PyCompileError
+ exception will be raised.
+ :return: Path to the resulting byte compiled file.
Note that it isn't necessary to byte-compile Python modules for
execution efficiency -- Python itself byte-compiles a module when
@@ -102,7 +103,6 @@
See compileall.py for a script/module that uses this module to
byte-compile all installed files (or all files in selected
directories).
-
"""
with open(file, "rb") as f:
encoding = tokenize.detect_encoding(f.readline)[0]
@@ -122,7 +122,12 @@
sys.stderr.write(py_exc.msg + '\n')
return
if cfile is None:
- cfile = file + (__debug__ and 'c' or 'o')
+ cfile = imp.cache_from_source(file)
+ try:
+ os.mkdir(os.path.dirname(cfile))
+ except OSError as error:
+ if error.errno != errno.EEXIST:
+ raise
with open(cfile, 'wb') as fc:
fc.write(b'\0\0\0\0')
wr_long(fc, timestamp)
@@ -130,6 +135,7 @@
fc.flush()
fc.seek(0, 0)
fc.write(MAGIC)
+ return cfile
def main(args=None):
"""Compile several source files.
Modified: python/branches/py3k/Lib/pydoc.py
==============================================================================
--- python/branches/py3k/Lib/pydoc.py (original)
+++ python/branches/py3k/Lib/pydoc.py Sat Apr 17 02:19:56 2010
@@ -159,7 +159,8 @@
"""Decide whether to show documentation on a variable."""
# Certain special names are redundant.
_hidden_names = ('__builtins__', '__doc__', '__file__', '__path__',
- '__module__', '__name__', '__slots__', '__package__')
+ '__module__', '__name__', '__slots__', '__package__',
+ '__cached__')
if name in _hidden_names: return 0
# Private names are hidden, but special names are displayed.
if name.startswith('__') and name.endswith('__'): return 1
Modified: python/branches/py3k/Lib/runpy.py
==============================================================================
--- python/branches/py3k/Lib/runpy.py (original)
+++ python/branches/py3k/Lib/runpy.py Sat Apr 17 02:19:56 2010
@@ -67,6 +67,7 @@
run_globals.update(init_globals)
run_globals.update(__name__ = mod_name,
__file__ = mod_fname,
+ __cached__ = None,
__loader__ = mod_loader,
__package__ = pkg_name)
exec(code, run_globals)
@@ -130,6 +131,7 @@
At the very least, these variables in __main__ will be overwritten:
__name__
__file__
+ __cached__
__loader__
__package__
"""
Modified: python/branches/py3k/Lib/site.py
==============================================================================
--- python/branches/py3k/Lib/site.py (original)
+++ python/branches/py3k/Lib/site.py Sat Apr 17 02:19:56 2010
@@ -74,15 +74,19 @@
return dir, os.path.normcase(dir)
-def abs__file__():
- """Set all module' __file__ attribute to an absolute path"""
+def abs_paths():
+ """Set all module __file__ and __cached__ attributes to an absolute path"""
for m in set(sys.modules.values()):
if hasattr(m, '__loader__'):
continue # don't mess with a PEP 302-supplied __file__
try:
m.__file__ = os.path.abspath(m.__file__)
except AttributeError:
- continue
+ pass
+ try:
+ m.__cached__ = os.path.abspath(m.__cached__)
+ except AttributeError:
+ pass
def removeduppaths():
@@ -518,7 +522,7 @@
def main():
global ENABLE_USER_SITE
- abs__file__()
+ abs_paths()
known_paths = removeduppaths()
if (os.name == "posix" and sys.path and
os.path.basename(sys.path[-1]) == "Modules"):
Modified: python/branches/py3k/Lib/test/script_helper.py
==============================================================================
--- python/branches/py3k/Lib/test/script_helper.py (original)
+++ python/branches/py3k/Lib/test/script_helper.py Sat Apr 17 02:19:56 2010
@@ -11,6 +11,9 @@
import shutil
import zipfile
+from imp import source_from_cache
+from test.support import make_legacy_pyc
+
# Executing the interpreter in a subprocess
def python_exit_code(*args):
cmd_line = [sys.executable, '-E']
@@ -62,20 +65,18 @@
script_file.close()
return script_name
-def compile_script(script_name):
- py_compile.compile(script_name, doraise=True)
- if __debug__:
- compiled_name = script_name + 'c'
- else:
- compiled_name = script_name + 'o'
- return compiled_name
-
def make_zip_script(zip_dir, zip_basename, script_name, name_in_zip=None):
zip_filename = zip_basename+os.extsep+'zip'
zip_name = os.path.join(zip_dir, zip_filename)
zip_file = zipfile.ZipFile(zip_name, 'w')
if name_in_zip is None:
- name_in_zip = os.path.basename(script_name)
+ parts = script_name.split(os.sep)
+ if len(parts) >= 2 and parts[-2] == '__pycache__':
+ legacy_pyc = make_legacy_pyc(source_from_cache(script_name))
+ name_in_zip = os.path.basename(legacy_pyc)
+ script_name = legacy_pyc
+ else:
+ name_in_zip = os.path.basename(script_name)
zip_file.write(script_name, name_in_zip)
zip_file.close()
#if test.test_support.verbose:
@@ -98,8 +99,8 @@
script_name = make_script(zip_dir, script_basename, source)
unlink.append(script_name)
if compiled:
- init_name = compile_script(init_name)
- script_name = compile_script(script_name)
+ init_name = py_compile(init_name, doraise=True)
+ script_name = py_compile(script_name, doraise=True)
unlink.extend((init_name, script_name))
pkg_names = [os.sep.join([pkg_name]*i) for i in range(1, depth+1)]
script_name_in_zip = os.path.join(pkg_names[-1], os.path.basename(script_name))
Modified: python/branches/py3k/Lib/test/support.py
==============================================================================
--- python/branches/py3k/Lib/test/support.py (original)
+++ python/branches/py3k/Lib/test/support.py Sat Apr 17 02:19:56 2010
@@ -17,22 +17,25 @@
import importlib
import collections
import re
+import imp
import time
-__all__ = ["Error", "TestFailed", "ResourceDenied", "import_module",
- "verbose", "use_resources", "max_memuse", "record_original_stdout",
- "get_original_stdout", "unload", "unlink", "rmtree", "forget",
- "is_resource_enabled", "requires", "find_unused_port", "bind_port",
- "fcmp", "is_jython", "TESTFN", "HOST", "FUZZ", "SAVEDCWD", "temp_cwd",
- "findfile", "sortdict", "check_syntax_error", "open_urlresource",
- "check_warnings", "CleanImport", "EnvironmentVarGuard",
- "TransientResource", "captured_output", "captured_stdout",
- "time_out", "socket_peer_reset", "ioerror_peer_reset",
- "run_with_locale",
- "set_memlimit", "bigmemtest", "bigaddrspacetest", "BasicTestRunner",
- "run_unittest", "run_doctest", "threading_setup", "threading_cleanup",
- "reap_children", "cpython_only", "check_impl_detail", "get_attribute",
- "swap_item", "swap_attr"]
+__all__ = [
+ "Error", "TestFailed", "ResourceDenied", "import_module",
+ "verbose", "use_resources", "max_memuse", "record_original_stdout",
+ "get_original_stdout", "unload", "unlink", "rmtree", "forget",
+ "is_resource_enabled", "requires", "find_unused_port", "bind_port",
+ "fcmp", "is_jython", "TESTFN", "HOST", "FUZZ", "SAVEDCWD", "temp_cwd",
+ "findfile", "sortdict", "check_syntax_error", "open_urlresource",
+ "check_warnings", "CleanImport", "EnvironmentVarGuard",
+ "TransientResource", "captured_output", "captured_stdout",
+ "time_out", "socket_peer_reset", "ioerror_peer_reset",
+ "run_with_locale", 'temp_umask',
+ "set_memlimit", "bigmemtest", "bigaddrspacetest", "BasicTestRunner",
+ "run_unittest", "run_doctest", "threading_setup", "threading_cleanup",
+ "reap_children", "cpython_only", "check_impl_detail", "get_attribute",
+ "swap_item", "swap_attr",
+ ]
class Error(Exception):
@@ -177,27 +180,50 @@
def unlink(filename):
try:
os.unlink(filename)
- except OSError:
- pass
+ except OSError as error:
+ # The filename need not exist.
+ if error.errno != errno.ENOENT:
+ raise
def rmtree(path):
try:
shutil.rmtree(path)
- except OSError as e:
+ except OSError as error:
# Unix returns ENOENT, Windows returns ESRCH.
- if e.errno not in (errno.ENOENT, errno.ESRCH):
+ if error.errno not in (errno.ENOENT, errno.ESRCH):
raise
+def make_legacy_pyc(source):
+ """Move a PEP 3147 pyc/pyo file to its legacy pyc/pyo location.
+
+ The choice of .pyc or .pyo extension is done based on the __debug__ flag
+ value.
+
+ :param source: The file system path to the source file. The source file
+ does not need to exist, however the PEP 3147 pyc file must exist.
+ :return: The file system path to the legacy pyc file.
+ """
+ pyc_file = imp.cache_from_source(source)
+ up_one = os.path.dirname(os.path.abspath(source))
+ legacy_pyc = os.path.join(up_one, source + ('c' if __debug__ else 'o'))
+ os.rename(pyc_file, legacy_pyc)
+ return legacy_pyc
+
def forget(modname):
- '''"Forget" a module was ever imported by removing it from sys.modules and
- deleting any .pyc and .pyo files.'''
+ """'Forget' a module was ever imported.
+
+ This removes the module from sys.modules and deletes any PEP 3147 or
+ legacy .pyc and .pyo files.
+ """
unload(modname)
for dirname in sys.path:
- unlink(os.path.join(dirname, modname + '.pyc'))
- # Deleting the .pyo file cannot be within the 'try' for the .pyc since
- # the chance exists that there is no .pyc (and thus the 'try' statement
- # is exited) but there is a .pyo file.
- unlink(os.path.join(dirname, modname + '.pyo'))
+ source = os.path.join(dirname, modname + '.py')
+ # It doesn't matter if they exist or not, unlink all possible
+ # combinations of PEP 3147 and legacy pyc and pyo files.
+ unlink(source + 'c')
+ unlink(source + 'o')
+ unlink(imp.cache_from_source(source, debug_override=True))
+ unlink(imp.cache_from_source(source, debug_override=False))
def is_resource_enabled(resource):
"""Test whether a resource is enabled. Known resources are set by
@@ -208,7 +234,9 @@
"""Raise ResourceDenied if the specified resource is not available.
If the caller's module is __main__ then automatically return True. The
- possibility of False being returned occurs when regrtest.py is executing."""
+ possibility of False being returned occurs when regrtest.py is
+ executing.
+ """
# see if the caller's module is __main__ - if so, treat as if
# the resource was set
if sys._getframe(1).f_globals.get("__name__") == "__main__":
@@ -405,6 +433,16 @@
rmtree(name)
+ at contextlib.contextmanager
+def temp_umask(umask):
+ """Context manager that temporarily sets the process umask."""
+ oldmask = os.umask(umask)
+ try:
+ yield
+ finally:
+ os.umask(oldmask)
+
+
def findfile(file, here=__file__, subdir=None):
"""Try to find a file on sys.path and the working directory. If it is not
found the argument passed to the function is returned (this does not
Modified: python/branches/py3k/Lib/test/test_cmd_line_script.py
==============================================================================
--- python/branches/py3k/Lib/test/test_cmd_line_script.py (original)
+++ python/branches/py3k/Lib/test/test_cmd_line_script.py Sat Apr 17 02:19:56 2010
@@ -1,12 +1,14 @@
-# Tests command line execution of scripts
+# tests command line execution of scripts
import unittest
import os
import os.path
+import py_compile
+
import test.support
-from test.script_helper import (run_python,
- temp_dir, make_script, compile_script,
- make_pkg, make_zip_script, make_zip_pkg)
+from test.script_helper import (
+ make_pkg, make_script, make_zip_pkg, make_zip_script, run_python,
+ temp_dir)
verbose = test.support.verbose
@@ -28,6 +30,7 @@
# Check population of magic variables
assertEqual(__name__, '__main__')
print('__file__==%r' % __file__)
+assertEqual(__cached__, None)
print('__package__==%r' % __package__)
# Check the sys module
import sys
@@ -101,9 +104,10 @@
def test_script_compiled(self):
with temp_dir() as script_dir:
script_name = _make_test_script(script_dir, 'script')
- compiled_name = compile_script(script_name)
+ compiled_name = py_compile.compile(script_name, doraise=True)
os.remove(script_name)
- self._check_script(compiled_name, compiled_name, compiled_name, None)
+ self._check_script(compiled_name, compiled_name,
+ compiled_name, None)
def test_directory(self):
with temp_dir() as script_dir:
@@ -113,9 +117,10 @@
def test_directory_compiled(self):
with temp_dir() as script_dir:
script_name = _make_test_script(script_dir, '__main__')
- compiled_name = compile_script(script_name)
+ compiled_name = py_compile.compile(script_name, doraise=True)
os.remove(script_name)
- self._check_script(script_dir, compiled_name, script_dir, '')
+ pyc_file = test.support.make_legacy_pyc(script_name)
+ self._check_script(script_dir, pyc_file, script_dir, '')
def test_directory_error(self):
with temp_dir() as script_dir:
@@ -131,7 +136,7 @@
def test_zipfile_compiled(self):
with temp_dir() as script_dir:
script_name = _make_test_script(script_dir, '__main__')
- compiled_name = compile_script(script_name)
+ compiled_name = py_compile.compile(script_name, doraise=True)
zip_name, run_name = make_zip_script(script_dir, 'test_zip', compiled_name)
self._check_script(zip_name, run_name, zip_name, '')
@@ -176,11 +181,12 @@
pkg_dir = os.path.join(script_dir, 'test_pkg')
make_pkg(pkg_dir)
script_name = _make_test_script(pkg_dir, '__main__')
- compiled_name = compile_script(script_name)
+ compiled_name = py_compile.compile(script_name, doraise=True)
os.remove(script_name)
+ pyc_file = test.support.make_legacy_pyc(script_name)
launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg')
- self._check_script(launch_name, compiled_name,
- compiled_name, 'test_pkg')
+ self._check_script(launch_name, pyc_file,
+ pyc_file, 'test_pkg')
def test_package_error(self):
with temp_dir() as script_dir:
Modified: python/branches/py3k/Lib/test/test_compileall.py
==============================================================================
--- python/branches/py3k/Lib/test/test_compileall.py (original)
+++ python/branches/py3k/Lib/test/test_compileall.py Sat Apr 17 02:19:56 2010
@@ -5,22 +5,23 @@
import py_compile
import shutil
import struct
+import subprocess
import tempfile
-from test import support
import unittest
import io
+from test import support
class CompileallTests(unittest.TestCase):
def setUp(self):
self.directory = tempfile.mkdtemp()
self.source_path = os.path.join(self.directory, '_test.py')
- self.bc_path = self.source_path + ('c' if __debug__ else 'o')
+ self.bc_path = imp.cache_from_source(self.source_path)
with open(self.source_path, 'w') as file:
file.write('x = 123\n')
self.source_path2 = os.path.join(self.directory, '_test2.py')
- self.bc_path2 = self.source_path2 + ('c' if __debug__ else 'o')
+ self.bc_path2 = imp.cache_from_source(self.source_path2)
shutil.copyfile(self.source_path, self.source_path2)
def tearDown(self):
@@ -65,17 +66,19 @@
except:
pass
compileall.compile_file(self.source_path, force=False, quiet=True)
- self.assertTrue(os.path.isfile(self.bc_path) \
- and not os.path.isfile(self.bc_path2))
+ self.assertTrue(os.path.isfile(self.bc_path) and
+ not os.path.isfile(self.bc_path2))
os.unlink(self.bc_path)
compileall.compile_dir(self.directory, force=False, quiet=True)
- self.assertTrue(os.path.isfile(self.bc_path) \
- and os.path.isfile(self.bc_path2))
+ self.assertTrue(os.path.isfile(self.bc_path) and
+ os.path.isfile(self.bc_path2))
os.unlink(self.bc_path)
os.unlink(self.bc_path2)
+
class EncodingTest(unittest.TestCase):
- 'Issue 6716: compileall should escape source code when printing errors to stdout.'
+ """Issue 6716: compileall should escape source code when printing errors
+ to stdout."""
def setUp(self):
self.directory = tempfile.mkdtemp()
@@ -95,9 +98,65 @@
finally:
sys.stdout = orig_stdout
+class CommandLineTests(unittest.TestCase):
+ """Test some aspects of compileall's CLI."""
+
+ def setUp(self):
+ self.addCleanup(self._cleanup)
+ self.directory = tempfile.mkdtemp()
+ self.pkgdir = os.path.join(self.directory, 'foo')
+ os.mkdir(self.pkgdir)
+ # Touch the __init__.py and a package module.
+ with open(os.path.join(self.pkgdir, '__init__.py'), 'w'):
+ pass
+ with open(os.path.join(self.pkgdir, 'bar.py'), 'w'):
+ pass
+ sys.path.insert(0, self.directory)
+
+ def _cleanup(self):
+ support.rmtree(self.directory)
+ assert sys.path[0] == self.directory, 'Missing path'
+ del sys.path[0]
+
+ def test_pep3147_paths(self):
+ # Ensure that the default behavior of compileall's CLI is to create
+ # PEP 3147 pyc/pyo files.
+ retcode = subprocess.call(
+ (sys.executable, '-m', 'compileall', '-q', self.pkgdir))
+ self.assertEqual(retcode, 0)
+ # Verify the __pycache__ directory contents.
+ cachedir = os.path.join(self.pkgdir, '__pycache__')
+ self.assertTrue(os.path.exists(cachedir))
+ ext = ('pyc' if __debug__ else 'pyo')
+ expected = sorted(base.format(imp.get_tag(), ext) for base in
+ ('__init__.{}.{}', 'bar.{}.{}'))
+ self.assertEqual(sorted(os.listdir(cachedir)), expected)
+ # Make sure there are no .pyc files in the source directory.
+ self.assertFalse([pyc_file for pyc_file in os.listdir(self.pkgdir)
+ if pyc_file.endswith(ext)])
+
+ def test_legacy_paths(self):
+ # Ensure that with the proper switch, compileall leaves legacy
+ # pyc/pyo files, and no __pycache__ directory.
+ retcode = subprocess.call(
+ (sys.executable, '-m', 'compileall', '-b', '-q', self.pkgdir))
+ self.assertEqual(retcode, 0)
+ # Verify the __pycache__ directory contents.
+ cachedir = os.path.join(self.pkgdir, '__pycache__')
+ self.assertFalse(os.path.exists(cachedir))
+ ext = ('pyc' if __debug__ else 'pyo')
+ expected = [base.format(ext) for base in ('__init__.{}', 'bar.{}')]
+ expected.extend(['__init__.py', 'bar.py'])
+ expected.sort()
+ self.assertEqual(sorted(os.listdir(self.pkgdir)), expected)
+
+
def test_main():
- support.run_unittest(CompileallTests,
- EncodingTest)
+ support.run_unittest(
+ CommandLineTests,
+ CompileallTests,
+ EncodingTest,
+ )
if __name__ == "__main__":
Modified: python/branches/py3k/Lib/test/test_frozen.py
==============================================================================
--- python/branches/py3k/Lib/test/test_frozen.py (original)
+++ python/branches/py3k/Lib/test/test_frozen.py Sat Apr 17 02:19:56 2010
@@ -11,7 +11,7 @@
except ImportError as x:
self.fail("import __hello__ failed:" + str(x))
self.assertEqual(__hello__.initialized, True)
- self.assertEqual(len(dir(__hello__)), 6, dir(__hello__))
+ self.assertEqual(len(dir(__hello__)), 7, dir(__hello__))
try:
import __phello__
@@ -19,9 +19,9 @@
self.fail("import __phello__ failed:" + str(x))
self.assertEqual(__phello__.initialized, True)
if not "__phello__.spam" in sys.modules:
- self.assertEqual(len(dir(__phello__)), 7, dir(__phello__))
- else:
self.assertEqual(len(dir(__phello__)), 8, dir(__phello__))
+ else:
+ self.assertEqual(len(dir(__phello__)), 9, dir(__phello__))
self.assertEquals(__phello__.__path__, [__phello__.__name__])
try:
@@ -29,8 +29,8 @@
except ImportError as x:
self.fail("import __phello__.spam failed:" + str(x))
self.assertEqual(__phello__.spam.initialized, True)
- self.assertEqual(len(dir(__phello__.spam)), 6)
- self.assertEqual(len(dir(__phello__)), 8)
+ self.assertEqual(len(dir(__phello__.spam)), 7)
+ self.assertEqual(len(dir(__phello__)), 9)
try:
import __phello__.foo
Modified: python/branches/py3k/Lib/test/test_imp.py
==============================================================================
--- python/branches/py3k/Lib/test/test_imp.py (original)
+++ python/branches/py3k/Lib/test/test_imp.py Sat Apr 17 02:19:56 2010
@@ -1,6 +1,7 @@
import imp
import os
import os.path
+import shutil
import sys
import unittest
from test import support
@@ -139,7 +140,8 @@
mod = imp.load_source(temp_mod_name, temp_mod_name + '.py')
self.assertEqual(mod.a, 1)
- mod = imp.load_compiled(temp_mod_name, temp_mod_name + '.pyc')
+ mod = imp.load_compiled(
+ temp_mod_name, imp.cache_from_source(temp_mod_name + '.py'))
self.assertEqual(mod.a, 1)
if not os.path.exists(test_package_name):
@@ -184,11 +186,132 @@
imp.reload(marshal)
+class PEP3147Tests(unittest.TestCase):
+ """Tests of PEP 3147."""
+
+ tag = imp.get_tag()
+
+ def test_cache_from_source(self):
+ # Given the path to a .py file, return the path to its PEP 3147
+ # defined .pyc file (i.e. under __pycache__).
+ self.assertEqual(
+ imp.cache_from_source('/foo/bar/baz/qux.py', True),
+ '/foo/bar/baz/__pycache__/qux.{}.pyc'.format(self.tag))
+
+ def test_cache_from_source_optimized(self):
+ # Given the path to a .py file, return the path to its PEP 3147
+ # defined .pyo file (i.e. under __pycache__).
+ self.assertEqual(
+ imp.cache_from_source('/foo/bar/baz/qux.py', False),
+ '/foo/bar/baz/__pycache__/qux.{}.pyo'.format(self.tag))
+
+ def test_cache_from_source_cwd(self):
+ self.assertEqual(imp.cache_from_source('foo.py', True),
+ os.sep.join(('__pycache__',
+ 'foo.{}.pyc'.format(self.tag))))
+
+ def test_cache_from_source_override(self):
+ # When debug_override is not None, it can be any true-ish or false-ish
+ # value.
+ self.assertEqual(
+ imp.cache_from_source('/foo/bar/baz.py', []),
+ '/foo/bar/__pycache__/baz.{}.pyo'.format(self.tag))
+ self.assertEqual(
+ imp.cache_from_source('/foo/bar/baz.py', [17]),
+ '/foo/bar/__pycache__/baz.{}.pyc'.format(self.tag))
+ # However if the bool-ishness can't be determined, the exception
+ # propagates.
+ class Bearish:
+ def __bool__(self): raise RuntimeError
+ self.assertRaises(
+ RuntimeError,
+ imp.cache_from_source, '/foo/bar/baz.py', Bearish())
+
+ @unittest.skipIf(os.altsep is None,
+ 'test meaningful only where os.altsep is defined')
+ def test_altsep_cache_from_source(self):
+ # Windows path and PEP 3147.
+ self.assertEqual(
+ imp.cache_from_source('\\foo\\bar\\baz\\qux.py', True),
+ '\\foo\\bar\\baz\\__pycache__\\qux.{}.pyc'.format(self.tag))
+
+ @unittest.skipIf(os.altsep is None,
+ 'test meaningful only where os.altsep is defined')
+ def test_altsep_and_sep_cache_from_source(self):
+ # Windows path and PEP 3147 where altsep is right of sep.
+ self.assertEqual(
+ imp.cache_from_source('\\foo\\bar/baz\\qux.py', True),
+ '\\foo\\bar/baz\\__pycache__\\qux.{}.pyc'.format(self.tag))
+
+ @unittest.skipIf(os.altsep is None,
+ 'test meaningful only where os.altsep is defined')
+ def test_sep_altsep_and_sep_cache_from_source(self):
+ # Windows path and PEP 3147 where sep is right of altsep.
+ self.assertEqual(
+ imp.cache_from_source('\\foo\\bar\\baz/qux.py', True),
+ '\\foo\\bar\\baz/__pycache__/qux.{}.pyc'.format(self.tag))
+
+ def test_source_from_cache(self):
+ # Given the path to a PEP 3147 defined .pyc file, return the path to
+ # its source. This tests the good path.
+ self.assertEqual(imp.source_from_cache(
+ '/foo/bar/baz/__pycache__/qux.{}.pyc'.format(self.tag)),
+ '/foo/bar/baz/qux.py')
+
+ def test_source_from_cache_bad_path(self):
+ # When the path to a pyc file is not in PEP 3147 format, a ValueError
+ # is raised.
+ self.assertRaises(
+ ValueError, imp.source_from_cache, '/foo/bar/bazqux.pyc')
+
+ def test_source_from_cache_no_slash(self):
+ # No slashes at all in path -> ValueError
+ self.assertRaises(
+ ValueError, imp.source_from_cache, 'foo.cpython-32.pyc')
+
+ def test_source_from_cache_too_few_dots(self):
+ # Too few dots in final path component -> ValueError
+ self.assertRaises(
+ ValueError, imp.source_from_cache, '__pycache__/foo.pyc')
+
+ def test_source_from_cache_too_many_dots(self):
+ # Too many dots in final path component -> ValueError
+ self.assertRaises(
+ ValueError, imp.source_from_cache,
+ '__pycache__/foo.cpython-32.foo.pyc')
+
+ def test_source_from_cache_no__pycache__(self):
+ # Another problem with the path -> ValueError
+ self.assertRaises(
+ ValueError, imp.source_from_cache,
+ '/foo/bar/foo.cpython-32.foo.pyc')
+
+ def test_package___file__(self):
+ # Test that a package's __file__ points to the right source directory.
+ os.mkdir('pep3147')
+ sys.path.insert(0, os.curdir)
+ def cleanup():
+ if sys.path[0] == os.curdir:
+ del sys.path[0]
+ shutil.rmtree('pep3147')
+ self.addCleanup(cleanup)
+ # Touch the __init__.py file.
+ with open('pep3147/__init__.py', 'w'):
+ pass
+ m = __import__('pep3147')
+ # Ensure we load the pyc file.
+ support.forget('pep3147')
+ m = __import__('pep3147')
+ self.assertEqual(m.__file__,
+ os.sep.join(('.', 'pep3147', '__init__.py')))
+
+
def test_main():
tests = [
ImportTests,
+ PEP3147Tests,
ReloadTests,
- ]
+ ]
try:
import _thread
except ImportError:
Modified: python/branches/py3k/Lib/test/test_import.py
==============================================================================
--- python/branches/py3k/Lib/test/test_import.py (original)
+++ python/branches/py3k/Lib/test/test_import.py Sat Apr 17 02:19:56 2010
@@ -1,4 +1,5 @@
import builtins
+import errno
import imp
import marshal
import os
@@ -8,8 +9,11 @@
import stat
import sys
import unittest
-from test.support import (unlink, TESTFN, unload, run_unittest, is_jython,
- check_warnings, EnvironmentVarGuard, swap_attr, swap_item)
+
+from test.support import (
+ EnvironmentVarGuard, TESTFN, check_warnings, forget, is_jython,
+ make_legacy_pyc, rmtree, run_unittest, swap_attr, swap_item, temp_umask,
+ unlink, unload)
def remove_files(name):
@@ -19,12 +23,18 @@
name + ".pyw",
name + "$py.class"):
unlink(f)
+ try:
+ shutil.rmtree('__pycache__')
+ except OSError as error:
+ if error.errno != errno.ENOENT:
+ raise
class ImportTests(unittest.TestCase):
def tearDown(self):
unload(TESTFN)
+
setUp = tearDown
def test_case_sensitivity(self):
@@ -53,8 +63,8 @@
pyc = TESTFN + ".pyc"
with open(source, "w") as f:
- print("# This tests Python's ability to import a", ext, "file.",
- file=f)
+ print("# This tests Python's ability to import a",
+ ext, "file.", file=f)
a = random.randrange(1000)
b = random.randrange(1000)
print("a =", a, file=f)
@@ -73,10 +83,10 @@
self.assertEqual(mod.b, b,
"module loaded (%s) but contents invalid" % mod)
finally:
+ forget(TESTFN)
unlink(source)
unlink(pyc)
unlink(pyo)
- unload(TESTFN)
sys.path.insert(0, os.curdir)
try:
@@ -87,32 +97,31 @@
finally:
del sys.path[0]
- @unittest.skipUnless(os.name == 'posix', "test meaningful only on posix systems")
+ @unittest.skipUnless(os.name == 'posix',
+ "test meaningful only on posix systems")
def test_execute_bit_not_copied(self):
# Issue 6070: under posix .pyc files got their execute bit set if
# the .py file had the execute bit set, but they aren't executable.
- oldmask = os.umask(0o022)
- sys.path.insert(0, os.curdir)
- try:
- fname = TESTFN + os.extsep + "py"
- f = open(fname, 'w').close()
- os.chmod(fname, (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH |
- stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH))
- __import__(TESTFN)
- fn = fname + 'c'
- if not os.path.exists(fn):
- fn = fname + 'o'
+ with temp_umask(0o022):
+ sys.path.insert(0, os.curdir)
+ try:
+ fname = TESTFN + os.extsep + "py"
+ f = open(fname, 'w').close()
+ os.chmod(fname, (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH |
+ stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH))
+ __import__(TESTFN)
+ fn = imp.cache_from_source(fname)
if not os.path.exists(fn):
self.fail("__import__ did not result in creation of "
"either a .pyc or .pyo file")
- s = os.stat(fn)
- self.assertEqual(stat.S_IMODE(s.st_mode),
- stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
- finally:
- os.umask(oldmask)
- remove_files(TESTFN)
- unload(TESTFN)
- del sys.path[0]
+ s = os.stat(fn)
+ self.assertEqual(
+ stat.S_IMODE(s.st_mode),
+ stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
+ finally:
+ del sys.path[0]
+ remove_files(TESTFN)
+ unload(TESTFN)
def test_imp_module(self):
# Verify that the imp module can correctly load and find .py files
@@ -144,10 +153,12 @@
f.write('"",\n')
f.write(']')
- # Compile & remove .py file, we only need .pyc (or .pyo).
+ # Compile & remove .py file, we only need .pyc (or .pyo), but that
+ # must be relocated to the PEP 3147 bytecode-only location.
with open(filename, 'r') as f:
py_compile.compile(filename)
unlink(filename)
+ make_legacy_pyc(filename)
# Need to be able to load from current dir.
sys.path.append('')
@@ -247,8 +258,9 @@
self.assertTrue(mod.__file__.endswith('.py'))
os.remove(source)
del sys.modules[TESTFN]
+ make_legacy_pyc(source)
mod = __import__(TESTFN)
- ext = mod.__file__[-4:]
+ base, ext = os.path.splitext(mod.__file__)
self.assertIn(ext, ('.pyc', '.pyo'))
finally:
del sys.path[0]
@@ -298,7 +310,7 @@
"""
dir_name = os.path.abspath(TESTFN)
file_name = os.path.join(dir_name, module_name) + os.extsep + "py"
- compiled_name = file_name + ("c" if __debug__ else "o")
+ compiled_name = imp.cache_from_source(file_name)
def setUp(self):
self.sys_path = sys.path[:]
@@ -346,8 +358,9 @@
target = "another_module.py"
py_compile.compile(self.file_name, dfile=target)
os.remove(self.file_name)
+ pyc_file = make_legacy_pyc(self.file_name)
mod = self.import_module()
- self.assertEqual(mod.module_filename, self.compiled_name)
+ self.assertEqual(mod.module_filename, pyc_file)
self.assertEqual(mod.code_filename, target)
self.assertEqual(mod.func_filename, target)
@@ -476,10 +489,143 @@
self.assertEqual(foo(), os)
+class PycacheTests(unittest.TestCase):
+ # Test the various PEP 3147 related behaviors.
+
+ tag = imp.get_tag()
+
+ def _clean(self):
+ forget(TESTFN)
+ rmtree('__pycache__')
+ unlink(self.source)
+
+ def setUp(self):
+ self.source = TESTFN + '.py'
+ self._clean()
+ with open(self.source, 'w') as fp:
+ print('# This is a test file written by test_import.py', file=fp)
+ sys.path.insert(0, os.curdir)
+
+ def tearDown(self):
+ assert sys.path[0] == os.curdir, 'Unexpected sys.path[0]'
+ del sys.path[0]
+ self._clean()
+
+ def test_import_pyc_path(self):
+ self.assertFalse(os.path.exists('__pycache__'))
+ __import__(TESTFN)
+ self.assertTrue(os.path.exists('__pycache__'))
+ self.assertTrue(os.path.exists(os.path.join(
+ '__pycache__', '{}.{}.pyc'.format(TESTFN, self.tag))))
+
+ @unittest.skipUnless(os.name == 'posix',
+ "test meaningful only on posix systems")
+ def test_unwritable_directory(self):
+ # When the umask causes the new __pycache__ directory to be
+ # unwritable, the import still succeeds but no .pyc file is written.
+ with temp_umask(0o222):
+ __import__(TESTFN)
+ self.assertTrue(os.path.exists('__pycache__'))
+ self.assertFalse(os.path.exists(os.path.join(
+ '__pycache__', '{}.{}.pyc'.format(TESTFN, self.tag))))
+
+ def test_missing_source(self):
+ # With PEP 3147 cache layout, removing the source but leaving the pyc
+ # file does not satisfy the import.
+ __import__(TESTFN)
+ pyc_file = imp.cache_from_source(self.source)
+ self.assertTrue(os.path.exists(pyc_file))
+ os.remove(self.source)
+ forget(TESTFN)
+ self.assertRaises(ImportError, __import__, TESTFN)
+
+ def test_missing_source_legacy(self):
+ # Like test_missing_source() except that for backward compatibility,
+ # when the pyc file lives where the py file would have been (and named
+ # without the tag), it is importable. The __file__ of the imported
+ # module is the pyc location.
+ __import__(TESTFN)
+ # pyc_file gets removed in _clean() via tearDown().
+ pyc_file = make_legacy_pyc(self.source)
+ os.remove(self.source)
+ unload(TESTFN)
+ m = __import__(TESTFN)
+ self.assertEqual(m.__file__,
+ os.path.join(os.curdir, os.path.relpath(pyc_file)))
+
+ def test___cached__(self):
+ # Modules now also have an __cached__ that points to the pyc file.
+ m = __import__(TESTFN)
+ pyc_file = imp.cache_from_source(TESTFN + '.py')
+ self.assertEqual(m.__cached__, os.path.join(os.curdir, pyc_file))
+
+ def test___cached___legacy_pyc(self):
+ # Like test___cached__() except that for backward compatibility,
+ # when the pyc file lives where the py file would have been (and named
+ # without the tag), it is importable. The __cached__ of the imported
+ # module is the pyc location.
+ __import__(TESTFN)
+ # pyc_file gets removed in _clean() via tearDown().
+ pyc_file = make_legacy_pyc(self.source)
+ os.remove(self.source)
+ unload(TESTFN)
+ m = __import__(TESTFN)
+ self.assertEqual(m.__cached__,
+ os.path.join(os.curdir, os.path.relpath(pyc_file)))
+
+ def test_package___cached__(self):
+ # Like test___cached__ but for packages.
+ def cleanup():
+ shutil.rmtree('pep3147')
+ os.mkdir('pep3147')
+ self.addCleanup(cleanup)
+ # Touch the __init__.py
+ with open(os.path.join('pep3147', '__init__.py'), 'w'):
+ pass
+ with open(os.path.join('pep3147', 'foo.py'), 'w'):
+ pass
+ unload('pep3147.foo')
+ unload('pep3147')
+ m = __import__('pep3147.foo')
+ init_pyc = imp.cache_from_source(
+ os.path.join('pep3147', '__init__.py'))
+ self.assertEqual(m.__cached__, os.path.join(os.curdir, init_pyc))
+ foo_pyc = imp.cache_from_source(os.path.join('pep3147', 'foo.py'))
+ self.assertEqual(sys.modules['pep3147.foo'].__cached__,
+ os.path.join(os.curdir, foo_pyc))
+
+ def test_package___cached___from_pyc(self):
+ # Like test___cached__ but ensuring __cached__ when imported from a
+ # PEP 3147 pyc file.
+ def cleanup():
+ shutil.rmtree('pep3147')
+ os.mkdir('pep3147')
+ self.addCleanup(cleanup)
+ unload('pep3147.foo')
+ unload('pep3147')
+ # Touch the __init__.py
+ with open(os.path.join('pep3147', '__init__.py'), 'w'):
+ pass
+ with open(os.path.join('pep3147', 'foo.py'), 'w'):
+ pass
+ m = __import__('pep3147.foo')
+ unload('pep3147.foo')
+ unload('pep3147')
+ m = __import__('pep3147.foo')
+ init_pyc = imp.cache_from_source(
+ os.path.join('pep3147', '__init__.py'))
+ self.assertEqual(m.__cached__, os.path.join(os.curdir, init_pyc))
+ foo_pyc = imp.cache_from_source(os.path.join('pep3147', 'foo.py'))
+ self.assertEqual(sys.modules['pep3147.foo'].__cached__,
+ os.path.join(os.curdir, foo_pyc))
+
+
def test_main(verbose=None):
- run_unittest(ImportTests, PycRewritingTests, PathsTests, RelativeImportTests,
+ run_unittest(ImportTests, PycacheTests,
+ PycRewritingTests, PathsTests, RelativeImportTests,
OverridingImportBuiltinTests)
+
if __name__ == '__main__':
# Test needs to be a package, so we can do relative imports.
from test.test_import import test_main
Modified: python/branches/py3k/Lib/test/test_pkg.py
==============================================================================
--- python/branches/py3k/Lib/test/test_pkg.py (original)
+++ python/branches/py3k/Lib/test/test_pkg.py Sat Apr 17 02:19:56 2010
@@ -196,14 +196,14 @@
import t5
self.assertEqual(fixdir(dir(t5)),
- ['__doc__', '__file__', '__name__',
+ ['__cached__', '__doc__', '__file__', '__name__',
'__package__', '__path__', 'foo', 'string', 't5'])
self.assertEqual(fixdir(dir(t5.foo)),
- ['__doc__', '__file__', '__name__', '__package__',
- 'string'])
+ ['__cached__', '__doc__', '__file__', '__name__',
+ '__package__', 'string'])
self.assertEqual(fixdir(dir(t5.string)),
- ['__doc__', '__file__', '__name__','__package__',
- 'spam'])
+ ['__cached__', '__doc__', '__file__', '__name__',
+ '__package__', 'spam'])
def test_6(self):
hier = [
@@ -218,13 +218,13 @@
import t6
self.assertEqual(fixdir(dir(t6)),
- ['__all__', '__doc__', '__file__',
+ ['__all__', '__cached__', '__doc__', '__file__',
'__name__', '__package__', '__path__'])
s = """
import t6
from t6 import *
self.assertEqual(fixdir(dir(t6)),
- ['__all__', '__doc__', '__file__',
+ ['__all__', '__cached__', '__doc__', '__file__',
'__name__', '__package__', '__path__',
'eggs', 'ham', 'spam'])
self.assertEqual(dir(), ['eggs', 'ham', 'self', 'spam', 't6'])
@@ -252,18 +252,18 @@
t7, sub, subsub = None, None, None
import t7 as tas
self.assertEqual(fixdir(dir(tas)),
- ['__doc__', '__file__', '__name__',
+ ['__cached__', '__doc__', '__file__', '__name__',
'__package__', '__path__'])
self.assertFalse(t7)
from t7 import sub as subpar
self.assertEqual(fixdir(dir(subpar)),
- ['__doc__', '__file__', '__name__',
+ ['__cached__', '__doc__', '__file__', '__name__',
'__package__', '__path__'])
self.assertFalse(t7)
self.assertFalse(sub)
from t7.sub import subsub as subsubsub
self.assertEqual(fixdir(dir(subsubsub)),
- ['__doc__', '__file__', '__name__',
+ ['__cached__', '__doc__', '__file__', '__name__',
'__package__', '__path__', 'spam'])
self.assertFalse(t7)
self.assertFalse(sub)
Modified: python/branches/py3k/Lib/test/test_pkgimport.py
==============================================================================
--- python/branches/py3k/Lib/test/test_pkgimport.py (original)
+++ python/branches/py3k/Lib/test/test_pkgimport.py Sat Apr 17 02:19:56 2010
@@ -1,5 +1,12 @@
-import os, sys, string, random, tempfile, unittest
+import os
+import sys
+import shutil
+import string
+import random
+import tempfile
+import unittest
+from imp import cache_from_source
from test.support import run_unittest
class TestImport(unittest.TestCase):
@@ -26,22 +33,17 @@
self.module_path = os.path.join(self.package_dir, 'foo.py')
def tearDown(self):
- for file in os.listdir(self.package_dir):
- os.remove(os.path.join(self.package_dir, file))
- os.rmdir(self.package_dir)
- os.rmdir(self.test_dir)
+ shutil.rmtree(self.test_dir)
self.assertNotEqual(sys.path.count(self.test_dir), 0)
sys.path.remove(self.test_dir)
self.remove_modules()
def rewrite_file(self, contents):
- for extension in "co":
- compiled_path = self.module_path + extension
- if os.path.exists(compiled_path):
- os.remove(compiled_path)
- f = open(self.module_path, 'w')
- f.write(contents)
- f.close()
+ compiled_path = cache_from_source(self.module_path)
+ if os.path.exists(compiled_path):
+ os.remove(compiled_path)
+ with open(self.module_path, 'w') as f:
+ f.write(contents)
def test_package_import__semantics(self):
Modified: python/branches/py3k/Lib/test/test_pydoc.py
==============================================================================
--- python/branches/py3k/Lib/test/test_pydoc.py (original)
+++ python/branches/py3k/Lib/test/test_pydoc.py Sat Apr 17 02:19:56 2010
@@ -19,8 +19,7 @@
if hasattr(pydoc_mod, "__loader__"):
del pydoc_mod.__loader__
-expected_text_pattern = \
-"""
+expected_text_pattern = """
NAME
test.pydoc_mod - This is a test module for test_pydoc
@@ -87,8 +86,7 @@
Nobody
""".strip()
-expected_html_pattern = \
-"""
+expected_html_pattern = """
<table width="100%%" cellspacing=0 cellpadding=2 border=0 summary="heading">
<tr bgcolor="#7799ee">
<td valign=bottom> <br>
@@ -186,7 +184,7 @@
\x20\x20\x20\x20
<tr><td bgcolor="#7799ee"><tt> </tt></td><td> </td>
<td width="100%%">Nobody</td></tr></table>
-""".strip()
+""".strip() # ' <- emacs turd
# output pattern for missing module
@@ -287,7 +285,8 @@
('i_am_not_here', 'i_am_not_here'),
('test.i_am_not_here_either', 'i_am_not_here_either'),
('test.i_am_not_here.neither_am_i', 'i_am_not_here.neither_am_i'),
- ('i_am_not_here.{}'.format(modname), 'i_am_not_here.{}'.format(modname)),
+ ('i_am_not_here.{}'.format(modname),
+ 'i_am_not_here.{}'.format(modname)),
('test.{}'.format(modname), modname),
)
@@ -304,9 +303,8 @@
fullmodname = os.path.join(TESTFN, modname)
sourcefn = fullmodname + os.extsep + "py"
for importstring, expectedinmsg in testpairs:
- f = open(sourcefn, 'w')
- f.write("import {}\n".format(importstring))
- f.close()
+ with open(sourcefn, 'w') as f:
+ f.write("import {}\n".format(importstring))
try:
result = run_pydoc(modname).decode("ascii")
finally:
Modified: python/branches/py3k/Lib/test/test_runpy.py
==============================================================================
--- python/branches/py3k/Lib/test/test_runpy.py (original)
+++ python/branches/py3k/Lib/test/test_runpy.py Sat Apr 17 02:19:56 2010
@@ -5,9 +5,10 @@
import sys
import re
import tempfile
-from test.support import verbose, run_unittest, forget
-from test.script_helper import (temp_dir, make_script, compile_script,
- make_pkg, make_zip_script, make_zip_pkg)
+import py_compile
+from test.support import forget, make_legacy_pyc, run_unittest, verbose
+from test.script_helper import (
+ make_pkg, make_script, make_zip_pkg, make_zip_script, temp_dir)
from runpy import _run_code, _run_module_code, run_module, run_path
@@ -45,6 +46,7 @@
self.assertEqual(d["result"], self.expected_result)
self.assertIs(d["__name__"], None)
self.assertIs(d["__file__"], None)
+ self.assertIs(d["__cached__"], None)
self.assertIs(d["__loader__"], None)
self.assertIs(d["__package__"], None)
self.assertIs(d["run_argv0"], saved_argv0)
@@ -73,6 +75,7 @@
self.assertTrue(d2["run_name_in_sys_modules"])
self.assertTrue(d2["module_in_sys_modules"])
self.assertIs(d2["__file__"], file)
+ self.assertIs(d2["__cached__"], None)
self.assertIs(d2["run_argv0"], file)
self.assertIs(d2["__loader__"], loader)
self.assertIs(d2["__package__"], package)
@@ -170,6 +173,7 @@
del d1 # Ensure __loader__ entry doesn't keep file open
__import__(mod_name)
os.remove(mod_fname)
+ make_legacy_pyc(mod_fname)
if verbose: print("Running from compiled:", mod_name)
d2 = run_module(mod_name) # Read from bytecode
self.assertIn("x", d2)
@@ -192,6 +196,7 @@
del d1 # Ensure __loader__ entry doesn't keep file open
__import__(mod_name)
os.remove(mod_fname)
+ make_legacy_pyc(mod_fname)
if verbose: print("Running from compiled:", pkg_name)
d2 = run_module(pkg_name) # Read from bytecode
self.assertIn("x", d2)
@@ -246,6 +251,7 @@
del d1 # Ensure __loader__ entry doesn't keep file open
__import__(mod_name)
os.remove(mod_fname)
+ make_legacy_pyc(mod_fname)
if verbose: print("Running from compiled:", mod_name)
d2 = run_module(mod_name, run_name=run_name) # Read from bytecode
self.assertIn("__package__", d2)
@@ -313,6 +319,7 @@
result = run_path(script_name)
self.assertEqual(result["__name__"], expected_name)
self.assertEqual(result["__file__"], expected_file)
+ self.assertEqual(result["__cached__"], None)
self.assertIn("argv0", result)
self.assertEqual(result["argv0"], expected_argv0)
self.assertEqual(result["__package__"], expected_package)
@@ -332,7 +339,7 @@
with temp_dir() as script_dir:
mod_name = 'script'
script_name = self._make_test_script(script_dir, mod_name)
- compiled_name = compile_script(script_name)
+ compiled_name = py_compile.compile(script_name, doraise=True)
os.remove(script_name)
self._check_script(compiled_name, "<run_path>", compiled_name,
compiled_name, None)
@@ -348,9 +355,10 @@
with temp_dir() as script_dir:
mod_name = '__main__'
script_name = self._make_test_script(script_dir, mod_name)
- compiled_name = compile_script(script_name)
+ compiled_name = py_compile.compile(script_name, doraise=True)
os.remove(script_name)
- self._check_script(script_dir, "<run_path>", compiled_name,
+ legacy_pyc = make_legacy_pyc(script_name)
+ self._check_script(script_dir, "<run_path>", legacy_pyc,
script_dir, '')
def test_directory_error(self):
@@ -371,8 +379,9 @@
with temp_dir() as script_dir:
mod_name = '__main__'
script_name = self._make_test_script(script_dir, mod_name)
- compiled_name = compile_script(script_name)
- zip_name, fname = make_zip_script(script_dir, 'test_zip', compiled_name)
+ compiled_name = py_compile.compile(script_name, doraise=True)
+ zip_name, fname = make_zip_script(script_dir, 'test_zip',
+ compiled_name)
self._check_script(zip_name, "<run_path>", fname, zip_name, '')
def test_zipfile_error(self):
Modified: python/branches/py3k/Lib/test/test_site.py
==============================================================================
--- python/branches/py3k/Lib/test/test_site.py (original)
+++ python/branches/py3k/Lib/test/test_site.py Sat Apr 17 02:19:56 2010
@@ -258,19 +258,38 @@
"""Restore sys.path"""
sys.path[:] = self.sys_path
- def test_abs__file__(self):
- # Make sure all imported modules have their __file__ attribute
- # as an absolute path.
- # Handled by abs__file__()
- site.abs__file__()
- for module in (sys, os, builtins):
- try:
- self.assertTrue(os.path.isabs(module.__file__), repr(module))
- except AttributeError:
- continue
- # We could try everything in sys.modules; however, when regrtest.py
- # runs something like test_frozen before test_site, then we will
- # be testing things loaded *after* test_site did path normalization
+ def test_abs_paths(self):
+ # Make sure all imported modules have their __file__ and __cached__
+ # attributes as absolute paths. Arranging to put the Lib directory on
+ # PYTHONPATH would cause the os module to have a relative path for
+ # __file__ if abs_paths() does not get run. sys and builtins (the
+ # only other modules imported before site.py runs) do not have
+ # __file__ or __cached__ because they are built-in.
+ parent = os.path.relpath(os.path.dirname(os.__file__))
+ env = os.environ.copy()
+ env['PYTHONPATH'] = parent
+ command = 'import os; print(os.__file__, os.__cached__)'
+ # First, prove that with -S (no 'import site'), the paths are
+ # relative.
+ proc = subprocess.Popen([sys.executable, '-S', '-c', command],
+ env=env,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ stdout, stderr = proc.communicate()
+ self.assertEqual(proc.returncode, 0)
+ os__file__, os__cached__ = stdout.split()
+ self.assertFalse(os.path.isabs(os__file__))
+ self.assertFalse(os.path.isabs(os__cached__))
+ # Now, with 'import site', it works.
+ proc = subprocess.Popen([sys.executable, '-c', command],
+ env=env,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ stdout, stderr = proc.communicate()
+ self.assertEqual(proc.returncode, 0)
+ os__file__, os__cached__ = stdout.split()
+ self.assertTrue(os.path.isabs(os__file__))
+ self.assertTrue(os.path.isabs(os__cached__))
def test_no_duplicate_paths(self):
# No duplicate paths should exist in sys.path
Modified: python/branches/py3k/Lib/test/test_zipfile.py
==============================================================================
--- python/branches/py3k/Lib/test/test_zipfile.py (original)
+++ python/branches/py3k/Lib/test/test_zipfile.py Sat Apr 17 02:19:56 2010
@@ -6,6 +6,7 @@
import io
import os
+import imp
import time
import shutil
import struct
@@ -587,7 +588,13 @@
with zipfile.PyZipFile(TemporaryFile(), "w") as zipfp:
fn = __file__
if fn.endswith('.pyc') or fn.endswith('.pyo'):
- fn = fn[:-1]
+ path_split = fn.split(os.sep)
+ if os.altsep is not None:
+ path_split.extend(fn.split(os.altsep))
+ if '__pycache__' in path_split:
+ fn = imp.source_from_cache(fn)
+ else:
+ fn = fn[:-1]
zipfp.writepy(fn)
Modified: python/branches/py3k/Lib/test/test_zipimport.py
==============================================================================
--- python/branches/py3k/Lib/test/test_zipimport.py (original)
+++ python/branches/py3k/Lib/test/test_zipimport.py Sat Apr 17 02:19:56 2010
@@ -48,17 +48,14 @@
test_pyc = make_pyc(test_co, NOW)
-if __debug__:
- pyc_ext = ".pyc"
-else:
- pyc_ext = ".pyo"
-
-
TESTMOD = "ziptestmodule"
TESTPACK = "ziptestpackage"
TESTPACK2 = "ziptestpackage2"
TEMP_ZIP = os.path.abspath("junk95142.zip")
+pyc_file = imp.cache_from_source(TESTMOD + '.py')
+pyc_ext = ('.pyc' if __debug__ else '.pyo')
+
class UncompressedZipImportTestCase(ImportHooksBaseTestCase):
@@ -83,14 +80,11 @@
stuff = kw.get("stuff", None)
if stuff is not None:
# Prepend 'stuff' to the start of the zipfile
- f = open(TEMP_ZIP, "rb")
- data = f.read()
- f.close()
-
- f = open(TEMP_ZIP, "wb")
- f.write(stuff)
- f.write(data)
- f.close()
+ with open(TEMP_ZIP, "rb") as f:
+ data = f.read()
+ with open(TEMP_ZIP, "wb") as f:
+ f.write(stuff)
+ f.write(data)
sys.path.insert(0, TEMP_ZIP)
@@ -180,8 +174,9 @@
def testBadMTime(self):
badtime_pyc = bytearray(test_pyc)
- badtime_pyc[7] ^= 0x02 # flip the second bit -- not the first as that one
- # isn't stored in the .py's mtime in the zip archive.
+ # flip the second bit -- not the first as that one isn't stored in the
+ # .py's mtime in the zip archive.
+ badtime_pyc[7] ^= 0x02
files = {TESTMOD + ".py": (NOW, test_src),
TESTMOD + pyc_ext: (NOW, badtime_pyc)}
self.doTest(".py", files, TESTMOD)
@@ -232,7 +227,8 @@
self.assertEquals(zi.get_source(TESTPACK), None)
self.assertEquals(zi.get_source(mod_path), None)
self.assertEquals(zi.get_filename(mod_path), mod.__file__)
- # To pass in the module name instead of the path, we must use the right importer
+ # To pass in the module name instead of the path, we must use the
+ # right importer
loader = mod.__loader__
self.assertEquals(loader.get_source(mod_name), None)
self.assertEquals(loader.get_filename(mod_name), mod.__file__)
@@ -266,8 +262,10 @@
mod = zi.load_module(TESTPACK2)
self.assertEquals(zi.get_filename(TESTPACK2), mod.__file__)
- self.assertEquals(zi.is_package(TESTPACK2 + os.sep + '__init__'), False)
- self.assertEquals(zi.is_package(TESTPACK2 + os.sep + TESTMOD), False)
+ self.assertEquals(
+ zi.is_package(TESTPACK2 + os.sep + '__init__'), False)
+ self.assertEquals(
+ zi.is_package(TESTPACK2 + os.sep + TESTMOD), False)
mod_path = TESTPACK2 + os.sep + TESTMOD
mod_name = module_path_to_dotted_name(mod_path)
@@ -276,7 +274,8 @@
self.assertEquals(zi.get_source(TESTPACK2), None)
self.assertEquals(zi.get_source(mod_path), None)
self.assertEquals(zi.get_filename(mod_path), mod.__file__)
- # To pass in the module name instead of the path, we must use the right importer
+ # To pass in the module name instead of the path, we must use the
+ # right importer
loader = mod.__loader__
self.assertEquals(loader.get_source(mod_name), None)
self.assertEquals(loader.get_filename(mod_name), mod.__file__)
Modified: python/branches/py3k/Lib/zipfile.py
==============================================================================
--- python/branches/py3k/Lib/zipfile.py (original)
+++ python/branches/py3k/Lib/zipfile.py Sat Apr 17 02:19:56 2010
@@ -3,10 +3,17 @@
XXX references to utf-8 need further investigation.
"""
-import struct, os, time, sys, shutil
-import binascii, io, stat
import io
+import os
import re
+import imp
+import sys
+import time
+import stat
+import shutil
+import struct
+import binascii
+
try:
import zlib # We may need its compression method
@@ -1303,22 +1310,42 @@
file_py = pathname + ".py"
file_pyc = pathname + ".pyc"
file_pyo = pathname + ".pyo"
- if os.path.isfile(file_pyo) and \
- os.stat(file_pyo).st_mtime >= os.stat(file_py).st_mtime:
- fname = file_pyo # Use .pyo file
- elif not os.path.isfile(file_pyc) or \
- os.stat(file_pyc).st_mtime < os.stat(file_py).st_mtime:
+ pycache_pyc = imp.cache_from_source(file_py, True)
+ pycache_pyo = imp.cache_from_source(file_py, False)
+ if (os.path.isfile(file_pyo) and
+ os.stat(file_pyo).st_mtime >= os.stat(file_py).st_mtime):
+ # Use .pyo file.
+ arcname = fname = file_pyo
+ elif (os.path.isfile(file_pyc) and
+ os.stat(file_pyc).st_mtime >= os.stat(file_py).st_mtime):
+ # Use .pyc file.
+ arcname = fname = file_pyc
+ elif (os.path.isfile(pycache_pyc) and
+ os.stat(pycache_pyc).st_mtime >= os.stat(file_py).st_mtime):
+ # Use the __pycache__/*.pyc file, but write it to the legacy pyc
+ # file name in the archive.
+ fname = pycache_pyc
+ arcname = file_pyc
+ elif (os.path.isfile(pycache_pyo) and
+ os.stat(pycache_pyo).st_mtime >= os.stat(file_py).st_mtime):
+ # Use the __pycache__/*.pyo file, but write it to the legacy pyo
+ # file name in the archive.
+ fname = pycache_pyo
+ arcname = file_pyo
+ else:
+ # Compile py into PEP 3147 pyc file.
import py_compile
if self.debug:
print("Compiling", file_py)
try:
- py_compile.compile(file_py, file_pyc, None, True)
- except py_compile.PyCompileError as err:
+ py_compile.compile(file_py, doraise=True)
+ except py_compile.PyCompileError as error:
print(err.msg)
- fname = file_pyc
- else:
- fname = file_pyc
- archivename = os.path.split(fname)[1]
+ fname = file_py
+ else:
+ fname = (pycache_pyc if __debug__ else pycache_pyo)
+ arcname = (file_pyc if __debug__ else file_pyo)
+ archivename = os.path.split(arcname)[1]
if basename:
archivename = "%s/%s" % (basename, archivename)
return (fname, archivename)
Modified: python/branches/py3k/Makefile.pre.in
==============================================================================
--- python/branches/py3k/Makefile.pre.in (original)
+++ python/branches/py3k/Makefile.pre.in Sat Apr 17 02:19:56 2010
@@ -1161,6 +1161,7 @@
# files, which clobber removes as well
pycremoval:
find $(srcdir) -name '*.py[co]' -exec rm -f {} ';'
+ find $(srcdir) -name '__pycache__' | xargs rmdir
rmtestturds:
-rm -f *BAD *GOOD *SKIPPED
Modified: python/branches/py3k/Python/import.c
==============================================================================
--- python/branches/py3k/Python/import.c (original)
+++ python/branches/py3k/Python/import.c Sat Apr 17 02:19:56 2010
@@ -43,6 +43,15 @@
The current working scheme is to increment the previous value by
10.
+ Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic
+ number also includes a new "magic tag", i.e. a human readable string used
+ to represent the magic number in __pycache__ directories. When you change
+ the magic number, you must also set a new unique magic tag. Generally this
+ can be named after the Python major version of the magic number bump, but
+ it can really be anything, as long as it's different than anything else
+ that's come before. The tags are included in the following table, starting
+ with Python 3.2a0.
+
Known values:
Python 1.5: 20121
Python 1.5.1: 20121
@@ -91,11 +100,18 @@
Python 3.1a0: 3151 (optimize conditional branches:
introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE)
Python 3.2a0: 3160 (add SETUP_WITH)
+ tag: cpython-32
*/
+/* If you change MAGIC, you must change TAG and you must insert the old value
+ into _PyMagicNumberTags below.
+*/
#define MAGIC (3160 | ((long)'\r'<<16) | ((long)'\n'<<24))
-/* Magic word as global */
+#define TAG "cpython-32"
+#define CACHEDIR "__pycache__"
+/* Current magic word and string tag as globals. */
static long pyc_magic = MAGIC;
+static const char *pyc_tag = TAG;
/* See _PyImport_FixupExtension() below */
static PyObject *extensions = NULL;
@@ -517,7 +533,7 @@
}
-/* Helper for pythonrun.c -- return magic number */
+/* Helper for pythonrun.c -- return magic number and tag. */
long
PyImport_GetMagicNumber(void)
@@ -526,6 +542,12 @@
}
+const char *
+PyImport_GetMagicTag(void)
+{
+ return pyc_tag;
+}
+
/* Magic for extension modules (built-in as well as dynamically
loaded). To prevent initializing an extension module more than
once, we keep a static dictionary 'extensions' keyed by module name
@@ -671,7 +693,10 @@
"sys.modules failed");
}
-static PyObject * get_sourcefile(const char *file);
+static PyObject * get_sourcefile(char *file);
+static char *make_source_pathname(char *pathname, char *buf);
+static char *make_compiled_pathname(char *pathname, char *buf, size_t buflen,
+ int debug);
/* Execute a code object in a module and return the module object
* WITH INCREMENTED REFERENCE COUNT. If an error occurs, name is
@@ -679,16 +704,28 @@
* in sys.modules. The caller may wish to restore the original
* module object (if any) in this case; PyImport_ReloadModule is an
* example.
+ *
+ * Note that PyImport_ExecCodeModuleWithPathnames() is the preferred, richer
+ * interface. The other two exist primarily for backward compatibility.
*/
PyObject *
PyImport_ExecCodeModule(char *name, PyObject *co)
{
- return PyImport_ExecCodeModuleEx(name, co, (char *)NULL);
+ return PyImport_ExecCodeModuleWithPathnames(
+ name, co, (char *)NULL, (char *)NULL);
}
PyObject *
PyImport_ExecCodeModuleEx(char *name, PyObject *co, char *pathname)
{
+ return PyImport_ExecCodeModuleWithPathnames(
+ name, co, pathname, (char *)NULL);
+}
+
+PyObject *
+PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname,
+ char *cpathname)
+{
PyObject *modules = PyImport_GetModuleDict();
PyObject *m, *d, *v;
@@ -718,6 +755,20 @@
PyErr_Clear(); /* Not important enough to report */
Py_DECREF(v);
+ /* Remember the pyc path name as the __cached__ attribute. */
+ if (cpathname == NULL) {
+ v = Py_None;
+ Py_INCREF(v);
+ }
+ else if ((v = PyUnicode_FromString(cpathname)) == NULL) {
+ PyErr_Clear(); /* Not important enough to report */
+ v = Py_None;
+ Py_INCREF(v);
+ }
+ if (PyDict_SetItemString(d, "__cached__", v) != 0)
+ PyErr_Clear(); /* Not important enough to report */
+ Py_DECREF(v);
+
v = PyEval_EvalCode((PyCodeObject *)co, d, d);
if (v == NULL)
goto error;
@@ -740,32 +791,189 @@
}
+/* Like strrchr(string, '/') but searches for the rightmost of either SEP
+ or ALTSEP, if the latter is defined.
+*/
+static char *
+rightmost_sep(char *s)
+{
+ char *found, c;
+ for (found = NULL; (c = *s); s++) {
+ if (c == SEP
+#ifdef ALTSEP
+ || c == ALTSEP
+#endif
+ )
+ {
+ found = s;
+ }
+ }
+ return found;
+}
+
+
/* Given a pathname for a Python source file, fill a buffer with the
pathname for the corresponding compiled file. Return the pathname
for the compiled file, or NULL if there's no space in the buffer.
Doesn't set an exception. */
static char *
-make_compiled_pathname(char *pathname, char *buf, size_t buflen)
+make_compiled_pathname(char *pathname, char *buf, size_t buflen, int debug)
{
+ /* foo.py -> __pycache__/foo.<tag>.pyc */
size_t len = strlen(pathname);
- if (len+2 > buflen)
+ size_t i, save;
+ char *pos;
+ int sep = SEP;
+
+ /* Sanity check that the buffer has roughly enough space to hold what
+ will eventually be the full path to the compiled file. The 5 extra
+ bytes include the slash afer __pycache__, the two extra dots, the
+ extra trailing character ('c' or 'o') and null. This isn't exact
+ because the contents of the buffer can affect how many actual
+ characters of the string get into the buffer. We'll do a final
+ sanity check before writing the extension to ensure we do not
+ overflow the buffer.
+ */
+ if (len + strlen(CACHEDIR) + strlen(pyc_tag) + 5 > buflen)
return NULL;
-#ifdef MS_WINDOWS
- /* Treat .pyw as if it were .py. The case of ".pyw" must match
- that used in _PyImport_StandardFiletab. */
- if (len >= 4 && strcmp(&pathname[len-4], ".pyw") == 0)
- --len; /* pretend 'w' isn't there */
-#endif
- memcpy(buf, pathname, len);
- buf[len] = Py_OptimizeFlag ? 'o' : 'c';
- buf[len+1] = '\0';
+ /* Find the last path separator and copy everything from the start of
+ the source string up to and including the separator.
+ */
+ if ((pos = rightmost_sep(pathname)) == NULL) {
+ i = 0;
+ }
+ else {
+ sep = *pos;
+ i = pos - pathname + 1;
+ strncpy(buf, pathname, i);
+ }
+ save = i;
+ buf[i++] = '\0';
+ /* Add __pycache__/ */
+ strcat(buf, CACHEDIR);
+ i += strlen(CACHEDIR) - 1;
+ buf[i++] = sep;
+ buf[i++] = '\0';
+ /* Add the base filename, but remove the .py or .pyw extension, since
+ the tag name must go before the extension.
+ */
+ strcat(buf, pathname + save);
+ if ((pos = strrchr(buf, '.')) != NULL)
+ *++pos = '\0';
+ strcat(buf, pyc_tag);
+ /* The length test above assumes that we're only adding one character
+ to the end of what would normally be the extension. What if there
+ is no extension, or the string ends in '.' or '.p', and otherwise
+ fills the buffer? By appending 4 more characters onto the string
+ here, we could overrun the buffer.
+
+ As a simple example, let's say buflen=32 and the input string is
+ 'xxx.py'. strlen() would be 6 and the test above would yield:
+
+ (6 + 11 + 10 + 5 == 32) > 32
+
+ which is false and so the name mangling would continue. This would
+ be fine because we'd end up with this string in buf:
+
+ __pycache__/xxx.cpython-32.pyc\0
+
+ strlen(of that) == 30 + the nul fits inside a 32 character buffer.
+ We can even handle an input string of say 'xxxxx' above because
+ that's (5 + 11 + 10 + 5 == 31) > 32 which is also false. Name
+ mangling that yields:
+
+ __pycache__/xxxxxcpython-32.pyc\0
+
+ which is 32 characters including the nul, and thus fits in the
+ buffer. However, an input string of 'xxxxxx' would yield a result
+ string of:
+
+ __pycache__/xxxxxxcpython-32.pyc\0
+
+ which is 33 characters long (including the nul), thus overflowing
+ the buffer, even though the first test would fail, i.e.: the input
+ string is also 6 characters long, so 32 > 32 is false.
+
+ The reason the first test fails but we still overflow the buffer is
+ that the test above only expects to add one extra character to be
+ added to the extension, and here we're adding three (pyc). We
+ don't add the first dot, so that reclaims one of expected
+ positions, leaving us overflowing by 1 byte (3 extra - 1 reclaimed
+ dot - 1 expected extra == 1 overflowed).
+
+ The best we can do is ensure that we still have enough room in the
+ target buffer before we write the extension. Because it's always
+ only the extension that can cause the overflow, and never the other
+ path bytes we've written, it's sufficient to just do one more test
+ here. Still, the assertion that follows can't hurt.
+ */
+#if 0
+ printf("strlen(buf): %d; buflen: %d\n", (int)strlen(buf), (int)buflen);
+#endif
+ if (strlen(buf) + 5 > buflen)
+ return NULL;
+ strcat(buf, debug ? ".pyc" : ".pyo");
+ assert(strlen(buf) < buflen);
return buf;
}
+/* Given a pathname to a Python byte compiled file, return the path to the
+ source file, if the path matches the PEP 3147 format. This does not check
+ for any file existence, however, if the pyc file name does not match PEP
+ 3147 style, NULL is returned. buf must be at least as big as pathname;
+ the resulting path will always be shorter. */
+
+static char *
+make_source_pathname(char *pathname, char *buf)
+{
+ /* __pycache__/foo.<tag>.pyc -> foo.py */
+ size_t i, j;
+ char *left, *right, *dot0, *dot1, sep;
+
+ /* Look back two slashes from the end. In between these two slashes
+ must be the string __pycache__ or this is not a PEP 3147 style
+ path. It's possible for there to be only one slash.
+ */
+ if ((right = rightmost_sep(pathname)) == NULL)
+ return NULL;
+ sep = *right;
+ *right = '\0';
+ left = rightmost_sep(pathname);
+ *right = sep;
+ if (left == NULL)
+ left = pathname;
+ else
+ left++;
+ if (right-left != strlen(CACHEDIR) ||
+ strncmp(left, CACHEDIR, right-left) != 0)
+ return NULL;
+
+ /* Now verify that the path component to the right of the last slash
+ has two dots in it.
+ */
+ if ((dot0 = strchr(right + 1, '.')) == NULL)
+ return NULL;
+ if ((dot1 = strchr(dot0 + 1, '.')) == NULL)
+ return NULL;
+ /* Too many dots? */
+ if (strchr(dot1 + 1, '.') != NULL)
+ return NULL;
+
+ /* This is a PEP 3147 path. Start by copying everything from the
+ start of pathname up to and including the leftmost slash. Then
+ copy the file's basename, removing the magic tag and adding a .py
+ suffix.
+ */
+ strncpy(buf, pathname, (i=left-pathname));
+ strncpy(buf+i, right+1, (j=dot0-right));
+ strcpy(buf+i+j, "py");
+ return buf;
+}
+
/* Given a pathname for a Python source file, its time of last
modification, and a pathname for a compiled file, check whether the
compiled file represents the same version of the source. If so,
@@ -846,7 +1054,8 @@
if (Py_VerboseFlag)
PySys_WriteStderr("import %s # precompiled from %s\n",
name, cpathname);
- m = PyImport_ExecCodeModuleEx(name, (PyObject *)co, cpathname);
+ m = PyImport_ExecCodeModuleWithPathnames(
+ name, (PyObject *)co, cpathname, cpathname);
Py_DECREF(co);
return m;
@@ -919,12 +1128,41 @@
write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat)
{
FILE *fp;
+ char *dirpath;
time_t mtime = srcstat->st_mtime;
#ifdef MS_WINDOWS /* since Windows uses different permissions */
mode_t mode = srcstat->st_mode & ~S_IEXEC;
+ mode_t dirmode = srcstat->st_mode | S_IEXEC; /* XXX Is this correct
+ for Windows?
+ 2010-04-07 BAW */
#else
mode_t mode = srcstat->st_mode & ~S_IXUSR & ~S_IXGRP & ~S_IXOTH;
-#endif
+ mode_t dirmode = (srcstat->st_mode |
+ S_IXUSR | S_IXGRP | S_IXOTH |
+ S_IWUSR | S_IWGRP | S_IWOTH);
+#endif
+ int saved;
+
+ /* Ensure that the __pycache__ directory exists. */
+ dirpath = rightmost_sep(cpathname);
+ if (dirpath == NULL) {
+ if (Py_VerboseFlag)
+ PySys_WriteStderr(
+ "# no %s path found %s\n",
+ CACHEDIR, cpathname);
+ return;
+ }
+ saved = *dirpath;
+ *dirpath = '\0';
+ /* XXX call os.mkdir() or maybe CreateDirectoryA() on Windows? */
+ if (mkdir(cpathname, dirmode) < 0 && errno != EEXIST) {
+ *dirpath = saved;
+ if (Py_VerboseFlag)
+ PySys_WriteStderr(
+ "# cannot create cache dir %s\n", cpathname);
+ return;
+ }
+ *dirpath = saved;
fp = open_exclusive(cpathname, mode);
if (fp == NULL) {
@@ -1032,8 +1270,8 @@
return NULL;
}
#endif
- cpathname = make_compiled_pathname(pathname, buf,
- (size_t)MAXPATHLEN + 1);
+ cpathname = make_compiled_pathname(
+ pathname, buf, (size_t)MAXPATHLEN + 1, !Py_OptimizeFlag);
if (cpathname != NULL &&
(fpc = check_compiled_module(pathname, st.st_mtime, cpathname))) {
co = read_compiled_module(cpathname, fpc);
@@ -1060,7 +1298,8 @@
write_compiled_module(co, cpathname, &st);
}
}
- m = PyImport_ExecCodeModuleEx(name, (PyObject *)co, pathname);
+ m = PyImport_ExecCodeModuleWithPathnames(
+ name, (PyObject *)co, pathname, cpathname);
Py_DECREF(co);
return m;
@@ -1070,7 +1309,7 @@
* Returns the path to the py file if available, else the given path
*/
static PyObject *
-get_sourcefile(const char *file)
+get_sourcefile(char *file)
{
char py[MAXPATHLEN + 1];
Py_ssize_t len;
@@ -1087,8 +1326,15 @@
return PyUnicode_DecodeFSDefault(file);
}
- strncpy(py, file, len-1);
- py[len-1] = '\0';
+ /* Start by trying to turn PEP 3147 path into source path. If that
+ * fails, just chop off the trailing character, i.e. legacy pyc path
+ * to py.
+ */
+ if (make_source_pathname(file, py) == NULL) {
+ strncpy(py, file, len-1);
+ py[len-1] = '\0';
+ }
+
if (stat(py, &statbuf) == 0 &&
S_ISREG(statbuf.st_mode)) {
u = PyUnicode_DecodeFSDefault(py);
@@ -2813,16 +3059,28 @@
*/
static PyObject *
-imp_get_magic(PyObject *self, PyObject *noargs)
+imp_make_magic(long magic)
{
char buf[4];
- buf[0] = (char) ((pyc_magic >> 0) & 0xff);
- buf[1] = (char) ((pyc_magic >> 8) & 0xff);
- buf[2] = (char) ((pyc_magic >> 16) & 0xff);
- buf[3] = (char) ((pyc_magic >> 24) & 0xff);
+ buf[0] = (char) ((magic >> 0) & 0xff);
+ buf[1] = (char) ((magic >> 8) & 0xff);
+ buf[2] = (char) ((magic >> 16) & 0xff);
+ buf[3] = (char) ((magic >> 24) & 0xff);
return PyBytes_FromStringAndSize(buf, 4);
+};
+
+static PyObject *
+imp_get_magic(PyObject *self, PyObject *noargs)
+{
+ return imp_make_magic(pyc_magic);
+}
+
+static PyObject *
+imp_get_tag(PyObject *self, PyObject *noargs)
+{
+ return PyUnicode_FromString(pyc_tag);
}
static PyObject *
@@ -3190,6 +3448,75 @@
\n\
Reload the module. The module must have been successfully imported before.");
+static PyObject *
+imp_cache_from_source(PyObject *self, PyObject *args, PyObject *kws)
+{
+ static char *kwlist[] = {"path", "debug_override", NULL};
+
+ char buf[MAXPATHLEN+1];
+ char *pathname, *cpathname;
+ PyObject *debug_override = Py_None;
+ int debug = !Py_OptimizeFlag;
+
+ if (!PyArg_ParseTupleAndKeywords(
+ args, kws, "es|O", kwlist,
+ Py_FileSystemDefaultEncoding, &pathname, &debug_override))
+ return NULL;
+
+ if (debug_override != Py_None)
+ if ((debug = PyObject_IsTrue(debug_override)) < 0)
+ return NULL;
+
+ cpathname = make_compiled_pathname(pathname, buf, MAXPATHLEN+1, debug);
+ PyMem_Free(pathname);
+
+ if (cpathname == NULL) {
+ PyErr_Format(PyExc_SystemError, "path buffer too short");
+ return NULL;
+ }
+ return PyUnicode_FromString(buf);
+}
+
+PyDoc_STRVAR(doc_cache_from_source,
+"Given the path to a .py file, return the path to its .pyc/.pyo file.\n\
+\n\
+The .py file does not need to exist; this simply returns the path to the\n\
+.pyc/.pyo file calculated as if the .py file were imported. The extension\n\
+will be .pyc unless __debug__ is not defined, then it will be .pyo.\n\
+\n\
+If debug_override is not None, then it must be a boolean and is taken as\n\
+the value of __debug__ instead.");
+
+static PyObject *
+imp_source_from_cache(PyObject *self, PyObject *args, PyObject *kws)
+{
+ static char *kwlist[] = {"path", NULL};
+
+ char *pathname;
+ char buf[MAXPATHLEN+1];
+
+ if (!PyArg_ParseTupleAndKeywords(
+ args, kws, "es", kwlist,
+ Py_FileSystemDefaultEncoding, &pathname))
+ return NULL;
+
+ if (make_source_pathname(pathname, buf) == NULL) {
+ PyErr_Format(PyExc_ValueError, "Not a PEP 3147 pyc path: %s",
+ pathname);
+ PyMem_Free(pathname);
+ return NULL;
+ }
+ PyMem_Free(pathname);
+ return PyUnicode_FromString(buf);
+}
+
+PyDoc_STRVAR(doc_source_from_cache,
+"Given the path to a .pyc./.pyo file, return the path to its .py file.\n\
+\n\
+The .pyc/.pyo file does not need to exist; this simply returns the path to\n\
+the .py file calculated to correspond to the .pyc/.pyo file. If path\n\
+does not conform to PEP 3147 format, ValueError will be raised.");
+
/* Doc strings */
PyDoc_STRVAR(doc_imp,
@@ -3212,6 +3539,10 @@
"get_magic() -> string\n\
Return the magic number for .pyc or .pyo files.");
+PyDoc_STRVAR(doc_get_tag,
+"get_tag() -> string\n\
+Return the magic tag for .pyc or .pyo files.");
+
PyDoc_STRVAR(doc_get_suffixes,
"get_suffixes() -> [(suffix, mode, type), ...]\n\
Return a list of (suffix, mode, type) tuples describing the files\n\
@@ -3242,6 +3573,7 @@
static PyMethodDef imp_methods[] = {
{"find_module", imp_find_module, METH_VARARGS, doc_find_module},
{"get_magic", imp_get_magic, METH_NOARGS, doc_get_magic},
+ {"get_tag", imp_get_tag, METH_NOARGS, doc_get_tag},
{"get_suffixes", imp_get_suffixes, METH_NOARGS, doc_get_suffixes},
{"load_module", imp_load_module, METH_VARARGS, doc_load_module},
{"new_module", imp_new_module, METH_VARARGS, doc_new_module},
@@ -3249,6 +3581,10 @@
{"acquire_lock", imp_acquire_lock, METH_NOARGS, doc_acquire_lock},
{"release_lock", imp_release_lock, METH_NOARGS, doc_release_lock},
{"reload", imp_reload, METH_O, doc_reload},
+ {"cache_from_source", (PyCFunction)imp_cache_from_source,
+ METH_VARARGS | METH_KEYWORDS, doc_cache_from_source},
+ {"source_from_cache", (PyCFunction)imp_source_from_cache,
+ METH_VARARGS | METH_KEYWORDS, doc_source_from_cache},
/* The rest are obsolete */
{"get_frozen_object", imp_get_frozen_object, METH_VARARGS},
{"is_frozen_package", imp_is_frozen_package, METH_VARARGS},
@@ -3436,7 +3772,6 @@
failure:
Py_XDECREF(m);
return NULL;
-
}
Modified: python/branches/py3k/Python/pythonrun.c
==============================================================================
--- python/branches/py3k/Python/pythonrun.c (original)
+++ python/branches/py3k/Python/pythonrun.c Sat Apr 17 02:19:56 2010
@@ -1155,6 +1155,8 @@
Py_DECREF(f);
return -1;
}
+ if (PyDict_SetItemString(d, "__cached__", Py_None) < 0)
+ return -1;
set_file_name = 1;
Py_DECREF(f);
}
More information about the Python-checkins
mailing list