[pypy-commit] pypy py3k: merge the app_main-refactor branch; tests are stil broken because we can't import autopath and in general the pypy tree in python3
antocuni
noreply at buildbot.pypy.org
Mon Jun 11 16:01:13 CEST 2012
Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r55579:d5c33e36323f
Date: 2012-06-11 14:19 +0200
http://bitbucket.org/pypy/pypy/changeset/d5c33e36323f/
Log: merge the app_main-refactor branch; tests are stil broken because we
can't import autopath and in general the pypy tree in python3
diff --git a/pypy/annotation/binaryop.py b/pypy/annotation/binaryop.py
--- a/pypy/annotation/binaryop.py
+++ b/pypy/annotation/binaryop.py
@@ -660,7 +660,7 @@
def mul((str1, int2)): # xxx do we want to support this
getbookkeeper().count("str_mul", str1, int2)
- return SomeString()
+ return SomeString(no_nul=str1.no_nul)
class __extend__(pairtype(SomeUnicodeString, SomeInteger)):
def getitem((str1, int2)):
diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -2138,6 +2138,15 @@
assert isinstance(s, annmodel.SomeString)
assert s.no_nul
+ def test_mul_str0(self):
+ def f(s):
+ return s*10
+ a = self.RPythonAnnotator()
+ s = a.build_types(f, [annmodel.SomeString(no_nul=True)])
+ assert isinstance(s, annmodel.SomeString)
+ assert s.no_nul
+
+
def test_non_none_and_none_with_isinstance(self):
class A(object):
pass
@@ -3780,6 +3789,26 @@
e = py.test.raises(Exception, a.build_types, f, [])
assert 'object with a __call__ is not RPython' in str(e.value)
+ def test_os_getcwd(self):
+ import os
+ def fn():
+ return os.getcwd()
+ a = self.RPythonAnnotator()
+ s = a.build_types(fn, [])
+ assert isinstance(s, annmodel.SomeString)
+ assert s.no_nul
+
+ def test_os_getenv(self):
+ import os
+ def fn():
+ return os.environ.get('PATH')
+ a = self.RPythonAnnotator()
+ s = a.build_types(fn, [])
+ assert isinstance(s, annmodel.SomeString)
+ assert s.no_nul
+
+
+
def g(n):
return [0,1,2,n]
diff --git a/pypy/bin/py.py b/pypy/bin/py.py
--- a/pypy/bin/py.py
+++ b/pypy/bin/py.py
@@ -89,12 +89,12 @@
space.setitem(space.sys.w_dict, space.wrap('executable'),
space.wrap(argv[0]))
- # call pypy_initial_path: the side-effect is that it sets sys.prefix and
+ # call pypy_find_stdlib: the side-effect is that it sets sys.prefix and
# sys.exec_prefix
- srcdir = os.path.dirname(os.path.dirname(pypy.__file__))
- space.appexec([space.wrap(srcdir)], """(srcdir):
+ executable = argv[0]
+ space.appexec([space.wrap(executable)], """(executable):
import sys
- sys.pypy_initial_path(srcdir)
+ sys.pypy_find_stdlib(executable)
""")
# set warning control options (if any)
diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
--- a/pypy/doc/cppyy.rst
+++ b/pypy/doc/cppyy.rst
@@ -71,10 +71,14 @@
.. _`recent snapshot`: http://cern.ch/wlav/reflex-2012-05-02.tar.bz2
.. _`gccxml`: http://www.gccxml.org
-Next, get the `PyPy sources`_, select the reflex-support branch, and build
-pypy-c.
+Next, get the `PyPy sources`_, select the reflex-support branch, and build.
For the build to succeed, the ``$ROOTSYS`` environment variable must point to
-the location of your ROOT (or standalone Reflex) installation::
+the location of your ROOT (or standalone Reflex) installation, or the
+``root-config`` utility must be accessible through ``PATH`` (e.g. by adding
+``$ROOTSYS/bin`` to ``PATH``).
+In case of the former, include files are expected under ``$ROOTSYS/include``
+and libraries under ``$ROOTSYS/lib``.
+Then run the translation to build ``pypy-c``::
$ hg clone https://bitbucket.org/pypy/pypy
$ cd pypy
@@ -115,7 +119,7 @@
code::
$ genreflex MyClass.h
- $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyClass_rflx.cpp -o libMyClassDict.so
+ $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyClass_rflx.cpp -o libMyClassDict.so -L$ROOTSYS/lib -lReflex
Now you're ready to use the bindings.
Since the bindings are designed to look pythonistic, it should be
@@ -139,6 +143,51 @@
That's all there is to it!
+Automatic class loader
+======================
+There is one big problem in the code above, that prevents its use in a (large
+scale) production setting: the explicit loading of the reflection library.
+Clearly, if explicit load statements such as these show up in code downstream
+from the ``MyClass`` package, then that prevents the ``MyClass`` author from
+repackaging or even simply renaming the dictionary library.
+
+The solution is to make use of an automatic class loader, so that downstream
+code never has to call ``load_reflection_info()`` directly.
+The class loader makes use of so-called rootmap files, which ``genreflex``
+can produce.
+These files contain the list of available C++ classes and specify the library
+that needs to be loaded for their use.
+By convention, the rootmap files should be located next to the reflection info
+libraries, so that they can be found through the normal shared library search
+path.
+They can be concatenated together, or consist of a single rootmap file per
+library.
+For example::
+
+ $ genreflex MyClass.h --rootmap=libMyClassDict.rootmap --rootmap-lib=libMyClassDict.so
+ $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyClass_rflx.cpp -o libMyClassDict.so -L$ROOTSYS/lib -lReflex
+
+where the first option (``--rootmap``) specifies the output file name, and the
+second option (``--rootmap-lib``) the name of the reflection library where
+``MyClass`` will live.
+It is necessary to provide that name explicitly, since it is only in the
+separate linking step where this name is fixed.
+If the second option is not given, the library is assumed to be libMyClass.so,
+a name that is derived from the name of the header file.
+
+With the rootmap file in place, the above example can be rerun without explicit
+loading of the reflection info library::
+
+ $ pypy-c
+ >>>> import cppyy
+ >>>> myinst = cppyy.gbl.MyClass(42)
+ >>>> print myinst.GetMyInt()
+ 42
+ >>>> # etc. ...
+
+As a caveat, note that the class loader is currently limited to classes only.
+
+
Advanced example
================
The following snippet of C++ is very contrived, to allow showing that such
@@ -171,7 +220,7 @@
std::string m_name;
};
- Base1* BaseFactory(const std::string& name, int i, double d) {
+ Base2* BaseFactory(const std::string& name, int i, double d) {
return new Derived(name, i, d);
}
@@ -213,7 +262,7 @@
Now the reflection info can be generated and compiled::
$ genreflex MyAdvanced.h --selection=MyAdvanced.xml
- $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyAdvanced_rflx.cpp -o libAdvExDict.so
+ $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyAdvanced_rflx.cpp -o libAdvExDict.so -L$ROOTSYS/lib -lReflex
and subsequently be used from PyPy::
@@ -237,7 +286,7 @@
A couple of things to note, though.
If you look back at the C++ definition of the ``BaseFactory`` function,
-you will see that it declares the return type to be a ``Base1``, yet the
+you will see that it declares the return type to be a ``Base2``, yet the
bindings return an object of the actual type ``Derived``?
This choice is made for a couple of reasons.
First, it makes method dispatching easier: if bound objects are always their
@@ -434,7 +483,9 @@
int m_i;
};
- template class std::vector<MyClass>;
+ #ifdef __GCCXML__
+ template class std::vector<MyClass>; // explicit instantiation
+ #endif
If you know for certain that all symbols will be linked in from other sources,
you can also declare the explicit template instantiation ``extern``.
@@ -445,8 +496,9 @@
internal namespace, rather than in the iterator classes.
One way to handle this, is to deal with this once in a macro, then reuse that
macro for all ``vector`` classes.
-Thus, the header above needs this, instead of just the explicit instantiation
-of the ``vector<MyClass>``::
+Thus, the header above needs this (again protected with
+``#ifdef __GCCXML__``), instead of just the explicit instantiation of the
+``vector<MyClass>``::
#define STLTYPES_EXPLICIT_INSTANTIATION_DECL(STLTYPE, TTYPE) \
template class std::STLTYPE< TTYPE >; \
@@ -467,11 +519,9 @@
$ cat MyTemplate.xml
<lcgdict>
<class pattern="std::vector<*>" />
- <class pattern="__gnu_cxx::__normal_iterator<*>" />
- <class pattern="__gnu_cxx::new_allocator<*>" />
+ <class pattern="std::vector<*>::iterator" />
<class pattern="std::_Vector_base<*>" />
<class pattern="std::_Vector_base<*>::_Vector_impl" />
- <class pattern="std::allocator<*>" />
<function name="__gnu_cxx::operator=="/>
<function name="__gnu_cxx::operator!="/>
@@ -480,8 +530,8 @@
Run the normal ``genreflex`` and compilation steps::
- $ genreflex MyTemplate.h --selection=MyTemplate.xm
- $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyTemplate_rflx.cpp -o libTemplateDict.so
+ $ genreflex MyTemplate.h --selection=MyTemplate.xml
+ $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyTemplate_rflx.cpp -o libTemplateDict.so -L$ROOTSYS/lib -lReflex
Note: this is a dirty corner that clearly could do with some automation,
even if the macro already helps.
@@ -555,7 +605,9 @@
There are a couple of minor differences between PyCintex and cppyy, most to do
with naming.
The one that you will run into directly, is that PyCintex uses a function
-called ``loadDictionary`` rather than ``load_reflection_info``.
+called ``loadDictionary`` rather than ``load_reflection_info`` (it has the
+same rootmap-based class loader functionality, though, making this point
+somewhat moot).
The reason for this is that Reflex calls the shared libraries that contain
reflection info "dictionaries."
However, in python, the name `dictionary` already has a well-defined meaning,
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -324,5 +324,10 @@
type and vice versa. For builtin types, a dictionary will be returned that
cannot be changed (but still looks and behaves like a normal dictionary).
+* the ``__len__`` or ``__length_hint__`` special methods are sometimes
+ called by CPython to get a length estimate to preallocate internal arrays.
+ So far, PyPy never calls ``__len__`` for this purpose, and never calls
+ ``__length_hint__`` at all.
+
.. include:: _ref.txt
diff --git a/pypy/doc/release-1.9.0.rst b/pypy/doc/release-1.9.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-1.9.0.rst
@@ -0,0 +1,107 @@
+====================
+PyPy 1.9 - Yard Wolf
+====================
+
+We're pleased to announce the 1.9 release of PyPy. This release brings mostly
+bugfixes, performance improvements, other small improvements and overall
+progress on the `numpypy`_ effort.
+It also brings an improved situation on windows and OS X.
+
+You can download the PyPy 1.9 release here:
+
+ http://pypy.org/download.html
+
+.. _`numpypy`: http://pypy.org/numpydonate.html
+
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`pypy 1.9 and cpython 2.7.2`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+This release supports x86 machines running Linux 32/64, Mac OS X 64 or
+Windows 32. Windows 64 work is still stalling, we would welcome a volunteer
+to handle that.
+
+.. _`pypy 1.9 and cpython 2.7.2`: http://speed.pypy.org
+
+
+Thanks to our donators
+======================
+
+But first of all, we would like to say thank you to all people who
+donated some money to one of our four calls:
+
+ * `NumPy in PyPy`_ (got so far $44502 out of $60000, 74%)
+
+ * `Py3k (Python 3)`_ (got so far $43563 out of $105000, 41%)
+
+ * `Software Transactional Memory`_ (got so far $21791 of $50400, 43%)
+
+ * as well as our general PyPy pot.
+
+Thank you all for proving that it is indeed possible for a small team of
+programmers to get funded like that, at least for some
+time. We want to include this thank you in the present release
+announcement even though most of the work is not finished yet. More
+precisely, neither Py3k nor STM are ready to make it an official release
+yet: people interested in them need to grab and (attempt to) translate
+PyPy from the corresponding branches (respectively ``py3k`` and
+``stm-thread``).
+
+.. _`NumPy in PyPy`: http://pypy.org/numpydonate.html
+.. _`Py3k (Python 3)`: http://pypy.org/py3donate.html
+.. _`Software Transactional Memory`: http://pypy.org/tmdonate.html
+
+Highlights
+==========
+
+* This release still implements Python 2.7, the standard library has been
+ upgraded to CPython 2.7.2.
+
+* Many bugs were corrected for Windows 32 bit. This includes new
+ functionality to test the validity of file descriptors; and
+ correct handling of the calling convensions for ctypes. (Still not
+ much progress on Win64.) A lot of work on this has been done by Matti Picus
+ and Amaury Forgeot d'Arc.
+
+* Improvements in ``cpyext``, our emulator for CPython C extension modules.
+ For example PyOpenSSL should now work.
+
+* Sets now have strategies just like dictionaries. This means for example
+ that a set containing only ints will be more compact (and faster).
+
+* A lot of progress on various aspects of ``numpypy``. See `numpy-status`_
+ page for the automatic report.
+
+* It is now possible to create and manipulate C-like structures using the
+ PyPy-only ``_ffi`` module. The advantage over using e.g. ``ctypes`` is that
+ ``_ffi`` is very JIT-friendly, and getting/setting of fields is translated
+ to few assembler instructions by the JIT. However, this is mostly intended
+ as a low-level backend to be used by more user-friendly FFI packages, and
+ the API might change in the future. Use it at your own risk.
+
+* The non-x86 backends for the JIT are progressing but are still not
+ merged (ARMv7 and PPC64).
+
+* JIT hooks for inspecting the created assembler code has been improved.
+ See `JIT hooks documentation`_ for details.
+
+* ``select.kqueue`` has been added.
+
+* Handling of keyword arguments has been drastically improved in the best-case
+ scenario.
+
+* List comprehension has been improved.
+
+JitViewer
+=========
+
+There is a corresponding 1.9 release of JitViewer which is guaranteed to work
+with PyPy 1.9. See `JitViewer docs`_ for details.
+
+.. _`numpy status`: http://buildbot.pypy.org/numpy-status/latest.html
+.. _`JitViewer docs`: http://bitbucket.org/pypy/jitviewer
+.. _`JIT hooks documentation`: http://doc.pypy.org/en/latest/jit-hooks.html
diff --git a/pypy/jit/codewriter/policy.py b/pypy/jit/codewriter/policy.py
--- a/pypy/jit/codewriter/policy.py
+++ b/pypy/jit/codewriter/policy.py
@@ -48,8 +48,6 @@
mod = func.__module__ or '?'
if mod.startswith('pypy.rpython.module.'):
return True
- if mod == 'pypy.translator.goal.nanos': # more helpers
- return True
return False
def look_inside_graph(self, graph):
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -44,6 +44,8 @@
'list_strategy' : 'interp_magic.list_strategy',
'validate_fd' : 'interp_magic.validate_fd',
}
+ if sys.platform == 'win32':
+ interpleveldefs['get_console_cp'] = 'interp_magic.get_console_cp'
submodules = {
"builders": BuildersModule,
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -85,3 +85,10 @@
rposix.validate_fd(fd)
except OSError, e:
raise wrap_oserror(space, e)
+
+def get_console_cp(space):
+ from pypy.rlib import rwin32 # Windows only
+ return space.newtuple([
+ space.wrap('cp%d' % rwin32.GetConsoleCP()),
+ space.wrap('cp%d' % rwin32.GetConsoleOutputCP()),
+ ])
diff --git a/pypy/module/pypyjit/test_pypy_c/test__ffi.py b/pypy/module/pypyjit/test_pypy_c/test__ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test__ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test__ffi.py
@@ -82,7 +82,7 @@
#
if os.name == 'nt':
from _ffi import WinDLL, types
- libc = WinDLL(libc_name)
+ libc = WinDLL('Kernel32.dll')
sleep = libc.getfunc('Sleep', [types.uint], types.uint)
delays = [0]*n + [1000]
else:
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -38,7 +38,9 @@
'abiflags' : 'space.wrap("")',
'builtin_module_names' : 'space.w_None',
'pypy_getudir' : 'state.pypy_getudir', # not translated
- 'pypy_initial_path' : 'state.pypy_initial_path',
+ 'pypy_find_stdlib' : 'initpath.pypy_find_stdlib',
+ 'pypy_find_executable' : 'initpath.pypy_find_executable',
+ 'pypy_resolvedirof' : 'initpath.pypy_resolvedirof',
'_getframe' : 'vm._getframe',
'_current_frames' : 'currentframes._current_frames',
diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/sys/initpath.py
@@ -0,0 +1,155 @@
+"""
+Logic to find sys.executable and the initial sys.path containing the stdlib
+"""
+
+import sys
+import os
+import stat
+import errno
+from pypy.rlib import rpath
+from pypy.rlib.objectmodel import we_are_translated
+from pypy.interpreter.gateway import unwrap_spec
+from pypy.module.sys.state import get as get_state
+
+platform = sys.platform
+IS_WINDOWS = sys.platform == 'win32'
+
+def find_executable(executable):
+ """
+ Return the absolute path of the executable, by looking into PATH and the
+ current directory. If it cannot be found, return ''.
+ """
+ if we_are_translated() and IS_WINDOWS and not executable.lower().endswith('.exe'):
+ executable += '.exe'
+ if os.sep in executable or (IS_WINDOWS and ':' in executable):
+ pass # the path is already more than just an executable name
+ else:
+ path = os.environ.get('PATH')
+ if path:
+ for dir in path.split(os.pathsep):
+ fn = os.path.join(dir, executable)
+ if os.path.isfile(fn):
+ executable = fn
+ break
+ executable = rpath.rabspath(executable)
+ #
+ # 'sys.executable' should not end up being an non-existing file;
+ # just use '' in this case. (CPython issue #7774)
+ if not os.path.isfile(executable):
+ executable = ''
+ return executable
+
+
+def readlink_maybe(filename):
+ if not IS_WINDOWS:
+ return os.readlink(filename)
+ raise NotImplementedError
+
+def resolvedirof(filename):
+ try:
+ filename = rpath.rabspath(filename)
+ except OSError:
+ pass
+ dirname = rpath.rabspath(os.path.join(filename, '..'))
+ if os.path.islink(filename):
+ try:
+ link = readlink_maybe(filename)
+ except OSError:
+ pass
+ else:
+ return resolvedirof(os.path.join(dirname, link))
+ return dirname
+
+def find_stdlib(state, executable):
+ """
+ Find and compute the stdlib path, starting from the directory where
+ ``executable`` is and going one level up until we find it. Return a tuple
+ (path, prefix), where ``prefix`` is the root directory which contains the
+ stdlib.
+ If it cannot be found, return (None, None).
+ """
+ search = executable
+ while True:
+ dirname = resolvedirof(search)
+ if dirname == search:
+ return None, None # not found :-(
+ newpath = compute_stdlib_path_maybe(state, dirname)
+ if newpath is not None:
+ return newpath, dirname
+ search = dirname # walk to the parent directory
+
+
+
+def checkdir(path):
+ st = os.stat(path)
+ if not stat.S_ISDIR(st[0]):
+ raise OSError(errno.ENOTDIR, path)
+
+def compute_stdlib_path(state, prefix):
+ """
+ Compute the paths for the stdlib rooted at ``prefix``. ``prefix`` must at
+ least contain a directory called ``lib-python/X.Y`` and another one called
+ ``lib_pypy``. If they cannot be found, it raises OSError.
+ """
+ from pypy.module.sys.version import CPYTHON_VERSION
+ dirname = '%d.%d' % (CPYTHON_VERSION[0],
+ CPYTHON_VERSION[1])
+ lib_python = os.path.join(prefix, 'lib-python')
+ python_std_lib = os.path.join(lib_python, dirname)
+ checkdir(python_std_lib)
+
+ lib_pypy = os.path.join(prefix, 'lib_pypy')
+ checkdir(lib_pypy)
+
+ importlist = []
+ #
+ if state is not None: # 'None' for testing only
+ lib_extensions = os.path.join(lib_pypy, '__extensions__')
+ state.w_lib_extensions = state.space.wrap(lib_extensions)
+ importlist.append(lib_extensions)
+ #
+ importlist.append(lib_pypy)
+ importlist.append(python_std_lib)
+ #
+ lib_tk = os.path.join(python_std_lib, 'lib-tk')
+ importlist.append(lib_tk)
+ #
+ # List here the extra platform-specific paths.
+ if platform != 'win32':
+ importlist.append(os.path.join(python_std_lib, 'plat-'+platform))
+ if platform == 'darwin':
+ platmac = os.path.join(python_std_lib, 'plat-mac')
+ importlist.append(platmac)
+ importlist.append(os.path.join(platmac, 'lib-scriptpackages'))
+ #
+ return importlist
+
+def compute_stdlib_path_maybe(state, prefix):
+ """
+ Return the stdlib path rooted at ``prefix``, or None if it cannot be
+ found.
+ """
+ try:
+ return compute_stdlib_path(state, prefix)
+ except OSError:
+ return None
+
+ at unwrap_spec(executable='str0')
+def pypy_find_executable(space, executable):
+ return space.wrap(find_executable(executable))
+
+ at unwrap_spec(filename='str0')
+def pypy_resolvedirof(space, filename):
+ return space.wrap(resolvedirof(filename))
+
+ at unwrap_spec(executable='str0')
+def pypy_find_stdlib(space, executable):
+ path, prefix = find_stdlib(get_state(space), executable)
+ if path is None:
+ return space.w_None
+ else:
+ space.setitem(space.sys.w_dict, space.wrap('prefix'),
+ space.wrap(prefix))
+ space.setitem(space.sys.w_dict, space.wrap('exec_prefix'),
+ space.wrap(prefix))
+ return space.newlist([space.wrap(p) for p in path])
diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py
--- a/pypy/module/sys/state.py
+++ b/pypy/module/sys/state.py
@@ -1,11 +1,8 @@
"""
Implementation of interpreter-level 'sys' routines.
"""
+import os
import pypy
-from pypy.interpreter.error import OperationError
-from pypy.interpreter.gateway import unwrap_spec
-
-import sys, os, stat, errno
# ____________________________________________________________
#
@@ -20,67 +17,14 @@
self.w_argv = space.newlist([])
self.setinitialpath(space)
- def setinitialpath(self, space):
+ def setinitialpath(self, space):
+ from pypy.module.sys.initpath import compute_stdlib_path
# Initialize the default path
pypydir = os.path.dirname(os.path.abspath(pypy.__file__))
srcdir = os.path.dirname(pypydir)
- path = getinitialpath(self, srcdir)
+ path = compute_stdlib_path(self, srcdir)
self.w_path = space.newlist([space.wrap(p) for p in path])
-def checkdir(path):
- st = os.stat(path)
- if not stat.S_ISDIR(st[0]):
- raise OSError(errno.ENOTDIR, path)
-
-
-platform = sys.platform
-
-def getinitialpath(state, prefix):
- from pypy.module.sys.version import CPYTHON_VERSION
- dirname = '%d.%d' % (CPYTHON_VERSION[0],
- CPYTHON_VERSION[1])
- lib_python = os.path.join(prefix, 'lib-python')
- python_std_lib = os.path.join(lib_python, dirname)
- checkdir(python_std_lib)
-
- lib_pypy = os.path.join(prefix, 'lib_pypy')
- checkdir(lib_pypy)
-
- importlist = []
- #
- if state is not None: # 'None' for testing only
- lib_extensions = os.path.join(lib_pypy, '__extensions__')
- state.w_lib_extensions = state.space.wrap(lib_extensions)
- importlist.append(lib_extensions)
- #
- importlist.append(lib_pypy)
- importlist.append(python_std_lib)
- #
- lib_tk = os.path.join(python_std_lib, 'lib-tk')
- importlist.append(lib_tk)
- #
- # List here the extra platform-specific paths.
- if platform != 'win32':
- importlist.append(os.path.join(python_std_lib, 'plat-'+platform))
- if platform == 'darwin':
- platmac = os.path.join(python_std_lib, 'plat-mac')
- importlist.append(platmac)
- importlist.append(os.path.join(platmac, 'lib-scriptpackages'))
- #
- return importlist
-
- at unwrap_spec(srcdir='str0')
-def pypy_initial_path(space, srcdir):
- try:
- path = getinitialpath(get(space), srcdir)
- except OSError:
- return space.w_None
- else:
- space.setitem(space.sys.w_dict, space.wrap('prefix'),
- space.wrap(srcdir))
- space.setitem(space.sys.w_dict, space.wrap('exec_prefix'),
- space.wrap(srcdir))
- return space.newlist([space.wrap(p) for p in path])
def get(space):
return space.fromcache(State)
@@ -90,3 +34,4 @@
(should be removed from interpleveldefs before translation)"""
from pypy.tool.udir import udir
return space.wrap(str(udir))
+
diff --git a/pypy/module/sys/test/test_initialpath.py b/pypy/module/sys/test/test_initpath.py
rename from pypy/module/sys/test/test_initialpath.py
rename to pypy/module/sys/test/test_initpath.py
--- a/pypy/module/sys/test/test_initialpath.py
+++ b/pypy/module/sys/test/test_initpath.py
@@ -1,5 +1,7 @@
import py
-from pypy.module.sys.state import getinitialpath
+import os.path
+from pypy.module.sys.initpath import (compute_stdlib_path, find_executable, find_stdlib,
+ resolvedirof)
from pypy.module.sys.version import PYPY_VERSION, CPYTHON_VERSION
def build_hierarchy(prefix):
@@ -8,15 +10,86 @@
b = prefix.join('lib-python', dirname).ensure(dir=1)
return a, b
+def test_find_stdlib(tmpdir):
+ bin_dir = tmpdir.join('bin').ensure(dir=True)
+ pypy = bin_dir.join('pypy').ensure(file=True)
+ build_hierarchy(tmpdir)
+ path, prefix = find_stdlib(None, str(pypy))
+ assert prefix == tmpdir
-def test_stdlib_in_prefix(tmpdir):
+ at py.test.mark.skipif('not hasattr(os, "symlink")')
+def test_find_stdlib_follow_symlink(tmpdir):
+ pypydir = tmpdir.join('opt', 'pypy-xxx')
+ pypy = pypydir.join('bin', 'pypy').ensure(file=True)
+ build_hierarchy(pypydir)
+ pypy_sym = tmpdir.join('pypy_sym')
+ os.symlink(str(pypy), str(pypy_sym))
+ path, prefix = find_stdlib(None, str(pypy_sym))
+ assert prefix == pypydir
+
+
+def test_compute_stdlib_path(tmpdir):
dirs = build_hierarchy(tmpdir)
- path = getinitialpath(None, str(tmpdir))
+ path = compute_stdlib_path(None, str(tmpdir))
# we get at least 'dirs', and maybe more (e.g. plat-linux2)
assert path[:len(dirs)] == map(str, dirs)
def test_include_libtk(tmpdir):
lib_pypy, lib_python = build_hierarchy(tmpdir)
lib_tk = lib_python.join('lib-tk')
- path = getinitialpath(None, str(tmpdir))
+ path = compute_stdlib_path(None, str(tmpdir))
assert lib_tk in path
+
+
+def test_find_executable(tmpdir, monkeypatch):
+ from pypy.module.sys import initpath
+ # /tmp/a/pypy
+ # /tmp/b/pypy
+ # /tmp/c
+ a = tmpdir.join('a').ensure(dir=True)
+ b = tmpdir.join('b').ensure(dir=True)
+ c = tmpdir.join('c').ensure(dir=True)
+ a.join('pypy').ensure(file=True)
+ b.join('pypy').ensure(file=True)
+ #
+ # if there is already a slash, don't do anything
+ monkeypatch.chdir(tmpdir)
+ assert find_executable('a/pypy') == a.join('pypy')
+ #
+ # if path is None, try abspath (if the file exists)
+ monkeypatch.setenv('PATH', None)
+ monkeypatch.chdir(a)
+ assert find_executable('pypy') == a.join('pypy')
+ monkeypatch.chdir(tmpdir) # no pypy there
+ assert find_executable('pypy') == ''
+ #
+ # find it in path
+ monkeypatch.setenv('PATH', str(a))
+ assert find_executable('pypy') == a.join('pypy')
+ #
+ # find it in the first dir in path
+ monkeypatch.setenv('PATH', '%s%s%s' % (b, os.pathsep, a))
+ assert find_executable('pypy') == b.join('pypy')
+ #
+ # find it in the second, because in the first it's not there
+ monkeypatch.setenv('PATH', '%s%s%s' % (c, os.pathsep, a))
+ assert find_executable('pypy') == a.join('pypy')
+ # if pypy is found but it's not a file, ignore it
+ c.join('pypy').ensure(dir=True)
+ assert find_executable('pypy') == a.join('pypy')
+ #
+ monkeypatch.setattr(initpath, 'we_are_translated', lambda: True)
+ monkeypatch.setattr(initpath, 'IS_WINDOWS', True)
+ monkeypatch.setenv('PATH', str(a))
+ a.join('pypy.exe').ensure(file=True)
+ assert find_executable('pypy') == a.join('pypy.exe')
+
+def test_resolvedirof(tmpdir):
+ foo = tmpdir.join('foo').ensure(dir=True)
+ bar = tmpdir.join('bar').ensure(dir=True)
+ myfile = foo.join('myfile').ensure(file=True)
+ assert resolvedirof(str(myfile)) == foo
+ if hasattr(myfile, 'mksymlinkto'):
+ myfile2 = bar.join('myfile')
+ myfile2.mksymlinkto(myfile)
+ assert resolvedirof(str(myfile2)) == foo
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -161,6 +161,8 @@
if isinstance(x, r_singlefloat):
self._wrap_not_rpython(x)
if isinstance(x, list):
+ if x == []: # special case: it is used e.g. in sys/__init__.py
+ return w_some_obj()
self._wrap_not_rpython(x)
return w_some_obj()
wrap._annspecialcase_ = "specialize:argtype(1)"
diff --git a/pypy/rlib/rpath.py b/pypy/rlib/rpath.py
new file mode 100644
--- /dev/null
+++ b/pypy/rlib/rpath.py
@@ -0,0 +1,20 @@
+"""
+Minimal (and limited) RPython version of some functions contained in os.path.
+"""
+
+import os.path
+from pypy.rlib import rposix
+
+if os.name == 'posix':
+ # the posix version is already RPython, just use it
+ rabspath = os.path.abspath
+elif os.name == 'nt':
+ def rabspath(path):
+ if path == '':
+ path = os.getcwd()
+ try:
+ return rposix._getfullpathname(path)
+ except OSError:
+ return path
+else:
+ raise ImportError('Unsupported os: %s' % os.name)
diff --git a/pypy/rlib/rwin32.py b/pypy/rlib/rwin32.py
--- a/pypy/rlib/rwin32.py
+++ b/pypy/rlib/rwin32.py
@@ -367,6 +367,14 @@
'GetCurrentProcessId', [], DWORD)
def GetCurrentProcessId():
return rffi.cast(lltype.Signed, _GetCurrentProcessId())
+
+ _GetConsoleCP = winexternal('GetConsoleCP', [], DWORD)
+ _GetConsoleOutputCP = winexternal('GetConsoleOutputCP', [], DWORD)
+ def GetConsoleCP():
+ return rffi.cast(lltype.Signed, _GetConsoleCP())
+ def GetConsoleOutputCP():
+ return rffi.cast(lltype.Signed, _GetConsoleOutputCP())
+
def os_kill(pid, sig):
if sig == CTRL_C_EVENT or sig == CTRL_BREAK_EVENT:
if GenerateConsoleCtrlEvent(sig, pid) == 0:
diff --git a/pypy/rlib/test/test_rpath.py b/pypy/rlib/test/test_rpath.py
new file mode 100644
--- /dev/null
+++ b/pypy/rlib/test/test_rpath.py
@@ -0,0 +1,18 @@
+import py
+import os
+from pypy.rlib import rpath
+
+IS_WINDOWS = os.name == 'nt'
+
+def test_rabspath_relative(tmpdir):
+ tmpdir.chdir()
+ assert rpath.rabspath('foo') == tmpdir.join('foo')
+
+ at py.test.mark.skipif("IS_WINDOWS")
+def test_rabspath_absolute_posix():
+ assert rpath.rabspath('/foo') == '/foo'
+
+ at py.test.mark.skipif("not IS_WINDOWS")
+def test_rabspath_absolute_nt():
+ curdrive, _ = os.path.splitdrive(os.getcwd())
+ assert rpath.rabspath('\\foo') == '%s\\foo' % curdrive
diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py
--- a/pypy/rpython/module/ll_os.py
+++ b/pypy/rpython/module/ll_os.py
@@ -1114,7 +1114,7 @@
def os_getcwd_oofakeimpl():
return OOSupport.to_rstr(os.getcwd())
- return extdef([], str,
+ return extdef([], str0,
"ll_os.ll_os_getcwd", llimpl=os_getcwd_llimpl,
oofakeimpl=os_getcwd_oofakeimpl)
diff --git a/pypy/rpython/module/ll_os_environ.py b/pypy/rpython/module/ll_os_environ.py
--- a/pypy/rpython/module/ll_os_environ.py
+++ b/pypy/rpython/module/ll_os_environ.py
@@ -67,7 +67,7 @@
rffi.free_charp(l_name)
return result
-register_external(r_getenv, [str0], annmodel.SomeString(can_be_None=True),
+register_external(r_getenv, [str0], annmodel.SomeString(can_be_None=True, no_nul=True),
export_name='ll_os.ll_os_getenv',
llimpl=getenv_llimpl)
diff --git a/pypy/test_all.py b/pypy/test_all.py
--- a/pypy/test_all.py
+++ b/pypy/test_all.py
@@ -3,9 +3,16 @@
PyPy Test runner interface
--------------------------
-Running test_all.py is equivalent to running py.test
-which you independently install, see
-http://pytest.org/getting-started.html
+Running pytest.py starts py.test, the testing tool
+we use in PyPy. It is distributed along with PyPy,
+but you may get more information about it at
+http://pytest.org/.
+
+Note that it makes no sense to run all tests at once.
+You need to pick a particular subdirectory and run
+
+ cd pypy/.../test
+ ../../../pytest.py [options]
For more information, use test_all.py -h.
"""
diff --git a/pypy/translator/goal/app_main.py b/pypy/translator/goal/app_main.py
--- a/pypy/translator/goal/app_main.py
+++ b/pypy/translator/goal/app_main.py
@@ -210,70 +210,19 @@
# ____________________________________________________________
# Main entry point
-# see nanos.py for explainment why we do not import os here
-# CAUTION!
-# remember to update nanos.py if you are using more functions
-# from os or os.path!
-# Running test/test_nanos.py might be helpful as well.
-
def we_are_translated():
# app-level, very different from pypy.rlib.objectmodel.we_are_translated
return hasattr(sys, 'pypy_translation_info')
if 'nt' in sys.builtin_module_names:
IS_WINDOWS = True
- DRIVE_LETTER_SEP = ':'
else:
IS_WINDOWS = False
-def get_library_path(executable):
- search = executable
- while 1:
- dirname = resolvedirof(search)
- if dirname == search:
- # not found! let's hope that the compiled-in path is ok
- print("""\
-debug: WARNING: Library path not found, using compiled-in sys.path.
-debug: WARNING: 'sys.prefix' will not be set.
-debug: WARNING: Make sure the pypy binary is kept inside its tree of files.
-debug: WARNING: It is ok to create a symlink to it from somewhere else.""",
- file=sys.stderr)
- newpath = sys.path[:]
- break
- newpath = sys.pypy_initial_path(dirname)
- if newpath is None:
- search = dirname # walk to the parent directory
- continue
- break # found!
- return newpath
-def setup_sys_executable(executable, nanos):
- # a substituted os if we are translated
- global os
- os = nanos
- # find the full path to the executable, assuming that if there is no '/'
- # in the provided one then we must look along the $PATH
- if we_are_translated() and IS_WINDOWS and not executable.lower().endswith('.exe'):
- executable += '.exe'
- if os.sep in executable or (IS_WINDOWS and DRIVE_LETTER_SEP in executable):
- pass # the path is already more than just an executable name
- else:
- path = os.getenv('PATH')
- if path:
- for dir in path.split(os.pathsep):
- fn = os.path.join(dir, executable)
- if os.path.isfile(fn):
- executable = fn
- break
- sys.executable = os.path.abspath(executable)
- #
- # 'sys.executable' should not end up being an non-existing file;
- # just use '' in this case. (CPython issue #7774)
- if not os.path.isfile(sys.executable):
- sys.executable = ''
-
-def setup_initial_paths(ignore_environment=False, **extra):
- newpath = get_library_path(sys.executable)
+def setup_and_fix_paths(ignore_environment=False, **extra):
+ import os
+ newpath = sys.path[:]
readenv = not ignore_environment
path = readenv and os.getenv('PYTHONPATH')
if path:
@@ -340,23 +289,30 @@
line_buffering=line_buffering)
return stream
-def set_io_encoding(io_encoding):
+def set_io_encoding(io_encoding, io_encoding_output, errors, overridden):
try:
import _file
except ImportError:
if sys.version_info < (2, 7):
return
- import ctypes # HACK: while running on top of CPython
+ # HACK: while running on top of CPython, and make sure to import
+ # CPython's ctypes (because at this point sys.path has already been
+ # set to the pypy one)
+ pypy_path = sys.path
+ try:
+ sys.path = sys.cpython_path
+ import ctypes
+ finally:
+ sys.path = pypy_path
set_file_encoding = ctypes.pythonapi.PyFile_SetEncodingAndErrors
set_file_encoding.argtypes = [ctypes.py_object, ctypes.c_char_p, ctypes.c_char_p]
else:
set_file_encoding = _file.set_file_encoding
- if ":" in io_encoding:
- encoding, errors = io_encoding.split(":", 1)
- else:
- encoding, errors = io_encoding, None
- for f in [sys.stdin, sys.stdout, sys.stderr]:
- set_file_encoding(f, encoding, errors)
+ for f, encoding in [(sys.stdin, io_encoding),
+ (sys.stdout, io_encoding_output),
+ (sys.stderr, io_encoding_output)]:
+ if isinstance(f, file) and (overridden or f.isatty()):
+ set_file_encoding(f, encoding, errors)
# Order is significant!
sys_flags = (
@@ -470,6 +426,7 @@
def parse_command_line(argv):
+ import os
options = default_options.copy()
options['warnoptions'] = []
#
@@ -554,6 +511,7 @@
# but we need more in the translated PyPy for the compiler package
if '__pypy__' not in sys.builtin_module_names:
sys.setrecursionlimit(5000)
+ import os
readenv = not ignore_environment
io_encoding = readenv and os.getenv("PYTHONIOENCODING")
@@ -568,6 +526,22 @@
except:
print("'import site' failed", file=sys.stderr)
+ readenv = not ignore_environment
+ io_encoding = readenv and os.getenv("PYTHONIOENCODING")
+ if io_encoding:
+ errors = None
+ if ":" in io_encoding:
+ io_encoding, errors = io_encoding.split(":", 1)
+ set_io_encoding(io_encoding, io_encoding, errors, True)
+ else:
+ if IS_WINDOWS:
+ import __pypy__
+ io_encoding, io_encoding_output = __pypy__.get_console_cp()
+ else:
+ io_encoding = io_encoding_output = sys.getfilesystemencoding()
+ if io_encoding:
+ set_io_encoding(io_encoding, io_encoding_output, None, False)
+
pythonwarnings = readenv and os.getenv('PYTHONWARNINGS')
if pythonwarnings:
warnoptions.extend(pythonwarnings.split(','))
@@ -665,7 +639,7 @@
# on the command-line.
filename = sys.argv[0]
mainmodule.__file__ = filename
- sys.path.insert(0, resolvedirof(filename))
+ sys.path.insert(0, sys.pypy_resolvedirof(filename))
# assume it's a pyc file only if its name says so.
# CPython goes to great lengths to detect other cases
# of pyc file format, but I think it's ok not to care.
@@ -714,28 +688,46 @@
return status
-def resolvedirof(filename):
- try:
- filename = os.path.abspath(filename)
- except OSError:
- pass
- dirname = os.path.dirname(filename)
- if os.path.islink(filename):
- try:
- link = os.readlink(filename)
- except OSError:
- pass
- else:
- return resolvedirof(os.path.join(dirname, link))
- return dirname
-
def print_banner():
print('Python %s on %s' % (sys.version, sys.platform))
print('Type "help", "copyright", "credits" or '
'"license" for more information.')
-def entry_point(executable, argv, nanos):
- setup_sys_executable(executable, nanos)
+STDLIB_WARNING = """\
+debug: WARNING: Library path not found, using compiled-in sys.path.
+debug: WARNING: 'sys.prefix' will not be set.
+debug: WARNING: Make sure the pypy binary is kept inside its tree of files.
+debug: WARNING: It is ok to create a symlink to it from somewhere else."""
+
+def setup_bootstrap_path(executable):
+ """
+ Try to to as little as possible and to have the stdlib in sys.path. In
+ particular, we cannot use any unicode at this point, because lots of
+ unicode operations require to be able to import encodings.
+ """
+ # at this point, sys.path is set to the compiled-in one, based on the
+ # location where pypy was compiled. This is set during the objspace
+ # initialization by module.sys.state.State.setinitialpath.
+ #
+ # Now, we try to find the absolute path of the executable and the stdlib
+ # path
+ executable = sys.pypy_find_executable(executable)
+ stdlib_path = sys.pypy_find_stdlib(executable)
+ if stdlib_path is None:
+ print >> sys.stderr, STDLIB_WARNING
+ else:
+ sys.path[:] = stdlib_path
+ # from this point on, we are free to use all the unicode stuff we want,
+ # This is important for py3k
+ sys.executable = executable
+
+def entry_point(executable, argv):
+ # note that before calling setup_bootstrap_path, we are limited because we
+ # cannot import stdlib modules. In particular, we cannot use unicode
+ # stuffs (because we need to be able to import encodings) and we cannot
+ # import os, which is used a bit everywhere in app_main, but only imported
+ # *after* setup_bootstrap_path
+ setup_bootstrap_path(executable)
try:
cmdline = parse_command_line(argv)
except CommandLineError as e:
@@ -744,14 +736,22 @@
return 2
except SystemExit as e:
return e.code or 0
- setup_initial_paths(**cmdline)
+ setup_and_fix_paths(**cmdline)
return run_command_line(**cmdline)
if __name__ == '__main__':
"For unit tests only"
- import os as nanos
- import os
+ import autopath
+ # we need to import pypy.translator.platform early, before we start to
+ # mess up the env variables. In particular, during the import we spawn a
+ # couple of processes which gets confused if PYTHONINSPECT is set (e.g.,
+ # hg to get the version and the hack in tool.runsubprocess to prevent
+ # out-of-memory for late os.fork())
+ import pypy.translator.platform
+ # obscure! try removing the following line, see how it crashes, and
+ # guess why...
+ ImStillAroundDontForgetMe = sys.modules['__main__']
if len(sys.argv) > 1 and sys.argv[1] == '--argparse-only':
import io
@@ -780,13 +780,24 @@
# To avoid this we make a copy of our __main__ module.
sys.modules['__cpython_main__'] = sys.modules['__main__']
- def pypy_initial_path(s):
- from os.path import abspath, join, dirname as dn
- thisfile = os.path.abspath(__file__)
- root = dn(dn(dn(dn(thisfile))))
- return [join(root, 'lib-python', '3.2'),
- join(root, 'lib_pypy')]
- sys.pypy_initial_path = pypy_initial_path
+ # debugging only
+ def pypy_find_executable(s):
+ from pypy.module.sys.initpath import find_executable
+ return find_executable(s)
+
+ def pypy_find_stdlib(s):
+ from pypy.module.sys.initpath import find_stdlib
+ path, prefix = find_stdlib(None, s)
+ if path is None:
+ return None
+ # contrarily to the interp-level version, we don't set sys.prefix
+ # here, else CPythno stops to work (and e.g. test_proper_sys_path
+ # fails)
+ return path
+
+ def pypy_resolvedirof(s):
+ from pypy.module.sys.initpath import resolvedirof
+ return resolvedirof(s)
# add an emulator for these pypy-only or 2.7-only functions
# (for test_pyc_commandline_argument)
@@ -809,14 +820,16 @@
imp._run_compiled_module = _run_compiled_module
imp._getimporter = _getimporter
- # stick the current sys.path into $PYTHONPATH, so that CPython still
- # finds its own extension modules :-/
import os
- os.environ['PYTHONPATH'] = ':'.join(sys.path)
reset = []
if 'PYTHONINSPECT_' in os.environ:
reset.append(('PYTHONINSPECT', os.environ.get('PYTHONINSPECT', '')))
os.environ['PYTHONINSPECT'] = os.environ['PYTHONINSPECT_']
+ if 'PYTHONWARNINGS_' in os.environ:
+ reset.append(('PYTHONWARNINGS', os.environ.get('PYTHONWARNINGS', '')))
+ os.environ['PYTHONWARNINGS'] = os.environ['PYTHONWARNINGS_']
+ del os # make sure that os is not available globally, because this is what
+ # happens in "real life" outside the tests
# no one should change to which lists sys.argv and sys.path are bound
old_argv = sys.argv
@@ -825,8 +838,13 @@
old_streams = sys.stdin, sys.stdout, sys.stderr
del sys.stdin, sys.stdout, sys.stderr
+ sys.pypy_find_executable = pypy_find_executable
+ sys.pypy_find_stdlib = pypy_find_stdlib
+ sys.pypy_resolvedirof = pypy_resolvedirof
+ sys.cpython_path = sys.path[:]
+
try:
- sys.exit(int(entry_point(sys.argv[0], sys.argv[1:], os)))
+ sys.exit(int(entry_point(sys.argv[0], sys.argv[1:])))
finally:
# restore the normal prompt (which was changed by _pypy_interact), in
# case we are dropping to CPython's prompt
diff --git a/pypy/translator/goal/nanos.py b/pypy/translator/goal/nanos.py
deleted file mode 100644
--- a/pypy/translator/goal/nanos.py
+++ /dev/null
@@ -1,290 +0,0 @@
-"""
-Not An os or Nano os :-)
-
-Implementation of a few methods needed for starting up
-PyPy in app_main without importing os there.
-
-At startup time, app_main wants to find out about itself,
-sys.executable, the path to its library etc.
-This is done the easiest using os.py, but since this
-is a python module, we have a recurrence problem.
-
-Importing it at compile time would work, partially,
-but the way os is initialized will cause os.getenv
-to malfunction, due to caching problems.
-
-The solution taken here implements a minimal os using
-app-level code that is created at compile-time. Only
-the few needed functions are implemented in a tiny os module
-that contains a tiny path module.
-
-os.getenv got a direct implementation to overcome the caching
-problem.
-
-Please adjust the applevel code below, if you need to support
-more from os and os.path.
-"""
-
-from pypy.interpreter.gateway import applevel, interp2app
-import os, py
-
-if os.name == 'posix':
- # code copied from posixpath.py
- app_os_path = applevel("""
- def dirname(p):
- i = p.rfind('/') + 1
- head = p[:i]
- if head and head != '/'*len(head):
- head = head.rstrip('/')
- return head
-
- def join(path, b):
- if b.startswith('/'):
- path = b
- elif path == '' or path.endswith('/'):
- path += b
- else:
- path += '/' + b
- return path
-
- def normpath(path):
- if path == '':
- return '.'
- initial_slashes = path.startswith('/')
- # POSIX allows one or two initial slashes, but treats three or more
- # as single slash.
- if (initial_slashes and
- path.startswith('//') and not path.startswith('///')):
- initial_slashes = 2
- comps = path.split('/')
- new_comps = []
- for comp in comps:
- if comp in ('', '.'):
- continue
- if (comp != '..' or (not initial_slashes and not new_comps) or
- (new_comps and new_comps[-1] == '..')):
- new_comps.append(comp)
- elif new_comps:
- new_comps.pop()
- comps = new_comps
- path = '/'.join(comps)
- if initial_slashes:
- path = '/'*initial_slashes + path
- return path or '.'
-
- def abspath(path):
- if not path.startswith('/'):
- import posix
- cwd = posix.getcwd()
- path = join(cwd, path)
- return normpath(path)
-
- def isfile(path):
- import posix
- try:
- st = posix.stat(path)
- except posix.error:
- return False
- return (st.st_mode & 0o170000) == 0o100000 # S_ISREG
-
- def islink(path):
- import posix
- try:
- st = posix.lstat(path)
- except posix.error:
- return False
- return (st.st_mode & 0o170000) == 0o120000 # S_ISLNK
-
- """, filename=__file__)
-
- app_os = applevel("""
- sep = '/'
- pathsep = ':'
- name = 'posix'
-
- def readlink(fn):
- import posix
- return posix.readlink(fn)
- """, filename=__file__)
-
-elif os.name == 'nt':
- # code copied from ntpath.py
- app_os_path = applevel(r"""
- def splitdrive(p):
- if p[1:2] == ':':
- return p[0:2], p[2:]
- return '', p
-
- def isabs(s):
- s = splitdrive(s)[1]
- return s != '' and s[:1] in '/\\'
-
- def dirname(p):
- d, p = splitdrive(p)
- # set i to index beyond p's last slash
- i = len(p)
- while i and p[i-1] not in '/\\':
- i = i - 1
- head = p[:i]
- # remove trailing slashes from head, unless it's all slashes
- head2 = head
- while head2 and head2[-1] in '/\\':
- head2 = head2[:-1]
- head = head2 or head
- return d + head
-
- def join(path, b):
- b_wins = 0 # set to 1 iff b makes path irrelevant
- if path == "":
- b_wins = 1
-
- elif isabs(b):
- # This probably wipes out path so far. However, it's more
- # complicated if path begins with a drive letter:
- # 1. join('c:', '/a') == 'c:/a'
- # 2. join('c:/', '/a') == 'c:/a'
- # But
- # 3. join('c:/a', '/b') == '/b'
- # 4. join('c:', 'd:/') = 'd:/'
- # 5. join('c:/', 'd:/') = 'd:/'
- if path[1:2] != ":" or b[1:2] == ":":
- # Path doesn't start with a drive letter, or cases 4 and 5.
- b_wins = 1
-
- # Else path has a drive letter, and b doesn't but is absolute.
- elif len(path) > 3 or (len(path) == 3 and
- path[-1] not in "/\\"):
- # case 3
- b_wins = 1
-
- if b_wins:
- path = b
- else:
- # Join, and ensure there's a separator.
- assert len(path) > 0
- if path[-1] in "/\\":
- if b and b[0] in "/\\":
- path += b[1:]
- else:
- path += b
- elif path[-1] == ":":
- path += b
- elif b:
- if b[0] in "/\\":
- path += b
- else:
- path += "\\" + b
- else:
- # path is not empty and does not end with a backslash,
- # but b is empty; since, e.g., split('a/') produces
- # ('a', ''), it's best if join() adds a backslash in
- # this case.
- path += '\\'
-
- return path
-
- def normpath(path):
- if path.startswith(('\\\\.\\', '\\\\?\\')):
- # in the case of paths with these prefixes:
- # \\.\ -> device names
- # \\?\ -> literal paths
- # do not do any normalization, but return the path unchanged
- return path
- path = path.replace('/', '\\')
- prefix, path = splitdrive(path)
- # We need to be careful here. If the prefix is empty, and
- # the path starts with a backslash, it could either be an
- # absolute path on the current drive (\dir1\dir2\file) or a
- # UNC filename (\\server\mount\dir1\file). It is therefore
- # imperative NOT to collapse multiple backslashes blindly in
- # that case. The code below preserves multiple backslashes
- # when there is no drive letter. This means that the invalid
- # filename \\\a\b is preserved unchanged, where a\\\b is
- # normalised to a\b. It's not clear that there is any better
- # behaviour for such edge cases.
- if prefix == '':
- # No drive letter - preserve initial backslashes
- while path[:1] == "\\":
- prefix = prefix + '\\'
- path = path[1:]
- else:
- # We have a drive letter - collapse initial backslashes
- if path.startswith("\\"):
- prefix = prefix + '\\'
- path = path.lstrip("\\")
- comps = path.split("\\")
- i = 0
- while i < len(comps):
- if comps[i] in ('.', ''):
- del comps[i]
- elif comps[i] == '..':
- if i > 0 and comps[i-1] != '..':
- del comps[i-1:i+1]
- i -= 1
- elif i == 0 and prefix.endswith("\\"):
- del comps[i]
- else:
- i += 1
- else:
- i += 1
- # If the path is now empty, substitute '.'
- if not prefix and not comps:
- comps.append('.')
- return prefix + '\\'.join(comps)
-
- def abspath(path):
- import nt
- if path: # Empty path must return current working directory.
- try:
- path = nt._getfullpathname(path)
- except WindowsError:
- pass # Bad path - return unchanged.
- else:
- path = nt.getcwd()
- return normpath(path)
-
- def isfile(path):
- import nt
- try:
- st = nt.stat(path)
- except nt.error:
- return False
- return (st.st_mode & 0o170000) == 0o100000 # S_ISREG
-
- def islink(path):
- return False
-
- """, filename=__file__)
-
- app_os = applevel(r"""
- sep = '\\'
- pathsep = ';'
- name = 'nt'
- """, filename=__file__)
-
-else:
- raise NotImplementedError("os.name == %r" % (os.name,))
-
-def getenv(space, w_name):
- name = space.str0_w(w_name)
- return space.wrap(os.environ.get(name))
-getenv_w = interp2app(getenv)
-
-def setup_nanos(space):
- w_os = space.wrap(app_os.buildmodule(space, 'os'))
- w_os_path = space.wrap(app_os_path.buildmodule(space, 'path'))
- space.setattr(w_os, space.wrap('path'), w_os_path)
- space.setattr(w_os, space.wrap('getenv'), space.wrap(getenv_w))
- return w_os
-
-
-# in order to be able to test app_main without the pypy interpreter
-# we create the nanos module with the same names here like it would
-# be created while translation
-path_module_for_testing = type(os)("os.path")
-os_module_for_testing = type(os)("os")
-os_module_for_testing.path = path_module_for_testing
-os_module_for_testing.getenv = os.getenv
-eval(py.code.Source(app_os_path.source).compile(), path_module_for_testing.__dict__)
-eval(py.code.Source(app_os.source).compile(), os_module_for_testing.__dict__)
-
diff --git a/pypy/translator/goal/targetpypystandalone.py b/pypy/translator/goal/targetpypystandalone.py
--- a/pypy/translator/goal/targetpypystandalone.py
+++ b/pypy/translator/goal/targetpypystandalone.py
@@ -8,7 +8,6 @@
from pypy.config.config import Config, to_optparse, make_dict, SUPPRESS_USAGE
from pypy.config.config import ConflictConfigError
from pypy.tool.option import make_objspace
-from pypy.translator.goal.nanos import setup_nanos
thisdir = py.path.local(__file__).dirpath()
@@ -27,7 +26,6 @@
w_run_toplevel = space.getitem(w_dict, space.wrap('run_toplevel'))
w_call_finish_gateway = space.wrap(gateway.interp2app(call_finish))
w_call_startup_gateway = space.wrap(gateway.interp2app(call_startup))
- w_os = setup_nanos(space)
withjit = space.config.objspace.usemodules.pypyjit
def entry_point(argv):
@@ -56,7 +54,7 @@
w_executable = space.wrap(argv[0])
w_argv = space.newlist([space.wrap(s) for s in argv[1:]])
space.timer.start("w_entry_point")
- w_exitcode = space.call_function(w_entry_point, w_executable, w_argv, w_os)
+ w_exitcode = space.call_function(w_entry_point, w_executable, w_argv)
space.timer.stop("w_entry_point")
exitcode = space.int_w(w_exitcode)
# try to pull it all in
diff --git a/pypy/translator/goal/test2/test_app_main.py b/pypy/translator/goal/test2/test_app_main.py
--- a/pypy/translator/goal/test2/test_app_main.py
+++ b/pypy/translator/goal/test2/test_app_main.py
@@ -237,7 +237,7 @@
pexpect.__version__,))
kwds.setdefault('timeout', 10)
- print 'SPAWN:', args, kwds
+ print 'SPAWN:', ' '.join([args[0]] + args[1]), kwds
child = pexpect.spawn(*args, **kwds)
child.logfile = sys.stdout
return child
@@ -337,7 +337,8 @@
child.sendline('import sys; sys.argv')
child.expect(re.escape("['-c']"))
- def test_options_i_c_crashing(self):
+ def test_options_i_c_crashing(self, monkeypatch):
+ monkeypatch.setenv('PYTHONPATH', None)
child = self.spawn(['-i', '-c', 'x=666;foobar'])
child.expect('NameError')
idx = child.expect(['>>> ', re.escape(banner)])
@@ -369,28 +370,25 @@
sys.stdin = old
child.expect('foobye')
- def test_pythonstartup(self):
- old = os.environ.get('PYTHONSTARTUP', '')
- try:
- os.environ['PYTHONSTARTUP'] = crashing_demo_script
- child = self.spawn([])
- child.expect(re.escape(banner))
- child.expect('Traceback')
- child.expect('NameError')
- child.expect('>>> ')
- child.sendline('[myvalue2]')
- child.expect(re.escape('[11]'))
- child.expect('>>> ')
+ def test_pythonstartup(self, monkeypatch):
+ monkeypatch.setenv('PYTHONPATH', None)
+ monkeypatch.setenv('PYTHONSTARTUP', crashing_demo_script)
+ child = self.spawn([])
+ child.expect(re.escape(banner))
+ child.expect('Traceback')
+ child.expect('NameError')
+ child.expect('>>> ')
+ child.sendline('[myvalue2]')
+ child.expect(re.escape('[11]'))
+ child.expect('>>> ')
- child = self.spawn(['-i', demo_script])
- for line in ['hello', 'goodbye', '>>> ']:
- idx = child.expect([line, 'Hello2'])
- assert idx == 0 # no PYTHONSTARTUP run here
- child.sendline('myvalue2')
- child.expect('Traceback')
- child.expect('NameError')
- finally:
- os.environ['PYTHONSTARTUP'] = old
+ child = self.spawn(['-i', demo_script])
+ for line in ['hello', 'goodbye', '>>> ']:
+ idx = child.expect([line, 'Hello2'])
+ assert idx == 0 # no PYTHONSTARTUP run here
+ child.sendline('myvalue2')
+ child.expect('Traceback')
+ child.expect('NameError')
def test_ignore_python_startup(self):
old = os.environ.get('PYTHONSTARTUP', '')
@@ -627,26 +625,21 @@
data = self.run('-c "print(6**5)"')
assert '7776' in data
- def test_no_pythonstartup(self):
- old = os.environ.get('PYTHONSTARTUP', '')
- try:
- os.environ['PYTHONSTARTUP'] = crashing_demo_script
- data = self.run('"%s"' % (demo_script,))
- assert 'Hello2' not in data
- data = self.run('-c pass')
- assert 'Hello2' not in data
- finally:
- os.environ['PYTHONSTARTUP'] = old
+ def test_no_pythonstartup(self, monkeypatch):
+ monkeypatch.setenv('PYTHONSTARTUP', crashing_demo_script)
+ data = self.run('"%s"' % (demo_script,))
+ assert 'Hello2' not in data
+ data = self.run('-c pass')
+ assert 'Hello2' not in data
- def test_pythonwarnings(self):
- old = os.environ.get('PYTHONWARNINGS', '')
- try:
- os.environ['PYTHONWARNINGS'] = "once,error"
- data = self.run('-W ignore -W default '
- '-c "import sys; print(sys.warnoptions)"')
- assert "['ignore', 'default', 'once', 'error']" in data
- finally:
- os.environ['PYTHONWARNINGS'] = old
+ def test_pythonwarnings(self, monkeypatch):
+ # PYTHONWARNINGS_ is special cased by app_main: we cannot directly set
+ # PYTHONWARNINGS because else the warnings raised from within pypy are
+ # turned in errors.
+ monkeypatch.setenv('PYTHONWARNINGS_', "once,error")
+ data = self.run('-W ignore -W default '
+ '-c "import sys; print(sys.warnoptions)"')
+ assert "['ignore', 'default', 'once', 'error']" in data
def test_option_m(self):
if not hasattr(runpy, '_run_module_as_main'):
@@ -760,6 +753,7 @@
assert data == p + os.sep + '\n'
def test_getfilesystemencoding(self):
+ py.test.skip("this has been failing since forever, but it's not tested nightly because buildbot uses python2.6 :-(")
if sys.version_info < (2, 7):
skip("test requires Python >= 2.7")
p = getscript_in_dir("""
@@ -841,9 +835,9 @@
class AppTestAppMain:
def setup_class(self):
- # ------------------------------------
- # setup code for test_get_library_path
- # ------------------------------------
+ # ----------------------------------------
+ # setup code for test_setup_bootstrap_path
+ # ----------------------------------------
from pypy.module.sys.version import CPYTHON_VERSION, PYPY_VERSION
cpy_ver = '%d.%d' % CPYTHON_VERSION[:2]
@@ -860,37 +854,58 @@
self.w_fake_exe = self.space.wrap(str(fake_exe))
self.w_expected_path = self.space.wrap(expected_path)
self.w_trunkdir = self.space.wrap(os.path.dirname(autopath.pypydir))
+ #
+ foo_py = prefix.join('foo.py').write("pass")
+ self.w_foo_py = self.space.wrap(str(foo_py))
- def test_get_library_path(self):
+ def test_setup_bootstrap_path(self):
import sys
import os
+ old_sys_path = sys.path[:]
sys.path.append(self.goal_dir)
try:
import app_main
- app_main.os = os
- newpath = app_main.get_library_path('/tmp/pypy-c') # stdlib not found
- assert newpath == sys.path
- newpath = app_main.get_library_path(self.fake_exe)
+ app_main.setup_bootstrap_path('/tmp/pypy-c') # stdlib not found
+ sys.path == old_sys_path
+ assert sys.executable == ''
+ #
+ app_main.setup_bootstrap_path(self.fake_exe)
+ assert sys.executable == self.fake_exe
+ newpath = sys.path[:]
if newpath[0].endswith('__extensions__'):
newpath = newpath[1:]
# we get at least 'expected_path', and maybe more (e.g.plat-linux2)
assert newpath[:len(self.expected_path)] == self.expected_path
finally:
- sys.path.pop()
+ sys.path[:] = old_sys_path
def test_trunk_can_be_prefix(self):
import sys
import os
+ old_sys_path = sys.path[:]
sys.path.append(self.goal_dir)
try:
import app_main
- app_main.os = os
pypy_c = os.path.join(self.trunkdir, 'pypy', 'translator', 'goal', 'pypy-c')
- newpath = app_main.get_library_path(pypy_c)
+ app_main.setup_bootstrap_path(pypy_c)
+ newpath = sys.path[:]
# we get at least lib_pypy
# lib-python/X.Y.Z, and maybe more (e.g. plat-linux2)
assert len(newpath) >= 2
for p in newpath:
assert p.startswith(self.trunkdir)
finally:
- sys.path.pop()
+ sys.path[:] = old_sys_path
+
+ def test_entry_point(self):
+ import sys
+ import os
+ old_sys_path = sys.path[:]
+ sys.path.append(self.goal_dir)
+ try:
+ import app_main
+ pypy_c = os.path.join(self.trunkdir, 'pypy', 'translator', 'goal', 'pypy-c')
+ app_main.entry_point(pypy_c, [self.foo_py])
+ # assert it did not crash
+ finally:
+ sys.path[:] = old_sys_path
diff --git a/pypy/translator/goal/test2/test_nanos.py b/pypy/translator/goal/test2/test_nanos.py
deleted file mode 100644
--- a/pypy/translator/goal/test2/test_nanos.py
+++ /dev/null
@@ -1,85 +0,0 @@
-"""
-Tests for the entry point of pypy-c, whether nanos.py is supplying
-the needed names for app_main.py.
-"""
-import os
-
-from pypy.translator.goal import app_main
-this_dir = os.path.dirname(app_main.__file__)
-
-from pypy.objspace.std import Space
-from pypy.translator.goal.targetpypystandalone import create_entry_point
-from pypy.tool.udir import udir
-
-
-class TestNanos:
- def getnanos(self):
- from pypy.translator.goal.nanos import os_module_for_testing
- return os_module_for_testing
-
- def test_exists(self):
- os1 = self.getnanos()
- assert os1.name == os.name
- assert os1.sep == os.sep
- assert os1.pathsep == os.pathsep
-
- def test_dirname(self):
- p1 = os.path
- p2 = self.getnanos().path
- path = str(udir.join('baz'))
- assert p1.dirname(path) == p2.dirname(path)
- assert p1.dirname(path + os.sep) == p2.dirname(path + os.sep)
- assert p1.dirname(path + 2*os.sep) == p2.dirname(path + 2*os.sep)
- assert p1.dirname(p1.dirname(path)) == p2.dirname(p2.dirname(path))
-
- def test_join(self):
- p1 = os.path
- p2 = self.getnanos().path
- base = str(udir)
- assert p1.join(base, '') == p2.join(base, '')
- assert p1.join(base, 'baz') == p2.join(base, 'baz')
- assert p1.join(base + os.sep, 'baz') == p2.join(base + os.sep, 'baz')
- assert p1.join(base, 'baz' + os.sep) == p2.join(base, 'baz' + os.sep)
- assert p1.join(base, base) == p2.join(base, base)
-
- def test_abspath(self):
- p2 = self.getnanos().path
- base = str(udir)
- assert p2.abspath(base) == base
- assert p2.abspath('x') == os.path.join(os.getcwd(), 'x')
-
- def test_abspath_uses_normpath(self):
- p1 = os.path
- p2 = self.getnanos().path
- base = str(udir)
- assert p2.abspath(p1.join(base, '.')) == base
- assert p2.abspath(p1.join(base, '.', '.', '.')) == base
- assert p2.abspath(p1.join(base, 'foo', '..')) == base
-
- def test_isfile(self):
- p2 = self.getnanos().path
- udir.join('test_isfile').write('\n')
- base = str(udir)
- assert p2.isfile(p2.join(base, 'test_isfile'))
- assert not p2.isfile(p2.join(base, 'test_isfile.DOES.NOT.EXIST'))
- assert not p2.isfile(base)
-
-
-def test_nanos():
- space = Space()
- # manually imports app_main.py
- filename = os.path.join(this_dir, 'app_main.py')
- w_dict = space.newdict()
- space.exec_(open(filename).read(), w_dict, w_dict)
- entry_point = create_entry_point(space, w_dict)
-
- # check that 'os' is not in sys.modules
- assert not space.is_true(
- space.call_method(space.sys.get('modules'),
- '__contains__', space.wrap('os')))
- # But that 'sys' is still present
- assert space.is_true(
- space.call_method(space.sys.get('modules'),
- '__contains__', space.wrap('sys')))
-
- entry_point(['', '-c', 'print 42'])
diff --git a/pytest.py b/pytest.py
--- a/pytest.py
+++ b/pytest.py
@@ -1,6 +1,20 @@
#!/usr/bin/env python
"""
-unit and functional testing with Python.
+PyPy Test runner interface
+--------------------------
+
+Running pytest.py starts py.test, the testing tool
+we use in PyPy. It is distributed along with PyPy,
+but you may get more information about it at
+http://pytest.org/.
+
+Note that it makes no sense to run all tests at once.
+You need to pick a particular subdirectory and run
+
+ cd pypy/.../test
+ ../../../pytest.py [options]
+
+For more information, use pytest.py -h.
"""
__all__ = ['main']
@@ -23,6 +37,11 @@
from _pytest import __version__
if __name__ == '__main__': # if run as a script or by 'python -m pytest'
+ import os
+ if len(sys.argv) == 1 and os.path.dirname(sys.argv[0]) in '.':
+ print >> sys.stderr, __doc__
+ sys.exit(2)
+
#XXX: sync to upstream later
import pytest_cov
raise SystemExit(main(plugins=[pytest_cov]))
More information about the pypy-commit
mailing list