[pypy-commit] pypy default: merge
fijal
noreply at buildbot.pypy.org
Mon Jun 18 15:56:10 CEST 2012
Author: Maciej Fijalkowski <fijall at gmail.com>
Branch:
Changeset: r55705:bc644ee035cd
Date: 2012-06-18 15:55 +0200
http://bitbucket.org/pypy/pypy/changeset/bc644ee035cd/
Log: merge
diff --git a/ctypes_configure/cbuild.py b/ctypes_configure/cbuild.py
--- a/ctypes_configure/cbuild.py
+++ b/ctypes_configure/cbuild.py
@@ -372,7 +372,7 @@
self.library_dirs = list(eci.library_dirs)
self.compiler_exe = compiler_exe
self.profbased = profbased
- if not sys.platform in ('win32', 'darwin'): # xxx
+ if not sys.platform in ('win32', 'darwin', 'cygwin'): # xxx
if 'm' not in self.libraries:
self.libraries.append('m')
if 'pthread' not in self.libraries:
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -39,11 +39,10 @@
If 'prefix' is supplied, use it instead of sys.prefix or
sys.exec_prefix -- i.e., ignore 'plat_specific'.
"""
- if standard_lib:
- raise DistutilsPlatformError(
- "calls to get_python_lib(standard_lib=1) cannot succeed")
if prefix is None:
prefix = PREFIX
+ if standard_lib:
+ return os.path.join(prefix, "lib-python", get_python_version())
return os.path.join(prefix, 'site-packages')
diff --git a/lib-python/stdlib-upgrade.txt b/lib-python/stdlib-upgrade.txt
new file mode 100644
--- /dev/null
+++ b/lib-python/stdlib-upgrade.txt
@@ -0,0 +1,19 @@
+Process for upgrading the stdlib to a new cpython version
+==========================================================
+
+.. note::
+
+ overly detailed
+
+1. check out the branch vendor/stdlib
+2. upgrade the files there
+3. update stdlib-versions.txt with the output of hg -id from the cpython repo
+4. commit
+5. update to default/py3k
+6. create a integration branch for the new stdlib
+ (just hg branch stdlib-$version)
+7. merge vendor/stdlib
+8. commit
+10. fix issues
+11. commit --close-branch
+12. merge to default
diff --git a/lib_pypy/_ctypes/primitive.py b/lib_pypy/_ctypes/primitive.py
--- a/lib_pypy/_ctypes/primitive.py
+++ b/lib_pypy/_ctypes/primitive.py
@@ -249,6 +249,13 @@
self._buffer[0] = value
result.value = property(_getvalue, _setvalue)
+ elif tp == '?': # regular bool
+ def _getvalue(self):
+ return bool(self._buffer[0])
+ def _setvalue(self, value):
+ self._buffer[0] = bool(value)
+ result.value = property(_getvalue, _setvalue)
+
elif tp == 'v': # VARIANT_BOOL type
def _getvalue(self):
return bool(self._buffer[0])
diff --git a/pypy/annotation/binaryop.py b/pypy/annotation/binaryop.py
--- a/pypy/annotation/binaryop.py
+++ b/pypy/annotation/binaryop.py
@@ -659,7 +659,7 @@
def mul((str1, int2)): # xxx do we want to support this
getbookkeeper().count("str_mul", str1, int2)
- return SomeString()
+ return SomeString(no_nul=str1.no_nul)
class __extend__(pairtype(SomeUnicodeString, SomeInteger)):
def getitem((str1, int2)):
diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -2138,6 +2138,15 @@
assert isinstance(s, annmodel.SomeString)
assert s.no_nul
+ def test_mul_str0(self):
+ def f(s):
+ return s*10
+ a = self.RPythonAnnotator()
+ s = a.build_types(f, [annmodel.SomeString(no_nul=True)])
+ assert isinstance(s, annmodel.SomeString)
+ assert s.no_nul
+
+
def test_non_none_and_none_with_isinstance(self):
class A(object):
pass
@@ -3780,6 +3789,26 @@
e = py.test.raises(Exception, a.build_types, f, [])
assert 'object with a __call__ is not RPython' in str(e.value)
+ def test_os_getcwd(self):
+ import os
+ def fn():
+ return os.getcwd()
+ a = self.RPythonAnnotator()
+ s = a.build_types(fn, [])
+ assert isinstance(s, annmodel.SomeString)
+ assert s.no_nul
+
+ def test_os_getenv(self):
+ import os
+ def fn():
+ return os.environ.get('PATH')
+ a = self.RPythonAnnotator()
+ s = a.build_types(fn, [])
+ assert isinstance(s, annmodel.SomeString)
+ assert s.no_nul
+
+
+
def g(n):
return [0,1,2,n]
diff --git a/pypy/bin/py.py b/pypy/bin/py.py
--- a/pypy/bin/py.py
+++ b/pypy/bin/py.py
@@ -89,12 +89,12 @@
space.setitem(space.sys.w_dict, space.wrap('executable'),
space.wrap(argv[0]))
- # call pypy_initial_path: the side-effect is that it sets sys.prefix and
+ # call pypy_find_stdlib: the side-effect is that it sets sys.prefix and
# sys.exec_prefix
- srcdir = os.path.dirname(os.path.dirname(pypy.__file__))
- space.appexec([space.wrap(srcdir)], """(srcdir):
+ executable = argv[0]
+ space.appexec([space.wrap(executable)], """(executable):
import sys
- sys.pypy_initial_path(srcdir)
+ sys.pypy_find_stdlib(executable)
""")
# set warning control options (if any)
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -79,6 +79,7 @@
module_dependencies = {
'_multiprocessing': [('objspace.usemodules.rctime', True),
('objspace.usemodules.thread', True)],
+ 'cpyext': [('objspace.usemodules.array', True)],
}
module_suggests = {
# the reason you want _rawffi is for ctypes, which
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -85,13 +85,6 @@
_winreg
- Note that only some of these modules are built-in in a typical
- CPython installation, and the rest is from non built-in extension
- modules. This means that e.g. ``import parser`` will, on CPython,
- find a local file ``parser.py``, while ``import sys`` will not find a
- local file ``sys.py``. In PyPy the difference does not exist: all
- these modules are built-in.
-
* Supported by being rewritten in pure Python (possibly using ``ctypes``):
see the `lib_pypy/`_ directory. Examples of modules that we
support this way: ``ctypes``, ``cPickle``, ``cmath``, ``dbm``, ``datetime``...
diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -23,7 +23,9 @@
some of the next updates may be done before or after branching; make
sure things are ported back to the trunk and to the branch as
necessary
-* update pypy/doc/contributor.txt (and possibly LICENSE)
+* update pypy/doc/contributor.rst (and possibly LICENSE)
+* rename pypy/doc/whatsnew_head.rst to whatsnew_VERSION.rst
+ and create a fresh whatsnew_head.rst after the release
* update README
* change the tracker to have a new release tag to file bugs against
* go to pypy/tool/release and run:
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-head.rst
@@ -0,0 +1,14 @@
+======================
+What's new in PyPy xxx
+======================
+
+.. this is the revision of the last merge from default to release-1.9.x
+.. startrev: 8d567513d04d
+
+.. branch: default
+.. branch: app_main-refactor
+.. branch: win-ordinal
+
+
+.. "uninteresting" branches that we should just ignore for the whatsnew:
+.. branch: slightly-shorter-c
diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -1105,6 +1105,17 @@
assert isinstance(s, ast.Str)
assert space.eq_w(s.s, space.wrap(sentence))
+ def test_string_bug(self):
+ space = self.space
+ source = '# -*- encoding: utf8 -*-\nstuff = "x \xc3\xa9 \\n"\n'
+ info = pyparse.CompileInfo("<test>", "exec")
+ tree = self.parser.parse_source(source, info)
+ assert info.encoding == "utf8"
+ s = ast_from_node(space, tree, info).body[0].value
+ assert isinstance(s, ast.Str)
+ expected = ['x', ' ', chr(0xc3), chr(0xa9), ' ', '\n']
+ assert space.eq_w(s.s, space.wrap(''.join(expected)))
+
def test_number(self):
def get_num(s):
node = self.get_first_expr(s)
diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -97,7 +97,8 @@
return space.wrap(v)
need_encoding = (encoding is not None and
- encoding != "utf-8" and encoding != "iso-8859-1")
+ encoding != "utf-8" and encoding != "utf8" and
+ encoding != "iso-8859-1")
assert 0 <= ps <= q
substr = s[ps : q]
if rawmode or '\\' not in s[ps:]:
@@ -129,19 +130,18 @@
builder = StringBuilder(len(s))
ps = 0
end = len(s)
- while 1:
- ps2 = ps
- while ps < end and s[ps] != '\\':
+ while ps < end:
+ if s[ps] != '\\':
+ # note that the C code has a label here.
+ # the logic is the same.
if recode_encoding and ord(s[ps]) & 0x80:
w, ps = decode_utf8(space, s, ps, end, recode_encoding)
+ # Append bytes to output buffer.
builder.append(w)
- ps2 = ps
else:
+ builder.append(s[ps])
ps += 1
- if ps > ps2:
- builder.append_slice(s, ps2, ps)
- if ps == end:
- break
+ continue
ps += 1
if ps == end:
diff --git a/pypy/interpreter/pyparser/test/test_parsestring.py b/pypy/interpreter/pyparser/test/test_parsestring.py
--- a/pypy/interpreter/pyparser/test/test_parsestring.py
+++ b/pypy/interpreter/pyparser/test/test_parsestring.py
@@ -84,3 +84,10 @@
s = '"""' + '\\' + '\n"""'
w_ret = parsestring.parsestr(space, None, s)
assert space.str_w(w_ret) == ''
+
+ def test_bug1(self):
+ space = self.space
+ expected = ['x', ' ', chr(0xc3), chr(0xa9), ' ', '\n']
+ input = ["'", 'x', ' ', chr(0xc3), chr(0xa9), ' ', chr(92), 'n', "'"]
+ w_ret = parsestring.parsestr(space, 'utf8', ''.join(input))
+ assert space.str_w(w_ret) == ''.join(expected)
diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -659,10 +659,11 @@
def _check_valid_gc(self):
# we need the hybrid or minimark GC for rgc._make_sure_does_not_move()
- # to work
- if self.gcdescr.config.translation.gc not in ('hybrid', 'minimark'):
+ # to work. Additionally, 'hybrid' is missing some stuff like
+ # jit_remember_young_pointer() for now.
+ if self.gcdescr.config.translation.gc not in ('minimark',):
raise NotImplementedError("--gc=%s not implemented with the JIT" %
- (gcdescr.config.translation.gc,))
+ (self.gcdescr.config.translation.gc,))
def _make_gcrootmap(self):
# to find roots in the assembler, make a GcRootMap
diff --git a/pypy/jit/backend/llsupport/test/test_gc.py b/pypy/jit/backend/llsupport/test/test_gc.py
--- a/pypy/jit/backend/llsupport/test/test_gc.py
+++ b/pypy/jit/backend/llsupport/test/test_gc.py
@@ -296,7 +296,7 @@
class TestFramework(object):
- gc = 'hybrid'
+ gc = 'minimark'
def setup_method(self, meth):
class config_(object):
diff --git a/pypy/jit/backend/llsupport/test/test_rewrite.py b/pypy/jit/backend/llsupport/test/test_rewrite.py
--- a/pypy/jit/backend/llsupport/test/test_rewrite.py
+++ b/pypy/jit/backend/llsupport/test/test_rewrite.py
@@ -205,7 +205,7 @@
def setup_method(self, meth):
class config_(object):
class translation(object):
- gc = 'hybrid'
+ gc = 'minimark'
gcrootfinder = 'asmgcc'
gctransformer = 'framework'
gcremovetypeptr = False
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -1110,6 +1110,79 @@
def test_virtual_ref_finish(self):
pass # VIRTUAL_REF_FINISH must not reach the backend nowadays
+ def test_arguments_to_execute_token(self):
+ # this test checks that execute_token() can be called with any
+ # variant of ints and floats as arguments
+ if self.cpu.supports_floats:
+ numkinds = 2
+ else:
+ numkinds = 1
+ seed = random.randrange(0, 10000)
+ print 'Seed is', seed # or choose it by changing the previous line
+ r = random.Random()
+ r.seed(seed)
+ for nb_args in range(50):
+ print 'Passing %d arguments to execute_token...' % nb_args
+ #
+ inputargs = []
+ values = []
+ for k in range(nb_args):
+ kind = r.randrange(0, numkinds)
+ if kind == 0:
+ inputargs.append(BoxInt())
+ values.append(r.randrange(-100000, 100000))
+ else:
+ inputargs.append(BoxFloat())
+ values.append(longlong.getfloatstorage(r.random()))
+ #
+ looptoken = JitCellToken()
+ faildescr = BasicFailDescr(42)
+ operations = []
+ retboxes = []
+ retvalues = []
+ #
+ ks = range(nb_args)
+ random.shuffle(ks)
+ for k in ks:
+ if isinstance(inputargs[k], BoxInt):
+ newbox = BoxInt()
+ x = r.randrange(-100000, 100000)
+ operations.append(
+ ResOperation(rop.INT_ADD, [inputargs[k],
+ ConstInt(x)], newbox)
+ )
+ y = values[k] + x
+ else:
+ newbox = BoxFloat()
+ x = r.random()
+ operations.append(
+ ResOperation(rop.FLOAT_ADD, [inputargs[k],
+ constfloat(x)], newbox)
+ )
+ y = longlong.getrealfloat(values[k]) + x
+ y = longlong.getfloatstorage(y)
+ kk = r.randrange(0, len(retboxes)+1)
+ retboxes.insert(kk, newbox)
+ retvalues.insert(kk, y)
+ #
+ operations.append(
+ ResOperation(rop.FINISH, retboxes, None, descr=faildescr)
+ )
+ print inputargs
+ for op in operations:
+ print op
+ self.cpu.compile_loop(inputargs, operations, looptoken)
+ #
+ fail = self.cpu.execute_token(looptoken, *values)
+ assert fail.identifier == 42
+ #
+ for k in range(len(retvalues)):
+ if isinstance(retboxes[k], BoxInt):
+ got = self.cpu.get_latest_value_int(k)
+ else:
+ got = self.cpu.get_latest_value_float(k)
+ assert got == retvalues[k]
+
def test_jump(self):
# this test generates small loops where the JUMP passes many
# arguments of various types, shuffling them around.
diff --git a/pypy/jit/backend/x86/tool/viewcode.py b/pypy/jit/backend/x86/tool/viewcode.py
--- a/pypy/jit/backend/x86/tool/viewcode.py
+++ b/pypy/jit/backend/x86/tool/viewcode.py
@@ -253,7 +253,7 @@
self.logentries[addr] = pieces[3]
elif line.startswith('SYS_EXECUTABLE '):
filename = line[len('SYS_EXECUTABLE '):].strip()
- if filename != self.executable_name:
+ if filename != self.executable_name and filename != '??':
self.symbols.update(load_symbols(filename))
self.executable_name = filename
diff --git a/pypy/jit/codewriter/policy.py b/pypy/jit/codewriter/policy.py
--- a/pypy/jit/codewriter/policy.py
+++ b/pypy/jit/codewriter/policy.py
@@ -48,8 +48,6 @@
mod = func.__module__ or '?'
if mod.startswith('pypy.rpython.module.'):
return True
- if mod == 'pypy.translator.goal.nanos': # more helpers
- return True
return False
def look_inside_graph(self, graph):
diff --git a/pypy/module/_ffi/interp_funcptr.py b/pypy/module/_ffi/interp_funcptr.py
--- a/pypy/module/_ffi/interp_funcptr.py
+++ b/pypy/module/_ffi/interp_funcptr.py
@@ -15,6 +15,51 @@
from pypy.rlib.objectmodel import we_are_translated
from pypy.module._ffi.type_converter import FromAppLevelConverter, ToAppLevelConverter
+import os
+if os.name == 'nt':
+ def _getfunc(space, CDLL, w_name, w_argtypes, w_restype):
+ argtypes_w, argtypes, w_restype, restype = unpack_argtypes(
+ space, w_argtypes, w_restype)
+ if space.isinstance_w(w_name, space.w_str):
+ name = space.str_w(w_name)
+ try:
+ func = CDLL.cdll.getpointer(name, argtypes, restype,
+ flags = CDLL.flags)
+ except KeyError:
+ raise operationerrfmt(
+ space.w_AttributeError,
+ "No symbol %s found in library %s", name, CDLL.name)
+
+ return W_FuncPtr(func, argtypes_w, w_restype)
+ elif space.isinstance_w(w_name, space.w_int):
+ ordinal = space.int_w(w_name)
+ try:
+ func = CDLL.cdll.getpointer_by_ordinal(
+ ordinal, argtypes, restype,
+ flags = CDLL.flags)
+ except KeyError:
+ raise operationerrfmt(
+ space.w_AttributeError,
+ "No ordinal %d found in library %s", ordinal, CDLL.name)
+ return W_FuncPtr(func, argtypes_w, w_restype)
+ else:
+ raise OperationError(space.w_TypeError, space.wrap(
+ 'function name must be a string or integer'))
+else:
+ @unwrap_spec(name=str)
+ def _getfunc(space, CDLL, w_name, w_argtypes, w_restype):
+ name = space.str_w(w_name)
+ argtypes_w, argtypes, w_restype, restype = unpack_argtypes(
+ space, w_argtypes, w_restype)
+ try:
+ func = CDLL.cdll.getpointer(name, argtypes, restype,
+ flags = CDLL.flags)
+ except KeyError:
+ raise operationerrfmt(
+ space.w_AttributeError,
+ "No symbol %s found in library %s", name, CDLL.name)
+
+ return W_FuncPtr(func, argtypes_w, w_restype)
def unwrap_ffitype(space, w_argtype, allow_void=False):
res = w_argtype.get_ffitype()
@@ -271,19 +316,8 @@
raise operationerrfmt(space.w_OSError, '%s: %s', self.name,
e.msg or 'unspecified error')
- @unwrap_spec(name=str)
- def getfunc(self, space, name, w_argtypes, w_restype):
- argtypes_w, argtypes, w_restype, restype = unpack_argtypes(space,
- w_argtypes,
- w_restype)
- try:
- func = self.cdll.getpointer(name, argtypes, restype,
- flags = self.flags)
- except KeyError:
- raise operationerrfmt(space.w_AttributeError,
- "No symbol %s found in library %s", name, self.name)
-
- return W_FuncPtr(func, argtypes_w, w_restype)
+ def getfunc(self, space, w_name, w_argtypes, w_restype):
+ return _getfunc(space, self, w_name, w_argtypes, w_restype)
@unwrap_spec(name=str)
def getaddressindll(self, space, name):
@@ -291,8 +325,9 @@
address_as_uint = rffi.cast(lltype.Unsigned,
self.cdll.getaddressindll(name))
except KeyError:
- raise operationerrfmt(space.w_ValueError,
- "No symbol %s found in library %s", name, self.name)
+ raise operationerrfmt(
+ space.w_ValueError,
+ "No symbol %s found in library %s", name, self.name)
return space.wrap(address_as_uint)
@unwrap_spec(name='str_or_None', mode=int)
diff --git a/pypy/module/_ffi/interp_struct.py b/pypy/module/_ffi/interp_struct.py
--- a/pypy/module/_ffi/interp_struct.py
+++ b/pypy/module/_ffi/interp_struct.py
@@ -56,8 +56,7 @@
class W__StructDescr(Wrappable):
- def __init__(self, space, name):
- self.space = space
+ def __init__(self, name):
self.w_ffitype = W_FFIType('struct %s' % name, clibffi.FFI_TYPE_NULL,
w_structdescr=self)
self.fields_w = None
@@ -69,7 +68,6 @@
raise operationerrfmt(space.w_ValueError,
"%s's fields has already been defined",
self.w_ffitype.name)
- space = self.space
fields_w = space.fixedview(w_fields)
# note that the fields_w returned by compute_size_and_alignement has a
# different annotation than the original: list(W_Root) vs list(W_Field)
@@ -104,11 +102,11 @@
return W__StructInstance(self, allocate=False, autofree=True, rawmem=rawmem)
@jit.elidable_promote('0')
- def get_type_and_offset_for_field(self, name):
+ def get_type_and_offset_for_field(self, space, name):
try:
w_field = self.name2w_field[name]
except KeyError:
- raise operationerrfmt(self.space.w_AttributeError, '%s', name)
+ raise operationerrfmt(space.w_AttributeError, '%s', name)
return w_field.w_ffitype, w_field.offset
@@ -116,7 +114,7 @@
@unwrap_spec(name=str)
def descr_new_structdescr(space, w_type, name, w_fields=None):
- descr = W__StructDescr(space, name)
+ descr = W__StructDescr(name)
if w_fields is not space.w_None:
descr.define_fields(space, w_fields)
return descr
@@ -185,13 +183,15 @@
@unwrap_spec(name=str)
def getfield(self, space, name):
- w_ffitype, offset = self.structdescr.get_type_and_offset_for_field(name)
+ w_ffitype, offset = self.structdescr.get_type_and_offset_for_field(
+ space, name)
field_getter = GetFieldConverter(space, self.rawmem, offset)
return field_getter.do_and_wrap(w_ffitype)
@unwrap_spec(name=str)
def setfield(self, space, name, w_value):
- w_ffitype, offset = self.structdescr.get_type_and_offset_for_field(name)
+ w_ffitype, offset = self.structdescr.get_type_and_offset_for_field(
+ space, name)
field_setter = SetFieldConverter(space, self.rawmem, offset)
field_setter.unwrap_and_do(w_ffitype, w_value)
diff --git a/pypy/module/_ffi/test/test_funcptr.py b/pypy/module/_ffi/test/test_funcptr.py
--- a/pypy/module/_ffi/test/test_funcptr.py
+++ b/pypy/module/_ffi/test/test_funcptr.py
@@ -627,4 +627,17 @@
types.void, FUNCFLAG_STDCALL)
sleep(10)
-
+ def test_by_ordinal(self):
+ """
+ int DLLEXPORT AAA_first_ordinal_function()
+ {
+ return 42;
+ }
+ """
+ if not self.iswin32:
+ skip("windows specific")
+ from _ffi import CDLL, types
+ libfoo = CDLL(self.libfoo_name)
+ f_name = libfoo.getfunc('AAA_first_ordinal_function', [], types.sint)
+ f_ordinal = libfoo.getfunc(1, [], types.sint)
+ assert f_name.getaddr() == f_ordinal.getaddr()
diff --git a/pypy/module/_ffi/test/test_ztranslation.py b/pypy/module/_ffi/test/test_ztranslation.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_ffi/test/test_ztranslation.py
@@ -0,0 +1,4 @@
+from pypy.objspace.fake.checkmodule import checkmodule
+
+def test__ffi_translates():
+ checkmodule('_ffi', '_rawffi')
diff --git a/pypy/module/_ssl/__init__.py b/pypy/module/_ssl/__init__.py
--- a/pypy/module/_ssl/__init__.py
+++ b/pypy/module/_ssl/__init__.py
@@ -31,5 +31,6 @@
def startup(self, space):
from pypy.rlib.ropenssl import init_ssl
init_ssl()
- from pypy.module._ssl.interp_ssl import setup_ssl_threads
- setup_ssl_threads()
+ if space.config.objspace.usemodules.thread:
+ from pypy.module._ssl.thread_lock import setup_ssl_threads
+ setup_ssl_threads()
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -789,7 +789,11 @@
def _ssl_seterror(space, ss, ret):
assert ret <= 0
- if ss and ss.ssl:
+ if ss is None:
+ errval = libssl_ERR_peek_last_error()
+ errstr = rffi.charp2str(libssl_ERR_error_string(errval, None))
+ return ssl_error(space, errstr, errval)
+ elif ss.ssl:
err = libssl_SSL_get_error(ss.ssl, ret)
else:
err = SSL_ERROR_SSL
@@ -880,38 +884,3 @@
libssl_X509_free(x)
finally:
libssl_BIO_free(cert)
-
-# this function is needed to perform locking on shared data
-# structures. (Note that OpenSSL uses a number of global data
-# structures that will be implicitly shared whenever multiple threads
-# use OpenSSL.) Multi-threaded applications will crash at random if
-# it is not set.
-#
-# locking_function() must be able to handle up to CRYPTO_num_locks()
-# different mutex locks. It sets the n-th lock if mode & CRYPTO_LOCK, and
-# releases it otherwise.
-#
-# filename and line are the file number of the function setting the
-# lock. They can be useful for debugging.
-_ssl_locks = []
-
-def _ssl_thread_locking_function(mode, n, filename, line):
- n = intmask(n)
- if n < 0 or n >= len(_ssl_locks):
- return
-
- if intmask(mode) & CRYPTO_LOCK:
- _ssl_locks[n].acquire(True)
- else:
- _ssl_locks[n].release()
-
-def _ssl_thread_id_function():
- from pypy.module.thread import ll_thread
- return rffi.cast(rffi.LONG, ll_thread.get_ident())
-
-def setup_ssl_threads():
- from pypy.module.thread import ll_thread
- for i in range(libssl_CRYPTO_num_locks()):
- _ssl_locks.append(ll_thread.allocate_lock())
- libssl_CRYPTO_set_locking_callback(_ssl_thread_locking_function)
- libssl_CRYPTO_set_id_callback(_ssl_thread_id_function)
diff --git a/pypy/module/_ssl/thread_lock.py b/pypy/module/_ssl/thread_lock.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_ssl/thread_lock.py
@@ -0,0 +1,78 @@
+from pypy.rlib.ropenssl import *
+from pypy.rpython.lltypesystem import lltype, rffi
+from pypy.translator.tool.cbuild import ExternalCompilationInfo
+
+# CRYPTO_set_locking_callback:
+#
+# this function is needed to perform locking on shared data
+# structures. (Note that OpenSSL uses a number of global data
+# structures that will be implicitly shared whenever multiple threads
+# use OpenSSL.) Multi-threaded applications will crash at random if
+# it is not set.
+#
+# locking_function() must be able to handle up to CRYPTO_num_locks()
+# different mutex locks. It sets the n-th lock if mode & CRYPTO_LOCK, and
+# releases it otherwise.
+#
+# filename and line are the file number of the function setting the
+# lock. They can be useful for debugging.
+
+
+# This logic is moved to C code so that the callbacks can be invoked
+# without caring about the GIL.
+
+separate_module_source = """
+
+#include <openssl/crypto.h>
+
+static unsigned int _ssl_locks_count = 0;
+static struct RPyOpaque_ThreadLock *_ssl_locks;
+
+static unsigned long _ssl_thread_id_function(void) {
+ return RPyThreadGetIdent();
+}
+
+static void _ssl_thread_locking_function(int mode, int n, const char *file,
+ int line) {
+ if ((_ssl_locks == NULL) ||
+ (n < 0) || ((unsigned)n >= _ssl_locks_count))
+ return;
+
+ if (mode & CRYPTO_LOCK) {
+ RPyThreadAcquireLock(_ssl_locks + n, 1);
+ } else {
+ RPyThreadReleaseLock(_ssl_locks + n);
+ }
+}
+
+int _PyPy_SSL_SetupThreads(void)
+{
+ unsigned int i;
+ _ssl_locks_count = CRYPTO_num_locks();
+ _ssl_locks = calloc(_ssl_locks_count, sizeof(struct RPyOpaque_ThreadLock));
+ if (_ssl_locks == NULL)
+ return 0;
+ for (i=0; i<_ssl_locks_count; i++) {
+ if (RPyThreadLockInit(_ssl_locks + i) == 0)
+ return 0;
+ }
+ CRYPTO_set_locking_callback(_ssl_thread_locking_function);
+ CRYPTO_set_id_callback(_ssl_thread_id_function);
+ return 1;
+}
+"""
+
+
+eci = ExternalCompilationInfo(
+ separate_module_sources=[separate_module_source],
+ export_symbols=['_PyPy_SSL_SetupThreads'],
+)
+
+_PyPy_SSL_SetupThreads = rffi.llexternal('_PyPy_SSL_SetupThreads',
+ [], rffi.INT,
+ compilation_info=eci)
+
+def setup_ssl_threads():
+ result = _PyPy_SSL_SetupThreads()
+ if rffi.cast(lltype.Signed, result) == 0:
+ raise MemoryError
diff --git a/pypy/module/cStringIO/interp_stringio.py b/pypy/module/cStringIO/interp_stringio.py
--- a/pypy/module/cStringIO/interp_stringio.py
+++ b/pypy/module/cStringIO/interp_stringio.py
@@ -221,7 +221,8 @@
}
W_InputType.typedef = TypeDef(
- "cStringIO.StringI",
+ "StringI",
+ __module__ = "cStringIO",
__doc__ = "Simple type for treating strings as input file streams",
closed = GetSetProperty(descr_closed, cls=W_InputType),
softspace = GetSetProperty(descr_softspace,
@@ -232,7 +233,8 @@
)
W_OutputType.typedef = TypeDef(
- "cStringIO.StringO",
+ "StringO",
+ __module__ = "cStringIO",
__doc__ = "Simple type for output to strings.",
truncate = interp2app(W_OutputType.descr_truncate),
write = interp2app(W_OutputType.descr_write),
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -489,3 +489,4 @@
provides a subset of CPython's behavior.
"""
Py_DecRef(space, view.c_obj)
+ view.c_obj = lltype.nullptr(PyObject.TO)
diff --git a/pypy/module/cpyext/src/getargs.c b/pypy/module/cpyext/src/getargs.c
--- a/pypy/module/cpyext/src/getargs.c
+++ b/pypy/module/cpyext/src/getargs.c
@@ -24,14 +24,15 @@
/* Forward */
+typedef struct freelist_s freelist_t;
static int vgetargs1(PyObject *, const char *, va_list *, int);
static void seterror(int, const char *, int *, const char *, const char *);
static char *convertitem(PyObject *, const char **, va_list *, int, int *,
- char *, size_t, PyObject **);
+ char *, size_t, freelist_t **);
static char *converttuple(PyObject *, const char **, va_list *, int,
- int *, char *, size_t, int, PyObject **);
+ int *, char *, size_t, int, freelist_t **);
static char *convertsimple(PyObject *, const char **, va_list *, int, char *,
- size_t, PyObject **);
+ size_t, freelist_t **);
static Py_ssize_t convertbuffer(PyObject *, void **p, char **);
static int getbuffer(PyObject *, Py_buffer *, char**);
@@ -128,72 +129,45 @@
/* Handle cleanup of allocated memory in case of exception */
-#define GETARGS_CAPSULE_NAME_CLEANUP_PTR "getargs.cleanup_ptr"
-#define GETARGS_CAPSULE_NAME_CLEANUP_BUFFER "getargs.cleanup_buffer"
+typedef void (*cleanup_fn)(void *);
-static void
-cleanup_ptr(PyObject *self)
-{
- void *ptr = PyCapsule_GetPointer(self, GETARGS_CAPSULE_NAME_CLEANUP_PTR);
- if (ptr) {
- PyMem_FREE(ptr);
- }
-}
+struct freelist_s {
+ void *ptr;
+ cleanup_fn destr;
+ struct freelist_s *next;
+};
-static void
-cleanup_buffer(PyObject *self)
-{
- Py_buffer *ptr = (Py_buffer *)PyCapsule_GetPointer(self, GETARGS_CAPSULE_NAME_CLEANUP_BUFFER);
- if (ptr) {
- PyBuffer_Release(ptr);
- }
-}
+#define cleanup_ptr ((cleanup_fn)PyMem_FREE)
+#define cleanup_buffer ((cleanup_fn)PyBuffer_Release)
static int
-addcleanup(void *ptr, PyObject **freelist, PyCapsule_Destructor destr)
+addcleanup(void *ptr, freelist_t **freelist, cleanup_fn destr)
{
- PyObject *cobj;
- const char *name;
-
- if (!*freelist) {
- *freelist = PyList_New(0);
- if (!*freelist) {
- destr(ptr);
- return -1;
- }
- }
-
- if (destr == cleanup_ptr) {
- name = GETARGS_CAPSULE_NAME_CLEANUP_PTR;
- } else if (destr == cleanup_buffer) {
- name = GETARGS_CAPSULE_NAME_CLEANUP_BUFFER;
- } else {
- return -1;
- }
- cobj = PyCapsule_New(ptr, name, destr);
- if (!cobj) {
+ freelist_t *node = PyMem_MALLOC(sizeof(freelist_t));
+ if (!node) {
destr(ptr);
return -1;
}
- if (PyList_Append(*freelist, cobj)) {
- Py_DECREF(cobj);
- return -1;
- }
- Py_DECREF(cobj);
+ node->ptr = ptr;
+ node->destr = destr;
+ node->next = *freelist;
+ *freelist = node;
return 0;
}
static int
-cleanreturn(int retval, PyObject *freelist)
+cleanreturn(int retval, freelist_t *freelist)
{
- if (freelist && retval != 0) {
- /* We were successful, reset the destructors so that they
- don't get called. */
- Py_ssize_t len = PyList_GET_SIZE(freelist), i;
- for (i = 0; i < len; i++)
- PyCapsule_SetDestructor(PyList_GET_ITEM(freelist, i), NULL);
+ freelist_t *next;
+ while (freelist != NULL) {
+ if (retval == 0) {
+ /* Leaving with an error */
+ freelist->destr(freelist->ptr);
+ }
+ next = freelist->next;
+ PyMem_FREE(freelist);
+ freelist = next;
}
- Py_XDECREF(freelist);
return retval;
}
@@ -212,7 +186,7 @@
const char *formatsave = format;
Py_ssize_t i, len;
char *msg;
- PyObject *freelist = NULL;
+ freelist_t *freelist = NULL;
int compat = flags & FLAG_COMPAT;
assert(compat || (args != (PyObject*)NULL));
@@ -412,7 +386,7 @@
static char *
converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
int *levels, char *msgbuf, size_t bufsize, int toplevel,
- PyObject **freelist)
+ freelist_t **freelist)
{
int level = 0;
int n = 0;
@@ -488,7 +462,7 @@
static char *
convertitem(PyObject *arg, const char **p_format, va_list *p_va, int flags,
- int *levels, char *msgbuf, size_t bufsize, PyObject **freelist)
+ int *levels, char *msgbuf, size_t bufsize, freelist_t **freelist)
{
char *msg;
const char *format = *p_format;
@@ -569,7 +543,7 @@
static char *
convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
- char *msgbuf, size_t bufsize, PyObject **freelist)
+ char *msgbuf, size_t bufsize, freelist_t **freelist)
{
/* For # codes */
#define FETCH_SIZE int *q=NULL;Py_ssize_t *q2=NULL;\
@@ -1534,7 +1508,8 @@
const char *fname, *msg, *custom_msg, *keyword;
int min = INT_MAX;
int i, len, nargs, nkeywords;
- PyObject *freelist = NULL, *current_arg;
+ freelist_t *freelist = NULL;
+ PyObject *current_arg;
assert(args != NULL && PyTuple_Check(args));
assert(keywords == NULL || PyDict_Check(keywords));
diff --git a/pypy/module/cpyext/test/test_getargs.py b/pypy/module/cpyext/test/test_getargs.py
--- a/pypy/module/cpyext/test/test_getargs.py
+++ b/pypy/module/cpyext/test/test_getargs.py
@@ -144,6 +144,31 @@
assert 'foo\0bar\0baz' == pybuffer(buffer('foo\0bar\0baz'))
+ def test_pyarg_parse_string_fails(self):
+ """
+ Test the failing case of PyArg_ParseTuple(): it must not keep
+ a reference on the PyObject passed in.
+ """
+ pybuffer = self.import_parser(
+ '''
+ Py_buffer buf1, buf2, buf3;
+ PyObject *result;
+ if (!PyArg_ParseTuple(args, "s*s*s*", &buf1, &buf2, &buf3)) {
+ return NULL;
+ }
+ Py_FatalError("should not get there");
+ return NULL;
+ ''')
+ freed = []
+ class freestring(str):
+ def __del__(self):
+ freed.append('x')
+ raises(TypeError, pybuffer,
+ freestring("string"), freestring("other string"), 42)
+ import gc; gc.collect()
+ assert freed == ['x', 'x']
+
+
def test_pyarg_parse_charbuf_and_length(self):
"""
The `t#` format specifier can be used to parse a read-only 8-bit
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -363,6 +363,10 @@
* Py_buffer and the string should be released as well.
*/
PyBuffer_Release(&buf);
+ assert(!buf.obj);
+ PyBuffer_Release(&buf); /* call again, should not have any more effect */
+ PyBuffer_Release(&buf);
+ PyBuffer_Release(&buf);
Py_RETURN_NONE;
""")])
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -263,7 +263,7 @@
w_mod = check_sys_modules_w(space, rel_modulename)
if w_mod is not None and space.is_w(w_mod, space.w_None):
# if we already find space.w_None, it means that we
- # already tried and failed and falled back to the
+ # already tried and failed and fell back to the
# end of this function.
w_mod = None
else:
@@ -283,10 +283,16 @@
return w_mod
def absolute_import(space, modulename, baselevel, fromlist_w, tentative):
- # Short path: check in sys.modules
- w_mod = absolute_import_try(space, modulename, baselevel, fromlist_w)
- if w_mod is not None and not space.is_w(w_mod, space.w_None):
- return w_mod
+ # Short path: check in sys.modules, but only if there is no conflict
+ # on the import lock. In the situation of 'import' statements
+ # inside tight loops, this should be true, and absolute_import_try()
+ # should be followed by the JIT and turned into not much code. But
+ # if the import lock is currently held by another thread, then we
+ # have to wait, and so shouldn't use the fast path.
+ if not getimportlock(space).lock_held_by_someone_else():
+ w_mod = absolute_import_try(space, modulename, baselevel, fromlist_w)
+ if w_mod is not None and not space.is_w(w_mod, space.w_None):
+ return w_mod
return absolute_import_with_lock(space, modulename, baselevel,
fromlist_w, tentative)
@@ -741,6 +747,9 @@
self.lockowner = None
self.lockcounter = 0
+ def lock_held_by_someone_else(self):
+ return self.lockowner is not None and not self.lock_held()
+
def lock_held(self):
me = self.space.getexecutioncontext() # used as thread ident
return self.lockowner is me
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -1212,3 +1212,45 @@
"objspace.usepycfiles": True,
"objspace.lonepycfiles": True
}
+
+
+class AppTestMultithreadedImp(object):
+ def setup_class(cls):
+ #if not conftest.option.runappdirect:
+ # py.test.skip("meant as an -A test")
+ cls.space = gettestobjspace(usemodules=['thread', 'time'])
+ tmpfile = udir.join('test_multithreaded_imp.py')
+ tmpfile.write('''if 1:
+ x = 666
+ import time
+ for i in range(1000): time.sleep(0.001)
+ x = 42
+ ''')
+ cls.w_tmppath = cls.space.wrap(str(udir))
+
+ def test_multithreaded_import(self):
+ import sys, thread, time
+ oldpath = sys.path[:]
+ try:
+ sys.path.insert(0, self.tmppath)
+ got = []
+
+ def check():
+ import test_multithreaded_imp
+ got.append(getattr(test_multithreaded_imp, 'x', '?'))
+
+ for i in range(5):
+ thread.start_new_thread(check, ())
+
+ for n in range(100):
+ for i in range(105): time.sleep(0.001)
+ if len(got) == 5:
+ break
+ else:
+ raise AssertionError("got %r so far but still waiting" %
+ (got,))
+
+ assert got == [42] * 5, got
+
+ finally:
+ sys.path[:] = oldpath
diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py
--- a/pypy/module/posix/app_posix.py
+++ b/pypy/module/posix/app_posix.py
@@ -66,15 +66,10 @@
self.__dict__['st_ctime'] = self[9]
if osname == 'posix':
- def _validate_fd(fd):
- try:
- import fcntl
- except ImportError:
- return
- try:
- fcntl.fcntl(fd, fcntl.F_GETFD)
- except IOError, e:
- raise OSError(e.errno, e.strerror, e.filename)
+ # POSIX: we want to check the file descriptor when fdopen() is called,
+ # not later when we read or write data. So we call fstat(), letting
+ # it raise if fd is invalid.
+ _validate_fd = posix.fstat
else:
_validate_fd = validate_fd
diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py
--- a/pypy/module/signal/interp_signal.py
+++ b/pypy/module/signal/interp_signal.py
@@ -143,34 +143,37 @@
else:
self.reissue_signal_action = None
+ @jit.dont_look_inside
def perform(self, executioncontext, frame):
while True:
n = pypysig_poll()
if n < 0:
break
- if self.reissue_signal_action is None:
- # no threads: we can report the signal immediately
+ self.perform_signal(executioncontext, n)
+
+ @jit.dont_look_inside
+ def perform_signal(self, executioncontext, n):
+ if self.reissue_signal_action is None:
+ # no threads: we can report the signal immediately
+ self.report_signal(n)
+ else:
+ main_ec = self.space.threadlocals.getmainthreadvalue()
+ if executioncontext is main_ec:
+ # running in the main thread: we can report the
+ # signal immediately
self.report_signal(n)
else:
- main_ec = self.space.threadlocals.getmainthreadvalue()
- if executioncontext is main_ec:
- # running in the main thread: we can report the
- # signal immediately
- self.report_signal(n)
- else:
- # running in another thread: we need to hack a bit
- self.pending_signals[n] = None
- self.reissue_signal_action.fire_after_thread_switch()
+ # running in another thread: we need to hack a bit
+ self.pending_signals[n] = None
+ self.reissue_signal_action.fire_after_thread_switch()
+ @jit.dont_look_inside
def set_interrupt(self):
"Simulates the effect of a SIGINT signal arriving"
- n = cpy_signal.SIGINT
- if self.reissue_signal_action is None:
- self.report_signal(n)
- else:
- self.pending_signals[n] = None
- self.reissue_signal_action.fire_after_thread_switch()
+ ec = self.space.getexecutioncontext()
+ self.perform_signal(ec, cpy_signal.SIGINT)
+ @jit.dont_look_inside
def report_signal(self, n):
try:
w_handler = self.handlers_w[n]
@@ -184,6 +187,7 @@
w_frame = space.wrap(ec.gettopframe_nohidden())
space.call_function(w_handler, space.wrap(n), w_frame)
+ @jit.dont_look_inside
def report_pending_signals(self):
# XXX this logic isn't so complicated but I have no clue how
# to test it :-(
@@ -228,7 +232,7 @@
None -- if an unknown handler is in effect (XXX UNIMPLEMENTED)
anything else -- the callable Python object used as a handler
"""
- check_signum(space, signum)
+ check_signum_in_range(space, signum)
action = space.check_signal_action
if signum in action.handlers_w:
return action.handlers_w[signum]
@@ -254,12 +258,18 @@
c_pause()
return space.w_None
-def check_signum(space, signum):
+def check_signum_exists(space, signum):
if signum in signal_values:
return
raise OperationError(space.w_ValueError,
space.wrap("invalid signal value"))
+def check_signum_in_range(space, signum):
+ if 1 <= signum < NSIG:
+ return
+ raise OperationError(space.w_ValueError,
+ space.wrap("signal number out of range"))
+
@jit.dont_look_inside
@unwrap_spec(signum=int)
@@ -300,6 +310,7 @@
action.handlers_w[signum] = w_handler
return old_handler
+ at jit.dont_look_inside
@unwrap_spec(fd=int)
def set_wakeup_fd(space, fd):
"""Sets the fd to be written to (with '\0') when a signal
@@ -318,9 +329,10 @@
old_fd = pypysig_set_wakeup_fd(fd)
return space.wrap(intmask(old_fd))
+ at jit.dont_look_inside
@unwrap_spec(signum=int, flag=int)
def siginterrupt(space, signum, flag):
- check_signum(space, signum)
+ check_signum_exists(space, signum)
if rffi.cast(lltype.Signed, c_siginterrupt(signum, flag)) < 0:
errno = rposix.get_errno()
raise OperationError(space.w_RuntimeError, space.wrap(errno))
diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py
--- a/pypy/module/signal/test/test_signal.py
+++ b/pypy/module/signal/test/test_signal.py
@@ -54,10 +54,9 @@
if not hasattr(os, 'kill') or not hasattr(os, 'getpid'):
skip("requires os.kill() and os.getpid()")
signal = self.signal # the signal module to test
- if hasattr(signal,'SIGUSR1'):
- signum = signal.SIGUSR1
- else:
- signum = signal.CTRL_BREAK_EVENT
+ if not hasattr(signal, 'SIGUSR1'):
+ py.test.skip("requires SIGUSR1 in signal")
+ signum = signal.SIGUSR1
received = []
def myhandler(signum, frame):
@@ -154,7 +153,12 @@
raises(ValueError, getsignal, 4444)
raises(ValueError, signal, 4444, lambda *args: None)
- raises(ValueError, signal, 42, lambda *args: None)
+ import sys
+ if sys.platform == 'win32':
+ raises(ValueError, signal, 42, lambda *args: None)
+ else:
+ signal(42, lambda *args: None)
+ signal(42, SIG_DFL)
def test_alarm(self):
try:
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -44,7 +44,9 @@
'warnoptions' : 'state.get(space).w_warnoptions',
'builtin_module_names' : 'space.w_None',
'pypy_getudir' : 'state.pypy_getudir', # not translated
- 'pypy_initial_path' : 'state.pypy_initial_path',
+ 'pypy_find_stdlib' : 'initpath.pypy_find_stdlib',
+ 'pypy_find_executable' : 'initpath.pypy_find_executable',
+ 'pypy_resolvedirof' : 'initpath.pypy_resolvedirof',
'_getframe' : 'vm._getframe',
'_current_frames' : 'currentframes._current_frames',
diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/sys/initpath.py
@@ -0,0 +1,155 @@
+"""
+Logic to find sys.executable and the initial sys.path containing the stdlib
+"""
+
+import sys
+import os
+import stat
+import errno
+from pypy.rlib import rpath
+from pypy.rlib.objectmodel import we_are_translated
+from pypy.interpreter.gateway import unwrap_spec
+from pypy.module.sys.state import get as get_state
+
+platform = sys.platform
+IS_WINDOWS = sys.platform == 'win32'
+
+def find_executable(executable):
+ """
+ Return the absolute path of the executable, by looking into PATH and the
+ current directory. If it cannot be found, return ''.
+ """
+ if we_are_translated() and IS_WINDOWS and not executable.lower().endswith('.exe'):
+ executable += '.exe'
+ if os.sep in executable or (IS_WINDOWS and ':' in executable):
+ pass # the path is already more than just an executable name
+ else:
+ path = os.environ.get('PATH')
+ if path:
+ for dir in path.split(os.pathsep):
+ fn = os.path.join(dir, executable)
+ if os.path.isfile(fn):
+ executable = fn
+ break
+ executable = rpath.rabspath(executable)
+ #
+ # 'sys.executable' should not end up being an non-existing file;
+ # just use '' in this case. (CPython issue #7774)
+ if not os.path.isfile(executable):
+ executable = ''
+ return executable
+
+
+def readlink_maybe(filename):
+ if not IS_WINDOWS:
+ return os.readlink(filename)
+ raise NotImplementedError
+
+def resolvedirof(filename):
+ try:
+ filename = rpath.rabspath(filename)
+ except OSError:
+ pass
+ dirname = rpath.rabspath(os.path.join(filename, '..'))
+ if os.path.islink(filename):
+ try:
+ link = readlink_maybe(filename)
+ except OSError:
+ pass
+ else:
+ return resolvedirof(os.path.join(dirname, link))
+ return dirname
+
+def find_stdlib(state, executable):
+ """
+ Find and compute the stdlib path, starting from the directory where
+ ``executable`` is and going one level up until we find it. Return a tuple
+ (path, prefix), where ``prefix`` is the root directory which contains the
+ stdlib.
+ If it cannot be found, return (None, None).
+ """
+ search = executable
+ while True:
+ dirname = resolvedirof(search)
+ if dirname == search:
+ return None, None # not found :-(
+ newpath = compute_stdlib_path_maybe(state, dirname)
+ if newpath is not None:
+ return newpath, dirname
+ search = dirname # walk to the parent directory
+
+
+
+def checkdir(path):
+ st = os.stat(path)
+ if not stat.S_ISDIR(st[0]):
+ raise OSError(errno.ENOTDIR, path)
+
+def compute_stdlib_path(state, prefix):
+ """
+ Compute the paths for the stdlib rooted at ``prefix``. ``prefix`` must at
+ least contain a directory called ``lib-python/X.Y`` and another one called
+ ``lib_pypy``. If they cannot be found, it raises OSError.
+ """
+ from pypy.module.sys.version import CPYTHON_VERSION
+ dirname = '%d.%d' % (CPYTHON_VERSION[0],
+ CPYTHON_VERSION[1])
+ lib_python = os.path.join(prefix, 'lib-python')
+ python_std_lib = os.path.join(lib_python, dirname)
+ checkdir(python_std_lib)
+
+ lib_pypy = os.path.join(prefix, 'lib_pypy')
+ checkdir(lib_pypy)
+
+ importlist = []
+ #
+ if state is not None: # 'None' for testing only
+ lib_extensions = os.path.join(lib_pypy, '__extensions__')
+ state.w_lib_extensions = state.space.wrap(lib_extensions)
+ importlist.append(lib_extensions)
+ #
+ importlist.append(lib_pypy)
+ importlist.append(python_std_lib)
+ #
+ lib_tk = os.path.join(python_std_lib, 'lib-tk')
+ importlist.append(lib_tk)
+ #
+ # List here the extra platform-specific paths.
+ if platform != 'win32':
+ importlist.append(os.path.join(python_std_lib, 'plat-'+platform))
+ if platform == 'darwin':
+ platmac = os.path.join(python_std_lib, 'plat-mac')
+ importlist.append(platmac)
+ importlist.append(os.path.join(platmac, 'lib-scriptpackages'))
+ #
+ return importlist
+
+def compute_stdlib_path_maybe(state, prefix):
+ """
+ Return the stdlib path rooted at ``prefix``, or None if it cannot be
+ found.
+ """
+ try:
+ return compute_stdlib_path(state, prefix)
+ except OSError:
+ return None
+
+ at unwrap_spec(executable='str0')
+def pypy_find_executable(space, executable):
+ return space.wrap(find_executable(executable))
+
+ at unwrap_spec(filename='str0')
+def pypy_resolvedirof(space, filename):
+ return space.wrap(resolvedirof(filename))
+
+ at unwrap_spec(executable='str0')
+def pypy_find_stdlib(space, executable):
+ path, prefix = find_stdlib(get_state(space), executable)
+ if path is None:
+ return space.w_None
+ else:
+ space.setitem(space.sys.w_dict, space.wrap('prefix'),
+ space.wrap(prefix))
+ space.setitem(space.sys.w_dict, space.wrap('exec_prefix'),
+ space.wrap(prefix))
+ return space.newlist([space.wrap(p) for p in path])
diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py
--- a/pypy/module/sys/state.py
+++ b/pypy/module/sys/state.py
@@ -1,11 +1,8 @@
"""
Implementation of interpreter-level 'sys' routines.
"""
+import os
import pypy
-from pypy.interpreter.error import OperationError
-from pypy.interpreter.gateway import unwrap_spec
-
-import sys, os, stat, errno
# ____________________________________________________________
#
@@ -20,67 +17,14 @@
self.w_argv = space.newlist([])
self.setinitialpath(space)
- def setinitialpath(self, space):
+ def setinitialpath(self, space):
+ from pypy.module.sys.initpath import compute_stdlib_path
# Initialize the default path
pypydir = os.path.dirname(os.path.abspath(pypy.__file__))
srcdir = os.path.dirname(pypydir)
- path = getinitialpath(self, srcdir)
+ path = compute_stdlib_path(self, srcdir)
self.w_path = space.newlist([space.wrap(p) for p in path])
-def checkdir(path):
- st = os.stat(path)
- if not stat.S_ISDIR(st[0]):
- raise OSError(errno.ENOTDIR, path)
-
-
-platform = sys.platform
-
-def getinitialpath(state, prefix):
- from pypy.module.sys.version import CPYTHON_VERSION
- dirname = '%d.%d' % (CPYTHON_VERSION[0],
- CPYTHON_VERSION[1])
- lib_python = os.path.join(prefix, 'lib-python')
- python_std_lib = os.path.join(lib_python, dirname)
- checkdir(python_std_lib)
-
- lib_pypy = os.path.join(prefix, 'lib_pypy')
- checkdir(lib_pypy)
-
- importlist = []
- #
- if state is not None: # 'None' for testing only
- lib_extensions = os.path.join(lib_pypy, '__extensions__')
- state.w_lib_extensions = state.space.wrap(lib_extensions)
- importlist.append(lib_extensions)
- #
- importlist.append(lib_pypy)
- importlist.append(python_std_lib)
- #
- lib_tk = os.path.join(python_std_lib, 'lib-tk')
- importlist.append(lib_tk)
- #
- # List here the extra platform-specific paths.
- if platform != 'win32':
- importlist.append(os.path.join(python_std_lib, 'plat-'+platform))
- if platform == 'darwin':
- platmac = os.path.join(python_std_lib, 'plat-mac')
- importlist.append(platmac)
- importlist.append(os.path.join(platmac, 'lib-scriptpackages'))
- #
- return importlist
-
- at unwrap_spec(srcdir='str0')
-def pypy_initial_path(space, srcdir):
- try:
- path = getinitialpath(get(space), srcdir)
- except OSError:
- return space.w_None
- else:
- space.setitem(space.sys.w_dict, space.wrap('prefix'),
- space.wrap(srcdir))
- space.setitem(space.sys.w_dict, space.wrap('exec_prefix'),
- space.wrap(srcdir))
- return space.newlist([space.wrap(p) for p in path])
def get(space):
return space.fromcache(State)
@@ -115,3 +59,4 @@
(should be removed from interpleveldefs before translation)"""
from pypy.tool.udir import udir
return space.wrap(str(udir))
+
diff --git a/pypy/module/sys/test/test_initialpath.py b/pypy/module/sys/test/test_initialpath.py
deleted file mode 100644
--- a/pypy/module/sys/test/test_initialpath.py
+++ /dev/null
@@ -1,22 +0,0 @@
-import py
-from pypy.module.sys.state import getinitialpath
-from pypy.module.sys.version import PYPY_VERSION, CPYTHON_VERSION
-
-def build_hierarchy(prefix):
- dirname = '%d.%d' % CPYTHON_VERSION[:2]
- a = prefix.join('lib_pypy').ensure(dir=1)
- b = prefix.join('lib-python', dirname).ensure(dir=1)
- return a, b
-
-
-def test_stdlib_in_prefix(tmpdir):
- dirs = build_hierarchy(tmpdir)
- path = getinitialpath(None, str(tmpdir))
- # we get at least 'dirs', and maybe more (e.g. plat-linux2)
- assert path[:len(dirs)] == map(str, dirs)
-
-def test_include_libtk(tmpdir):
- lib_pypy, lib_python = build_hierarchy(tmpdir)
- lib_tk = lib_python.join('lib-tk')
- path = getinitialpath(None, str(tmpdir))
- assert lib_tk in path
diff --git a/pypy/module/sys/test/test_initpath.py b/pypy/module/sys/test/test_initpath.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/sys/test/test_initpath.py
@@ -0,0 +1,95 @@
+import py
+import os.path
+from pypy.module.sys.initpath import (compute_stdlib_path, find_executable, find_stdlib,
+ resolvedirof)
+from pypy.module.sys.version import PYPY_VERSION, CPYTHON_VERSION
+
+def build_hierarchy(prefix):
+ dirname = '%d.%d' % CPYTHON_VERSION[:2]
+ a = prefix.join('lib_pypy').ensure(dir=1)
+ b = prefix.join('lib-python', dirname).ensure(dir=1)
+ return a, b
+
+def test_find_stdlib(tmpdir):
+ bin_dir = tmpdir.join('bin').ensure(dir=True)
+ pypy = bin_dir.join('pypy').ensure(file=True)
+ build_hierarchy(tmpdir)
+ path, prefix = find_stdlib(None, str(pypy))
+ assert prefix == tmpdir
+
+ at py.test.mark.skipif('not hasattr(os, "symlink")')
+def test_find_stdlib_follow_symlink(tmpdir):
+ pypydir = tmpdir.join('opt', 'pypy-xxx')
+ pypy = pypydir.join('bin', 'pypy').ensure(file=True)
+ build_hierarchy(pypydir)
+ pypy_sym = tmpdir.join('pypy_sym')
+ os.symlink(str(pypy), str(pypy_sym))
+ path, prefix = find_stdlib(None, str(pypy_sym))
+ assert prefix == pypydir
+
+
+def test_compute_stdlib_path(tmpdir):
+ dirs = build_hierarchy(tmpdir)
+ path = compute_stdlib_path(None, str(tmpdir))
+ # we get at least 'dirs', and maybe more (e.g. plat-linux2)
+ assert path[:len(dirs)] == map(str, dirs)
+
+def test_include_libtk(tmpdir):
+ lib_pypy, lib_python = build_hierarchy(tmpdir)
+ lib_tk = lib_python.join('lib-tk')
+ path = compute_stdlib_path(None, str(tmpdir))
+ assert lib_tk in path
+
+
+def test_find_executable(tmpdir, monkeypatch):
+ from pypy.module.sys import initpath
+ # /tmp/a/pypy
+ # /tmp/b/pypy
+ # /tmp/c
+ a = tmpdir.join('a').ensure(dir=True)
+ b = tmpdir.join('b').ensure(dir=True)
+ c = tmpdir.join('c').ensure(dir=True)
+ a.join('pypy').ensure(file=True)
+ b.join('pypy').ensure(file=True)
+ #
+ # if there is already a slash, don't do anything
+ monkeypatch.chdir(tmpdir)
+ assert find_executable('a/pypy') == a.join('pypy')
+ #
+ # if path is None, try abspath (if the file exists)
+ monkeypatch.setenv('PATH', None)
+ monkeypatch.chdir(a)
+ assert find_executable('pypy') == a.join('pypy')
+ monkeypatch.chdir(tmpdir) # no pypy there
+ assert find_executable('pypy') == ''
+ #
+ # find it in path
+ monkeypatch.setenv('PATH', str(a))
+ assert find_executable('pypy') == a.join('pypy')
+ #
+ # find it in the first dir in path
+ monkeypatch.setenv('PATH', '%s%s%s' % (b, os.pathsep, a))
+ assert find_executable('pypy') == b.join('pypy')
+ #
+ # find it in the second, because in the first it's not there
+ monkeypatch.setenv('PATH', '%s%s%s' % (c, os.pathsep, a))
+ assert find_executable('pypy') == a.join('pypy')
+ # if pypy is found but it's not a file, ignore it
+ c.join('pypy').ensure(dir=True)
+ assert find_executable('pypy') == a.join('pypy')
+ #
+ monkeypatch.setattr(initpath, 'we_are_translated', lambda: True)
+ monkeypatch.setattr(initpath, 'IS_WINDOWS', True)
+ monkeypatch.setenv('PATH', str(a))
+ a.join('pypy.exe').ensure(file=True)
+ assert find_executable('pypy') == a.join('pypy.exe')
+
+def test_resolvedirof(tmpdir):
+ foo = tmpdir.join('foo').ensure(dir=True)
+ bar = tmpdir.join('bar').ensure(dir=True)
+ myfile = foo.join('myfile').ensure(file=True)
+ assert resolvedirof(str(myfile)) == foo
+ if hasattr(myfile, 'mksymlinkto'):
+ myfile2 = bar.join('myfile')
+ myfile2.mksymlinkto(myfile)
+ assert resolvedirof(str(myfile2)) == foo
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py b/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py
@@ -94,4 +94,13 @@
def test_cast_argumenterror(self):
param = c_uint(42)
py.test.raises(ArgumentError, "cast(param, c_void_p)")
-
+
+ def test_c_bool(self):
+ x = c_bool(42)
+ assert x.value is True
+ x = c_bool(0.0)
+ assert x.value is False
+ x = c_bool("")
+ assert x.value is False
+ x = c_bool(['yadda'])
+ assert x.value is True
diff --git a/pypy/objspace/fake/checkmodule.py b/pypy/objspace/fake/checkmodule.py
--- a/pypy/objspace/fake/checkmodule.py
+++ b/pypy/objspace/fake/checkmodule.py
@@ -2,13 +2,14 @@
from pypy.config.pypyoption import get_pypy_config
-def checkmodule(modname):
+def checkmodule(*modnames):
config = get_pypy_config(translating=True)
space = FakeObjSpace(config)
- mod = __import__('pypy.module.%s' % modname, None, None, ['__doc__'])
- # force computation and record what we wrap
- module = mod.Module(space, W_Root())
- for name in module.loaders:
- module._load_lazily(space, name)
+ for modname in modnames:
+ mod = __import__('pypy.module.%s' % modname, None, None, ['__doc__'])
+ # force computation and record what we wrap
+ module = mod.Module(space, W_Root())
+ for name in module.loaders:
+ module._load_lazily(space, name)
#
space.translates(**{'translation.list_comprehension_operations':True})
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -158,6 +158,8 @@
if isinstance(x, r_singlefloat):
self._wrap_not_rpython(x)
if isinstance(x, list):
+ if x == []: # special case: it is used e.g. in sys/__init__.py
+ return w_some_obj()
self._wrap_not_rpython(x)
return w_some_obj()
wrap._annspecialcase_ = "specialize:argtype(1)"
diff --git a/pypy/rlib/libffi.py b/pypy/rlib/libffi.py
--- a/pypy/rlib/libffi.py
+++ b/pypy/rlib/libffi.py
@@ -415,6 +415,11 @@
return Func(name, argtypes, restype, dlsym(self.lib, name),
flags=flags, keepalive=self)
+ def getpointer_by_ordinal(self, name, argtypes, restype,
+ flags=FUNCFLAG_CDECL):
+ return Func('by_ordinal', argtypes, restype,
+ dlsym_byordinal(self.lib, name),
+ flags=flags, keepalive=self)
def getaddressindll(self, name):
return dlsym(self.lib, name)
@@ -423,6 +428,11 @@
def getpointer(self, name, argtypes, restype, flags=FUNCFLAG_STDCALL):
return Func(name, argtypes, restype, dlsym(self.lib, name),
flags=flags, keepalive=self)
+ def getpointer_by_ordinal(self, name, argtypes, restype,
+ flags=FUNCFLAG_STDCALL):
+ return Func(name, argtypes, restype, dlsym_byordinal(self.lib, name),
+ flags=flags, keepalive=self)
+
# ======================================================================
@jit.oopspec('libffi_struct_getfield(ffitype, addr, offset)')
diff --git a/pypy/rlib/parsing/parsing.py b/pypy/rlib/parsing/parsing.py
--- a/pypy/rlib/parsing/parsing.py
+++ b/pypy/rlib/parsing/parsing.py
@@ -110,8 +110,7 @@
lastexpansion = len(rule.expansions) - 1
subsymbol = None
error = None
- for expansionindex in range(len(rule.expansions)):
- expansion = rule.expansions[expansionindex]
+ for expansion in rule.expansions:
curr = i
children = []
for j in range(len(expansion)):
diff --git a/pypy/rlib/rmmap.py b/pypy/rlib/rmmap.py
--- a/pypy/rlib/rmmap.py
+++ b/pypy/rlib/rmmap.py
@@ -14,6 +14,7 @@
_MS_WINDOWS = os.name == "nt"
_LINUX = "linux" in sys.platform
_64BIT = "64bit" in platform.architecture()[0]
+_CYGWIN = "cygwin" == sys.platform
class RValueError(Exception):
def __init__(self, message):
@@ -115,6 +116,10 @@
PTR = rffi.CCHARP
+if _CYGWIN:
+ c_malloc, _ = external('malloc', [size_t], PTR)
+ c_free, _ = external('free', [PTR], lltype.Void)
+
c_memmove, _ = external('memmove', [PTR, PTR, size_t], lltype.Void)
if _POSIX:
@@ -692,6 +697,14 @@
so the memory has the executable bit set and gets allocated
internally in case of a sandboxed process.
"""
+ if _CYGWIN:
+ # XXX: JIT memory should be using mmap MAP_PRIVATE with
+ # PROT_EXEC but Cygwin's fork() fails. mprotect()
+ # cannot be used, but seems to be unnecessary there.
+ res = c_malloc(map_size)
+ if res == rffi.cast(PTR, 0):
+ raise MemoryError
+ return res
flags = MAP_PRIVATE | MAP_ANONYMOUS
prot = PROT_EXEC | PROT_READ | PROT_WRITE
hintp = rffi.cast(PTR, hint.pos)
@@ -708,7 +721,10 @@
return res
alloc._annenforceargs_ = (int,)
- free = c_munmap_safe
+ if _CYGWIN:
+ free = c_free
+ else:
+ free = c_munmap_safe
elif _MS_WINDOWS:
def mmap(fileno, length, tagname="", access=_ACCESS_DEFAULT, offset=0):
diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -259,6 +259,7 @@
ssl_external('SSL_CIPHER_get_bits', [SSL_CIPHER, rffi.INTP], rffi.INT)
ssl_external('ERR_get_error', [], rffi.INT)
+ssl_external('ERR_peek_last_error', [], rffi.INT)
ssl_external('ERR_error_string', [rffi.ULONG, rffi.CCHARP], rffi.CCHARP)
ssl_external('SSL_free', [SSL], lltype.Void)
diff --git a/pypy/rlib/rpath.py b/pypy/rlib/rpath.py
new file mode 100644
--- /dev/null
+++ b/pypy/rlib/rpath.py
@@ -0,0 +1,20 @@
+"""
+Minimal (and limited) RPython version of some functions contained in os.path.
+"""
+
+import os.path
+from pypy.rlib import rposix
+
+if os.name == 'posix':
+ # the posix version is already RPython, just use it
+ rabspath = os.path.abspath
+elif os.name == 'nt':
+ def rabspath(path):
+ if path == '':
+ path = os.getcwd()
+ try:
+ return rposix._getfullpathname(path)
+ except OSError:
+ return path
+else:
+ raise ImportError('Unsupported os: %s' % os.name)
diff --git a/pypy/rlib/test/test_libffi.py b/pypy/rlib/test/test_libffi.py
--- a/pypy/rlib/test/test_libffi.py
+++ b/pypy/rlib/test/test_libffi.py
@@ -222,7 +222,7 @@
if meth.__doc__ is not None and '{' in meth.__doc__:
snippets.append(meth.__doc__)
import re
- for match in re.finditer(" ([a-z_]+)\(", meth.__doc__):
+ for match in re.finditer(" ([A-Za-z_]+)\(", meth.__doc__):
exports.append(match.group(1))
#
c_file.write(STANDARD_DEFINES + str(py.code.Source('\n'.join(snippets))))
@@ -557,10 +557,10 @@
if os.name == 'nt':
def test_stdcall_simple(self):
"""
- int __stdcall std_diff_xy(int x, Signed y)
- {
- return x - y;
- }
+ int __stdcall std_diff_xy(int x, Signed y)
+ {
+ return x - y;
+ }
"""
libfoo = self.get_libfoo()
func = (libfoo, 'std_diff_xy', [types.sint, types.signed], types.sint)
@@ -575,5 +575,36 @@
else:
assert 0, 'wrong calling convention should have raised'
+ def test_by_ordinal(self):
+ """
+ int AAA_first_ordinal_function()
+ {
+ return 42;
+ }
+ """
+ libfoo = self.get_libfoo()
+ f_by_name = libfoo.getpointer('AAA_first_ordinal_function' ,[],
+ types.uint)
+ f_by_ordinal = libfoo.getpointer_by_ordinal(1 ,[], types.uint)
+ print dir(f_by_name)
+ assert f_by_name.funcsym == f_by_ordinal.funcsym
+
+ def test_by_ordinal2(self):
+ """
+ int __stdcall BBB_second_ordinal_function()
+ {
+ return 24;
+ }
+ """
+ from pypy.rlib.libffi import WinDLL
+ dll = WinDLL(self.libfoo_name)
+ f_by_name = dll.getpointer('BBB_second_ordinal_function' ,[],
+ types.uint)
+ f_by_ordinal = dll.getpointer_by_ordinal(2 ,[], types.uint)
+ print dir(f_by_name)
+ assert f_by_name.funcsym == f_by_ordinal.funcsym
+ chain = ArgChain()
+ assert 24 == f_by_ordinal.call(chain, lltype.Signed, is_struct=False)
+
diff --git a/pypy/rlib/test/test_rpath.py b/pypy/rlib/test/test_rpath.py
new file mode 100644
--- /dev/null
+++ b/pypy/rlib/test/test_rpath.py
@@ -0,0 +1,18 @@
+import py
+import os
+from pypy.rlib import rpath
+
+IS_WINDOWS = os.name == 'nt'
+
+def test_rabspath_relative(tmpdir):
+ tmpdir.chdir()
+ assert rpath.rabspath('foo') == tmpdir.join('foo')
+
+ at py.test.mark.skipif("IS_WINDOWS")
+def test_rabspath_absolute_posix():
+ assert rpath.rabspath('/foo') == '/foo'
+
+ at py.test.mark.skipif("not IS_WINDOWS")
+def test_rabspath_absolute_nt():
+ curdrive, _ = os.path.splitdrive(os.getcwd())
+ assert rpath.rabspath('\\foo') == '%s\\foo' % curdrive
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -1234,6 +1234,8 @@
# upgrade to a more recent ctypes (e.g. 1.0.2) if you get
# an OverflowError on the following line.
cvalue = ctypes.cast(ctypes.c_void_p(cvalue), cresulttype)
+ elif RESTYPE == lltype.Bool:
+ cvalue = bool(cvalue)
else:
try:
cvalue = cresulttype(cvalue).value # mask high bits off if needed
diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py
--- a/pypy/rpython/lltypesystem/rffi.py
+++ b/pypy/rpython/lltypesystem/rffi.py
@@ -435,7 +435,8 @@
TYPES += ['signed char', 'unsigned char',
'long long', 'unsigned long long',
'size_t', 'time_t', 'wchar_t',
- 'uintptr_t', 'intptr_t']
+ 'uintptr_t', 'intptr_t',
+ 'void*'] # generic pointer type
_TYPES_ARE_UNSIGNED = set(['size_t', 'uintptr_t']) # plus "unsigned *"
if os.name != 'nt':
TYPES.append('mode_t')
@@ -903,7 +904,7 @@
size = llmemory.sizeof(tp) # a symbolic result in this case
return size
if isinstance(tp, lltype.Ptr) or tp is llmemory.Address:
- tp = lltype.Signed
+ return globals()['r_void*'].BITS/8
if tp is lltype.Char or tp is lltype.Bool:
return 1
if tp is lltype.UniChar:
@@ -934,11 +935,16 @@
offsetof._annspecialcase_ = 'specialize:memo'
# check that we have a sane configuration
-assert maxint == (1 << (8 * sizeof(lltype.Signed) - 1)) - 1, (
+assert maxint == (1 << (8 * sizeof(llmemory.Address) - 1)) - 1, (
"Mixed configuration of the word size of the machine:\n\t"
"the underlying Python was compiled with maxint=%d,\n\t"
- "but the C compiler says that 'long' is %d bytes" % (
- maxint, sizeof(lltype.Signed)))
+ "but the C compiler says that 'void *' is %d bytes" % (
+ maxint, sizeof(llmemory.Address)))
+assert sizeof(lltype.Signed) == sizeof(llmemory.Address), (
+ "Bad configuration: we should manage to get lltype.Signed "
+ "be an integer type of the same size as llmemory.Address, "
+ "but we got %s != %s" % (sizeof(lltype.Signed),
+ sizeof(llmemory.Address)))
# ********************** some helpers *******************
diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py
--- a/pypy/rpython/module/ll_os.py
+++ b/pypy/rpython/module/ll_os.py
@@ -1114,7 +1114,7 @@
def os_getcwd_oofakeimpl():
return OOSupport.to_rstr(os.getcwd())
- return extdef([], str,
+ return extdef([], str0,
"ll_os.ll_os_getcwd", llimpl=os_getcwd_llimpl,
oofakeimpl=os_getcwd_oofakeimpl)
@@ -1362,7 +1362,8 @@
os_isatty = self.llexternal(underscore_on_windows+'isatty', [rffi.INT], rffi.INT)
def isatty_llimpl(fd):
- rposix.validate_fd(fd)
+ if not rposix.is_valid_fd(fd):
+ return False
res = rffi.cast(lltype.Signed, os_isatty(rffi.cast(rffi.INT, fd)))
return res != 0
diff --git a/pypy/rpython/module/ll_os_environ.py b/pypy/rpython/module/ll_os_environ.py
--- a/pypy/rpython/module/ll_os_environ.py
+++ b/pypy/rpython/module/ll_os_environ.py
@@ -67,7 +67,7 @@
rffi.free_charp(l_name)
return result
-register_external(r_getenv, [str0], annmodel.SomeString(can_be_None=True),
+register_external(r_getenv, [str0], annmodel.SomeString(can_be_None=True, no_nul=True),
export_name='ll_os.ll_os_getenv',
llimpl=getenv_llimpl)
diff --git a/pypy/rpython/module/test/test_ll_os.py b/pypy/rpython/module/test/test_ll_os.py
--- a/pypy/rpython/module/test/test_ll_os.py
+++ b/pypy/rpython/module/test/test_ll_os.py
@@ -268,6 +268,14 @@
expected = -signal.SIGTERM
assert proc.wait() == expected
+def test_isatty():
+ try:
+ f = getllimpl(os.isatty)
+ except:
+ skip('No isatty in os')
+ assert f(-1) == False
+
+
class ExpectTestOs:
def setup_class(cls):
if not hasattr(os, 'ttyname'):
diff --git a/pypy/translator/c/funcgen.py b/pypy/translator/c/funcgen.py
--- a/pypy/translator/c/funcgen.py
+++ b/pypy/translator/c/funcgen.py
@@ -214,6 +214,10 @@
myblocknum = self.blocknum[block]
yield ''
yield 'block%d:' % myblocknum
+ if block in self.innerloops:
+ for line in self.gen_while_loop_hack(block):
+ yield line
+ continue
for i, op in enumerate(block.operations):
for line in self.gen_op(op):
yield line
@@ -236,9 +240,6 @@
assert len(block.exits) == 1
for op in self.gen_link(block.exits[0]):
yield op
- elif block in self.innerloops:
- for line in self.gen_while_loop_hack(block):
- yield line
else:
assert block.exitswitch != c_last_exception
# block ending in a switch on a value
@@ -341,11 +342,11 @@
# decision is) we produce code like this:
#
# headblock:
- # ...headblock operations...
- # while (cond) {
+ # while (1) {
+ # ...headblock operations...
+ # if (!cond) break;
# goto firstbodyblock;
- # headblock_back:
- # ...headblock operations...
+ # headblock_back: ;
# }
#
# The real body of the loop is not syntactically within the
@@ -366,19 +367,19 @@
i = list(headblock.exits).index(enterlink)
exitlink = headblock.exits[1 - i]
+ yield 'while (1) {'
+
+ for i, op in enumerate(headblock.operations):
+ for line in self.gen_op(op):
+ yield '\t' + line
+
expr = self.expr(headblock.exitswitch)
- if enterlink.exitcase == False:
+ if enterlink.exitcase == True:
expr = '!' + expr
- yield 'while (%s) {' % expr
+ yield '\tif (%s) break;' % expr
for op in self.gen_link(enterlink):
yield '\t' + op
- # the semicolon after the colon is needed in case no operation
- # produces any code after the label
- yield '\t block%d_back: ;' % self.blocknum[headblock]
- if headblock.operations:
- for i, op in enumerate(headblock.operations):
- for line in self.gen_op(op):
- yield '\t' + line
+ yield ' block%d_back: ;' % self.blocknum[headblock]
yield '}'
for op in self.gen_link(exitlink):
yield op
@@ -715,12 +716,14 @@
def OP_CAST_PRIMITIVE(self, op):
TYPE = self.lltypemap(op.result)
val = self.expr(op.args[0])
+ result = self.expr(op.result)
+ if TYPE == Bool:
+ return "%(result)s = !!%(val)s;" % locals()
ORIG = self.lltypemap(op.args[0])
if ORIG is Char:
val = "(unsigned char)%s" % val
elif ORIG is UniChar:
val = "(unsigned long)%s" % val
- result = self.expr(op.result)
typename = cdecl(self.db.gettype(TYPE), '')
return "%(result)s = (%(typename)s)(%(val)s);" % locals()
diff --git a/pypy/translator/c/gcc/trackgcroot.py b/pypy/translator/c/gcc/trackgcroot.py
--- a/pypy/translator/c/gcc/trackgcroot.py
+++ b/pypy/translator/c/gcc/trackgcroot.py
@@ -476,15 +476,17 @@
# floating-point operations cannot produce GC pointers
'f',
'cvt', 'ucomi', 'comi', 'subs', 'subp' , 'adds', 'addp', 'xorp',
- 'movap', 'movd', 'movlp', 'sqrtsd', 'movhpd',
+ 'movap', 'movd', 'movlp', 'movup', 'sqrt', 'rsqrt', 'movhpd',
'mins', 'minp', 'maxs', 'maxp', 'unpck', 'pxor', 'por', # sse2
'shufps', 'shufpd',
# arithmetic operations should not produce GC pointers
'inc', 'dec', 'not', 'neg', 'or', 'and', 'sbb', 'adc',
'shl', 'shr', 'sal', 'sar', 'rol', 'ror', 'mul', 'imul', 'div', 'idiv',
- 'bswap', 'bt', 'rdtsc',
- 'punpck', 'pshufd', 'pcmp', 'pand', 'psllw', 'pslld', 'psllq',
- 'paddq', 'pinsr', 'pmul', 'psrl',
+ 'bswap', 'bt', 'rdtsc', 'rounds',
+ 'pabs', 'pack', 'padd', 'palign', 'pand', 'pavg', 'pcmp', 'pextr',
+ 'phadd', 'phsub', 'pinsr', 'pmadd', 'pmax', 'pmin', 'pmovmsk',
+ 'pmul', 'por', 'psadb', 'pshuf', 'psign', 'psll', 'psra', 'psrl',
+ 'psub', 'punpck', 'pxor',
# all vectors don't produce pointers
'v',
# sign-extending moves should not produce GC pointers
@@ -492,7 +494,7 @@
# zero-extending moves should not produce GC pointers
'movz',
# locked operations should not move GC pointers, at least so far
- 'lock',
+ 'lock', 'pause',
])
# a partial list is hopefully good enough for now; it's all to support
diff --git a/pypy/translator/c/src/thread_nt.h b/pypy/translator/c/src/thread_nt.h
--- a/pypy/translator/c/src/thread_nt.h
+++ b/pypy/translator/c/src/thread_nt.h
@@ -33,6 +33,7 @@
VOID DeleteNonRecursiveMutex(PNRMUTEX mutex);
DWORD EnterNonRecursiveMutex(PNRMUTEX mutex, BOOL wait);
BOOL LeaveNonRecursiveMutex(PNRMUTEX mutex);
+int RPyThreadLockInit(struct RPyOpaque_ThreadLock *lock);
void RPyOpaqueDealloc_ThreadLock(struct RPyOpaque_ThreadLock *lock);
int RPyThreadAcquireLock(struct RPyOpaque_ThreadLock *lock, int waitflag);
void RPyThreadReleaseLock(struct RPyOpaque_ThreadLock *lock);
@@ -174,7 +175,7 @@
{
}
-int RPyThreadLockInit(struct RPyOpaque_ThreadLock * lock)
+int RPyThreadLockInit(struct RPyOpaque_ThreadLock *lock)
{
return InitializeNonRecursiveMutex(lock);
}
diff --git a/pypy/translator/c/src/thread_pthread.h b/pypy/translator/c/src/thread_pthread.h
--- a/pypy/translator/c/src/thread_pthread.h
+++ b/pypy/translator/c/src/thread_pthread.h
@@ -134,10 +134,15 @@
/* Jump through some hoops for Alpha OSF/1 */
threadid = pthread_self();
+#ifdef __CYGWIN__
+ /* typedef __uint32_t pthread_t; */
+ return (long) threadid;
+#else
if (sizeof(pthread_t) <= sizeof(long))
return (long) threadid;
else
return (long) *(long *) &threadid;
+#endif
}
static long _pypythread_stacksize = 0;
@@ -190,10 +195,15 @@
pthread_detach(th);
+#ifdef __CYGWIN__
+ /* typedef __uint32_t pthread_t; */
+ return (long) th;
+#else
if (sizeof(pthread_t) <= sizeof(long))
return (long) th;
else
return (long) *(long *) &th;
+#endif
}
long RPyThreadGetStackSize(void)
diff --git a/pypy/translator/c/test/test_typed.py b/pypy/translator/c/test/test_typed.py
--- a/pypy/translator/c/test/test_typed.py
+++ b/pypy/translator/c/test/test_typed.py
@@ -895,3 +895,12 @@
f = self.getcompiled(func, [int])
res = f(-2000000000)
assert res == -200000000000000
+
+ def test_bool_2(self):
+ from pypy.rpython.lltypesystem import lltype, rffi
+ def func(n):
+ x = rffi.cast(lltype.Bool, n)
+ return int(x)
+ f = self.getcompiled(func, [int])
+ res = f(2)
+ assert res == 1 # and not 2
diff --git a/pypy/translator/cli/opcodes.py b/pypy/translator/cli/opcodes.py
--- a/pypy/translator/cli/opcodes.py
+++ b/pypy/translator/cli/opcodes.py
@@ -97,6 +97,8 @@
'jit_force_quasi_immutable':Ignore,
'jit_force_virtualizable': Ignore,
'jit_force_virtual': DoNothing,
+ 'jit_force_quasi_immutable':Ignore,
+ 'jit_is_virtual': [PushPrimitive(ootype.Bool, False)],
}
# __________ numeric operations __________
diff --git a/pypy/translator/goal/app_main.py b/pypy/translator/goal/app_main.py
--- a/pypy/translator/goal/app_main.py
+++ b/pypy/translator/goal/app_main.py
@@ -213,69 +213,19 @@
# ____________________________________________________________
# Main entry point
-# see nanos.py for explainment why we do not import os here
-# CAUTION!
-# remember to update nanos.py if you are using more functions
-# from os or os.path!
-# Running test/test_nanos.py might be helpful as well.
-
def we_are_translated():
# app-level, very different from pypy.rlib.objectmodel.we_are_translated
return hasattr(sys, 'pypy_translation_info')
if 'nt' in sys.builtin_module_names:
IS_WINDOWS = True
- DRIVE_LETTER_SEP = ':'
else:
IS_WINDOWS = False
-def get_library_path(executable):
- search = executable
- while 1:
- dirname = resolvedirof(search)
- if dirname == search:
- # not found! let's hope that the compiled-in path is ok
- print >> sys.stderr, """\
-debug: WARNING: Library path not found, using compiled-in sys.path.
-debug: WARNING: 'sys.prefix' will not be set.
-debug: WARNING: Make sure the pypy binary is kept inside its tree of files.
-debug: WARNING: It is ok to create a symlink to it from somewhere else."""
- newpath = sys.path[:]
- break
- newpath = sys.pypy_initial_path(dirname)
- if newpath is None:
- search = dirname # walk to the parent directory
- continue
- break # found!
- return newpath
-def setup_sys_executable(executable, nanos):
- # a substituted os if we are translated
- global os
- os = nanos
- # find the full path to the executable, assuming that if there is no '/'
- # in the provided one then we must look along the $PATH
- if we_are_translated() and IS_WINDOWS and not executable.lower().endswith('.exe'):
- executable += '.exe'
- if os.sep in executable or (IS_WINDOWS and DRIVE_LETTER_SEP in executable):
- pass # the path is already more than just an executable name
- else:
- path = os.getenv('PATH')
- if path:
- for dir in path.split(os.pathsep):
- fn = os.path.join(dir, executable)
- if os.path.isfile(fn):
- executable = fn
- break
- sys.executable = os.path.abspath(executable)
- #
- # 'sys.executable' should not end up being an non-existing file;
- # just use '' in this case. (CPython issue #7774)
- if not os.path.isfile(sys.executable):
- sys.executable = ''
-
-def setup_initial_paths(ignore_environment=False, **extra):
- newpath = get_library_path(sys.executable)
+def setup_and_fix_paths(ignore_environment=False, **extra):
+ import os
+ newpath = sys.path[:]
readenv = not ignore_environment
path = readenv and os.getenv('PYTHONPATH')
if path:
@@ -288,13 +238,39 @@
sys.path.append(dir)
_seen[dir] = True
+def set_stdio_encodings(ignore_environment):
+ import os
+ readenv = not ignore_environment
+ io_encoding = readenv and os.getenv("PYTHONIOENCODING")
+ if io_encoding:
+ errors = None
+ if ":" in io_encoding:
+ io_encoding, errors = io_encoding.split(":", 1)
+ set_io_encoding(io_encoding, io_encoding, errors, True)
+ else:
+ if IS_WINDOWS:
+ import __pypy__
+ io_encoding, io_encoding_output = __pypy__.get_console_cp()
+ else:
+ io_encoding = io_encoding_output = sys.getfilesystemencoding()
+ if io_encoding:
+ set_io_encoding(io_encoding, io_encoding_output, None, False)
+
def set_io_encoding(io_encoding, io_encoding_output, errors, overridden):
try:
import _file
except ImportError:
if sys.version_info < (2, 7):
return
- import ctypes # HACK: while running on top of CPython
+ # HACK: while running on top of CPython, and make sure to import
+ # CPython's ctypes (because at this point sys.path has already been
+ # set to the pypy one)
+ pypy_path = sys.path
+ try:
+ sys.path = sys.cpython_path
+ import ctypes
+ finally:
+ sys.path = pypy_path
set_file_encoding = ctypes.pythonapi.PyFile_SetEncodingAndErrors
set_file_encoding.argtypes = [ctypes.py_object, ctypes.c_char_p, ctypes.c_char_p]
else:
@@ -417,6 +393,7 @@
def parse_command_line(argv):
+ import os
options = default_options.copy()
options['warnoptions'] = []
#
@@ -457,13 +434,13 @@
if PYTHON26 and not options["ignore_environment"]:
if os.getenv('PYTHONNOUSERSITE'):
- options["no_user_site"] = True
+ options["no_user_site"] = 1
if os.getenv('PYTHONDONTWRITEBYTECODE'):
- options["dont_write_bytecode"] = True
+ options["dont_write_bytecode"] = 1
if (options["interactive"] or
(not options["ignore_environment"] and os.getenv('PYTHONINSPECT'))):
- options["inspect"] = True
+ options["inspect"] = 1
if PYTHON26 and we_are_translated():
flags = [options[flag] for flag in sys_flags]
@@ -496,6 +473,7 @@
# but we need more in the translated PyPy for the compiler package
if '__pypy__' not in sys.builtin_module_names:
sys.setrecursionlimit(5000)
+ import os
if unbuffered:
set_unbuffered_io()
@@ -511,22 +489,9 @@
except:
print >> sys.stderr, "'import site' failed"
+ set_stdio_encodings(ignore_environment)
+
readenv = not ignore_environment
- io_encoding = readenv and os.getenv("PYTHONIOENCODING")
- if io_encoding:
- errors = None
- if ":" in io_encoding:
- io_encoding, errors = io_encoding.split(":", 1)
- set_io_encoding(io_encoding, io_encoding, errors, True)
- else:
- if IS_WINDOWS:
- import __pypy__
- io_encoding, io_encoding_output = __pypy__.get_console_cp()
- else:
- io_encoding = io_encoding_output = sys.getfilesystemencoding()
- if io_encoding:
- set_io_encoding(io_encoding, io_encoding_output, None, False)
-
pythonwarnings = readenv and os.getenv('PYTHONWARNINGS')
if pythonwarnings:
warnoptions.extend(pythonwarnings.split(','))
@@ -624,7 +589,7 @@
# on the command-line.
filename = sys.argv[0]
mainmodule.__file__ = filename
- sys.path.insert(0, resolvedirof(filename))
+ sys.path.insert(0, sys.pypy_resolvedirof(filename))
# assume it's a pyc file only if its name says so.
# CPython goes to great lengths to detect other cases
# of pyc file format, but I think it's ok not to care.
@@ -669,28 +634,46 @@
return status
-def resolvedirof(filename):
- try:
- filename = os.path.abspath(filename)
- except OSError:
- pass
- dirname = os.path.dirname(filename)
- if os.path.islink(filename):
- try:
- link = os.readlink(filename)
- except OSError:
- pass
- else:
- return resolvedirof(os.path.join(dirname, link))
- return dirname
-
def print_banner():
print 'Python %s on %s' % (sys.version, sys.platform)
print ('Type "help", "copyright", "credits" or '
'"license" for more information.')
-def entry_point(executable, argv, nanos):
- setup_sys_executable(executable, nanos)
+STDLIB_WARNING = """\
+debug: WARNING: Library path not found, using compiled-in sys.path.
+debug: WARNING: 'sys.prefix' will not be set.
+debug: WARNING: Make sure the pypy binary is kept inside its tree of files.
+debug: WARNING: It is ok to create a symlink to it from somewhere else."""
+
+def setup_bootstrap_path(executable):
+ """
+ Try to to as little as possible and to have the stdlib in sys.path. In
+ particular, we cannot use any unicode at this point, because lots of
+ unicode operations require to be able to import encodings.
+ """
+ # at this point, sys.path is set to the compiled-in one, based on the
+ # location where pypy was compiled. This is set during the objspace
+ # initialization by module.sys.state.State.setinitialpath.
+ #
+ # Now, we try to find the absolute path of the executable and the stdlib
+ # path
+ executable = sys.pypy_find_executable(executable)
+ stdlib_path = sys.pypy_find_stdlib(executable)
+ if stdlib_path is None:
+ print >> sys.stderr, STDLIB_WARNING
+ else:
+ sys.path[:] = stdlib_path
+ # from this point on, we are free to use all the unicode stuff we want,
+ # This is important for py3k
+ sys.executable = executable
+
+def entry_point(executable, argv):
+ # note that before calling setup_bootstrap_path, we are limited because we
+ # cannot import stdlib modules. In particular, we cannot use unicode
+ # stuffs (because we need to be able to import encodings) and we cannot
+ # import os, which is used a bit everywhere in app_main, but only imported
+ # *after* setup_bootstrap_path
+ setup_bootstrap_path(executable)
try:
cmdline = parse_command_line(argv)
except CommandLineError, e:
@@ -698,24 +681,33 @@
return 2
except SystemExit, e:
return e.code or 0
- setup_initial_paths(**cmdline)
+ setup_and_fix_paths(**cmdline)
return run_command_line(**cmdline)
if __name__ == '__main__':
- import autopath
- import nanos
# obscure! try removing the following line, see how it crashes, and
# guess why...
ImStillAroundDontForgetMe = sys.modules['__main__']
# debugging only
- def pypy_initial_path(s):
- from pypy.module.sys.state import getinitialpath
- try:
- return getinitialpath(None, s)
- except OSError:
- return None
+ def pypy_find_executable(s):
+ import os
+ return os.path.abspath(s)
+
+ def pypy_find_stdlib(s):
+ from os.path import abspath, join, dirname as dn
+ thisfile = abspath(__file__)
+ root = dn(dn(dn(dn(thisfile))))
+ return [join(root, 'lib-python', '2.7'),
+ join(root, 'lib_pypy')]
+
+ def pypy_resolvedirof(s):
+ # we ignore the issue of symlinks; for tests, the executable is always
+ # translator/goal/app_main.py anyway
+ import os
+ return os.path.abspath(os.path.join(s, '..'))
+
# add an emulator for these pypy-only or 2.7-only functions
# (for test_pyc_commandline_argument)
@@ -738,25 +730,27 @@
imp._run_compiled_module = _run_compiled_module
imp._getimporter = _getimporter
- # stick the current sys.path into $PYTHONPATH, so that CPython still
- # finds its own extension modules :-/
import os
- os.environ['PYTHONPATH'] = ':'.join(sys.path)
reset = []
if 'PYTHONINSPECT_' in os.environ:
reset.append(('PYTHONINSPECT', os.environ.get('PYTHONINSPECT', '')))
os.environ['PYTHONINSPECT'] = os.environ['PYTHONINSPECT_']
+ if 'PYTHONWARNINGS_' in os.environ:
+ reset.append(('PYTHONWARNINGS', os.environ.get('PYTHONWARNINGS', '')))
+ os.environ['PYTHONWARNINGS'] = os.environ['PYTHONWARNINGS_']
+ del os # make sure that os is not available globally, because this is what
+ # happens in "real life" outside the tests
# no one should change to which lists sys.argv and sys.path are bound
old_argv = sys.argv
old_path = sys.path
- from pypy.module.sys.version import PYPY_VERSION
- sys.pypy_version_info = PYPY_VERSION
- sys.pypy_initial_path = pypy_initial_path
- os = nanos.os_module_for_testing
+ sys.pypy_find_executable = pypy_find_executable
+ sys.pypy_find_stdlib = pypy_find_stdlib
+ sys.pypy_resolvedirof = pypy_resolvedirof
+ sys.cpython_path = sys.path[:]
try:
- sys.exit(int(entry_point(sys.argv[0], sys.argv[1:], os)))
+ sys.exit(int(entry_point(sys.argv[0], sys.argv[1:])))
finally:
# restore the normal prompt (which was changed by _pypy_interact), in
# case we are dropping to CPython's prompt
diff --git a/pypy/translator/goal/nanos.py b/pypy/translator/goal/nanos.py
deleted file mode 100644
--- a/pypy/translator/goal/nanos.py
+++ /dev/null
@@ -1,290 +0,0 @@
-"""
-Not An os or Nano os :-)
-
-Implementation of a few methods needed for starting up
-PyPy in app_main without importing os there.
-
-At startup time, app_main wants to find out about itself,
-sys.executable, the path to its library etc.
-This is done the easiest using os.py, but since this
-is a python module, we have a recurrence problem.
-
-Importing it at compile time would work, partially,
-but the way os is initialized will cause os.getenv
-to malfunction, due to caching problems.
-
-The solution taken here implements a minimal os using
-app-level code that is created at compile-time. Only
-the few needed functions are implemented in a tiny os module
-that contains a tiny path module.
-
-os.getenv got a direct implementation to overcome the caching
-problem.
-
-Please adjust the applevel code below, if you need to support
-more from os and os.path.
-"""
-
-from pypy.interpreter.gateway import applevel, interp2app
-import os, py
-
-if os.name == 'posix':
- # code copied from posixpath.py
- app_os_path = applevel("""
- def dirname(p):
- i = p.rfind('/') + 1
- head = p[:i]
- if head and head != '/'*len(head):
- head = head.rstrip('/')
- return head
-
- def join(path, b):
- if b.startswith('/'):
- path = b
- elif path == '' or path.endswith('/'):
- path += b
- else:
- path += '/' + b
- return path
-
- def normpath(path):
- if path == '':
- return '.'
- initial_slashes = path.startswith('/')
- # POSIX allows one or two initial slashes, but treats three or more
- # as single slash.
- if (initial_slashes and
- path.startswith('//') and not path.startswith('///')):
- initial_slashes = 2
- comps = path.split('/')
- new_comps = []
- for comp in comps:
- if comp in ('', '.'):
- continue
- if (comp != '..' or (not initial_slashes and not new_comps) or
- (new_comps and new_comps[-1] == '..')):
- new_comps.append(comp)
- elif new_comps:
- new_comps.pop()
- comps = new_comps
- path = '/'.join(comps)
- if initial_slashes:
- path = '/'*initial_slashes + path
- return path or '.'
-
- def abspath(path):
- if not path.startswith('/'):
- import posix
- cwd = posix.getcwd()
- path = join(cwd, path)
- return normpath(path)
-
- def isfile(path):
- import posix
- try:
- st = posix.stat(path)
- except posix.error:
- return False
- return (st.st_mode & 0170000) == 0100000 # S_ISREG
-
- def islink(path):
- import posix
- try:
- st = posix.lstat(path)
- except posix.error:
- return False
- return (st.st_mode & 0170000) == 0120000 # S_ISLNK
-
- """, filename=__file__)
-
- app_os = applevel("""
- sep = '/'
- pathsep = ':'
- name = 'posix'
-
- def readlink(fn):
- import posix
- return posix.readlink(fn)
- """, filename=__file__)
-
-elif os.name == 'nt':
- # code copied from ntpath.py
- app_os_path = applevel(r"""
- def splitdrive(p):
- if p[1:2] == ':':
- return p[0:2], p[2:]
- return '', p
-
- def isabs(s):
- s = splitdrive(s)[1]
- return s != '' and s[:1] in '/\\'
-
- def dirname(p):
- d, p = splitdrive(p)
- # set i to index beyond p's last slash
- i = len(p)
- while i and p[i-1] not in '/\\':
- i = i - 1
- head = p[:i]
- # remove trailing slashes from head, unless it's all slashes
- head2 = head
- while head2 and head2[-1] in '/\\':
- head2 = head2[:-1]
- head = head2 or head
- return d + head
-
- def join(path, b):
- b_wins = 0 # set to 1 iff b makes path irrelevant
- if path == "":
- b_wins = 1
-
- elif isabs(b):
- # This probably wipes out path so far. However, it's more
- # complicated if path begins with a drive letter:
- # 1. join('c:', '/a') == 'c:/a'
- # 2. join('c:/', '/a') == 'c:/a'
- # But
- # 3. join('c:/a', '/b') == '/b'
- # 4. join('c:', 'd:/') = 'd:/'
- # 5. join('c:/', 'd:/') = 'd:/'
- if path[1:2] != ":" or b[1:2] == ":":
- # Path doesn't start with a drive letter, or cases 4 and 5.
- b_wins = 1
-
- # Else path has a drive letter, and b doesn't but is absolute.
- elif len(path) > 3 or (len(path) == 3 and
- path[-1] not in "/\\"):
- # case 3
- b_wins = 1
-
- if b_wins:
- path = b
- else:
- # Join, and ensure there's a separator.
- assert len(path) > 0
- if path[-1] in "/\\":
- if b and b[0] in "/\\":
- path += b[1:]
- else:
- path += b
- elif path[-1] == ":":
- path += b
- elif b:
- if b[0] in "/\\":
- path += b
- else:
- path += "\\" + b
- else:
- # path is not empty and does not end with a backslash,
- # but b is empty; since, e.g., split('a/') produces
- # ('a', ''), it's best if join() adds a backslash in
- # this case.
- path += '\\'
-
- return path
-
- def normpath(path):
- if path.startswith(('\\\\.\\', '\\\\?\\')):
- # in the case of paths with these prefixes:
- # \\.\ -> device names
- # \\?\ -> literal paths
- # do not do any normalization, but return the path unchanged
- return path
- path = path.replace('/', '\\')
- prefix, path = splitdrive(path)
- # We need to be careful here. If the prefix is empty, and
- # the path starts with a backslash, it could either be an
- # absolute path on the current drive (\dir1\dir2\file) or a
- # UNC filename (\\server\mount\dir1\file). It is therefore
- # imperative NOT to collapse multiple backslashes blindly in
- # that case. The code below preserves multiple backslashes
- # when there is no drive letter. This means that the invalid
- # filename \\\a\b is preserved unchanged, where a\\\b is
- # normalised to a\b. It's not clear that there is any better
- # behaviour for such edge cases.
- if prefix == '':
- # No drive letter - preserve initial backslashes
- while path[:1] == "\\":
- prefix = prefix + '\\'
- path = path[1:]
- else:
- # We have a drive letter - collapse initial backslashes
- if path.startswith("\\"):
- prefix = prefix + '\\'
- path = path.lstrip("\\")
- comps = path.split("\\")
- i = 0
- while i < len(comps):
- if comps[i] in ('.', ''):
- del comps[i]
- elif comps[i] == '..':
- if i > 0 and comps[i-1] != '..':
- del comps[i-1:i+1]
- i -= 1
- elif i == 0 and prefix.endswith("\\"):
- del comps[i]
- else:
- i += 1
- else:
- i += 1
- # If the path is now empty, substitute '.'
- if not prefix and not comps:
- comps.append('.')
- return prefix + '\\'.join(comps)
-
- def abspath(path):
- import nt
- if path: # Empty path must return current working directory.
- try:
- path = nt._getfullpathname(path)
- except WindowsError:
- pass # Bad path - return unchanged.
- else:
- path = nt.getcwd()
- return normpath(path)
-
- def isfile(path):
- import nt
- try:
- st = nt.stat(path)
- except nt.error:
- return False
- return (st.st_mode & 0170000) == 0100000 # S_ISREG
-
- def islink(path):
- return False
-
- """, filename=__file__)
-
- app_os = applevel(r"""
- sep = '\\'
- pathsep = ';'
- name = 'nt'
- """, filename=__file__)
-
-else:
- raise NotImplementedError("os.name == %r" % (os.name,))
-
-def getenv(space, w_name):
- name = space.str0_w(w_name)
- return space.wrap(os.environ.get(name))
-getenv_w = interp2app(getenv)
-
-def setup_nanos(space):
- w_os = space.wrap(app_os.buildmodule(space, 'os'))
- w_os_path = space.wrap(app_os_path.buildmodule(space, 'path'))
- space.setattr(w_os, space.wrap('path'), w_os_path)
- space.setattr(w_os, space.wrap('getenv'), space.wrap(getenv_w))
- return w_os
-
-
-# in order to be able to test app_main without the pypy interpreter
-# we create the nanos module with the same names here like it would
-# be created while translation
-path_module_for_testing = type(os)("os.path")
-os_module_for_testing = type(os)("os")
-os_module_for_testing.path = path_module_for_testing
-os_module_for_testing.getenv = os.getenv
-eval(py.code.Source(app_os_path.source).compile(), path_module_for_testing.__dict__)
-eval(py.code.Source(app_os.source).compile(), os_module_for_testing.__dict__)
-
diff --git a/pypy/translator/goal/targetpypystandalone.py b/pypy/translator/goal/targetpypystandalone.py
--- a/pypy/translator/goal/targetpypystandalone.py
+++ b/pypy/translator/goal/targetpypystandalone.py
@@ -8,7 +8,6 @@
from pypy.config.config import Config, to_optparse, make_dict, SUPPRESS_USAGE
from pypy.config.config import ConflictConfigError
from pypy.tool.option import make_objspace
-from pypy.translator.goal.nanos import setup_nanos
thisdir = py.path.local(__file__).dirpath()
@@ -27,7 +26,6 @@
w_run_toplevel = space.getitem(w_dict, space.wrap('run_toplevel'))
w_call_finish_gateway = space.wrap(gateway.interp2app(call_finish))
w_call_startup_gateway = space.wrap(gateway.interp2app(call_startup))
- w_os = setup_nanos(space)
withjit = space.config.objspace.usemodules.pypyjit
def entry_point(argv):
@@ -56,7 +54,7 @@
w_executable = space.wrap(argv[0])
w_argv = space.newlist([space.wrap(s) for s in argv[1:]])
space.timer.start("w_entry_point")
- w_exitcode = space.call_function(w_entry_point, w_executable, w_argv, w_os)
+ w_exitcode = space.call_function(w_entry_point, w_executable, w_argv)
space.timer.stop("w_entry_point")
exitcode = space.int_w(w_exitcode)
# try to pull it all in
diff --git a/pypy/translator/goal/test2/test_app_main.py b/pypy/translator/goal/test2/test_app_main.py
--- a/pypy/translator/goal/test2/test_app_main.py
+++ b/pypy/translator/goal/test2/test_app_main.py
@@ -23,8 +23,7 @@
def getscript(source):
p = _get_next_path()
p.write(str(py.code.Source(source)))
- # return relative path for testing purposes
- return py.path.local().bestrelpath(p)
+ return str(p)
def getscript_pyc(space, source):
p = _get_next_path()
@@ -200,6 +199,11 @@
http://pexpect.sourceforge.net/
"""
+ def setup_class(cls):
+ # some tests need to be able to import test2, change the cwd
+ goal_dir = os.path.abspath(os.path.join(autopath.this_dir, '..'))
+ os.chdir(goal_dir)
+
def _spawn(self, *args, **kwds):
try:
import pexpect
@@ -221,7 +225,7 @@
pexpect.__version__,))
kwds.setdefault('timeout', 10)
- print 'SPAWN:', args, kwds
+ print 'SPAWN:', ' '.join([args[0]] + args[1]), kwds
child = pexpect.spawn(*args, **kwds)
child.logfile = sys.stdout
return child
@@ -319,7 +323,8 @@
child.sendline('import sys; sys.argv')
child.expect(re.escape("['-c']"))
- def test_options_i_c_crashing(self):
+ def test_options_i_c_crashing(self, monkeypatch):
+ monkeypatch.setenv('PYTHONPATH', None)
child = self.spawn(['-i', '-c', 'x=666;foobar'])
child.expect('NameError')
idx = child.expect(['>>> ', re.escape(banner)])
@@ -351,28 +356,25 @@
sys.stdin = old
child.expect('foobye')
- def test_pythonstartup(self):
- old = os.environ.get('PYTHONSTARTUP', '')
- try:
- os.environ['PYTHONSTARTUP'] = crashing_demo_script
- child = self.spawn([])
- child.expect(re.escape(banner))
- child.expect('Traceback')
- child.expect('NameError')
- child.expect('>>> ')
- child.sendline('[myvalue2]')
- child.expect(re.escape('[11]'))
- child.expect('>>> ')
+ def test_pythonstartup(self, monkeypatch):
+ monkeypatch.setenv('PYTHONPATH', None)
+ monkeypatch.setenv('PYTHONSTARTUP', crashing_demo_script)
+ child = self.spawn([])
+ child.expect(re.escape(banner))
+ child.expect('Traceback')
+ child.expect('NameError')
+ child.expect('>>> ')
+ child.sendline('[myvalue2]')
+ child.expect(re.escape('[11]'))
+ child.expect('>>> ')
- child = self.spawn(['-i', demo_script])
- for line in ['hello', 'goodbye', '>>> ']:
- idx = child.expect([line, 'Hello2'])
- assert idx == 0 # no PYTHONSTARTUP run here
- child.sendline('myvalue2')
- child.expect('Traceback')
- child.expect('NameError')
- finally:
- os.environ['PYTHONSTARTUP'] = old
+ child = self.spawn(['-i', demo_script])
+ for line in ['hello', 'goodbye', '>>> ']:
+ idx = child.expect([line, 'Hello2'])
+ assert idx == 0 # no PYTHONSTARTUP run here
+ child.sendline('myvalue2')
+ child.expect('Traceback')
+ child.expect('NameError')
def test_ignore_python_startup(self):
old = os.environ.get('PYTHONSTARTUP', '')
@@ -420,7 +422,7 @@
p = os.path.join(autopath.this_dir, 'mymodule.py')
p = os.path.abspath(p)
child = self.spawn(['-i',
- '-m', 'pypy.translator.goal.test2.mymodule',
+ '-m', 'test2.mymodule',
'extra'])
child.expect('mymodule running')
child.expect('Name: __main__')
@@ -431,9 +433,9 @@
child.expect(re.escape(repr("foobar")))
child.expect('>>> ')
child.sendline('import sys')
- child.sendline('"pypy.translator.goal.test2" in sys.modules')
+ child.sendline('"test2" in sys.modules')
child.expect('True')
- child.sendline('"pypy.translator.goal.test2.mymodule" in sys.modules')
+ child.sendline('"test2.mymodule" in sys.modules')
child.expect('False')
child.sendline('sys.path[0]')
child.expect("''")
@@ -513,7 +515,7 @@
print 'A five ounce bird could not carry a one pound coconut.'
""")
py_py = os.path.join(autopath.pypydir, 'bin', 'py.py')
- child = self._spawn(sys.executable, [py_py, path])
+ child = self._spawn(sys.executable, [py_py, '-S', path])
child.expect('Are you suggesting coconuts migrate?', timeout=120)
child.sendline('Not at all. They could be carried.')
child.expect('A five ounce bird could not carry a one pound coconut.')
@@ -524,7 +526,7 @@
child = self.spawn(['-cprint "hel" + "lo"'])
child.expect('hello')
- child = self.spawn(['-mpypy.translator.goal.test2.mymodule'])
+ child = self.spawn(['-mtest2.mymodule'])
child.expect('mymodule running')
def test_ps1_only_if_interactive(self):
@@ -607,33 +609,28 @@
data = self.run('-c "print 6**5"')
assert '7776' in data
- def test_no_pythonstartup(self):
- old = os.environ.get('PYTHONSTARTUP', '')
- try:
- os.environ['PYTHONSTARTUP'] = crashing_demo_script
- data = self.run('"%s"' % (demo_script,))
- assert 'Hello2' not in data
- data = self.run('-c pass')
- assert 'Hello2' not in data
- finally:
- os.environ['PYTHONSTARTUP'] = old
+ def test_no_pythonstartup(self, monkeypatch):
+ monkeypatch.setenv('PYTHONSTARTUP', crashing_demo_script)
+ data = self.run('"%s"' % (demo_script,))
+ assert 'Hello2' not in data
+ data = self.run('-c pass')
+ assert 'Hello2' not in data
- def test_pythonwarnings(self):
- old = os.environ.get('PYTHONWARNINGS', '')
- try:
- os.environ['PYTHONWARNINGS'] = "once,error"
- data = self.run('-W ignore -W default '
- '-c "import sys; print sys.warnoptions"')
- assert "['ignore', 'default', 'once', 'error']" in data
- finally:
- os.environ['PYTHONWARNINGS'] = old
+ def test_pythonwarnings(self, monkeypatch):
+ # PYTHONWARNINGS_ is special cased by app_main: we cannot directly set
+ # PYTHONWARNINGS because else the warnings raised from within pypy are
+ # turned in errors.
+ monkeypatch.setenv('PYTHONWARNINGS_', "once,error")
+ data = self.run('-W ignore -W default '
+ '-c "import sys; print sys.warnoptions"')
+ assert "['ignore', 'default', 'once', 'error']" in data
def test_option_m(self):
if not hasattr(runpy, '_run_module_as_main'):
skip("requires CPython >= 2.6")
p = os.path.join(autopath.this_dir, 'mymodule.py')
p = os.path.abspath(p)
- data = self.run('-m pypy.translator.goal.test2.mymodule extra')
+ data = self.run('-m test2.mymodule extra')
assert 'mymodule running' in data
assert 'Name: __main__' in data
# ignoring case for windows. abspath behaves different from autopath
@@ -740,6 +737,7 @@
assert data == p + os.sep + '\n'
def test_getfilesystemencoding(self):
+ py.test.skip("this has been failing since forever, but it's not tested nightly because buildbot uses python2.6 :-(")
if sys.version_info < (2, 7):
skip("test requires Python >= 2.7")
p = getscript_in_dir("""
@@ -821,9 +819,9 @@
class AppTestAppMain:
def setup_class(self):
- # ------------------------------------
- # setup code for test_get_library_path
- # ------------------------------------
+ # ----------------------------------------
+ # setup code for test_setup_bootstrap_path
+ # ----------------------------------------
from pypy.module.sys.version import CPYTHON_VERSION, PYPY_VERSION
cpy_ver = '%d.%d' % CPYTHON_VERSION[:2]
@@ -840,37 +838,58 @@
self.w_fake_exe = self.space.wrap(str(fake_exe))
self.w_expected_path = self.space.wrap(expected_path)
self.w_trunkdir = self.space.wrap(os.path.dirname(autopath.pypydir))
+ #
+ foo_py = prefix.join('foo.py').write("pass")
+ self.w_foo_py = self.space.wrap(str(foo_py))
- def test_get_library_path(self):
+ def test_setup_bootstrap_path(self):
import sys
import os
+ old_sys_path = sys.path[:]
sys.path.append(self.goal_dir)
try:
import app_main
- app_main.os = os
- newpath = app_main.get_library_path('/tmp/pypy-c') # stdlib not found
- assert newpath == sys.path
- newpath = app_main.get_library_path(self.fake_exe)
+ app_main.setup_bootstrap_path('/tmp/pypy-c') # stdlib not found
+ sys.path == old_sys_path
+ assert sys.executable == ''
+ #
+ app_main.setup_bootstrap_path(self.fake_exe)
+ assert sys.executable == self.fake_exe
+ newpath = sys.path[:]
if newpath[0].endswith('__extensions__'):
newpath = newpath[1:]
# we get at least 'expected_path', and maybe more (e.g.plat-linux2)
assert newpath[:len(self.expected_path)] == self.expected_path
finally:
- sys.path.pop()
+ sys.path[:] = old_sys_path
def test_trunk_can_be_prefix(self):
import sys
import os
+ old_sys_path = sys.path[:]
sys.path.append(self.goal_dir)
try:
import app_main
- app_main.os = os
pypy_c = os.path.join(self.trunkdir, 'pypy', 'translator', 'goal', 'pypy-c')
- newpath = app_main.get_library_path(pypy_c)
+ app_main.setup_bootstrap_path(pypy_c)
+ newpath = sys.path[:]
# we get at least lib_pypy
# lib-python/X.Y.Z, and maybe more (e.g. plat-linux2)
assert len(newpath) >= 2
for p in newpath:
assert p.startswith(self.trunkdir)
finally:
- sys.path.pop()
+ sys.path[:] = old_sys_path
+
+ def test_entry_point(self):
+ import sys
+ import os
+ old_sys_path = sys.path[:]
+ sys.path.append(self.goal_dir)
+ try:
+ import app_main
+ pypy_c = os.path.join(self.trunkdir, 'pypy', 'translator', 'goal', 'pypy-c')
+ app_main.entry_point(pypy_c, [self.foo_py])
+ # assert it did not crash
+ finally:
+ sys.path[:] = old_sys_path
diff --git a/pypy/translator/goal/test2/test_nanos.py b/pypy/translator/goal/test2/test_nanos.py
deleted file mode 100644
--- a/pypy/translator/goal/test2/test_nanos.py
+++ /dev/null
@@ -1,85 +0,0 @@
-"""
-Tests for the entry point of pypy-c, whether nanos.py is supplying
-the needed names for app_main.py.
-"""
-import os
-
-from pypy.translator.goal import app_main
-this_dir = os.path.dirname(app_main.__file__)
-
-from pypy.objspace.std import Space
-from pypy.translator.goal.targetpypystandalone import create_entry_point
-from pypy.tool.udir import udir
-
-
-class TestNanos:
- def getnanos(self):
- from pypy.translator.goal.nanos import os_module_for_testing
- return os_module_for_testing
-
- def test_exists(self):
- os1 = self.getnanos()
- assert os1.name == os.name
- assert os1.sep == os.sep
- assert os1.pathsep == os.pathsep
-
- def test_dirname(self):
- p1 = os.path
- p2 = self.getnanos().path
- path = str(udir.join('baz'))
- assert p1.dirname(path) == p2.dirname(path)
- assert p1.dirname(path + os.sep) == p2.dirname(path + os.sep)
- assert p1.dirname(path + 2*os.sep) == p2.dirname(path + 2*os.sep)
- assert p1.dirname(p1.dirname(path)) == p2.dirname(p2.dirname(path))
-
- def test_join(self):
- p1 = os.path
- p2 = self.getnanos().path
- base = str(udir)
- assert p1.join(base, '') == p2.join(base, '')
- assert p1.join(base, 'baz') == p2.join(base, 'baz')
- assert p1.join(base + os.sep, 'baz') == p2.join(base + os.sep, 'baz')
- assert p1.join(base, 'baz' + os.sep) == p2.join(base, 'baz' + os.sep)
- assert p1.join(base, base) == p2.join(base, base)
-
- def test_abspath(self):
- p2 = self.getnanos().path
- base = str(udir)
- assert p2.abspath(base) == base
- assert p2.abspath('x') == os.path.join(os.getcwd(), 'x')
-
- def test_abspath_uses_normpath(self):
- p1 = os.path
- p2 = self.getnanos().path
- base = str(udir)
- assert p2.abspath(p1.join(base, '.')) == base
- assert p2.abspath(p1.join(base, '.', '.', '.')) == base
- assert p2.abspath(p1.join(base, 'foo', '..')) == base
-
- def test_isfile(self):
- p2 = self.getnanos().path
- udir.join('test_isfile').write('\n')
- base = str(udir)
- assert p2.isfile(p2.join(base, 'test_isfile'))
- assert not p2.isfile(p2.join(base, 'test_isfile.DOES.NOT.EXIST'))
- assert not p2.isfile(base)
-
-
-def test_nanos():
- space = Space()
- # manually imports app_main.py
- filename = os.path.join(this_dir, 'app_main.py')
- w_dict = space.newdict()
- space.exec_(open(filename).read(), w_dict, w_dict)
- entry_point = create_entry_point(space, w_dict)
-
- # check that 'os' is not in sys.modules
- assert not space.is_true(
- space.call_method(space.sys.get('modules'),
- '__contains__', space.wrap('os')))
- # But that 'sys' is still present
- assert space.is_true(
- space.call_method(space.sys.get('modules'),
- '__contains__', space.wrap('sys')))
-
- entry_point(['', '-c', 'print 42'])
diff --git a/pypy/translator/platform/distutils_platform.py b/pypy/translator/platform/distutils_platform.py
--- a/pypy/translator/platform/distutils_platform.py
+++ b/pypy/translator/platform/distutils_platform.py
@@ -52,7 +52,7 @@
self.compile_extra = list(eci.compile_extra)
self.link_extra = list(eci.link_extra)
self.frameworks = list(eci.frameworks)
- if not self.name in ('win32', 'darwin'): # xxx
+ if not self.name in ('win32', 'darwin', 'cygwin'): # xxx
if 'm' not in self.libraries:
self.libraries.append('m')
self.compile_extra += CFLAGS + ['-fomit-frame-pointer']
More information about the pypy-commit
mailing list