[pypy-commit] pypy latest-improve-doc: Merge pypy/default into latest-imporve-doc
groggi
noreply at buildbot.pypy.org
Sat Apr 5 09:39:16 CEST 2014
Author: Gregor Wegberg <code at gregorwegberg.com>
Branch: latest-improve-doc
Changeset: r70455:11ebaff897a8
Date: 2014-04-03 12:08 +0200
http://bitbucket.org/pypy/pypy/changeset/11ebaff897a8/
Log: Merge pypy/default into latest-imporve-doc
diff --git a/lib-python/2.7/test/test_argparse.py b/lib-python/2.7/test/test_argparse.py
--- a/lib-python/2.7/test/test_argparse.py
+++ b/lib-python/2.7/test/test_argparse.py
@@ -10,6 +10,7 @@
import tempfile
import unittest
import argparse
+import gc
from StringIO import StringIO
@@ -47,7 +48,11 @@
def tearDown(self):
os.chdir(self.old_dir)
- shutil.rmtree(self.temp_dir, True)
+ gc.collect()
+ for root, dirs, files in os.walk(self.temp_dir, topdown=False):
+ for name in files:
+ os.chmod(os.path.join(self.temp_dir, name), stat.S_IWRITE)
+ shutil.rmtree(self.temp_dir, True)
def create_readonly_file(self, filename):
file_path = os.path.join(self.temp_dir, filename)
diff --git a/lib-python/2.7/test/test_file.py b/lib-python/2.7/test/test_file.py
--- a/lib-python/2.7/test/test_file.py
+++ b/lib-python/2.7/test/test_file.py
@@ -301,6 +301,7 @@
self.fail("readlines() after next() with empty buffer "
"failed. Got %r, expected %r" % (line, testline))
# Reading after iteration hit EOF shouldn't hurt either
+ f.close()
f = self.open(TESTFN, 'rb')
try:
for line in f:
diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py
--- a/lib_pypy/_ctypes_test.py
+++ b/lib_pypy/_ctypes_test.py
@@ -1,3 +1,5 @@
+import imp, os
+
try:
import cpyext
except ImportError:
@@ -10,4 +12,12 @@
pass # obscure condition of _ctypes_test.py being imported by py.test
else:
import _pypy_testcapi
- _pypy_testcapi.compile_shared('_ctypes_test.c', '_ctypes_test')
+ cfile = '_ctypes_test.c'
+ thisdir = os.path.dirname(__file__)
+ output_dir = _pypy_testcapi.get_hashed_dir(os.path.join(thisdir, cfile))
+ try:
+ fp, filename, description = imp.find_module('_ctypes_test', path=[output_dir])
+ imp.load_module('_ctypes_test', fp, filename, description)
+ except ImportError:
+ print('could not find _ctypes_test in %s' % output_dir)
+ _pypy_testcapi.compile_shared('_ctypes_test.c', '_ctypes_test', output_dir)
diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py
--- a/lib_pypy/_pypy_testcapi.py
+++ b/lib_pypy/_pypy_testcapi.py
@@ -1,5 +1,22 @@
import os, sys, imp
-import tempfile
+import tempfile, binascii
+
+def get_hashed_dir(cfile):
+ with open(cfile,'r') as fid:
+ content = fid.read()
+ # from cffi's Verifier()
+ key = '\x00'.join([sys.version[:3], content])
+ if sys.version_info >= (3,):
+ key = key.encode('utf-8')
+ k1 = hex(binascii.crc32(key[0::2]) & 0xffffffff)
+ k1 = k1.lstrip('0x').rstrip('L')
+ k2 = hex(binascii.crc32(key[1::2]) & 0xffffffff)
+ k2 = k2.lstrip('0').rstrip('L')
+ output_dir = tempfile.gettempdir() + os.path.sep + 'tmp_%s%s' %(k1, k2)
+ if not os.path.exists(output_dir):
+ os.mkdir(output_dir)
+ return output_dir
+
def _get_c_extension_suffix():
for ext, mod, typ in imp.get_suffixes():
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_testcapi.py
@@ -1,33 +1,17 @@
-import sys, tempfile, imp, binascii, os
+import imp, os
try:
import cpyext
except ImportError:
raise ImportError("No module named '_testcapi'")
-def get_hashed_dir(cfile):
- with open(cfile,'r') as fid:
- content = fid.read()
- # from cffi's Verifier()
- key = '\x00'.join([sys.version[:3], content])
- if sys.version_info >= (3,):
- key = key.encode('utf-8')
- k1 = hex(binascii.crc32(key[0::2]) & 0xffffffff)
- k1 = k1.lstrip('0x').rstrip('L')
- k2 = hex(binascii.crc32(key[1::2]) & 0xffffffff)
- k2 = k2.lstrip('0').rstrip('L')
- output_dir = tempfile.gettempdir() + os.path.sep + 'tmp_%s%s' %(k1, k2)
- if not os.path.exists(output_dir):
- os.mkdir(output_dir)
- return output_dir
-
+import _pypy_testcapi
cfile = '_testcapimodule.c'
thisdir = os.path.dirname(__file__)
-output_dir = get_hashed_dir(os.path.join(thisdir, cfile))
+output_dir = _pypy_testcapi.get_hashed_dir(os.path.join(thisdir, cfile))
try:
fp, filename, description = imp.find_module('_testcapi', path=[output_dir])
imp.load_module('_testcapi', fp, filename, description)
except ImportError:
- import _pypy_testcapi
_pypy_testcapi.compile_shared(cfile, '_testcapi', output_dir)
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -51,6 +51,8 @@
.. function:: int pypy_execute_source_ptr(char* source, void* ptr);
+ .. note:: Not available in PyPy <= 2.2.1
+
Just like the above, except it registers a magic argument in the source
scope as ``c_argument``, where ``void*`` is encoded as Python int.
@@ -100,9 +102,29 @@
Worked!
+.. note:: If the compilation fails because of missing PyPy.h header file,
+ you are running PyPy <= 2.2.1, please see the section `Missing PyPy.h`_.
+
+Missing PyPy.h
+--------------
+
+.. note:: PyPy.h is in the nightly builds and goes to new PyPy releases (>2.2.1).
+
+For PyPy <= 2.2.1, you can download PyPy.h from PyPy repository (it has been added in commit c4cd6ec):
+
+.. code-block:: bash
+
+ cd /opt/pypy/include
+ wget https://bitbucket.org/pypy/pypy/raw/c4cd6eca9358066571500ac82aaacfdaa3889e8c/include/PyPy.h
+
+
More advanced example
---------------------
+.. note:: This example depends on pypy_execute_source_ptr which is not available
+ in PyPy <= 2.2.1. You might want to see the alternative example
+ below.
+
Typically we need something more to do than simply execute source. The following
is a fully fledged example, please consult cffi documentation for details.
It's a bit longish, but it captures a gist what can be done with the PyPy
@@ -161,6 +183,97 @@
is that we would pass a struct full of callbacks to ``pypy_execute_source_ptr``
and fill the structure from Python side for the future use.
+Alternative example
+-------------------
+
+As ``pypy_execute_source_ptr`` is not available in PyPy 2.2.1, you might want to try
+an alternative approach which relies on -export-dynamic flag to the GNU linker.
+The downside to this approach is that it is platform dependent.
+
+.. code-block:: c
+
+ #include "include/PyPy.h"
+ #include <stdio.h>
+
+ char source[] = "from cffi import FFI\n\
+ ffi = FFI()\n\
+ @ffi.callback('int(int)')\n\
+ def func(a):\n\
+ print 'Got from C %d' % a\n\
+ return a * 2\n\
+ ffi.cdef('int callback(int (*func)(int));')\n\
+ lib = ffi.verify('int callback(int (*func)(int));')\n\
+ lib.callback(func)\n\
+ print 'finished the Python part'\n\
+ ";
+
+ int callback(int (*func)(int))
+ {
+ printf("Calling to Python, result: %d\n", func(3));
+ }
+
+ int main()
+ {
+ int res;
+ void *lib, *func;
+
+ rpython_startup_code();
+ res = pypy_setup_home("/opt/pypy/bin/libpypy-c.so", 1);
+ if (res) {
+ printf("Error setting pypy home!\n");
+ return 1;
+ }
+ res = pypy_execute_source(source);
+ if (res) {
+ printf("Error calling pypy_execute_source!\n");
+ }
+ return res;
+ }
+
+
+Make sure to pass -export-dynamic flag when compiling::
+
+ $ gcc -g -o x x.c -lpypy-c -L. -export-dynamic
+ $ LD_LIBRARY_PATH=. ./x
+ Got from C 3
+ Calling to Python, result: 6
+ finished the Python part
+
+Finding pypy_home
+-----------------
+
+Function pypy_setup_home takes one parameter - the path to libpypy. There's
+currently no "clean" way (pkg-config comes to mind) how to find this path. You
+can try the following (GNU-specific) hack (don't forget to link against *dl*):
+
+.. code-block:: c
+
+ #if !(_GNU_SOURCE)
+ #define _GNU_SOURCE
+ #endif
+
+ #include <dlfcn.h>
+ #include <limits.h>
+ #include <stdlib.h>
+
+ // caller should free returned pointer to avoid memleaks
+ // returns NULL on error
+ char* guess_pypyhome() {
+ // glibc-only (dladdr is why we #define _GNU_SOURCE)
+ Dl_info info;
+ void *_rpython_startup_code = dlsym(0,"rpython_startup_code");
+ if (_rpython_startup_code == 0) {
+ return 0;
+ }
+ if (dladdr(_rpython_startup_code, &info) != 0) {
+ const char* lib_path = info.dli_fname;
+ char* lib_realpath = realpath(lib_path, 0);
+ return lib_realpath;
+ }
+ return 0;
+ }
+
+
Threading
---------
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -429,12 +429,27 @@
Could we use LLVM?
------------------
-There is a (static) translation backend using LLVM in the branch
-``llvm-translation-backend``. It can translate PyPy with or without the JIT on
-Linux.
+In theory yes. But we tried to use it 5 or 6 times already, as a
+translation backend or as a JIT backend --- and failed each time.
-Using LLVM as our JIT backend looks interesting as well -- we made an attempt,
-but it failed: LLVM has no way to patch the generated machine code.
+In more details: using LLVM as a (static) translation backend is
+pointless nowadays because you can generate C code and compile it with
+clang. (Note that compiling PyPy with clang gives a result that is not
+faster than compiling it with gcc.) We might in theory get extra
+benefits from LLVM's GC integration, but this requires more work on the
+LLVM side before it would be remotely useful. Anyway, it could be
+interfaced via a custom primitive in the C code. (The latest such
+experimental backend is in the branch ``llvm-translation-backend``,
+which can translate PyPy with or without the JIT on Linux.)
+
+On the other hand, using LLVM as our JIT backend looks interesting as
+well --- but again we made an attempt, and it failed: LLVM has no way to
+patch the generated machine code.
+
+So the position of the core PyPy developers is that if anyone wants to
+make an N+1'th attempt with LLVM, they are welcome, and will be happy to
+provide help in the IRC channel, but they are left with the burden of proof
+that (a) it works and (b) it gives important benefits.
----------------------
How do I compile PyPy?
diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/stm.rst
@@ -0,0 +1,244 @@
+======================
+Transactional Memory
+======================
+
+.. contents::
+
+
+This page is about ``pypy-stm``, a special in-development version of
+PyPy which can run multiple independent CPU-hungry threads in the same
+process in parallel. It is side-stepping what is known in the Python
+world as the "global interpreter lock (GIL)" problem.
+
+"STM" stands for Software Transactional Memory, the technique used
+internally. This page describes ``pypy-stm`` from the perspective of a
+user, describes work in progress, and finally gives references to more
+implementation details.
+
+This work was done by Remi Meier and Armin Rigo.
+
+
+Introduction and current status
+===============================
+
+``pypy-stm`` is a variant of the regular PyPy interpreter. With caveats
+listed below, it should be in theory within 25%-50% of the speed of
+PyPy, comparing the JITting version in both cases. It is called STM for
+Software Transactional Memory, which is the internal technique used (see
+`Reference to implementation details`_).
+
+**pypy-stm requires 64-bit Linux for now.**
+
+Development is done in the branch `stmgc-c7`_. If you are only
+interested in trying it out, you can download a Ubuntu 12.04 binary
+here__. The current version supports four "segments", which means that
+it will run up to four threads in parallel (in other words, you get a
+GIL effect again, but only if trying to execute more than 4 threads).
+
+To build a version from sources, you first need to compile a custom
+version of clang; we recommend downloading `llvm and clang like
+described here`__, but at revision 201645 (use ``svn co -r 201645 ...``
+for all checkouts). Then apply all the patches in `this directory`__:
+they are fixes for the very extensive usage that pypy-stm does of a
+clang-only feature (without them, you get crashes of clang). Then get
+the branch `stmgc-c7`_ of PyPy and run::
+
+ rpython/bin/rpython -Ojit --stm pypy/goal/targetpypystandalone.py
+
+.. _`stmgc-c7`: https://bitbucket.org/pypy/pypy/src/stmgc-c7/
+.. __: http://buildbot.pypy.org/nightly/stmgc-c7/
+.. __: http://clang.llvm.org/get_started.html
+.. __: https://bitbucket.org/pypy/stmgc/src/default/c7/llvmfix/
+
+
+Caveats:
+
+* It should generally work. Please do `report bugs`_ that manifest as a
+ crash or wrong behavior (markedly different from the behavior of a
+ regular PyPy). Performance bugs are likely to be known issues; we're
+ working on them.
+
+* The JIT warm-up time is abysmal (as opposed to the regular PyPy's,
+ which is "only" bad). Moreover, you should run it with a command like
+ ``pypy-stm --jit trace_limit=60000 args...``; the default value of
+ 6000 for ``trace_limit`` is currently too low (6000 should become
+ reasonable again as we improve). Also, in order to produce machine
+ code, the JIT needs to enter a special single-threaded mode for now.
+ This all means that you *will* get very bad performance results if
+ your program doesn't run for *many* seconds for now.
+
+* The GC is new; although clearly inspired by PyPy's regular GC, it
+ misses a number of optimizations for now. Programs allocating large
+ numbers of small objects that don't immediately die, as well as
+ programs that modify large lists or dicts, suffer from these missing
+ optimizations.
+
+* The GC has no support for destructors: the ``__del__`` method is
+ never called (including on file objects, which won't be closed for
+ you). This is of course temporary.
+
+* The STM system is based on very efficient read/write barriers, which
+ are mostly done (their placement could be improved a bit in
+ JIT-generated machine code). But the overall bookkeeping logic could
+ see more improvements (see Statistics_ below).
+
+* You can use `atomic sections`_, but the most visible missing thing is
+ that you don't get reports about the "conflicts" you get. This would
+ be the first thing that you need in order to start using atomic
+ sections more extensively. Also, for now: for better results, try to
+ explicitly force a transaction break just before (and possibly after)
+ each large atomic section, with ``time.sleep(0)``.
+
+* Forking the process is slow because the complete memory needs to be
+ copied manually right now.
+
+* Very long-running processes should eventually crash on an assertion
+ error because of a non-implemented overflow of an internal 29-bit
+ number, but this requires at the very least ten hours --- more
+ probably, several days or more.
+
+.. _`report bugs`: https://bugs.pypy.org/
+
+
+
+Statistics
+==========
+
+When a non-main thread finishes, you get statistics printed to stderr,
+looking like that::
+
+ thread 0x7f73377fe600:
+ outside transaction 42182 0.506 s
+ run current 85466 0.000 s
+ run committed 34262 3.178 s
+ run aborted write write 6982 0.083 s
+ run aborted write read 550 0.005 s
+ run aborted inevitable 388 0.010 s
+ run aborted other 0 0.000 s
+ wait free segment 0 0.000 s
+ wait write read 78 0.027 s
+ wait inevitable 887 0.490 s
+ wait other 0 0.000 s
+ bookkeeping 51418 0.606 s
+ minor gc 162970 1.135 s
+ major gc 1 0.019 s
+ sync pause 59173 1.738 s
+ spin loop 129512 0.094 s
+
+The first number is a counter; the second number gives the associated
+time (the amount of real time that the thread was in this state; the sum
+of all the times should be equal to the total time between the thread's
+start and the thread's end). The most important points are "run
+committed", which gives the amount of useful work, and "outside
+transaction", which should give the time spent e.g. in library calls
+(right now it seems to be a bit larger than that; to investigate).
+Everything else is overhead of various forms. (Short-, medium- and
+long-term future work involves reducing this overhead :-)
+
+These statistics are not printed out for the main thread, for now.
+
+
+Atomic sections
+===============
+
+While one of the goal of pypy-stm is to give a GIL-free but otherwise
+unmodified Python, the other goal is to push for a better way to use
+multithreading. For this, you (as the Python programmer) get an API
+in the ``__pypy__.thread`` submodule:
+
+* ``__pypy__.thread.atomic``: a context manager (i.e. you use it in
+ a ``with __pypy__.thread.atomic:`` statement). It runs the whole
+ block of code without breaking the current transaction --- from
+ the point of view of a regular CPython/PyPy, this is equivalent to
+ saying that the GIL will not be released at all between the start and
+ the end of this block of code.
+
+The obvious usage is to use atomic blocks in the same way as one would
+use locks: to protect changes to some shared data, you do them in a
+``with atomic`` block, just like you would otherwise do them in a ``with
+mylock`` block after ``mylock = thread.allocate_lock()``. This allows
+you not to care about acquiring the correct locks in the correct order;
+it is equivalent to having only one global lock. This is how
+transactional memory is `generally described`__: as a way to efficiently
+execute such atomic blocks, running them in parallel while giving the
+illusion that they run in some serial order.
+
+.. __: http://en.wikipedia.org/wiki/Transactional_memory
+
+However, the less obvious intended usage of atomic sections is as a
+wide-ranging replacement of explicit threads. You can turn a program
+that is not multi-threaded at all into a program that uses threads
+internally, together with large atomic sections to keep the behavior
+unchanged. This capability can be hidden in a library or in the
+framework you use; the end user's code does not need to be explicitly
+aware of using threads. For a simple example of this, see
+`lib_pypy/transaction.py`_. The idea is that if you have a program
+where the function ``f(key, value)`` runs on every item of some big
+dictionary, you can replace the loop with::
+
+ for key, value in bigdict.items():
+ transaction.add(f, key, value)
+ transaction.run()
+
+This code runs the various calls to ``f(key, value)`` using a thread
+pool, but every single call is done in an atomic section. The end
+result is that the behavior should be exactly equivalent: you don't get
+any extra multithreading issue.
+
+.. _`lib_pypy/transaction.py`: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/lib_pypy/transaction.py
+
+==================
+
+Other APIs in pypy-stm:
+
+* ``__pypy__.thread.getsegmentlimit()``: return the number of "segments"
+ in this pypy-stm. This is the limit above which more threads will not
+ be able to execute on more cores. (Right now it is limited to 4 due
+ to inter-segment overhead, but should be increased in the future. It
+ should also be settable, and the default value should depend on the
+ number of actual CPUs.)
+
+* ``__pypy__.thread.exclusive_atomic``: same as ``atomic``, but
+ raises an exception if you attempt to nest it inside another
+ ``atomic``.
+
+* ``__pypy__.thread.signals_enabled``: a context manager that runs
+ its block with signals enabled. By default, signals are only
+ enabled in the main thread; a non-main thread will not receive
+ signals (this is like CPython). Enabling signals in non-main threads
+ is useful for libraries where threads are hidden and the end user is
+ not expecting his code to run elsewhere than in the main thread.
+
+Note that all of this API is (or will be) implemented in a regular PyPy
+too: for example, ``with atomic`` will simply mean "don't release the
+GIL" and ``getsegmentlimit()`` will return 1.
+
+==================
+
+
+Reference to implementation details
+===================================
+
+The core of the implementation is in a separate C library called stmgc_,
+in the c7_ subdirectory. Please see the `README.txt`_ for more
+information.
+
+.. _stmgc: https://bitbucket.org/pypy/stmgc/src/default/
+.. _c7: https://bitbucket.org/pypy/stmgc/src/default/c7/
+.. _`README.txt`: https://bitbucket.org/pypy/stmgc/raw/default/c7/README.txt
+
+PyPy itself adds on top of it the automatic placement of read__ and write__
+barriers and of `"becomes-inevitable-now" barriers`__, the logic to
+`start/stop transactions as an RPython transformation`__ and as
+`supporting`__ `C code`__, and the support in the JIT (mostly as a
+`transformation step on the trace`__ and generation of custom assembler
+in `assembler.py`__).
+
+.. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/translator/stm/readbarrier.py
+.. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/memory/gctransform/stmframework.py
+.. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/translator/stm/inevitable.py
+.. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/translator/stm/jitdriver.py
+.. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/translator/stm/src_stm/stmgcintf.h
+.. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/translator/stm/src_stm/stmgcintf.c
+.. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/jit/backend/llsupport/stmrewrite.py
+.. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/jit/backend/x86/assembler.py
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -127,3 +127,10 @@
.. branch: win32-fixes4
fix more tests for win32
+
+.. branch: latest-improve-doc
+Fix broken links in documentation
+
+.. branch: ast-issue1673
+fix ast classes __dict__ are always empty problem and fix the ast deepcopy issue when
+there is missing field
\ No newline at end of file
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -690,7 +690,7 @@
def setup_bootstrap_path(executable):
"""
- Try to to as little as possible and to have the stdlib in sys.path. In
+ Try to do as little as possible and to have the stdlib in sys.path. In
particular, we cannot use any unicode at this point, because lots of
unicode operations require to be able to import encodings.
"""
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -49,13 +49,19 @@
w_type = space.type(self)
w_fields = w_type.getdictvalue(space, "_fields")
for w_name in space.fixedview(w_fields):
- space.setitem(w_dict, w_name,
+ try:
+ space.setitem(w_dict, w_name,
space.getattr(self, w_name))
+ except OperationError:
+ pass
w_attrs = space.findattr(w_type, space.wrap("_attributes"))
if w_attrs:
for w_name in space.fixedview(w_attrs):
- space.setitem(w_dict, w_name,
+ try:
+ space.setitem(w_dict, w_name,
space.getattr(self, w_name))
+ except OperationError:
+ pass
return space.newtuple([space.type(self),
space.newtuple([]),
w_dict])
@@ -3011,7 +3017,8 @@
w_self.setdictvalue(space, 'lineno', w_new_value)
w_self.initialization_state &= ~1
return
- w_self.deldictvalue(space, 'lineno')
+ # need to save the original object too
+ w_self.setdictvalue(space, 'lineno', w_new_value)
w_self.initialization_state |= 1
def stmt_del_lineno(space, w_self):
@@ -3038,7 +3045,8 @@
w_self.setdictvalue(space, 'col_offset', w_new_value)
w_self.initialization_state &= ~2
return
- w_self.deldictvalue(space, 'col_offset')
+ # need to save the original object too
+ w_self.setdictvalue(space, 'col_offset', w_new_value)
w_self.initialization_state |= 2
def stmt_del_col_offset(space, w_self):
@@ -3074,7 +3082,8 @@
w_self.setdictvalue(space, 'name', w_new_value)
w_self.initialization_state &= ~4
return
- w_self.deldictvalue(space, 'name')
+ # need to save the original object too
+ w_self.setdictvalue(space, 'name', w_new_value)
w_self.initialization_state |= 4
def FunctionDef_del_name(space, w_self):
@@ -3201,7 +3210,8 @@
w_self.setdictvalue(space, 'name', w_new_value)
w_self.initialization_state &= ~4
return
- w_self.deldictvalue(space, 'name')
+ # need to save the original object too
+ w_self.setdictvalue(space, 'name', w_new_value)
w_self.initialization_state |= 4
def ClassDef_del_name(space, w_self):
@@ -3665,7 +3675,8 @@
w_self.setdictvalue(space, 'nl', w_new_value)
w_self.initialization_state &= ~16
return
- w_self.deldictvalue(space, 'nl')
+ # need to save the original object too
+ w_self.setdictvalue(space, 'nl', w_new_value)
w_self.initialization_state |= 16
def Print_del_nl(space, w_self):
@@ -4571,7 +4582,8 @@
w_self.setdictvalue(space, 'module', w_new_value)
w_self.initialization_state &= ~4
return
- w_self.deldictvalue(space, 'module')
+ # need to save the original object too
+ w_self.setdictvalue(space, 'module', w_new_value)
w_self.initialization_state |= 4
def ImportFrom_del_module(space, w_self):
@@ -4620,7 +4632,8 @@
w_self.setdictvalue(space, 'level', w_new_value)
w_self.initialization_state &= ~16
return
- w_self.deldictvalue(space, 'level')
+ # need to save the original object too
+ w_self.setdictvalue(space, 'level', w_new_value)
w_self.initialization_state |= 16
def ImportFrom_del_level(space, w_self):
@@ -4938,7 +4951,8 @@
w_self.setdictvalue(space, 'lineno', w_new_value)
w_self.initialization_state &= ~1
return
- w_self.deldictvalue(space, 'lineno')
+ # need to save the original object too
+ w_self.setdictvalue(space, 'lineno', w_new_value)
w_self.initialization_state |= 1
def expr_del_lineno(space, w_self):
@@ -4965,7 +4979,8 @@
w_self.setdictvalue(space, 'col_offset', w_new_value)
w_self.initialization_state &= ~2
return
- w_self.deldictvalue(space, 'col_offset')
+ # need to save the original object too
+ w_self.setdictvalue(space, 'col_offset', w_new_value)
w_self.initialization_state |= 2
def expr_del_col_offset(space, w_self):
@@ -6292,7 +6307,8 @@
w_self.setdictvalue(space, 'n', w_new_value)
w_self.initialization_state &= ~4
return
- w_self.deldictvalue(space, 'n')
+ # need to save the original object too
+ w_self.setdictvalue(space, 'n', w_new_value)
w_self.initialization_state |= 4
def Num_del_n(space, w_self):
@@ -6343,7 +6359,8 @@
w_self.setdictvalue(space, 's', w_new_value)
w_self.initialization_state &= ~4
return
- w_self.deldictvalue(space, 's')
+ # need to save the original object too
+ w_self.setdictvalue(space, 's', w_new_value)
w_self.initialization_state |= 4
def Str_del_s(space, w_self):
@@ -6423,7 +6440,8 @@
w_self.setdictvalue(space, 'attr', w_new_value)
w_self.initialization_state &= ~8
return
- w_self.deldictvalue(space, 'attr')
+ # need to save the original object too
+ w_self.setdictvalue(space, 'attr', w_new_value)
w_self.initialization_state |= 8
def Attribute_del_attr(space, w_self):
@@ -6618,7 +6636,8 @@
w_self.setdictvalue(space, 'id', w_new_value)
w_self.initialization_state &= ~4
return
- w_self.deldictvalue(space, 'id')
+ # need to save the original object too
+ w_self.setdictvalue(space, 'id', w_new_value)
w_self.initialization_state |= 4
def Name_del_id(space, w_self):
@@ -6853,7 +6872,8 @@
w_self.setdictvalue(space, 'value', w_new_value)
w_self.initialization_state &= ~4
return
- w_self.deldictvalue(space, 'value')
+ # need to save the original object too
+ w_self.setdictvalue(space, 'value', w_new_value)
w_self.initialization_state |= 4
def Const_del_value(space, w_self):
@@ -7521,7 +7541,8 @@
w_self.setdictvalue(space, 'lineno', w_new_value)
w_self.initialization_state &= ~1
return
- w_self.deldictvalue(space, 'lineno')
+ # need to save the original object too
+ w_self.setdictvalue(space, 'lineno', w_new_value)
w_self.initialization_state |= 1
def excepthandler_del_lineno(space, w_self):
@@ -7548,7 +7569,8 @@
w_self.setdictvalue(space, 'col_offset', w_new_value)
w_self.initialization_state &= ~2
return
- w_self.deldictvalue(space, 'col_offset')
+ # need to save the original object too
+ w_self.setdictvalue(space, 'col_offset', w_new_value)
w_self.initialization_state |= 2
def excepthandler_del_col_offset(space, w_self):
@@ -7716,7 +7738,8 @@
w_self.setdictvalue(space, 'vararg', w_new_value)
w_self.initialization_state &= ~2
return
- w_self.deldictvalue(space, 'vararg')
+ # need to save the original object too
+ w_self.setdictvalue(space, 'vararg', w_new_value)
w_self.initialization_state |= 2
def arguments_del_vararg(space, w_self):
@@ -7746,7 +7769,8 @@
w_self.setdictvalue(space, 'kwarg', w_new_value)
w_self.initialization_state &= ~4
return
- w_self.deldictvalue(space, 'kwarg')
+ # need to save the original object too
+ w_self.setdictvalue(space, 'kwarg', w_new_value)
w_self.initialization_state |= 4
def arguments_del_kwarg(space, w_self):
@@ -7824,7 +7848,8 @@
w_self.setdictvalue(space, 'arg', w_new_value)
w_self.initialization_state &= ~1
return
- w_self.deldictvalue(space, 'arg')
+ # need to save the original object too
+ w_self.setdictvalue(space, 'arg', w_new_value)
w_self.initialization_state |= 1
def keyword_del_arg(space, w_self):
@@ -7905,7 +7930,8 @@
w_self.setdictvalue(space, 'name', w_new_value)
w_self.initialization_state &= ~1
return
- w_self.deldictvalue(space, 'name')
+ # need to save the original object too
+ w_self.setdictvalue(space, 'name', w_new_value)
w_self.initialization_state |= 1
def alias_del_name(space, w_self):
@@ -7935,7 +7961,8 @@
w_self.setdictvalue(space, 'asname', w_new_value)
w_self.initialization_state &= ~2
return
- w_self.deldictvalue(space, 'asname')
+ # need to save the original object too
+ w_self.setdictvalue(space, 'asname', w_new_value)
w_self.initialization_state |= 2
def alias_del_asname(space, w_self):
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -459,6 +459,7 @@
self.emit("raise OperationError(space.w_TypeError, "
"space.w_None)", 3)
else:
+ save_original_object = True
level = 2
if field.opt and field.type.value != "int":
self.emit("if space.is_w(w_new_value, space.w_None):", 2)
@@ -596,13 +597,19 @@
w_type = space.type(self)
w_fields = w_type.getdictvalue(space, "_fields")
for w_name in space.fixedview(w_fields):
- space.setitem(w_dict, w_name,
+ try:
+ space.setitem(w_dict, w_name,
space.getattr(self, w_name))
+ except OperationError:
+ pass
w_attrs = space.findattr(w_type, space.wrap("_attributes"))
if w_attrs:
for w_name in space.fixedview(w_attrs):
- space.setitem(w_dict, w_name,
+ try:
+ space.setitem(w_dict, w_name,
space.getattr(self, w_name))
+ except OperationError:
+ pass
return space.newtuple([space.type(self),
space.newtuple([]),
w_dict])
diff --git a/pypy/module/_ast/test/test_ast.py b/pypy/module/_ast/test/test_ast.py
--- a/pypy/module/_ast/test/test_ast.py
+++ b/pypy/module/_ast/test/test_ast.py
@@ -387,3 +387,40 @@
mod.body[0].body[0].handlers[0].lineno = 7
mod.body[0].body[0].handlers[1].lineno = 6
code = compile(mod, "<test>", "exec")
+
+ def test_dict_astNode(self):
+ import ast
+ num_node = ast.Num(n=2, lineno=2, col_offset=3)
+ dict_res = num_node.__dict__
+
+ assert dict_res == {'n':2, 'lineno':2, 'col_offset':3}
+
+ def test_issue1673_Num_notfullinit(self):
+ import ast
+ import copy
+ num_node = ast.Num(n=2,lineno=2)
+ assert num_node.n == 2
+ assert num_node.lineno == 2
+ num_node2 = copy.deepcopy(num_node)
+
+ def test_issue1673_Num_fullinit(self):
+ import ast
+ import copy
+ num_node = ast.Num(n=2,lineno=2,col_offset=3)
+ num_node2 = copy.deepcopy(num_node)
+ assert num_node.n == num_node2.n
+ assert num_node.lineno == num_node2.lineno
+ assert num_node.col_offset == num_node2.col_offset
+ dict_res = num_node2.__dict__
+ assert dict_res == {'n':2, 'lineno':2, 'col_offset':3}
+
+ def test_issue1673_Str(self):
+ import ast
+ import copy
+ str_node = ast.Str(n=2,lineno=2)
+ assert str_node.n == 2
+ assert str_node.lineno == 2
+ str_node2 = copy.deepcopy(str_node)
+ dict_res = str_node2.__dict__
+ assert dict_res == {'n':2, 'lineno':2}
+
\ No newline at end of file
diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -254,6 +254,13 @@
if '__pypy__' in sys.builtin_module_names:
assert repr(self.temppath) in g.getvalue()
+ def test_truncate(self):
+ f = self.file(self.temppath, "w")
+ f.write("foo")
+ f.close()
+ with self.file(self.temppath, 'r') as f:
+ raises(IOError, f.truncate, 100)
+
class AppTestNonblocking(object):
def setup_class(cls):
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -64,8 +64,6 @@
kwds["libraries"] = [api_library]
# '%s' undefined; assuming extern returning int
kwds["compile_extra"] = ["/we4013"]
- # tests are not strictly ansi C compliant, compile as C++
- kwds["compile_extra"].append("/TP")
# prevent linking with PythonXX.lib
w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2]
kwds["link_extra"] = ["/NODEFAULTLIB:Python%d%d.lib" %
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -246,9 +246,9 @@
("test_FromAny", "METH_NOARGS",
'''
npy_intp dims[2] ={2, 3};
- PyObject * obj1 = PyArray_SimpleNew(2, dims, 1);
+ PyObject * obj2, * obj1 = PyArray_SimpleNew(2, dims, 1);
PyArray_FILLWBYTE(obj1, 42);
- PyObject * obj2 = _PyArray_FromAny(obj1, NULL, 0, 0, 0, NULL);
+ obj2 = _PyArray_FromAny(obj1, NULL, 0, 0, 0, NULL);
Py_DECREF(obj1);
return obj2;
'''
@@ -256,9 +256,9 @@
("test_FromObject", "METH_NOARGS",
'''
npy_intp dims[2] ={2, 3};
- PyObject * obj1 = PyArray_SimpleNew(2, dims, 1);
+ PyObject * obj2, * obj1 = PyArray_SimpleNew(2, dims, 1);
PyArray_FILLWBYTE(obj1, 42);
- PyObject * obj2 = _PyArray_FromObject(obj1, 12, 0, 0);
+ obj2 = _PyArray_FromObject(obj1, 12, 0, 0);
Py_DECREF(obj1);
return obj2;
'''
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -238,8 +238,8 @@
module = self.import_extension('foo', [
("set_from_errno", "METH_NOARGS",
'''
+ PyObject *filenameObject = PyString_FromString("/path/to/file");
errno = EBADF;
- PyObject *filenameObject = PyString_FromString("/path/to/file");
PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, filenameObject);
Py_DECREF(filenameObject);
return NULL;
@@ -257,8 +257,8 @@
module = self.import_extension('foo', [
("set_from_errno", "METH_NOARGS",
'''
+ PyObject *intObject = PyInt_FromLong(3);
errno = EBADF;
- PyObject *intObject = PyInt_FromLong(3);
PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, intObject);
Py_DECREF(intObject);
return NULL;
@@ -276,8 +276,8 @@
module = self.import_extension('foo', [
("set_from_errno", "METH_NOARGS",
'''
+ PyObject *lst = Py_BuildValue("[iis]", 1, 2, "three");
errno = EBADF;
- PyObject *lst = Py_BuildValue("[iis]", 1, 2, "three");
PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, lst);
Py_DECREF(lst);
return NULL;
@@ -295,8 +295,8 @@
module = self.import_extension('foo', [
("set_from_errno", "METH_NOARGS",
'''
+ PyObject *tuple = Py_BuildValue("(iis)", 1, 2, "three");
errno = EBADF;
- PyObject *tuple = Py_BuildValue("(iis)", 1, 2, "three");
PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, tuple);
Py_DECREF(tuple);
return NULL;
@@ -314,8 +314,8 @@
module = self.import_extension('foo', [
("set_from_errno", "METH_NOARGS",
'''
+ PyObject *none = Py_BuildValue("");
errno = EBADF;
- PyObject *none = Py_BuildValue("");
PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, none);
Py_DECREF(none);
return NULL;
diff --git a/pypy/module/test_lib_pypy/test_site_extra.py b/pypy/module/test_lib_pypy/test_site_extra.py
--- a/pypy/module/test_lib_pypy/test_site_extra.py
+++ b/pypy/module/test_lib_pypy/test_site_extra.py
@@ -4,8 +4,11 @@
def test_preimported_modules():
lst = ['__builtin__', '_codecs', '_warnings', 'codecs', 'encodings',
'exceptions', 'signal', 'sys', 'zipimport']
- g = os.popen('"%s" -c "import sys; print sorted(sys.modules)"' %
- (sys.executable,))
+ if sys.platform == 'win32':
+ cmd = '%s' % (sys.executable,)
+ else:
+ cmd = '"%s"' % (sys.executable,)
+ g = os.popen(cmd + ' -c "import sys; print sorted(sys.modules)"')
real_data = g.read()
g.close()
for name in lst:
diff --git a/pypy/module/test_lib_pypy/test_testcapi.py b/pypy/module/test_lib_pypy/test_testcapi.py
--- a/pypy/module/test_lib_pypy/test_testcapi.py
+++ b/pypy/module/test_lib_pypy/test_testcapi.py
@@ -8,7 +8,7 @@
def test_get_hashed_dir():
import sys
# This should not compile _testcapi, so the output is empty
- script = "import _testcapi; assert 'get_hashed_dir' in dir(_testcapi)"
+ script = "import _testcapi; assert 'get_hashed_dir' not in dir(_testcapi)"
output = py.process.cmdexec('''"%s" -c "%s"''' %
(sys.executable, script))
assert output == ''
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -151,6 +151,9 @@
'*.c', '*.o'))
for file in ['LICENSE', 'README.rst']:
shutil.copy(str(basedir.join(file)), str(pypydir))
+ for file in ['_testcapimodule.c', '_ctypes_test.c']:
+ shutil.copyfile(str(basedir.join('lib_pypy', file)),
+ str(pypydir.join('lib_pypy', file)))
#
spdir = pypydir.ensure('site-packages', dir=True)
shutil.copy(str(basedir.join('site-packages', 'README')), str(spdir))
diff --git a/pypy/tool/release/test/test_package.py b/pypy/tool/release/test/test_package.py
--- a/pypy/tool/release/test/test_package.py
+++ b/pypy/tool/release/test/test_package.py
@@ -17,6 +17,8 @@
exe_name_in_archive = 'bin/pypy'
pypy_c = py.path.local(pypydir).join('goal', basename)
if not pypy_c.check():
+ if sys.platform == 'win32':
+ assert False, "test on win32 requires exe"
pypy_c.write("#!/bin/sh")
pypy_c.chmod(0755)
fake_pypy_c = True
@@ -81,6 +83,8 @@
package.USE_ZIPFILE_MODULE = prev
def test_fix_permissions(tmpdir):
+ if sys.platform == 'win32':
+ py.test.skip('needs to be more general for windows')
def check(f, mode):
assert f.stat().mode & 0777 == mode
#
diff --git a/rpython/flowspace/specialcase.py b/rpython/flowspace/specialcase.py
--- a/rpython/flowspace/specialcase.py
+++ b/rpython/flowspace/specialcase.py
@@ -49,6 +49,11 @@
from rpython.rlib.rfile import create_file
return ctx.appcall(create_file, *args_w)
+ at register_flow_sc(os.fdopen)
+def sc_os_fdopen(ctx, *args_w):
+ from rpython.rlib.rfile import create_fdopen_rfile
+ return ctx.appcall(create_fdopen_rfile, *args_w)
+
@register_flow_sc(os.tmpfile)
def sc_os_tmpfile(ctx):
from rpython.rlib.rfile import create_temp_rfile
diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py
--- a/rpython/jit/backend/arm/opassembler.py
+++ b/rpython/jit/backend/arm/opassembler.py
@@ -362,11 +362,18 @@
self.store_reg(self.mc, r.ip, r.fp, ofs, helper=r.lr)
if op.numargs() > 0 and op.getarg(0).type == REF:
if self._finish_gcmap:
- self._finish_gcmap[0] |= r_uint(0) # r0
+ # we're returning with a guard_not_forced_2, and
+ # additionally we need to say that r0 contains
+ # a reference too:
+ self._finish_gcmap[0] |= r_uint(0)
gcmap = self._finish_gcmap
else:
gcmap = self.gcmap_for_finish
self.push_gcmap(self.mc, gcmap, store=True)
+ elif self._finish_gcmap:
+ # we're returning with a guard_not_forced_2
+ gcmap = self._finish_gcmap
+ self.push_gcmap(self.mc, gcmap, store=True)
else:
# note that the 0 here is redundant, but I would rather
# keep that one and kill all the others
diff --git a/rpython/jit/backend/llsupport/test/test_gc_integration.py b/rpython/jit/backend/llsupport/test/test_gc_integration.py
--- a/rpython/jit/backend/llsupport/test/test_gc_integration.py
+++ b/rpython/jit/backend/llsupport/test/test_gc_integration.py
@@ -916,3 +916,73 @@
cpu.execute_token(token, 1, a))
assert getmap(frame).count('1') == 4
+ def test_finish_without_gcmap(self):
+ cpu = self.cpu
+
+ loop = self.parse("""
+ [i0]
+ finish(i0, descr=finaldescr)
+ """, namespace={'finaldescr': BasicFinalDescr(2)})
+
+ token = JitCellToken()
+ cpu.gc_ll_descr.init_nursery(100)
+ cpu.setup_once()
+ cpu.compile_loop(loop.inputargs, loop.operations, token)
+ frame = lltype.cast_opaque_ptr(JITFRAMEPTR,
+ cpu.execute_token(token, 10))
+ assert not frame.jf_gcmap
+
+ def test_finish_with_trivial_gcmap(self):
+ cpu = self.cpu
+
+ loop = self.parse("""
+ [p0]
+ finish(p0, descr=finaldescr)
+ """, namespace={'finaldescr': BasicFinalDescr(2)})
+
+ token = JitCellToken()
+ cpu.gc_ll_descr.init_nursery(100)
+ cpu.setup_once()
+ cpu.compile_loop(loop.inputargs, loop.operations, token)
+ n = lltype.nullptr(llmemory.GCREF.TO)
+ frame = lltype.cast_opaque_ptr(JITFRAMEPTR,
+ cpu.execute_token(token, n))
+ assert getmap(frame) == '1'
+
+ def test_finish_with_guard_not_forced_2_ref(self):
+ cpu = self.cpu
+
+ loop = self.parse("""
+ [p0, p1]
+ guard_not_forced_2(descr=faildescr) [p1]
+ finish(p0, descr=finaldescr)
+ """, namespace={'faildescr': BasicFailDescr(1),
+ 'finaldescr': BasicFinalDescr(2)})
+
+ token = JitCellToken()
+ cpu.gc_ll_descr.init_nursery(100)
+ cpu.setup_once()
+ cpu.compile_loop(loop.inputargs, loop.operations, token)
+ n = lltype.nullptr(llmemory.GCREF.TO)
+ frame = lltype.cast_opaque_ptr(JITFRAMEPTR,
+ cpu.execute_token(token, n, n))
+ assert getmap(frame).count('1') == 2
+
+ def test_finish_with_guard_not_forced_2_int(self):
+ cpu = self.cpu
+
+ loop = self.parse("""
+ [i0, p1]
+ guard_not_forced_2(descr=faildescr) [p1]
+ finish(i0, descr=finaldescr)
+ """, namespace={'faildescr': BasicFailDescr(1),
+ 'finaldescr': BasicFinalDescr(2)})
+
+ token = JitCellToken()
+ cpu.gc_ll_descr.init_nursery(100)
+ cpu.setup_once()
+ cpu.compile_loop(loop.inputargs, loop.operations, token)
+ n = lltype.nullptr(llmemory.GCREF.TO)
+ frame = lltype.cast_opaque_ptr(JITFRAMEPTR,
+ cpu.execute_token(token, 10, n))
+ assert getmap(frame).count('1') == 1
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1857,11 +1857,18 @@
arglist = op.getarglist()
if arglist and arglist[0].type == REF:
if self._finish_gcmap:
- self._finish_gcmap[0] |= r_uint(1) # rax
+ # we're returning with a guard_not_forced_2, and
+ # additionally we need to say that eax/rax contains
+ # a reference too:
+ self._finish_gcmap[0] |= r_uint(1)
gcmap = self._finish_gcmap
else:
gcmap = self.gcmap_for_finish
self.push_gcmap(self.mc, gcmap, store=True)
+ elif self._finish_gcmap:
+ # we're returning with a guard_not_forced_2
+ gcmap = self._finish_gcmap
+ self.push_gcmap(self.mc, gcmap, store=True)
else:
# note that the 0 here is redundant, but I would rather
# keep that one and kill all the others
diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py
--- a/rpython/rlib/rfile.py
+++ b/rpython/rlib/rfile.py
@@ -47,6 +47,7 @@
rffi.INT)
c_tmpfile = llexternal('tmpfile', [], lltype.Ptr(FILE))
c_fileno = llexternal(fileno, [lltype.Ptr(FILE)], rffi.INT)
+c_fdopen = llexternal('fdopen', [rffi.INT, rffi.CCHARP], lltype.Ptr(FILE))
c_ftell = llexternal('ftell', [lltype.Ptr(FILE)], rffi.LONG)
c_fflush = llexternal('fflush', [lltype.Ptr(FILE)], rffi.INT)
c_ftruncate = llexternal(ftruncate, [rffi.INT, OFF_T], rffi.INT, macro=True)
@@ -93,6 +94,17 @@
raise OSError(errno, os.strerror(errno))
return RFile(res)
+def create_fdopen_rfile(fd, mode="r"):
+ assert mode is not None
+ ll_mode = rffi.str2charp(mode)
+ try:
+ ll_f = c_fdopen(rffi.cast(rffi.INT, fd), ll_mode)
+ if not ll_f:
+ errno = rposix.get_errno()
+ raise OSError(errno, os.strerror(errno))
+ finally:
+ lltype.free(ll_mode, flavor='raw')
+ return RFile(ll_f)
def create_popen_file(command, type):
ll_command = rffi.str2charp(command)
diff --git a/rpython/rlib/streamio.py b/rpython/rlib/streamio.py
--- a/rpython/rlib/streamio.py
+++ b/rpython/rlib/streamio.py
@@ -193,7 +193,7 @@
# Truncate. Note that this may grow the file!
handle = get_osfhandle(fd)
if not SetEndOfFile(handle):
- raise WindowsError(GetLastError(),
+ raise OSError(GetLastError(),
"Could not truncate file")
finally:
# we restore the file pointer position in any case
diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py
--- a/rpython/rlib/test/test_rfile.py
+++ b/rpython/rlib/test/test_rfile.py
@@ -79,6 +79,22 @@
f()
self.interpret(f, [])
+ def test_fdopen(self):
+ fname = str(self.tmpdir.join('file_4a'))
+
+ def f():
+ f = open(fname, "w")
+ new_fno = os.dup(f.fileno())
+ f2 = os.fdopen(new_fno, "w")
+ f.close()
+ f2.write("xxx")
+ f2.close()
+
+ f()
+ assert open(fname).read() == "xxx"
+ self.interpret(f, [])
+ assert open(fname).read() == "xxx"
+
def test_fileno(self):
fname = str(self.tmpdir.join('file_5'))
diff --git a/rpython/tool/logparser.py b/rpython/tool/logparser.py
--- a/rpython/tool/logparser.py
+++ b/rpython/tool/logparser.py
@@ -410,7 +410,7 @@
total = sum([b for a, b in l])
for a, b in l:
if a is None:
- a = 'interpret'
+ a = 'normal-execution'
s = " " * (50 - len(a))
print >>outfile, a, s, str(b*100/total) + "%"
if out != '-':
diff --git a/rpython/tool/runsubprocess.py b/rpython/tool/runsubprocess.py
--- a/rpython/tool/runsubprocess.py
+++ b/rpython/tool/runsubprocess.py
@@ -11,6 +11,9 @@
def run_subprocess(executable, args, env=None, cwd=None):
return _run(executable, args, env, cwd)
+shell_default = False
+if sys.platform == 'win32':
+ shell_default = True
def _run(executable, args, env, cwd): # unless overridden below
if isinstance(args, str):
@@ -21,7 +24,9 @@
args = [str(executable)]
else:
args = [str(executable)] + args
- shell = False
+ # shell=True on unix-like is a known security vulnerability, but
+ # on windows shell=True does not properly propogate the env dict
+ shell = shell_default
# Just before spawning the subprocess, do a gc.collect(). This
# should help if we are running on top of PyPy, if the subprocess
diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -414,7 +414,8 @@
try:
returncode, stdout, stderr = _run_subprocess(
'nmake',
- ['/nologo', '/f', str(path.join('Makefile'))] + extra_opts)
+ ['/nologo', '/f', str(path.join('Makefile'))] + extra_opts,
+ env = self.c_environ)
finally:
oldcwd.chdir()
More information about the pypy-commit
mailing list