[pypy-commit] pypy py3k-memoryview: hg merge py3k
Manuel Jacob
noreply at buildbot.pypy.org
Thu May 22 04:32:22 CEST 2014
Author: Manuel Jacob
Branch: py3k-memoryview
Changeset: r71666:6f35f19922e2
Date: 2014-05-22 04:29 +0200
http://bitbucket.org/pypy/pypy/changeset/6f35f19922e2/
Log: hg merge py3k
diff too long, truncating to 2000 out of 5503 lines
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -6,3 +6,7 @@
9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm
+20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0
+20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0
+0000000000000000000000000000000000000000 release-2.3.0
+394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -44,31 +44,33 @@
Alex Gaynor
Michael Hudson
David Schneider
+ Matti Picus
+ Brian Kearns
+ Philip Jenvey
Holger Krekel
Christian Tismer
Hakan Ardo
Benjamin Peterson
- Matti Picus
- Philip Jenvey
+ Manuel Jacob
Anders Chrigstrom
- Brian Kearns
Eric van Riet Paap
+ Wim Lavrijsen
+ Ronan Lamy
Richard Emslie
Alexander Schremmer
- Wim Lavrijsen
Dan Villiom Podlaski Christiansen
- Manuel Jacob
Lukas Diekmann
Sven Hager
Anders Lehmann
Aurelien Campeas
Niklaus Haldimann
- Ronan Lamy
Camillo Bruni
Laura Creighton
Toon Verwaest
+ Remi Meier
Leonardo Santagada
Seo Sanghyeon
+ Romain Guillebert
Justin Peel
Ronny Pfannschmidt
David Edelsohn
@@ -80,52 +82,61 @@
Daniel Roberts
Niko Matsakis
Adrien Di Mascio
+ Alexander Hesse
Ludovic Aubry
- Alexander Hesse
Jacob Hallen
- Romain Guillebert
Jason Creighton
Alex Martelli
Michal Bendowski
Jan de Mooij
+ stian
Michael Foord
Stephan Diehl
Stefan Schwarzer
Valentino Volonghi
Tomek Meka
Patrick Maupin
- stian
Bob Ippolito
Bruno Gola
Jean-Paul Calderone
Timo Paulssen
+ Squeaky
Alexandre Fayolle
Simon Burton
Marius Gedminas
John Witulski
+ Konstantin Lopuhin
Greg Price
Dario Bertini
Mark Pearse
Simon Cross
- Konstantin Lopuhin
Andreas Stührk
Jean-Philippe St. Pierre
Guido van Rossum
Pavel Vinogradov
+ Paweł Piotr Przeradowski
Paul deGrandis
Ilya Osadchiy
+ Tobias Oberstein
Adrian Kuhn
Boris Feigin
+ Stefano Rivera
tav
+ Taavi Burns
Georg Brandl
Bert Freudenberg
Stian Andreassen
- Stefano Rivera
+ Laurence Tratt
Wanja Saatkamp
Gerald Klix
Mike Blume
- Taavi Burns
Oscar Nierstrasz
+ Stefan H. Muller
+ Jeremy Thurgood
+ Gregor Wegberg
+ Rami Chowdhury
+ Tobias Pape
+ Edd Barrett
David Malcolm
Eugene Oden
Henry Mason
@@ -135,18 +146,16 @@
Dusty Phillips
Lukas Renggli
Guenter Jantzen
- Tobias Oberstein
- Remi Meier
Ned Batchelder
Amit Regmi
Ben Young
Nicolas Chauvat
Andrew Durdin
+ Andrew Chambers
Michael Schneider
Nicholas Riley
Jason Chu
Igor Trindade Oliveira
- Jeremy Thurgood
Rocco Moretti
Gintautas Miliauskas
Michael Twomey
@@ -159,18 +168,19 @@
Karl Bartel
Brian Dorsey
Victor Stinner
+ Andrews Medina
Stuart Williams
Jasper Schulz
+ Christian Hudon
Toby Watson
Antoine Pitrou
Aaron Iles
Michael Cheng
Justas Sadzevicius
+ Mikael Schönenberg
Gasper Zejn
Neil Shepperd
- Mikael Schönenberg
Elmo Mäntynen
- Tobias Pape
Jonathan David Riehl
Stanislaw Halik
Anders Qvist
@@ -182,19 +192,18 @@
Alexander Sedov
Corbin Simpson
Christopher Pope
- Laurence Tratt
- Guillebert Romain
+ wenzhuman
Christian Tismer
+ Marc Abramowitz
Dan Stromberg
Stefano Parmesan
- Christian Hudon
Alexis Daboville
Jens-Uwe Mager
Carl Meyer
Karl Ramm
Pieter Zieschang
Gabriel
- Paweł Piotr Przeradowski
+ Lukas Vacek
Andrew Dalke
Sylvain Thenault
Nathan Taylor
@@ -205,6 +214,7 @@
Travis Francis Athougies
Kristjan Valur Jonsson
Neil Blakey-Milner
+ anatoly techtonik
Lutz Paelike
Lucio Torre
Lars Wassermann
@@ -218,13 +228,14 @@
Martin Blais
Lene Wagner
Tomo Cocoa
- Andrews Medina
roberto at goyle
+ Yury V. Zaytsev
+ Anna Katrina Dominguez
William Leslie
Bobby Impollonia
timo at eistee.fritz.box
Andrew Thompson
- Yusei Tahara
+ Ben Darnell
Roberto De Ioris
Juan Francisco Cantero Hurtado
Godefroid Chappelle
@@ -235,27 +246,35 @@
Anders Sigfridsson
Yasir Suhail
Floris Bruynooghe
+ Laurens Van Houtven
Akira Li
Gustavo Niemeyer
Stephan Busemann
- Anna Katrina Dominguez
+ Rafał Gałczyński
+ Yusei Tahara
Christian Muirhead
James Lan
shoma hosaka
- Daniel Neuhäuser
+ Daniel Neuh?user
+ Matthew Miller
Buck Golemon
Konrad Delong
Dinu Gherman
Chris Lambacher
coolbutuseless at gmail.com
+ Rodrigo Araújo
+ w31rd0
Jim Baker
- Rodrigo Araújo
+ James Robert
Armin Ronacher
Brett Cannon
yrttyr
+ aliceinwire
+ OlivierBlanvillain
Zooko Wilcox-O Hearn
Tomer Chachamu
Christopher Groskopf
+ jiaaro
opassembler.py
Antony Lee
Jim Hunziker
@@ -263,12 +282,13 @@
Even Wiik Thomassen
jbs
soareschen
+ Kurt Griffiths
+ Mike Bayer
Flavio Percoco
Kristoffer Kleine
yasirs
Michael Chermside
Anna Ravencroft
- Andrew Chambers
Julien Phalip
Dan Loewenherz
diff --git a/lib-python/2.7/test/test_gdbm.py b/lib-python/2.7/test/test_gdbm.py
--- a/lib-python/2.7/test/test_gdbm.py
+++ b/lib-python/2.7/test/test_gdbm.py
@@ -74,6 +74,29 @@
size2 = os.path.getsize(filename)
self.assertTrue(size1 > size2 >= size0)
+ def test_sync(self):
+ # check if sync works at all, not sure how to check it
+ self.g = gdbm.open(filename, 'cf')
+ self.g['x'] = 'x' * 10000
+ self.g.sync()
+
+ def test_get_key(self):
+ self.g = gdbm.open(filename, 'cf')
+ self.g['x'] = 'x' * 10000
+ self.g.close()
+ self.g = gdbm.open(filename, 'r')
+ self.assertEquals(self.g['x'], 'x' * 10000)
+
+ def test_key_with_null_bytes(self):
+ key = 'a\x00b'
+ value = 'c\x00d'
+ self.g = gdbm.open(filename, 'cf')
+ self.g[key] = value
+ self.g.close()
+ self.g = gdbm.open(filename, 'r')
+ self.assertEquals(self.g[key], value)
+ self.assertTrue(key in self.g)
+ self.assertTrue(self.g.has_key(key))
def test_main():
run_unittest(TestGdbm)
diff --git a/lib_pypy/_tkinter/tklib.py b/lib_pypy/_tkinter/tklib.py
--- a/lib_pypy/_tkinter/tklib.py
+++ b/lib_pypy/_tkinter/tklib.py
@@ -121,6 +121,10 @@
incdirs = []
linklibs = ['tcl85', 'tk85']
libdirs = []
+elif sys.platform == 'darwin':
+ incdirs = ['/System/Library/Frameworks/Tk.framework/Versions/Current/Headers/']
+ linklibs = ['tcl', 'tk']
+ libdirs = []
else:
incdirs=['/usr/include/tcl']
linklibs=['tcl', 'tk']
diff --git a/lib_pypy/gdbm.py b/lib_pypy/gdbm.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/gdbm.py
@@ -0,0 +1,174 @@
+import cffi, os
+
+ffi = cffi.FFI()
+ffi.cdef('''
+#define GDBM_READER ...
+#define GDBM_WRITER ...
+#define GDBM_WRCREAT ...
+#define GDBM_NEWDB ...
+#define GDBM_FAST ...
+#define GDBM_SYNC ...
+#define GDBM_NOLOCK ...
+#define GDBM_REPLACE ...
+
+void* gdbm_open(char *, int, int, int, void (*)());
+void gdbm_close(void*);
+
+typedef struct {
+ char *dptr;
+ int dsize;
+} datum;
+
+datum gdbm_fetch(void*, datum);
+int gdbm_delete(void*, datum);
+int gdbm_store(void*, datum, datum, int);
+int gdbm_exists(void*, datum);
+
+int gdbm_reorganize(void*);
+
+datum gdbm_firstkey(void*);
+datum gdbm_nextkey(void*, datum);
+void gdbm_sync(void*);
+
+char* gdbm_strerror(int);
+int gdbm_errno;
+
+void free(void*);
+''')
+
+try:
+ lib = ffi.verify('''
+ #include "gdbm.h"
+ ''', libraries=['gdbm'])
+except cffi.VerificationError as e:
+ # distutils does not preserve the actual message,
+ # but the verification is simple enough that the
+ # failure must be due to missing gdbm dev libs
+ raise ImportError('%s: %s' %(e.__class__.__name__, e))
+
+class error(Exception):
+ pass
+
+def _fromstr(key):
+ if not isinstance(key, str):
+ raise TypeError("gdbm mappings have string indices only")
+ return {'dptr': ffi.new("char[]", key), 'dsize': len(key)}
+
+class gdbm(object):
+ ll_dbm = None
+
+ def __init__(self, filename, iflags, mode):
+ res = lib.gdbm_open(filename, 0, iflags, mode, ffi.NULL)
+ self.size = -1
+ if not res:
+ self._raise_from_errno()
+ self.ll_dbm = res
+
+ def close(self):
+ if self.ll_dbm:
+ lib.gdbm_close(self.ll_dbm)
+ self.ll_dbm = None
+
+ def _raise_from_errno(self):
+ if ffi.errno:
+ raise error(os.strerror(ffi.errno))
+ raise error(lib.gdbm_strerror(lib.gdbm_errno))
+
+ def __len__(self):
+ if self.size < 0:
+ self.size = len(self.keys())
+ return self.size
+
+ def __setitem__(self, key, value):
+ self._check_closed()
+ self._size = -1
+ r = lib.gdbm_store(self.ll_dbm, _fromstr(key), _fromstr(value),
+ lib.GDBM_REPLACE)
+ if r < 0:
+ self._raise_from_errno()
+
+ def __delitem__(self, key):
+ self._check_closed()
+ res = lib.gdbm_delete(self.ll_dbm, _fromstr(key))
+ if res < 0:
+ raise KeyError(key)
+
+ def __contains__(self, key):
+ self._check_closed()
+ return lib.gdbm_exists(self.ll_dbm, _fromstr(key))
+ has_key = __contains__
+
+ def __getitem__(self, key):
+ self._check_closed()
+ drec = lib.gdbm_fetch(self.ll_dbm, _fromstr(key))
+ if not drec.dptr:
+ raise KeyError(key)
+ res = str(ffi.buffer(drec.dptr, drec.dsize))
+ lib.free(drec.dptr)
+ return res
+
+ def keys(self):
+ self._check_closed()
+ l = []
+ key = lib.gdbm_firstkey(self.ll_dbm)
+ while key.dptr:
+ l.append(str(ffi.buffer(key.dptr, key.dsize)))
+ nextkey = lib.gdbm_nextkey(self.ll_dbm, key)
+ lib.free(key.dptr)
+ key = nextkey
+ return l
+
+ def firstkey(self):
+ self._check_closed()
+ key = lib.gdbm_firstkey(self.ll_dbm)
+ if key.dptr:
+ res = str(ffi.buffer(key.dptr, key.dsize))
+ lib.free(key.dptr)
+ return res
+
+ def nextkey(self, key):
+ self._check_closed()
+ key = lib.gdbm_nextkey(self.ll_dbm, _fromstr(key))
+ if key.dptr:
+ res = str(ffi.buffer(key.dptr, key.dsize))
+ lib.free(key.dptr)
+ return res
+
+ def reorganize(self):
+ self._check_closed()
+ if lib.gdbm_reorganize(self.ll_dbm) < 0:
+ self._raise_from_errno()
+
+ def _check_closed(self):
+ if not self.ll_dbm:
+ raise error("GDBM object has already been closed")
+
+ __del__ = close
+
+ def sync(self):
+ self._check_closed()
+ lib.gdbm_sync(self.ll_dbm)
+
+def open(filename, flags='r', mode=0o666):
+ if flags[0] == 'r':
+ iflags = lib.GDBM_READER
+ elif flags[0] == 'w':
+ iflags = lib.GDBM_WRITER
+ elif flags[0] == 'c':
+ iflags = lib.GDBM_WRCREAT
+ elif flags[0] == 'n':
+ iflags = lib.GDBM_NEWDB
+ else:
+ raise error("First flag must be one of 'r', 'w', 'c' or 'n'")
+ for flag in flags[1:]:
+ if flag == 'f':
+ iflags |= lib.GDBM_FAST
+ elif flag == 's':
+ iflags |= lib.GDBM_SYNC
+ elif flag == 'u':
+ iflags |= lib.GDBM_NOLOCK
+ else:
+ raise error("Flag '%s' not supported" % flag)
+ return gdbm(filename, iflags, mode)
+
+open_flags = "rwcnfsu"
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -45,9 +45,9 @@
# built documents.
#
# The short X.Y version.
-version = '2.2'
+version = '2.3'
# The full version, including alpha/beta/rc tags.
-release = '2.2.1'
+release = '2.3.0'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -28,10 +28,6 @@
pypy/doc/tool/makecontributor.py generates the list of contributors
* rename pypy/doc/whatsnew_head.rst to whatsnew_VERSION.rst
and create a fresh whatsnew_head.rst after the release
-* merge PYPY_IRC_TOPIC environment variable handling from previous release
- in pypy/doc/getting-started-dev.rst, pypy/doc/man/pypy.1.rst, and
- pypy/interpreter/app_main.py so release versions will not print a random
- IRC topic by default.
* change the tracker to have a new release tag to file bugs against
* go to pypy/tool/release and run:
force-builds.py <release branch>
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -40,7 +40,7 @@
* `FAQ`_: some frequently asked questions.
-* `Release 2.2.1`_: the latest official release
+* `Release 2.3.0`_: the latest official release
* `PyPy Blog`_: news and status info about PyPy
@@ -110,7 +110,7 @@
.. _`Getting Started`: getting-started.html
.. _`Papers`: extradoc.html
.. _`Videos`: video-index.html
-.. _`Release 2.2.1`: http://pypy.org/download.html
+.. _`Release 2.3.0`: http://pypy.org/download.html
.. _`speed.pypy.org`: http://speed.pypy.org
.. _`RPython toolchain`: translation.html
.. _`potential project ideas`: project-ideas.html
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy.1.rst
@@ -100,6 +100,8 @@
``debug_start``/``debug_stop`` but not any nested
``debug_print``.
*fname* can be ``-`` to log to *stderr*.
+ Note that using a : in fname is a bad idea, Windows
+ users, beware.
``:``\ *fname*
Full logging, including ``debug_print``.
@@ -113,6 +115,11 @@
generate a log suitable for *jitviewer*, a tool for debugging
performance issues under PyPy.
+``PYPY_IRC_TOPIC``
+ If set to a non-empty value, print a random #pypy IRC
+ topic at startup of interactive mode.
+
+
.. include:: ../gc_info.rst
:start-line: 7
diff --git a/pypy/doc/release-2.3.0.rst b/pypy/doc/release-2.3.0.rst
--- a/pypy/doc/release-2.3.0.rst
+++ b/pypy/doc/release-2.3.0.rst
@@ -93,7 +93,7 @@
* Fix handling of tp_name for type objects
.. _`HippyVM`: http://www.hippyvm.com
-.. _`whats-new`: :http://doc.pypy.org/en/latest/whatsnew-2.3.0.html
+.. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.0.html
New Platforms and Features
diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst
--- a/pypy/doc/stm.rst
+++ b/pypy/doc/stm.rst
@@ -1,70 +1,78 @@
-======================
-Transactional Memory
-======================
+
+=============================
+Software Transactional Memory
+=============================
.. contents::
This page is about ``pypy-stm``, a special in-development version of
PyPy which can run multiple independent CPU-hungry threads in the same
-process in parallel. It is side-stepping what is known in the Python
-world as the "global interpreter lock (GIL)" problem.
+process in parallel. It is a solution to what is known in the Python
+world as the "global interpreter lock (GIL)" problem --- it is an
+implementation of Python without the GIL.
-"STM" stands for Software Transactional Memory, the technique used
+"STM" stands for Software `Transactional Memory`_, the technique used
internally. This page describes ``pypy-stm`` from the perspective of a
user, describes work in progress, and finally gives references to more
implementation details.
-This work was done mostly by Remi Meier and Armin Rigo. Thanks to all
-donors for crowd-funding the work so far! Please have a look at the
-`2nd call for donation`_.
+This work was done by Remi Meier and Armin Rigo. Thanks to all donors
+for crowd-funding the work so far! Please have a look at the `2nd call
+for donation`_.
+.. _`Transactional Memory`: http://en.wikipedia.org/wiki/Transactional_memory
.. _`2nd call for donation`: http://pypy.org/tmdonate2.html
Introduction
============
-``pypy-stm`` is a variant of the regular PyPy interpreter. With caveats
-listed below, it should be in theory within 25%-50% slower than a
-regular PyPy, comparing the JIT version in both cases. It is called
+``pypy-stm`` is a variant of the regular PyPy interpreter. With caveats_
+listed below, it should be in theory within 20%-50% slower than a
+regular PyPy, comparing the JIT version in both cases (but see below!).
+It is called
STM for Software Transactional Memory, which is the internal technique
used (see `Reference to implementation details`_).
-What you get in exchange for this slow-down is that ``pypy-stm`` runs
-any multithreaded Python program on multiple CPUs at once. Programs
-running two threads or more in parallel should ideally run faster than
-in a regular PyPy, either now or soon as issues are fixed. In one way,
-that's all there is to it: this is a GIL-less Python, feel free to
-`download and try it`__. However, the deeper idea behind the
-``pypy-stm`` project is to improve what is so far the state-of-the-art
-for using multiple CPUs, which for cases where separate processes don't
-work is done by writing explicitly multi-threaded programs. Instead,
-``pypy-stm`` is pushing forward an approach to *hide* the threads, as
-described below in `atomic sections`_.
+The benefit is that the resulting ``pypy-stm`` can execute multiple
+threads of Python code in parallel. Programs running two threads or
+more in parallel should ideally run faster than in a regular PyPy
+(either now, or soon as bugs are fixed).
+* ``pypy-stm`` is fully compatible with a GIL-based PyPy; you can use
+ it as a drop-in replacement and multithreaded programs will run on
+ multiple cores.
-.. __:
+* ``pypy-stm`` does not impose any special API to the user, but it
+ provides a new pure Python module called `transactional_memory`_ with
+ features to inspect the state or debug conflicts_ that prevent
+ parallelization. This module can also be imported on top of a non-STM
+ PyPy or CPython.
-Current status
-==============
+* Building on top of the way the GIL is removed, we will talk
+ about `Atomic sections, Transactions, etc.: a better way to write
+ parallel programs`_.
+
+
+Getting Started
+===============
**pypy-stm requires 64-bit Linux for now.**
Development is done in the branch `stmgc-c7`_. If you are only
-interested in trying it out, you can download a Ubuntu 12.04 binary
-here__ (``pypy-2.2.x-stm*.tar.bz2``; this version is a release mode,
-but not stripped of debug symbols). The current version supports four
-"segments", which means that it will run up to four threads in parallel,
-in other words it is running a thread pool up to 4 threads emulating normal
-threads.
+interested in trying it out, you can download a Ubuntu binary here__
+(``pypy-2.3.x-stm*.tar.bz2``, Ubuntu 12.04-14.04; these versions are
+release mode, but not stripped of debug symbols). The current version
+supports four "segments", which means that it will run up to four
+threads in parallel.
To build a version from sources, you first need to compile a custom
-version of clang; we recommend downloading `llvm and clang like
-described here`__, but at revision 201645 (use ``svn co -r 201645 ...``
+version of clang(!); we recommend downloading `llvm and clang like
+described here`__, but at revision 201645 (use ``svn co -r 201645 <path>``
for all checkouts). Then apply all the patches in `this directory`__:
-they are fixes for the very extensive usage that pypy-stm does of a
-clang-only feature (without them, you get crashes of clang). Then get
+they are fixes for a clang-only feature that hasn't been used so heavily
+in the past (without the patches, you get crashes of clang). Then get
the branch `stmgc-c7`_ of PyPy and run::
rpython/bin/rpython -Ojit --stm pypy/goal/targetpypystandalone.py
@@ -75,23 +83,31 @@
.. __: https://bitbucket.org/pypy/stmgc/src/default/c7/llvmfix/
-Caveats:
+.. _caveats:
-* So far, small examples work fine, but there are still a number of
- bugs. We're busy fixing them.
+Current status
+--------------
+
+* So far, small examples work fine, but there are still a few bugs.
+ We're busy fixing them as we find them; feel free to `report bugs`_.
+
+* It runs with an overhead as low as 20% on examples like "richards".
+ There are also other examples with higher overheads --up to 10x for
+ "translate.py"-- which we are still trying to understand. One suspect
+ is our partial GC implementation, see below.
* Currently limited to 1.5 GB of RAM (this is just a parameter in
- `core.h`__). Memory overflows are not detected correctly, so may
- cause segmentation faults.
+ `core.h`__). Memory overflows are not correctly handled; they cause
+ segfaults.
-* The JIT warm-up time is abysmal (as opposed to the regular PyPy's,
- which is "only" bad). Moreover, you should run it with a command like
- ``pypy-stm --jit trace_limit=60000 args...``; the default value of
- 6000 for ``trace_limit`` is currently too low (6000 should become
- reasonable again as we improve). Also, in order to produce machine
- code, the JIT needs to enter a special single-threaded mode for now.
- This all means that you *will* get very bad performance results if
- your program doesn't run for *many* seconds for now.
+* The JIT warm-up time improved recently but is still bad. In order to
+ produce machine code, the JIT needs to enter a special single-threaded
+ mode for now. This means that you will get bad performance results if
+ your program doesn't run for several seconds, where *several* can mean
+ *many.* When trying benchmarks, be sure to check that you have
+ reached the warmed state, i.e. the performance is not improving any
+ more. This should be clear from the fact that as long as it's
+ producing more machine code, ``pypy-stm`` will run on a single core.
* The GC is new; although clearly inspired by PyPy's regular GC, it
misses a number of optimizations for now. Programs allocating large
@@ -108,111 +124,197 @@
* The STM system is based on very efficient read/write barriers, which
are mostly done (their placement could be improved a bit in
JIT-generated machine code). But the overall bookkeeping logic could
- see more improvements (see Statistics_ below).
-
-* You can use `atomic sections`_, but the most visible missing thing is
- that you don't get reports about the "conflicts" you get. This would
- be the first thing that you need in order to start using atomic
- sections more extensively. Also, for now: for better results, try to
- explicitly force a transaction break just before (and possibly after)
- each large atomic section, with ``time.sleep(0)``.
+ see more improvements (see `Low-level statistics`_ below).
* Forking the process is slow because the complete memory needs to be
- copied manually right now.
+ copied manually. A warning is printed to this effect.
-* Very long-running processes should eventually crash on an assertion
- error because of a non-implemented overflow of an internal 29-bit
- number, but this requires at the very least ten hours --- more
- probably, several days or more.
+* Very long-running processes (on the order of days) will eventually
+ crash on an assertion error because of a non-implemented overflow of
+ an internal 29-bit number.
.. _`report bugs`: https://bugs.pypy.org/
.. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/translator/stm/src_stm/stm/core.h
-Statistics
+User Guide
==========
+
-When a non-main thread finishes, you get statistics printed to stderr,
-looking like that::
+Drop-in replacement
+-------------------
- thread 0x7f73377fe600:
- outside transaction 42182 0.506 s
- run current 85466 0.000 s
- run committed 34262 3.178 s
- run aborted write write 6982 0.083 s
- run aborted write read 550 0.005 s
- run aborted inevitable 388 0.010 s
- run aborted other 0 0.000 s
- wait free segment 0 0.000 s
- wait write read 78 0.027 s
- wait inevitable 887 0.490 s
- wait other 0 0.000 s
- bookkeeping 51418 0.606 s
- minor gc 162970 1.135 s
- major gc 1 0.019 s
- sync pause 59173 1.738 s
- spin loop 129512 0.094 s
+Multithreaded, CPU-intensive Python programs should work unchanged on
+``pypy-stm``. They will run using multiple CPU cores in parallel.
-The first number is a counter; the second number gives the associated
-time (the amount of real time that the thread was in this state; the sum
-of all the times should be equal to the total time between the thread's
-start and the thread's end). The most important points are "run
-committed", which gives the amount of useful work, and "outside
-transaction", which should give the time spent e.g. in library calls
-(right now it seems to be a bit larger than that; to investigate).
-Everything else is overhead of various forms. (Short-, medium- and
-long-term future work involves reducing this overhead :-)
+The existing semantics of the GIL (Global Interpreter Lock) are
+unchanged: although running on multiple cores in parallel, ``pypy-stm``
+gives the illusion that threads are run serially, with switches only
+occurring between bytecodes, not in the middle of them. Programs can
+rely on this: using ``shared_list.append()/pop()`` or
+``shared_dict.setdefault()`` as synchronization mecanisms continues to
+work as expected.
-These statistics are not printed out for the main thread, for now.
+This works by internally considering the points where a standard PyPy or
+CPython would release the GIL, and replacing them with the boundaries of
+"transaction". Like their database equivalent, multiple transactions
+can execute in parallel, but will commit in some serial order. They
+appear to behave as if they were completely run in this serialization
+order.
Atomic sections
-===============
+---------------
-While one of the goal of pypy-stm is to give a GIL-free but otherwise
-unmodified Python, the other goal is to push for a better way to use
-multithreading. For this, you (as the Python programmer) get an API
-in the ``__pypy__.thread`` submodule:
+PyPy supports *atomic sections,* which are blocks of code which you want
+to execute without "releasing the GIL". *This is experimental and may
+be removed in the future.* In STM terms, this means blocks of code that
+are executed while guaranteeing that the transaction is not interrupted
+in the middle.
-* ``__pypy__.thread.atomic``: a context manager (i.e. you use it in
- a ``with __pypy__.thread.atomic:`` statement). It runs the whole
- block of code without breaking the current transaction --- from
- the point of view of a regular CPython/PyPy, this is equivalent to
- saying that the GIL will not be released at all between the start and
- the end of this block of code.
+Here is a usage example::
-The obvious usage is to use atomic blocks in the same way as one would
-use locks: to protect changes to some shared data, you do them in a
-``with atomic`` block, just like you would otherwise do them in a ``with
-mylock`` block after ``mylock = thread.allocate_lock()``. This allows
-you not to care about acquiring the correct locks in the correct order;
-it is equivalent to having only one global lock. This is how
-transactional memory is `generally described`__: as a way to efficiently
-execute such atomic blocks, running them in parallel while giving the
-illusion that they run in some serial order.
+ with __pypy__.thread.atomic:
+ assert len(lst1) == 10
+ x = lst1.pop(0)
+ lst1.append(x)
-.. __: http://en.wikipedia.org/wiki/Transactional_memory
+In this (bad) example, we are sure that the item popped off one end of
+the list is appened again at the other end atomically. It means that
+another thread can run ``len(lst1)`` or ``x in lst1`` without any
+particular synchronization, and always see the same results,
+respectively ``10`` and ``True``. It will never see the intermediate
+state where ``lst1`` only contains 9 elements. Atomic sections are
+similar to re-entrant locks (they can be nested), but additionally they
+protect against the concurrent execution of *any* code instead of just
+code that happens to be protected by the same lock in other threads.
-However, the less obvious intended usage of atomic sections is as a
-wide-ranging replacement of explicit threads. You can turn a program
-that is not multi-threaded at all into a program that uses threads
-internally, together with large atomic sections to keep the behavior
-unchanged. This capability can be hidden in a library or in the
-framework you use; the end user's code does not need to be explicitly
-aware of using threads. For a simple example of this, see
-`transaction.py`_ in ``lib_pypy``. The idea is that if you have a
-program where the function ``f(key, value)`` runs on every item of some
-big dictionary, you can replace the loop with::
+Note that the notion of atomic sections is very strong. If you write
+code like this::
+
+ with __pypy__.thread.atomic:
+ time.sleep(10)
+
+then, if you think about it as if we had a GIL, you are executing a
+10-seconds-long atomic transaction without releasing the GIL at all.
+This prevents all other threads from progressing at all. While it is
+not strictly true in ``pypy-stm``, the exact rules for when other
+threads can progress or not are rather complicated; you have to consider
+it likely that such a piece of code will eventually block all other
+threads anyway.
+
+Note that if you want to experiment with ``atomic``, you may have to add
+manually a transaction break just before the atomic block. This is
+because the boundaries of the block are not guaranteed to be the
+boundaries of the transaction: the latter is at least as big as the
+block, but maybe bigger. Therefore, if you run a big atomic block, it
+is a good idea to break the transaction just before. This can be done
+e.g. by the hack of calling ``time.sleep(0)``. (This may be fixed at
+some point.)
+
+There are also issues with the interaction of locks and atomic blocks.
+This can be seen if you write to files (which have locks), including
+with a ``print`` to standard output. If one thread tries to acquire a
+lock while running in an atomic block, and another thread has got the
+same lock, then the former may fail with a ``thread.error``. The reason
+is that "waiting" for some condition to become true --while running in
+an atomic block-- does not really make sense. For now you can work
+around it by making sure that, say, all your prints are either in an
+``atomic`` block or none of them are. (This kind of issue is
+theoretically hard to solve.)
+
+
+Locks
+-----
+
+**Not Implemented Yet**
+
+The thread module's locks have their basic semantic unchanged. However,
+using them (e.g. in ``with my_lock:`` blocks) starts an alternative
+running mode, called `Software lock elision`_. This means that PyPy
+will try to make sure that the transaction extends until the point where
+the lock is released, and if it succeeds, then the acquiring and
+releasing of the lock will be "elided". This means that in this case,
+the whole transaction will technically not cause any write into the lock
+object --- it was unacquired before, and is still unacquired after the
+transaction.
+
+This is specially useful if two threads run ``with my_lock:`` blocks
+with the same lock. If they each run a transaction that is long enough
+to contain the whole block, then all writes into the lock will be elided
+and the two transactions will not conflict with each other. As usual,
+they will be serialized in some order: one of the two will appear to run
+before the other. Simply, each of them executes an "acquire" followed
+by a "release" in the same transaction. As explained above, the lock
+state goes from "unacquired" to "unacquired" and can thus be left
+unchanged.
+
+This approach can gracefully fail: unlike atomic sections, there is no
+guarantee that the transaction runs until the end of the block. If you
+perform any input/output while you hold the lock, the transaction will
+end as usual just before the input/output operation. If this occurs,
+then the lock elision mode is cancelled and the lock's "acquired" state
+is really written.
+
+Even if the lock is really acquired already, a transaction doesn't have
+to wait for it to become free again. It can enter the elision-mode anyway
+and tentatively execute the content of the block. It is only at the end,
+when trying to commit, that the thread will pause. As soon as the real
+value stored in the lock is switched back to "unacquired", it can then
+proceed and attempt to commit its already-executed transaction (which
+can fail and abort and restart from the scratch, as usual).
+
+Note that this is all *not implemented yet,* but we expect it to work
+even if you acquire and release several locks. The elision-mode
+transaction will extend until the first lock you acquired is released,
+or until the code performs an input/output or a wait operation (for
+example, waiting for another lock that is currently not free). In the
+common case of acquiring several locks in nested order, they will all be
+elided by the same transaction.
+
+.. _`software lock elision`: https://www.repository.cam.ac.uk/handle/1810/239410
+
+
+Atomic sections, Transactions, etc.: a better way to write parallel programs
+----------------------------------------------------------------------------
+
+(This section is based on locks as we plan to implement them, but also
+works with the existing atomic sections.)
+
+In the cases where elision works, the block of code can run in parallel
+with other blocks of code *even if they are protected by the same lock.*
+You still get the illusion that the blocks are run sequentially. This
+works even for multiple threads that run each a series of such blocks
+and nothing else, protected by one single global lock. This is
+basically the Python application-level equivalent of what was done with
+the interpreter in ``pypy-stm``: while you think you are writing
+thread-unfriendly code because of this global lock, actually the
+underlying system is able to make it run on multiple cores anyway.
+
+This capability can be hidden in a library or in the framework you use;
+the end user's code does not need to be explicitly aware of using
+threads. For a simple example of this, there is `transaction.py`_ in
+``lib_pypy``. The idea is that you write, or already have, some program
+where the function ``f(key, value)`` runs on every item of some big
+dictionary, say::
+
+ for key, value in bigdict.items():
+ f(key, value)
+
+Then you simply replace the loop with::
for key, value in bigdict.items():
transaction.add(f, key, value)
transaction.run()
This code runs the various calls to ``f(key, value)`` using a thread
-pool, but every single call is done in an atomic section. The end
-result is that the behavior should be exactly equivalent: you don't get
-any extra multithreading issue.
+pool, but every single call is executed under the protection of a unique
+lock. The end result is that the behavior is exactly equivalent --- in
+fact it makes little sense to do it in this way on a non-STM PyPy or on
+CPython. But on ``pypy-stm``, the various locked calls to ``f(key,
+value)`` can tentatively be executed in parallel, even if the observable
+result is as if they were executed in some serial order.
This approach hides the notion of threads from the end programmer,
including all the hard multithreading-related issues. This is not the
@@ -223,41 +325,176 @@
only requires that the end programmer identifies where this parallelism
is likely to be found, and communicates it to the system, using for
example the ``transaction.add()`` scheme.
-
+
.. _`transaction.py`: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/lib_pypy/transaction.py
.. _OpenMP: http://en.wikipedia.org/wiki/OpenMP
-==================
-Other APIs in pypy-stm:
+.. _`transactional_memory`:
-* ``__pypy__.thread.getsegmentlimit()``: return the number of "segments"
- in this pypy-stm. This is the limit above which more threads will not
- be able to execute on more cores. (Right now it is limited to 4 due
- to inter-segment overhead, but should be increased in the future. It
+API of transactional_memory
+---------------------------
+
+The new pure Python module ``transactional_memory`` runs on both CPython
+and PyPy, both with and without STM. It contains:
+
+* ``getsegmentlimit()``: return the number of "segments" in
+ this pypy-stm. This is the limit above which more threads will not be
+ able to execute on more cores. (Right now it is limited to 4 due to
+ inter-segment overhead, but should be increased in the future. It
should also be settable, and the default value should depend on the
- number of actual CPUs.)
+ number of actual CPUs.) If STM is not available, this returns 1.
-* ``__pypy__.thread.exclusive_atomic``: same as ``atomic``, but
- raises an exception if you attempt to nest it inside another
- ``atomic``.
+* ``print_abort_info(minimum_time=0.0)``: debugging help. Each thread
+ remembers the longest abort or pause it did because of cross-thread
+ contention_. This function prints it to ``stderr`` if the time lost
+ is greater than ``minimum_time`` seconds. The record is then
+ cleared, to make it ready for new events. This function returns
+ ``True`` if it printed a report, and ``False`` otherwise.
-* ``__pypy__.thread.signals_enabled``: a context manager that runs
- its block with signals enabled. By default, signals are only
- enabled in the main thread; a non-main thread will not receive
- signals (this is like CPython). Enabling signals in non-main threads
- is useful for libraries where threads are hidden and the end user is
- not expecting his code to run elsewhere than in the main thread.
-Note that all of this API is (or will be) implemented in a regular PyPy
-too: for example, ``with atomic`` will simply mean "don't release the
-GIL" and ``getsegmentlimit()`` will return 1.
+API of __pypy__.thread
+----------------------
-==================
+The ``__pypy__.thread`` submodule is a built-in module of PyPy that
+contains a few internal built-in functions used by the
+``transactional_memory`` module, plus the following:
+
+* ``__pypy__.thread.atomic``: a context manager to run a block in
+ fully atomic mode, without "releasing the GIL". (May be eventually
+ removed?)
+
+* ``__pypy__.thread.signals_enabled``: a context manager that runs its
+ block with signals enabled. By default, signals are only enabled in
+ the main thread; a non-main thread will not receive signals (this is
+ like CPython). Enabling signals in non-main threads is useful for
+ libraries where threads are hidden and the end user is not expecting
+ his code to run elsewhere than in the main thread.
+
+
+.. _contention:
+
+Conflicts
+---------
+
+Based on Software Transactional Memory, the ``pypy-stm`` solution is
+prone to "conflicts". To repeat the basic idea, threads execute their code
+speculatively, and at known points (e.g. between bytecodes) they
+coordinate with each other to agree on which order their respective
+actions should be "committed", i.e. become globally visible. Each
+duration of time between two commit-points is called a transaction.
+
+A conflict occurs when there is no consistent ordering. The classical
+example is if two threads both tried to change the value of the same
+global variable. In that case, only one of them can be allowed to
+proceed, and the other one must be either paused or aborted (restarting
+the transaction). If this occurs too often, parallelization fails.
+
+How much actual parallelization a multithreaded program can see is a bit
+subtle. Basically, a program not using ``__pypy__.thread.atomic`` or
+eliding locks, or doing so for very short amounts of time, will
+parallelize almost freely (as long as it's not some artificial example
+where, say, all threads try to increase the same global counter and do
+nothing else).
+
+However, using if the program requires longer transactions, it comes
+with less obvious rules. The exact details may vary from version to
+version, too, until they are a bit more stabilized. Here is an
+overview.
+
+Parallelization works as long as two principles are respected. The
+first one is that the transactions must not *conflict* with each other.
+The most obvious sources of conflicts are threads that all increment a
+global shared counter, or that all store the result of their
+computations into the same list --- or, more subtly, that all ``pop()``
+the work to do from the same list, because that is also a mutation of
+the list. (It is expected that some STM-aware library will eventually
+be designed to help with conflict problems, like a STM-aware queue.)
+
+A conflict occurs as follows: when a transaction commits (i.e. finishes
+successfully) it may cause other transactions that are still in progress
+to abort and retry. This is a waste of CPU time, but even in the worst
+case senario it is not worse than a GIL, because at least one
+transaction succeeds (so we get at worst N-1 CPUs doing useless jobs and
+1 CPU doing a job that commits successfully).
+
+Conflicts do occur, of course, and it is pointless to try to avoid them
+all. For example they can be abundant during some warm-up phase. What
+is important is to keep them rare enough in total.
+
+Another issue is that of avoiding long-running so-called "inevitable"
+transactions ("inevitable" is taken in the sense of "which cannot be
+avoided", i.e. transactions which cannot abort any more). Transactions
+like that should only occur if you use ``__pypy__.thread.atomic``,
+generally become of I/O in atomic blocks. They work, but the
+transaction is turned inevitable before the I/O is performed. For all
+the remaining execution time of the atomic block, they will impede
+parallel work. The best is to organize the code so that such operations
+are done completely outside ``__pypy__.thread.atomic``.
+
+(This is related to the fact that blocking I/O operations are
+discouraged with Twisted, and if you really need them, you should do
+them on their own separate thread.)
+
+In case of lock elision, we don't get long-running inevitable
+transactions, but a different problem can occur: doing I/O cancels lock
+elision, and the lock turns into a real lock, preventing other threads
+from committing if they also need this lock. (More about it when lock
+elision is implemented and tested.)
+
+
+
+Implementation
+==============
+
+XXX this section mostly empty for now
+
+
+Low-level statistics
+--------------------
+
+When a non-main thread finishes, you get low-level statistics printed to
+stderr, looking like that::
+
+ thread 0x7f73377fe600:
+ outside transaction 42182 0.506 s
+ run current 85466 0.000 s
+ run committed 34262 3.178 s
+ run aborted write write 6982 0.083 s
+ run aborted write read 550 0.005 s
+ run aborted inevitable 388 0.010 s
+ run aborted other 0 0.000 s
+ wait free segment 0 0.000 s
+ wait write read 78 0.027 s
+ wait inevitable 887 0.490 s
+ wait other 0 0.000 s
+ sync commit soon 1 0.000 s
+ bookkeeping 51418 0.606 s
+ minor gc 162970 1.135 s
+ major gc 1 0.019 s
+ sync pause 59173 1.738 s
+ longest recordered marker 0.000826 s
+ "File "x.py", line 5, in f"
+
+On each line, the first number is a counter, and the second number gives
+the associated time --- the amount of real time that the thread was in
+this state. The sum of all the times should be equal to the total time
+between the thread's start and the thread's end. The most important
+points are "run committed", which gives the amount of useful work, and
+"outside transaction", which should give the time spent e.g. in library
+calls (right now it seems to be larger than that; to investigate). The
+various "run aborted" and "wait" entries are time lost due to
+conflicts_. Everything else is overhead of various forms. (Short-,
+medium- and long-term future work involves reducing this overhead :-)
+
+The last two lines are special; they are an internal marker read by
+``transactional_memory.print_abort_info()``.
+
+These statistics are not printed out for the main thread, for now.
Reference to implementation details
-===================================
+-----------------------------------
The core of the implementation is in a separate C library called stmgc_,
in the c7_ subdirectory. Please see the `README.txt`_ for more
@@ -282,3 +519,15 @@
.. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/translator/stm/src_stm/stmgcintf.c
.. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/jit/backend/llsupport/stmrewrite.py
.. __: https://bitbucket.org/pypy/pypy/raw/stmgc-c7/rpython/jit/backend/x86/assembler.py
+
+
+
+See also
+========
+
+See also
+https://bitbucket.org/pypy/pypy/raw/default/pypy/doc/project-ideas.rst
+(section about STM).
+
+
+.. include:: _ref.txt
diff --git a/pypy/doc/whatsnew-2.3.0.rst b/pypy/doc/whatsnew-2.3.0.rst
--- a/pypy/doc/whatsnew-2.3.0.rst
+++ b/pypy/doc/whatsnew-2.3.0.rst
@@ -167,3 +167,6 @@
.. branch: fix-tpname
Changes hacks surrounding W_TypeObject.name to match CPython's tp_name
+
+.. branch: tkinter_osx_packaging
+OS/X specific header path
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -3,4 +3,4 @@
=======================
.. this is a revision shortly after release-2.3.x
-.. startrev: ec864bd08d50
+.. startrev: f556d32f8319
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -56,7 +56,10 @@
try:
space.call_function(w_run_toplevel, w_call_startup_gateway)
if rlocale.HAVE_LANGINFO:
- rlocale.setlocale(rlocale.LC_ALL, '')
+ try:
+ rlocale.setlocale(rlocale.LC_ALL, '')
+ except rlocale.LocaleError:
+ pass
w_executable = space.fsdecode(space.wrapbytes(argv[0]))
w_argv = space.newlist([space.fsdecode(space.wrapbytes(s))
for s in argv[1:]])
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -37,6 +37,9 @@
PYTHONPATH : %r-separated list of directories prefixed to the
default module search path. The result is sys.path.
PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.
+PYPY_IRC_TOPIC: if set to a non-empty value, print a random #pypy IRC
+ topic at startup of interactive mode.
+PYPYLOG: If set to a non-empty value, enable logging.
"""
try:
@@ -113,6 +116,7 @@
except BaseException as e:
try:
+ initstdio()
stderr = sys.stderr
print('Error calling sys.excepthook:', file=stderr)
originalexcepthook(type(e), e, e.__traceback__)
@@ -678,7 +682,11 @@
if inspect_requested():
try:
from _pypy_interact import interactive_console
- success = run_toplevel(interactive_console, mainmodule, quiet)
+ pypy_version_info = getattr(sys, 'pypy_version_info', sys.version_info)
+ irc_topic = pypy_version_info[3] != 'final' or (
+ readenv and os.getenv('PYPY_IRC_TOPIC'))
+ success = run_toplevel(interactive_console, mainmodule,
+ quiet=quiet or not irc_topic)
except SystemExit as e:
status = e.code
else:
diff --git a/pypy/interpreter/test/test_app_main.py b/pypy/interpreter/test/test_app_main.py
--- a/pypy/interpreter/test/test_app_main.py
+++ b/pypy/interpreter/test/test_app_main.py
@@ -7,11 +7,8 @@
from rpython.tool.udir import udir
from contextlib import contextmanager
from pypy.conftest import pypydir
-from pypy.module.sys.version import PYPY_VERSION
from lib_pypy._pypy_interact import irc_header
-is_release = PYPY_VERSION[3] == "final"
-
python3 = os.environ.get("PYTHON3", "python3")
@@ -21,7 +18,6 @@
stdout=subprocess.PIPE)
return p.stdout.read().rstrip()
banner = get_banner()
-print repr(banner)
app_main = os.path.join(os.path.realpath(os.path.dirname(__file__)), os.pardir, 'app_main.py')
app_main = os.path.abspath(app_main)
@@ -255,10 +251,6 @@
child = self.spawn([])
child.expect('Python ') # banner
child.expect('>>> ') # prompt
- if is_release:
- assert irc_header not in child.before
- else:
- assert irc_header in child.before
child.sendline('[6*7]')
child.expect(re.escape('[42]'))
child.sendline('def f(x):')
@@ -278,6 +270,22 @@
child.sendline("'' in sys.path")
child.expect("True")
+ def test_yes_irc_topic(self, monkeypatch):
+ monkeypatch.setenv('PYPY_IRC_TOPIC', '1')
+ child = self.spawn([])
+ child.expect(irc_header) # banner
+
+ def test_maybe_irc_topic(self):
+ import sys
+ pypy_version_info = getattr(sys, 'pypy_version_info', sys.version_info)
+ irc_topic = pypy_version_info[3] != 'final'
+ child = self.spawn([])
+ child.expect('>>>') # banner
+ if irc_topic:
+ assert irc_header in child.before
+ else:
+ assert irc_header not in child.before
+
def test_help(self):
# test that -h prints the usage, including the name of the executable
# which should be /full/path/to/app_main.py in this case
@@ -1048,6 +1056,7 @@
# ----------------------------------------
from pypy.module.sys.version import CPYTHON_VERSION, PYPY_VERSION
cpy_ver = '%d' % CPYTHON_VERSION[0]
+ from lib_pypy._pypy_interact import irc_header
goal_dir = os.path.dirname(app_main)
# build a directory hierarchy like which contains both bin/pypy-c and
@@ -1067,6 +1076,7 @@
self.w_fake_exe = self.space.wrap(str(fake_exe))
self.w_expected_path = self.space.wrap(expected_path)
self.w_trunkdir = self.space.wrap(os.path.dirname(pypydir))
+ self.w_is_release = self.space.wrap(PYPY_VERSION[3] == "final")
self.w_tmp_dir = self.space.wrap(tmp_dir)
@@ -1136,3 +1146,4 @@
# assert it did not crash
finally:
sys.path[:] = old_sys_path
+
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,7 +29,7 @@
#define PY_VERSION "3.2.5"
/* PyPy version as a string */
-#define PYPY_VERSION "2.3.0-alpha0"
+#define PYPY_VERSION "2.4.0-alpha0"
/* Subversion Revision number of this file (not of the repository).
* Empty since Mercurial migration. */
diff --git a/pypy/module/fcntl/interp_fcntl.py b/pypy/module/fcntl/interp_fcntl.py
--- a/pypy/module/fcntl/interp_fcntl.py
+++ b/pypy/module/fcntl/interp_fcntl.py
@@ -62,8 +62,8 @@
fcntl_int = external('fcntl', [rffi.INT, rffi.INT, rffi.INT], rffi.INT)
fcntl_str = external('fcntl', [rffi.INT, rffi.INT, rffi.CCHARP], rffi.INT)
fcntl_flock = external('fcntl', [rffi.INT, rffi.INT, _flock], rffi.INT)
-ioctl_int = external('ioctl', [rffi.INT, rffi.INT, rffi.INT], rffi.INT)
-ioctl_str = external('ioctl', [rffi.INT, rffi.INT, rffi.CCHARP], rffi.INT)
+ioctl_int = external('ioctl', [rffi.INT, rffi.UINT, rffi.INT], rffi.INT)
+ioctl_str = external('ioctl', [rffi.INT, rffi.UINT, rffi.CCHARP], rffi.INT)
has_flock = cConfig.has_flock
if has_flock:
diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py
--- a/pypy/module/fcntl/test/test_fcntl.py
+++ b/pypy/module/fcntl/test/test_fcntl.py
@@ -11,7 +11,9 @@
os.unlink(i)
class AppTestFcntl:
- spaceconfig = dict(usemodules=('fcntl', 'array', 'struct', 'termios', 'select', 'rctime'))
+ spaceconfig = dict(usemodules=('fcntl', 'array', 'struct', 'termios',
+ 'select', 'rctime'))
+
def setup_class(cls):
tmpprefix = str(udir.ensure('test_fcntl', dir=1).join('tmp_'))
cls.w_tmp = cls.space.wrap(tmpprefix)
@@ -257,6 +259,31 @@
os.close(mfd)
os.close(sfd)
+ def test_ioctl_signed_unsigned_code_param(self):
+ import fcntl
+ import os
+ import pty
+ import struct
+ import termios
+
+ mfd, sfd = pty.openpty()
+ try:
+ if termios.TIOCSWINSZ < 0:
+ set_winsz_opcode_maybe_neg = termios.TIOCSWINSZ
+ set_winsz_opcode_pos = termios.TIOCSWINSZ & 0xffffffff
+ else:
+ set_winsz_opcode_pos = termios.TIOCSWINSZ
+ set_winsz_opcode_maybe_neg, = struct.unpack("i",
+ struct.pack("I", termios.TIOCSWINSZ))
+
+ our_winsz = struct.pack("HHHH",80,25,0,0)
+ # test both with a positive and potentially negative ioctl code
+ new_winsz = fcntl.ioctl(mfd, set_winsz_opcode_pos, our_winsz)
+ new_winsz = fcntl.ioctl(mfd, set_winsz_opcode_maybe_neg, our_winsz)
+ finally:
+ os.close(mfd)
+ os.close(sfd)
+
def test_large_flag(self):
import sys
if any(plat in sys.platform
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -2,7 +2,7 @@
Implementation of the interpreter-level default import logic.
"""
-import sys, os, stat, genericpath
+import sys, os, stat
from pypy.interpreter.module import Module
from pypy.interpreter.gateway import interp2app, unwrap_spec
@@ -528,8 +528,7 @@
path = space.str0_w(w_pathitem)
filepart = os.path.join(path, partname)
- # os.path.isdir on win32 is not rpython when pywin32 installed
- if genericpath.isdir(filepart) and case_ok(filepart):
+ if os.path.isdir(filepart) and case_ok(filepart):
initfile = os.path.join(filepart, '__init__')
modtype, _, _ = find_modtype(space, initfile)
if modtype in (PY_SOURCE, PY_COMPILED):
diff --git a/pypy/module/struct/__init__.py b/pypy/module/struct/__init__.py
--- a/pypy/module/struct/__init__.py
+++ b/pypy/module/struct/__init__.py
@@ -45,7 +45,7 @@
The variable struct.error is an exception raised on errors."""
- applevel_name = '_struct'
+ applevel_name = "_struct"
interpleveldefs = {
'error': 'interp_struct.get_error(space)',
@@ -55,9 +55,9 @@
'pack_into': 'interp_struct.pack_into',
'unpack': 'interp_struct.unpack',
'unpack_from': 'interp_struct.unpack_from',
- '_clearcache': 'interp_struct.clearcache',
'Struct': 'interp_struct.W_Struct',
+ '_clearcache': 'interp_struct.clearcache',
}
appleveldefs = {
diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -21,11 +21,6 @@
return space.fromcache(Cache).error
- at unwrap_spec(format=str)
-def calcsize(space, format):
- return space.wrap(_calcsize(space, format))
-
-
def _calcsize(space, format):
fmtiter = CalcSizeFormatIterator()
try:
@@ -38,7 +33,11 @@
@unwrap_spec(format=str)
-def pack(space, format, args_w):
+def calcsize(space, format):
+ return space.wrap(_calcsize(space, format))
+
+
+def _pack(space, format, args_w):
if jit.isconstant(format):
size = _calcsize(space, format)
else:
@@ -50,13 +49,18 @@
raise OperationError(space.w_OverflowError, space.wrap(e.msg))
except StructError, e:
raise OperationError(get_error(space), space.wrap(e.msg))
- return space.wrapbytes(fmtiter.result.build())
+ return fmtiter.result.build()
+
+
+ at unwrap_spec(format=str)
+def pack(space, format, args_w):
+ return space.wrapbytes(_pack(space, format, args_w))
# XXX inefficient
@unwrap_spec(format=str, offset=int)
def pack_into(space, format, w_buffer, offset, args_w):
- res = pack(space, format, args_w).bytes_w(space)
+ res = _pack(space, format, args_w)
buf = space.writebuf_w(w_buffer)
if offset < 0:
offset += buf.getlength()
@@ -140,3 +144,6 @@
pack_into=interp2app(W_Struct.descr_pack_into),
unpack_from=interp2app(W_Struct.descr_unpack_from),
)
+
+def clearcache(space):
+ """No-op on PyPy"""
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
#XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h
-PYPY_VERSION = (2, 3, 0, "alpha", 0) #XXX # sync patchlevel.h
+PYPY_VERSION = (2, 4, 0, "alpha", 0) #XXX # sync patchlevel.h
if platform.name == 'msvc':
COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600)
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -158,7 +158,15 @@
# Good default if there are no replacements.
buf = StringBuilder(len("bytearray(b'')") + len(s))
- buf.append("bytearray(b'")
+ buf.append("bytearray(b")
+ quote = "'"
+ for c in s:
+ if c == '"':
+ quote = "'"
+ break
+ elif c == "'":
+ quote = '"'
+ buf.append(quote)
for i in range(len(s)):
c = s[i]
@@ -180,7 +188,8 @@
else:
buf.append(c)
- buf.append("')")
+ buf.append(quote)
+ buf.append(")")
return space.wrap(buf.build())
diff --git a/pypy/objspace/std/complextype.py b/pypy/objspace/std/complextype.py
--- a/pypy/objspace/std/complextype.py
+++ b/pypy/objspace/std/complextype.py
@@ -39,22 +39,20 @@
# ignore whitespace after bracket
while i < slen and s[i] == ' ':
i += 1
+ while slen > 0 and s[slen-1] == ' ':
+ slen -= 1
# extract first number
realstart = i
pc = s[i]
while i < slen and s[i] != ' ':
- if s[i] in ('+','-') and pc not in ('e','E') and i != realstart:
+ if s[i] in ('+', '-') and pc not in ('e', 'E') and i != realstart:
break
pc = s[i]
i += 1
realstop = i
- # ignore whitespace
- while i < slen and s[i] == ' ':
- i += 1
-
# return appropriate strings is only one number is there
if i >= slen:
newstop = realstop - 1
@@ -76,20 +74,17 @@
# find sign for imaginary part
if s[i] == '-' or s[i] == '+':
imagsign = s[i]
- if imagsign == ' ':
+ else:
raise ValueError
- i+=1
- # whitespace
- while i < slen and s[i] == ' ':
- i += 1
+ i += 1
if i >= slen:
raise ValueError
imagstart = i
pc = s[i]
while i < slen and s[i] != ' ':
- if s[i] in ('+','-') and pc not in ('e','E'):
+ if s[i] in ('+', '-') and pc not in ('e', 'E'):
break
pc = s[i]
i += 1
@@ -97,14 +92,12 @@
imagstop = i - 1
if imagstop < 0:
raise ValueError
- if s[imagstop] not in ('j','J'):
+ if s[imagstop] not in ('j', 'J'):
raise ValueError
if imagstop < imagstart:
raise ValueError
- while i<slen and s[i] == ' ':
- i += 1
- if i < slen:
+ if i < slen:
raise ValueError
realpart = s[realstart:realstop]
diff --git a/pypy/objspace/std/test/test_boolobject.py b/pypy/objspace/std/test/test_boolobject.py
--- a/pypy/objspace/std/test/test_boolobject.py
+++ b/pypy/objspace/std/test/test_boolobject.py
@@ -1,8 +1,4 @@
-
-
-
class TestW_BoolObject:
-
def setup_method(self,method):
self.true = self.space.w_True
self.false = self.space.w_False
@@ -29,6 +25,7 @@
def test_rbigint_w(self):
assert self.space.bigint_w(self.true)._digits == [1]
+
class AppTestAppBoolTest:
def test_bool_callable(self):
assert True == bool(1)
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -1,5 +1,6 @@
from pypy import conftest
+
class AppTestBytesArray:
def setup_class(cls):
cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect)
@@ -49,12 +50,15 @@
def test_repr(self):
assert repr(bytearray()) == "bytearray(b'')"
assert repr(bytearray(b'test')) == "bytearray(b'test')"
- assert repr(bytearray(b"d'oh")) == r"bytearray(b'd\'oh')"
+ assert repr(bytearray(b"d'oh")) == r'bytearray(b"d\'oh")'
+ assert repr(bytearray(b'd"oh')) == 'bytearray(b\'d"oh\')'
+ assert repr(bytearray(b'd"\'oh')) == 'bytearray(b\'d"\\\'oh\')'
+ assert repr(bytearray(b'd\'"oh')) == 'bytearray(b\'d\\\'"oh\')'
def test_str(self):
assert str(bytearray()) == "bytearray(b'')"
assert str(bytearray(b'test')) == "bytearray(b'test')"
- assert str(bytearray(b"d'oh")) == r"bytearray(b'd\'oh')"
+ assert str(bytearray(b"d'oh")) == r'bytearray(b"d\'oh")'
def test_getitem(self):
b = bytearray(b'test')
diff --git a/pypy/objspace/std/test/test_complexobject.py b/pypy/objspace/std/test/test_complexobject.py
--- a/pypy/objspace/std/test/test_complexobject.py
+++ b/pypy/objspace/std/test/test_complexobject.py
@@ -85,7 +85,7 @@
class AppTestAppComplexTest:
- spaceconfig = dict(usemodules=['binascii', 'rctime', 'unicodedata'])
+ spaceconfig = {"usemodules": ["binascii", "rctime", "unicodedata"]}
def w_check_div(self, x, y):
"""Compute complex z=x*y, and check that z/x==y and z/y==x."""
@@ -319,6 +319,8 @@
assert self.almost_equal(complex("-1"), -1)
assert self.almost_equal(complex("+1"), +1)
assert self.almost_equal(complex(" ( +3.14-6J ) "), 3.14-6j)
+ exc = raises(ValueError, complex, " ( +3.14- 6J ) ")
+ assert str(exc.value) == "complex() arg is a malformed string"
class complex2(complex):
pass
@@ -379,10 +381,10 @@
def test_constructor_unicode(self):
b1 = '\N{MATHEMATICAL BOLD DIGIT ONE}' # 𝟏
b2 = '\N{MATHEMATICAL BOLD DIGIT TWO}' # 𝟐
- s = '{0} + {1}j'.format(b1, b2)
+ s = '{0}+{1}j'.format(b1, b2)
assert complex(s) == 1+2j
assert complex('\N{EM SPACE}(\N{EN SPACE}1+1j ) ') == 1+1j
-
+
def test___complex___returning_non_complex(self):
import cmath
class Obj(object):
@@ -399,7 +401,7 @@
#
assert cmath.polar(1) == (1.0, 0.0)
raises(TypeError, "cmath.polar(Obj(1))")
-
+
def test_hash(self):
for x in range(-30, 30):
assert hash(x) == hash(complex(x, 0))
@@ -417,7 +419,9 @@
pass
assert j(100 + 0j) == 100 + 0j
assert isinstance(j(100), j)
- assert j("100 + 0j") == 100 + 0j
+ assert j("100+0j") == 100 + 0j
+ exc = raises(ValueError, j, "100 + 0j")
+ assert str(exc.value) == "complex() arg is a malformed string"
x = j(1+0j)
x.foo = 42
assert x.foo == 42
diff --git a/pypy/tool/release/force-builds.py b/pypy/tool/release/force-builds.py
--- a/pypy/tool/release/force-builds.py
+++ b/pypy/tool/release/force-builds.py
@@ -20,11 +20,12 @@
'own-linux-x86-32',
'own-linux-x86-64',
'own-linux-armhf',
+ 'own-win-x86-32',
# 'own-macosx-x86-32',
# 'pypy-c-app-level-linux-x86-32',
# 'pypy-c-app-level-linux-x86-64',
# 'pypy-c-stackless-app-level-linux-x86-32',
- 'pypy-c-app-level-win-x86-32',
+# 'pypy-c-app-level-win-x86-32',
'pypy-c-jit-linux-x86-32',
'pypy-c-jit-linux-x86-64',
'pypy-c-jit-macosx-x86-64',
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -74,6 +74,7 @@
if not sys.platform == 'win32':
subprocess.check_call([str(pypy_c), '-c', 'import _curses'])
subprocess.check_call([str(pypy_c), '-c', 'import syslog'])
+ subprocess.check_call([str(pypy_c), '-c', 'import gdbm'])
if not withouttk:
try:
subprocess.check_call([str(pypy_c), '-c', 'import _tkinter'])
diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py
--- a/rpython/annotator/annrpython.py
+++ b/rpython/annotator/annrpython.py
@@ -397,16 +397,15 @@
return repr(graph) + blk + opid
def flowin(self, graph, block):
- #print 'Flowing', block, [self.binding(a) for a in block.inputargs]
try:
- for i in range(len(block.operations)):
+ for i, op in enumerate(block.operations):
+ self.bookkeeper.enter((graph, block, i))
try:
- self.bookkeeper.enter((graph, block, i))
- self.consider_op(block, i)
+ self.consider_op(op)
finally:
self.bookkeeper.leave()
- except BlockedInference, e:
+ except BlockedInference as e:
if (e.op is block.operations[-1] and
block.exitswitch == c_last_exception):
# this is the case where the last operation of the block will
@@ -428,11 +427,16 @@
# other cases are problematic (but will hopefully be solved
# later by reflowing). Throw the BlockedInference up to
# processblock().
+ e.opindex = i
raise
except annmodel.HarmlesslyBlocked:
return
+ except annmodel.AnnotatorError as e: # note that UnionError is a subclass
+ e.source = gather_error(self, graph, block, i)
+ raise
+
else:
# dead code removal: don't follow all exits if the exitswitch
# is known
@@ -443,11 +447,6 @@
exits = [link for link in exits
if link.exitcase == s_exitswitch.const]
- # mapping (exitcase, variable) -> s_annotation
- # that can be attached to booleans, exitswitches
- knowntypedata = getattr(self.bindings.get(block.exitswitch),
- "knowntypedata", {})
-
# filter out those exceptions which cannot
# occour for this specific, typed operation.
if block.exitswitch == c_last_exception:
@@ -480,93 +479,12 @@
exits.append(link)
candidates = [c for c in candidates if c not in covered]
+ # mapping (exitcase, variable) -> s_annotation
+ # that can be attached to booleans, exitswitches
+ knowntypedata = getattr(self.bindings.get(block.exitswitch),
+ "knowntypedata", {})
for link in exits:
- in_except_block = False
-
- last_exception_var = link.last_exception # may be None for non-exception link
- last_exc_value_var = link.last_exc_value # may be None for non-exception link
-
- if isinstance(link.exitcase, (types.ClassType, type)) \
- and issubclass(link.exitcase, py.builtin.BaseException):
- assert last_exception_var and last_exc_value_var
- last_exc_value_object = self.bookkeeper.valueoftype(link.exitcase)
- last_exception_object = annmodel.SomeType()
- if isinstance(last_exception_var, Constant):
- last_exception_object.const = last_exception_var.value
- last_exception_object.is_type_of = [last_exc_value_var]
-
- if isinstance(last_exception_var, Variable):
- self.setbinding(last_exception_var, last_exception_object)
- if isinstance(last_exc_value_var, Variable):
- self.setbinding(last_exc_value_var, last_exc_value_object)
-
- last_exception_object = annmodel.SomeType()
- if isinstance(last_exception_var, Constant):
- last_exception_object.const = last_exception_var.value
- #if link.exitcase is Exception:
- # last_exc_value_object = annmodel.SomeObject()
- #else:
- last_exc_value_vars = []
- in_except_block = True
-
- ignore_link = False
- cells = []
- renaming = {}
- for a,v in zip(link.args,link.target.inputargs):
- renaming.setdefault(a, []).append(v)
- for a,v in zip(link.args,link.target.inputargs):
- if a == last_exception_var:
- assert in_except_block
- cells.append(last_exception_object)
- elif a == last_exc_value_var:
- assert in_except_block
- cells.append(last_exc_value_object)
- last_exc_value_vars.append(v)
- else:
- cell = self.binding(a)
- if (link.exitcase, a) in knowntypedata:
- knownvarvalue = knowntypedata[(link.exitcase, a)]
- cell = pair(cell, knownvarvalue).improve()
- # ignore links that try to pass impossible values
- if cell == annmodel.s_ImpossibleValue:
- ignore_link = True
-
- if hasattr(cell,'is_type_of'):
- renamed_is_type_of = []
- for v in cell.is_type_of:
- new_vs = renaming.get(v,[])
- renamed_is_type_of += new_vs
- assert cell.knowntype is type
- newcell = annmodel.SomeType()
- if cell.is_constant():
- newcell.const = cell.const
- cell = newcell
- cell.is_type_of = renamed_is_type_of
-
- if hasattr(cell, 'knowntypedata'):
- renamed_knowntypedata = {}
- for (value, v), s in cell.knowntypedata.items():
- new_vs = renaming.get(v, [])
- for new_v in new_vs:
- renamed_knowntypedata[value, new_v] = s
- assert isinstance(cell, annmodel.SomeBool)
- newcell = annmodel.SomeBool()
- if cell.is_constant():
- newcell.const = cell.const
- cell = newcell
- cell.set_knowntypedata(renamed_knowntypedata)
-
- cells.append(cell)
-
- if ignore_link:
- continue
-
- if in_except_block:
- last_exception_object.is_type_of = last_exc_value_vars
-
- self.links_followed[link] = True
- self.addpendingblock(graph, link.target, cells)
-
+ self.follow_link(graph, link, knowntypedata)
if block in self.notify:
# reflow from certain positions when this block is done
for callback in self.notify[block]:
@@ -575,39 +493,114 @@
else:
callback()
+ def follow_link(self, graph, link, knowntypedata):
+ in_except_block = False
+ last_exception_var = link.last_exception # may be None for non-exception link
+ last_exc_value_var = link.last_exc_value # may be None for non-exception link
+
+ if isinstance(link.exitcase, (types.ClassType, type)) \
+ and issubclass(link.exitcase, py.builtin.BaseException):
+ assert last_exception_var and last_exc_value_var
+ last_exc_value_object = self.bookkeeper.valueoftype(link.exitcase)
+ last_exception_object = annmodel.SomeType()
+ if isinstance(last_exception_var, Constant):
+ last_exception_object.const = last_exception_var.value
+ last_exception_object.is_type_of = [last_exc_value_var]
+
+ if isinstance(last_exception_var, Variable):
+ self.setbinding(last_exception_var, last_exception_object)
+ if isinstance(last_exc_value_var, Variable):
+ self.setbinding(last_exc_value_var, last_exc_value_object)
+
+ last_exception_object = annmodel.SomeType()
+ if isinstance(last_exception_var, Constant):
+ last_exception_object.const = last_exception_var.value
+ #if link.exitcase is Exception:
+ # last_exc_value_object = annmodel.SomeObject()
+ #else:
+ last_exc_value_vars = []
+ in_except_block = True
+
+ ignore_link = False
+ cells = []
+ renaming = {}
+ for a, v in zip(link.args, link.target.inputargs):
+ renaming.setdefault(a, []).append(v)
+ for a, v in zip(link.args, link.target.inputargs):
+ if a == last_exception_var:
+ assert in_except_block
+ cells.append(last_exception_object)
+ elif a == last_exc_value_var:
+ assert in_except_block
+ cells.append(last_exc_value_object)
+ last_exc_value_vars.append(v)
+ else:
+ cell = self.binding(a)
+ if (link.exitcase, a) in knowntypedata:
+ knownvarvalue = knowntypedata[(link.exitcase, a)]
+ cell = pair(cell, knownvarvalue).improve()
+ # ignore links that try to pass impossible values
+ if cell == annmodel.s_ImpossibleValue:
+ ignore_link = True
+
+ if hasattr(cell,'is_type_of'):
+ renamed_is_type_of = []
More information about the pypy-commit
mailing list