From noreply at buildbot.pypy.org Sun Feb 1 00:27:14 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 1 Feb 2015 00:27:14 +0100 (CET) Subject: [pypy-commit] stmgc hashtable-iter: maybe found a simpler and better way Message-ID: <20150131232714.969421C02B3@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: hashtable-iter Changeset: r1599:274a3dfec8ad Date: 2015-02-01 00:27 +0100 http://bitbucket.org/pypy/stmgc/changeset/274a3dfec8ad/ Log: maybe found a simpler and better way diff --git a/c7/stm/hashtable.c b/c7/stm/hashtable.c --- a/c7/stm/hashtable.c +++ b/c7/stm/hashtable.c @@ -25,6 +25,10 @@ transaction rather than the modifying transaction, which is probably what we want. +XXX NO: when creating new key/value objects, we should copy the read +marker from the hashtableobj to the new key/value object. I *think* +this gives a correct and better result XXX + Implementation -------------- @@ -231,7 +235,16 @@ perturb >>= PERTURB_SHIFT; } } - /* here, we didn't find the 'entry' with the correct index. */ + /* here, we didn't find the 'entry' with the correct index. Note + that even if the same 'table' is modified or resized by other + threads concurrently, any new item found from a race condition + would anyway contain NULL in the present segment (ensured by + the first write_fence() below). If the 'table' grows an entry + just after we checked above, then we go ahead and lock the + table; but after we get the lock, we will notice the new entry + (ensured by the second write_fence() below) and restart the + whole process. + */ intptr_t rc = VOLATILE_TABLE(table)->resize_counter; bool rc_must_be_negative = false; @@ -347,9 +360,25 @@ */ int j, my_segment = STM_SEGMENT->segment_num; for (j = 1; j <= NB_SEGMENTS; j++) { - if (j != my_segment) { - if (was_read_remote(get_segment_base(j), hashtableobj)) { - xxxxxxxxxxxx conflict xxxxxxxxxxx; + if (j == my_segment) + continue; + if (was_read_remote(get_segment_base(j), hashtableobj)) { + /* conflict! */ + table->resize_counter = rc; /* unlock */ + if (write_read_contention_management(j, hashtableobj)) { + /* If we reach this point, we didn't abort, but we + had to wait for the other thread to commit. If + we did, then we have to restart. */ + return true; + ...; + + } + /* we aborted the other transaction without waiting, so + we can just break out of this loop on + modified_old_objects and continue with the next + segment */ + + xxx; } } } @@ -458,14 +487,15 @@ static void _stm_compact_hashtable(stm_hashtable_t *hashtable) { stm_hashtable_table_t *table = hashtable->table; - assert(!IS_EVEN(table->resize_counter)); + intptr_t rc = table->resize_counter; + assert(!IS_EVEN(rc)); if ((hashtable->additions >> 8) * 4 > table->mask) { int segment_num = (hashtable->additions & 0xFF); if (!segment_num) segment_num = 1; hashtable->additions = segment_num; uintptr_t initial_rc = (table->mask + 1) * 4 + 1; - uintptr_t num_entries_times_6 = initial_rc - table->resize_counter; + uintptr_t num_entries_times_6 = initial_rc - (rc < 0 ? -rc : rc); uintptr_t count = INITIAL_HASHTABLE_SIZE; while (count * 4 < num_entries_times_6) count *= 2; @@ -475,7 +505,7 @@ dprintf(("compact with %ld items:\n", num_entries_times_6 / 6)); _stm_rehash_hashtable(hashtable, count, /*remove_unread=*/segment_num, - /*rc_must_be_negative=*/false); + /*rc_must_be_negative=*/rc < 0); } table = hashtable->table; From noreply at buildbot.pypy.org Sun Feb 1 01:28:54 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 1 Feb 2015 01:28:54 +0100 (CET) Subject: [pypy-commit] stmgc hashtable-iter: in-progress, getting there hopefully Message-ID: <20150201002854.42BA41C0141@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: hashtable-iter Changeset: r1600:06c31e9c06f8 Date: 2015-02-01 01:29 +0100 http://bitbucket.org/pypy/stmgc/changeset/06c31e9c06f8/ Log: in-progress, getting there hopefully diff --git a/c7/stm/core.h b/c7/stm/core.h --- a/c7/stm/core.h +++ b/c7/stm/core.h @@ -281,10 +281,9 @@ static stm_thread_local_t *abort_with_mutex_no_longjmp(void); static void abort_data_structures_from_segment_num(int segment_num); -static inline bool was_read_local(object_t *obj) -{ - return ((stm_read_marker_t *)(((uintptr_t)obj) >> 4))->rm == - STM_SEGMENT->transaction_read_version; +static inline struct stm_read_marker_s *get_read_marker(char *base, + object_t *obj) { + return (struct stm_read_marker_s *)(base + (((uintptr_t)obj) >> 4)); } static inline bool was_read_remote(char *base, object_t *obj) @@ -292,8 +291,7 @@ uint8_t other_transaction_read_version = ((struct stm_segment_info_s *)REAL_ADDRESS(base, STM_PSEGMENT)) ->transaction_read_version; - uint8_t rm = ((struct stm_read_marker_s *) - (base + (((uintptr_t)obj) >> 4)))->rm; + uint8_t rm = get_read_marker(base, obj)->rm; assert(rm <= other_transaction_read_version); return rm == other_transaction_read_version; } diff --git a/c7/stm/hashtable.c b/c7/stm/hashtable.c --- a/c7/stm/hashtable.c +++ b/c7/stm/hashtable.c @@ -12,22 +12,16 @@ collision). The main operations on a hashtable are reading or writing an object at a -given index. It also supports iterating its non-NULL entries. +given index. It also supports fetching the list of non-NULL entries. There are two markers for every index (a read and a write marker). This is unlike regular arrays, which have only two markers in total. Additionally, we use the read marker for the hashtable object itself -to mean "we are iterating". When a transaction has got this "global" -read marker and another transaction attempts to create a new key/value -pair via stm_hashtable_{lookup,read,write}, the call immediately fails -with a read/write conflict. This gives priority to the iterating -transaction rather than the modifying transaction, which is probably -what we want. - -XXX NO: when creating new key/value objects, we should copy the read -marker from the hashtableobj to the new key/value object. I *think* -this gives a correct and better result XXX +to mean "we have read the complete list of keys". This plays the role +of a "global" read marker: when any thread adds a new key/value object +to the hashtable, this new object's read marker is initialized with a +copy of the "global" read marker --- in all segments. Implementation @@ -66,12 +60,8 @@ The field 'resize_counter' also works as a write lock: changes go via the intermediate value RESIZING_LOCK (0). - - In addition, 'resize_counter' can be the negation of the odd - number that it would normally be: in this case it is "probably - write-protected" (see stm_hashtable_next()). */ - intptr_t resize_counter; + uintptr_t resize_counter; stm_hashtable_entry_t *items[INITIAL_HASHTABLE_SIZE]; } stm_hashtable_table_t; @@ -104,7 +94,7 @@ void stm_hashtable_free(stm_hashtable_t *hashtable) { - intptr_t rc = hashtable->initial_table.resize_counter; + uintptr_t rc = hashtable->initial_table.resize_counter; free(hashtable); while (IS_EVEN(rc)) { assert(rc != RESIZING_LOCK); @@ -119,9 +109,7 @@ { long i; for (i = 1; i <= NB_SEGMENTS; i++) { - char *remote_base = get_segment_base(i); - uint8_t remote_version = get_segment(i)->transaction_read_version; - if (was_read_remote(remote_base, obj, remote_version)) + if (was_read_remote(get_segment_base(i), obj)) return true; } return false; @@ -155,8 +143,7 @@ static void _stm_rehash_hashtable(stm_hashtable_t *hashtable, uintptr_t biggercount, - int remove_unread_from_seg, - bool rc_must_be_negative) + int remove_unread_from_seg) { dprintf(("rehash %p to %ld, remove_unread_from_seg=%d\n", hashtable, biggercount, remove_unread_from_seg)); @@ -167,7 +154,7 @@ assert(biggertable); // XXX stm_hashtable_table_t *table = hashtable->table; - table->resize_counter = (intptr_t)biggertable; + table->resize_counter = (uintptr_t)biggertable; /* ^^^ this unlocks the table by writing a non-zero value to table->resize_counter, but the new value is a pointer to the new bigger table, so IS_EVEN() is still true */ @@ -176,7 +163,7 @@ init_table(biggertable, biggercount); uintptr_t j, mask = table->mask; - intptr_t rc = biggertable->resize_counter; + uintptr_t rc = biggertable->resize_counter; char *segment_base = get_segment_base(remove_unread_from_seg); for (j = 0; j <= mask; j++) { stm_hashtable_entry_t *entry = table->items[j]; @@ -191,10 +178,10 @@ } } _insert_clean(biggertable, entry); + assert(rc > 6); rc -= 6; } - assert(rc > 0); - biggertable->resize_counter = rc_must_be_negative ? -rc : rc; + biggertable->resize_counter = rc; write_fence(); /* make sure that 'biggertable' is valid here, and make sure 'table->resize_counter' is updated @@ -246,8 +233,7 @@ whole process. */ - intptr_t rc = VOLATILE_TABLE(table)->resize_counter; - bool rc_must_be_negative = false; + uintptr_t rc = VOLATILE_TABLE(table)->resize_counter; /* if rc is RESIZING_LOCK (which is 0, so even), a concurrent thread is writing to the hashtable. Or, if rc is another even number, it is @@ -279,7 +265,6 @@ /* if rc is greater than 6, there is enough room for a new item in the current table. */ - retry_adding: if (rc > 6) { /* we can only enter here once! If we allocate stuff, we may run the GC, and so 'hashtableobj' might move afterwards. */ @@ -312,12 +297,22 @@ synchronization with other pieces of the code that may change. */ + + /* First fetch the read marker of 'hashtableobj' in all + segments, before allocate_outside_nursery_large() + which might trigger a GC */ + long j; + uint8_t readmarkers[NB_SEGMENTS]; + for (j = 1; j <= NB_SEGMENTS; j++) { + readmarkers[j - 1] = get_read_marker(get_segment_base(j), + hashtableobj)->rm; + } + acquire_privatization_lock(); char *p = allocate_outside_nursery_large( sizeof(stm_hashtable_entry_t)); entry = (stm_hashtable_entry_t *)(p - stm_object_pages); - long j; for (j = 0; j <= NB_SEGMENTS; j++) { struct stm_hashtable_entry_s *e; e = (struct stm_hashtable_entry_s *) @@ -329,16 +324,19 @@ } hashtable->additions += 0x100; release_privatization_lock(); + + for (j = 1; j <= NB_SEGMENTS; j++) { + get_read_marker(get_segment_base(j), (object_t *)entry)->rm = + readmarkers[j - 1]; + } } write_fence(); /* make sure 'entry' is fully initialized here */ table->items[i] = entry; write_fence(); /* make sure 'table->items' is written here */ - rc -= 6; - VOLATILE_TABLE(table)->resize_counter = ( - rc_must_be_negative ? -rc : rc); /* unlock */ + VOLATILE_TABLE(table)->resize_counter = rc - 6; /* unlock */ return entry; } - else if (rc > 0) { + else { /* if rc is smaller than 6, we must allocate a new bigger table. */ uintptr_t biggercount = table->mask + 1; @@ -346,52 +344,9 @@ biggercount *= 4; else biggercount *= 2; - _stm_rehash_hashtable(hashtable, biggercount, /*remove_unread=*/0, - rc_must_be_negative); + _stm_rehash_hashtable(hashtable, biggercount, /*remove_unread=*/0); goto restart; } - else { - assert(rc < 0); - assert(!_is_in_nursery(hashtableobj)); - - /* if rc is negative, the hashtable is probably write-protected. - Check if the read marker of the 'hashtableobj' is set in - another segment. - */ - int j, my_segment = STM_SEGMENT->segment_num; - for (j = 1; j <= NB_SEGMENTS; j++) { - if (j == my_segment) - continue; - if (was_read_remote(get_segment_base(j), hashtableobj)) { - /* conflict! */ - table->resize_counter = rc; /* unlock */ - if (write_read_contention_management(j, hashtableobj)) { - /* If we reach this point, we didn't abort, but we - had to wait for the other thread to commit. If - we did, then we have to restart. */ - return true; - ...; - - } - /* we aborted the other transaction without waiting, so - we can just break out of this loop on - modified_old_objects and continue with the next - segment */ - - xxx; - } - } - } - - /* if even in this thread's segment it was not read, then there - is no point in keeping it write-protected. So we set - 'rc_must_be_negative', i.e. keep it write-protected, iff - it was read locally. - */ - rc_must_be_negative = was_read_local(hashtableobj); - rc = -rc; - goto retry_adding; - } } object_t *stm_hashtable_read(object_t *hobj, stm_hashtable_t *hashtable, @@ -413,81 +368,86 @@ e->object = nvalue; } -struct stm_hashtable_entry_s * -stm_hashtable_next(object_t *hobj, stm_hashtable_t *hashtable, - uintptr_t *pposition, stm_thread_local_t *tl) +long stm_hashtable_length_upper_bound(stm_hashtable_t *hashtable) { - /* this assumes the simple c7 model whereby commits only occur with - all other transaction paused at a known point. */ + stm_hashtable_table_t *table; + uintptr_t rc; + + restart: + table = VOLATILE_HASHTABLE(hashtable)->table; + rc = VOLATILE_TABLE(table)->resize_counter; + if (IS_EVEN(rc)) { + spin_loop(); + goto restart; + } + + uintptr_t initial_rc = (table->mask + 1) * 4 + 1; + uintptr_t num_entries_times_6 = initial_rc - rc; + return num_entries_times_6 / 6; +} + +long stm_hashtable_list(object_t *hobj, stm_hashtable_t *hashtable, + stm_hashtable_entry_t **results) +{ stm_hashtable_table_t *table; intptr_t rc; - /* First set the read marker. It will be left as long as we're running - the same transaction. Note that this code assumes that nothing else - can set the read marker! Also, if 'hobj' is still in the nursery, - it was created by this transaction and there is nothing to do. + /* Set the read marker. It will be left as long as we're running + the same transaction. */ - if (!_is_in_nursery(hobj) && !was_read_local(hobj)) { + stm_read(hobj); - stm_read(hobj); + /* Acquire and immediately release the lock. We don't actually + need to do anything while we hold the lock, but the point is to + wait until the lock is available, and to synchronize other + threads with the stm_read() done above. + */ + restart: + table = VOLATILE_HASHTABLE(hashtable)->table; + rc = VOLATILE_TABLE(table)->resize_counter; + if (IS_EVEN(rc)) { + spin_loop(); + goto restart; + } + if (!__sync_bool_compare_and_swap(&table->resize_counter, rc, rc)) + goto restart; - /* Change the 'resize_counter' field to its negative value. This - must be done after we've set the read marker. */ - restart: - table = VOLATILE_HASHTABLE(hashtable)->table; - rc = VOLATILE_TABLE(table)->resize_counter; - if (IS_EVEN(rc)) { - spin_loop(); - goto restart; + /* Read all entries, check which ones are not NULL, count them, + and optionally list them in 'results'. + */ + uintptr_t i, mask = table->mask; + stm_hashtable_entry_t *entry; + long nresult = 0; + + if (results != NULL) { + /* collect the results in the provided list */ + for (i = 0; i <= mask; i++) { + entry = VOLATILE_TABLE(table)->items[i]; + if (entry != NULL) { + stm_read((object_t *)entry); + if (entry->object != NULL) + results[nresult++] = entry; + } } - if (!__sync_bool_compare_and_swap(&table->resize_counter, rc, - rc > 0 ? -rc : rc)) - goto restart; - /* Note that we did a compare-and-swap even if rc was already - negative. This is needed for its memory-ordering effect, - to ensure that from now on the other threads do see our - read marker set. */ } else { - /* Read marker already set. Assume (and assert) that we - already set a negative value into 'resize_counter'. - Changes of 'table' or 'resize_counter' under our feet - should not be possible here. - */ - table = hashtable->table; - - if (!_is_in_nursery(hobj)) { - assert(!IS_EVEN(table->resize_counter) && - table->resize_counter < 0); + /* don't collect, just get the exact number of results */ + for (i = 0; i <= mask; i++) { + entry = VOLATILE_TABLE(table)->items[i]; + if (entry != NULL) { + stm_read((object_t *)entry); + if (entry->object != NULL) + nresult++; + } } } - - /* At this point, the hashtable is write-protected: no other - thread may add new key/value objects nor grow/replace the - 'table'. The hashtable will remain write-protected as long as - this transaction is running. Note that *this* thread is - allowed to continue modifying the hashtable (unless another - thread did also set a write protection). - */ - uintptr_t position = *pposition; - uintptr_t mask = table->mask; - stm_hashtable_entry_t *entry; - - while (position <= mask) { - entry = table->items[position++]; - if (entry != NULL) { - *pposition = position; - return entry; - } - } - *pposition = (uintptr_t)-1; - return NULL; + return nresult; } static void _stm_compact_hashtable(stm_hashtable_t *hashtable) { stm_hashtable_table_t *table = hashtable->table; - intptr_t rc = table->resize_counter; + uintptr_t rc = table->resize_counter; assert(!IS_EVEN(rc)); if ((hashtable->additions >> 8) * 4 > table->mask) { @@ -495,7 +455,7 @@ if (!segment_num) segment_num = 1; hashtable->additions = segment_num; uintptr_t initial_rc = (table->mask + 1) * 4 + 1; - uintptr_t num_entries_times_6 = initial_rc - (rc < 0 ? -rc : rc); + uintptr_t num_entries_times_6 = initial_rc - rc; uintptr_t count = INITIAL_HASHTABLE_SIZE; while (count * 4 < num_entries_times_6) count *= 2; @@ -504,15 +464,14 @@ assert(count <= table->mask + 1); dprintf(("compact with %ld items:\n", num_entries_times_6 / 6)); - _stm_rehash_hashtable(hashtable, count, /*remove_unread=*/segment_num, - /*rc_must_be_negative=*/rc < 0); + _stm_rehash_hashtable(hashtable, count, /*remove_unread=*/segment_num); } table = hashtable->table; assert(!IS_EVEN(table->resize_counter)); if (table != &hashtable->initial_table) { - intptr_t rc = hashtable->initial_table.resize_counter; + uintptr_t rc = hashtable->initial_table.resize_counter; while (1) { assert(IS_EVEN(rc)); assert(rc != RESIZING_LOCK); @@ -523,7 +482,7 @@ rc = old_table->resize_counter; free(old_table); } - hashtable->initial_table.resize_counter = (intptr_t)table; + hashtable->initial_table.resize_counter = (uintptr_t)table; assert(IS_EVEN(hashtable->initial_table.resize_counter)); } } diff --git a/c7/stm/misc.c b/c7/stm/misc.c --- a/c7/stm/misc.c +++ b/c7/stm/misc.c @@ -31,7 +31,7 @@ bool _stm_was_read(object_t *obj) { - return was_read_local(obj); + return was_read_remote(STM_SEGMENT->segment_base, obj); } bool _stm_was_written(object_t *obj) diff --git a/c7/stmgc.h b/c7/stmgc.h --- a/c7/stmgc.h +++ b/c7/stmgc.h @@ -545,8 +545,9 @@ object_t *stm_hashtable_read(object_t *, stm_hashtable_t *, uintptr_t key); void stm_hashtable_write(object_t *, stm_hashtable_t *, uintptr_t key, object_t *nvalue, stm_thread_local_t *); -struct stm_hashtable_entry_s *stm_hashtable_next( - object_t *, stm_hashtable_t *, uintptr_t *pposition, stm_thread_local_t *); +long stm_hashtable_length_upper_bound(stm_hashtable_t *); +long stm_hashtable_list(object_t *, stm_hashtable_t *, + stm_hashtable_entry_t **results); extern uint32_t stm_hashtable_entry_userdata; void stm_hashtable_tracefn(stm_hashtable_t *, void (object_t **)); From noreply at buildbot.pypy.org Sun Feb 1 10:19:46 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 1 Feb 2015 10:19:46 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Do a single lookup in all methods of hashtable, stmset and stmdict. Message-ID: <20150201091946.F15921C039A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r75617:982b5fc67ac4 Date: 2015-02-01 10:19 +0100 http://bitbucket.org/pypy/pypy/changeset/982b5fc67ac4/ Log: Do a single lookup in all methods of hashtable, stmset and stmdict. diff --git a/pypy/module/pypystm/hashtable.py b/pypy/module/pypystm/hashtable.py --- a/pypy/module/pypystm/hashtable.py +++ b/pypy/module/pypystm/hashtable.py @@ -25,15 +25,15 @@ @unwrap_spec(key=int) def setitem_w(self, key, w_value): - gcref = cast_instance_to_gcref(w_value) - self.h.set(key, gcref) + entry = self.h.lookup(key) + entry.object = cast_instance_to_gcref(w_value) @unwrap_spec(key=int) def delitem_w(self, space, key): - gcref = self.h.get(key) - if not gcref: + entry = self.h.lookup(key) + if not entry.object: space.raise_key_error(space.wrap(key)) - self.h.set(key, rstm.NULL_GCREF) + entry.object = rstm.NULL_GCREF @unwrap_spec(key=int) def contains_w(self, space, key): @@ -49,10 +49,10 @@ @unwrap_spec(key=int, w_default=WrappedDefault(None)) def setdefault_w(self, space, key, w_default): - gcref = self.h.get(key) + entry = self.h.lookup(key) + gcref = entry.object if not gcref: - gcref = cast_instance_to_gcref(w_default) - self.h.set(key, gcref) + entry.object = cast_instance_to_gcref(w_default) return w_default return cast_gcref_to_instance(W_Root, gcref) diff --git a/pypy/module/pypystm/stmdict.py b/pypy/module/pypystm/stmdict.py --- a/pypy/module/pypystm/stmdict.py +++ b/pypy/module/pypystm/stmdict.py @@ -61,8 +61,8 @@ def setitem_w(self, space, w_key, w_value): hkey = space.hash_w(w_key) - gcref = self.h.get(hkey) - array = lltype.cast_opaque_ptr(PARRAY, gcref) + entry = self.h.lookup(hkey) + array = lltype.cast_opaque_ptr(PARRAY, entry.object) if array: i = find_equal_item(space, array, w_key) if i >= 0: @@ -77,13 +77,12 @@ L = 0 narray[L] = cast_instance_to_gcref(w_key) narray[L + 1] = cast_instance_to_gcref(w_value) - gcref = lltype.cast_opaque_ptr(llmemory.GCREF, narray) - self.h.set(hkey, gcref) + entry.object = lltype.cast_opaque_ptr(llmemory.GCREF, narray) def delitem_w(self, space, w_key): hkey = space.hash_w(w_key) - gcref = self.h.get(hkey) - array = lltype.cast_opaque_ptr(PARRAY, gcref) + entry = self.h.lookup(hkey) + array = lltype.cast_opaque_ptr(PARRAY, entry.object) if array: i = find_equal_item(space, array, w_key) if i >= 0: @@ -95,8 +94,7 @@ narray = lltype.malloc(ARRAY, L) ll_arraycopy(array, narray, 0, 0, i) ll_arraycopy(array, narray, i + 2, i, L - i) - gcref = lltype.cast_opaque_ptr(llmemory.GCREF, narray) - self.h.set(hkey, gcref) + entry.object = lltype.cast_opaque_ptr(llmemory.GCREF, narray) return space.raise_key_error(w_key) @@ -122,8 +120,8 @@ @unwrap_spec(w_default=WrappedDefault(None)) def setdefault_w(self, space, w_key, w_default): hkey = space.hash_w(w_key) - gcref = self.h.get(hkey) - array = lltype.cast_opaque_ptr(PARRAY, gcref) + entry = self.h.lookup(hkey) + array = lltype.cast_opaque_ptr(PARRAY, entry.object) if array: i = find_equal_item(space, array, w_key) if i >= 0: @@ -137,8 +135,7 @@ L = 0 narray[L] = cast_instance_to_gcref(w_key) narray[L + 1] = cast_instance_to_gcref(w_default) - gcref = lltype.cast_opaque_ptr(llmemory.GCREF, narray) - self.h.set(hkey, gcref) + entry.object = lltype.cast_opaque_ptr(llmemory.GCREF, narray) return w_default diff --git a/pypy/module/pypystm/stmset.py b/pypy/module/pypystm/stmset.py --- a/pypy/module/pypystm/stmset.py +++ b/pypy/module/pypystm/stmset.py @@ -53,8 +53,8 @@ def add_w(self, space, w_key): hkey = space.hash_w(w_key) - gcref = self.h.get(hkey) - array = lltype.cast_opaque_ptr(PARRAY, gcref) + entry = self.h.lookup(hkey) + array = lltype.cast_opaque_ptr(PARRAY, entry.object) if array: if find_equal_item(space, array, w_key) >= 0: return # already there @@ -65,13 +65,12 @@ narray = lltype.malloc(ARRAY, 1) L = 0 narray[L] = cast_instance_to_gcref(w_key) - gcref = lltype.cast_opaque_ptr(llmemory.GCREF, narray) - self.h.set(hkey, gcref) + entry.object = lltype.cast_opaque_ptr(llmemory.GCREF, narray) def try_remove(self, space, w_key): hkey = space.hash_w(w_key) - gcref = self.h.get(hkey) - array = lltype.cast_opaque_ptr(PARRAY, gcref) + entry = self.h.lookup(hkey) + array = lltype.cast_opaque_ptr(PARRAY, entry.object) if not array: return False i = find_equal_item(space, array, w_key) @@ -85,8 +84,7 @@ narray = lltype.malloc(ARRAY, L) ll_arraycopy(array, narray, 0, 0, i) ll_arraycopy(array, narray, i + 1, i, L - i) - gcref = lltype.cast_opaque_ptr(llmemory.GCREF, narray) - self.h.set(hkey, gcref) + entry.object = lltype.cast_opaque_ptr(llmemory.GCREF, narray) return True def remove_w(self, space, w_key): diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py --- a/rpython/rlib/rstm.py +++ b/rpython/rlib/rstm.py @@ -181,6 +181,7 @@ _STM_HASHTABLE_ENTRY = lltype.GcStruct('HASHTABLE_ENTRY', ('index', lltype.Unsigned), ('object', llmemory.GCREF)) +_STM_HASHTABLE_ENTRY_P = lltype.Ptr(_STM_HASHTABLE_ENTRY) @dont_look_inside def _ll_hashtable_get(h, key): @@ -191,11 +192,17 @@ def _ll_hashtable_set(h, key, value): llop.stm_hashtable_write(lltype.Void, h, h.ll_raw_hashtable, key, value) + at dont_look_inside +def _ll_hashtable_lookup(h, key): + return llop.stm_hashtable_lookup(_STM_HASHTABLE_ENTRY_P, + h, h.ll_raw_hashtable, key) + _HASHTABLE_OBJ = lltype.GcStruct('HASHTABLE_OBJ', ('ll_raw_hashtable', _STM_HASHTABLE_P), rtti=True, adtmeths={'get': _ll_hashtable_get, - 'set': _ll_hashtable_set}) + 'set': _ll_hashtable_set, + 'lookup': _ll_hashtable_lookup}) NULL_HASHTABLE = lltype.nullptr(_HASHTABLE_OBJ) def _ll_hashtable_trace(gc, obj, callback, arg): @@ -254,3 +261,19 @@ del self._content[key] except KeyError: pass + + def lookup(self, key): + assert type(key) is int + return EntryObjectForTest(self, key) + +class EntryObjectForTest(object): + def __init__(self, hashtable, key): + self.hashtable = hashtable + self.key = key + + def _getobj(self): + return self.hashtable.get(self.key) + def _setobj(self, nvalue): + self.hashtable.set(self.key, nvalue) + + object = property(_getobj, _setobj) diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -464,6 +464,7 @@ 'stm_hashtable_free': LLOp(), 'stm_hashtable_read': LLOp(), 'stm_hashtable_write': LLOp(), + 'stm_hashtable_lookup': LLOp(), 'stm_hashtable_tracefn': LLOp(), # __________ address operations __________ diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py --- a/rpython/translator/stm/funcgen.py +++ b/rpython/translator/stm/funcgen.py @@ -312,6 +312,14 @@ return ('stm_hashtable_write((object_t *)%s, %s, %s, (object_t *)%s, ' '&stm_thread_local);' % (arg0, arg1, arg2, arg3)) +def stm_hashtable_lookup(funcgen, op): + arg0 = funcgen.expr(op.args[0]) + arg1 = funcgen.expr(op.args[1]) + arg2 = funcgen.expr(op.args[2]) + result = funcgen.expr(op.result) + return '%s = stm_hashtable_lookup((object_t *)%s, %s, %s);' % ( + result, arg0, arg1, arg2) + def stm_hashtable_tracefn(funcgen, op): arg0 = funcgen.expr(op.args[0]) arg1 = funcgen.expr(op.args[1]) From noreply at buildbot.pypy.org Sun Feb 1 10:50:20 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 1 Feb 2015 10:50:20 +0100 (CET) Subject: [pypy-commit] stmgc hashtable-iter: Adding tests for stm_hashtable_list Message-ID: <20150201095020.C4CCF1C039A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: hashtable-iter Changeset: r1601:5e3761dd15b6 Date: 2015-02-01 10:50 +0100 http://bitbucket.org/pypy/stmgc/changeset/5e3761dd15b6/ Log: Adding tests for stm_hashtable_list diff --git a/c7/test/support.py b/c7/test/support.py --- a/c7/test/support.py +++ b/c7/test/support.py @@ -167,17 +167,23 @@ void (*stmcb_finalizer)(object_t *); typedef struct stm_hashtable_s stm_hashtable_t; +typedef ... stm_hashtable_entry_t; stm_hashtable_t *stm_hashtable_create(void); void stm_hashtable_free(stm_hashtable_t *); bool _check_hashtable_read(object_t *, stm_hashtable_t *, uintptr_t key); object_t *hashtable_read_result; bool _check_hashtable_write(object_t *, stm_hashtable_t *, uintptr_t key, object_t *nvalue, stm_thread_local_t *tl); +long stm_hashtable_length_upper_bound(stm_hashtable_t *); +long stm_hashtable_list(object_t *, stm_hashtable_t *, + stm_hashtable_entry_t **results); uint32_t stm_hashtable_entry_userdata; void stm_hashtable_tracefn(stm_hashtable_t *, void (object_t **)); void _set_hashtable(object_t *obj, stm_hashtable_t *h); stm_hashtable_t *_get_hashtable(object_t *obj); +uintptr_t _get_entry_index(stm_hashtable_entry_t *entry); +object_t *_get_entry_object(stm_hashtable_entry_t *entry); """) @@ -308,6 +314,18 @@ return *(stm_hashtable_t *TLPREFIX *)field_addr; } +uintptr_t _get_entry_index(stm_hashtable_entry_t *entry) +{ + stm_read((object_t *)entry); + return entry->index; +} + +object_t *_get_entry_object(stm_hashtable_entry_t *entry) +{ + stm_read((object_t *)entry); + return entry->object; +} + void _set_ptr(object_t *obj, int n, object_t *v) { long nrefs = (long)((myobj_t*)obj)->type_id - 421420; diff --git a/c7/test/test_hashtable.py b/c7/test/test_hashtable.py --- a/c7/test/test_hashtable.py +++ b/c7/test/test_hashtable.py @@ -16,6 +16,19 @@ if res: raise Conflict +def ht_length_upper_bound(o): + h = get_hashtable(o) + return lib.stm_hashtable_length_upper_bound(h) + +def htitems(o): + h = get_hashtable(o) + upper_bound = lib.stm_hashtable_length_upper_bound(h) + entries = ffi.new("stm_hashtable_entry_t *[]", upper_bound) + count = lib.stm_hashtable_list(o, h, entries) + assert count <= upper_bound + return [(lib._get_entry_index(entries[i]), + lib._get_entry_object(entries[i])) for i in range(count)] + class BaseTestHashtable(BaseTest): @@ -236,6 +249,28 @@ assert htget(h, 11) != ffi.NULL assert htget(h, 12) != ffi.NULL + def test_list_1(self): + self.start_transaction() + h = self.allocate_hashtable() + tl0 = self.tls[self.current_thread] + for i in range(32): + assert ht_length_upper_bound(h) == i + htset(h, 19 ^ i, stm_allocate(32), tl0) + assert ht_length_upper_bound(h) == 32 + + def test_list_2(self): + self.start_transaction() + h = self.allocate_hashtable() + tl0 = self.tls[self.current_thread] + expected = [] + for i in range(29): + lp = stm_allocate(32) + htset(h, 19 ^ i, lp, tl0) + expected.append((19 ^ i, lp)) + lst = htitems(h) + assert len(lst) == 29 + assert sorted(lst) == sorted(expected) + class TestRandomHashtable(BaseTestHashtable): From noreply at buildbot.pypy.org Sun Feb 1 10:52:10 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 1 Feb 2015 10:52:10 +0100 (CET) Subject: [pypy-commit] stmgc hashtable-iter: test stm_hashtable_list(..., NULL) Message-ID: <20150201095210.89EAC1C039A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: hashtable-iter Changeset: r1602:de27e8f11e38 Date: 2015-02-01 10:52 +0100 http://bitbucket.org/pypy/stmgc/changeset/de27e8f11e38/ Log: test stm_hashtable_list(..., NULL) diff --git a/c7/test/test_hashtable.py b/c7/test/test_hashtable.py --- a/c7/test/test_hashtable.py +++ b/c7/test/test_hashtable.py @@ -29,6 +29,11 @@ return [(lib._get_entry_index(entries[i]), lib._get_entry_object(entries[i])) for i in range(count)] +def htlen(o): + h = get_hashtable(o) + count = lib.stm_hashtable_list(o, h, ffi.NULL) + return count + class BaseTestHashtable(BaseTest): @@ -271,6 +276,14 @@ assert len(lst) == 29 assert sorted(lst) == sorted(expected) + def test_list_3(self): + self.start_transaction() + h = self.allocate_hashtable() + tl0 = self.tls[self.current_thread] + for i in range(29): + htset(h, 19 ^ i, stm_allocate(32), tl0) + assert htlen(h) == 29 + class TestRandomHashtable(BaseTestHashtable): From noreply at buildbot.pypy.org Sun Feb 1 11:21:26 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 1 Feb 2015 11:21:26 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: update to stmgc/de27e8f11e38 (branch hashtable-iter) Message-ID: <20150201102126.486171C024F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r75618:bf785d8257b2 Date: 2015-02-01 11:03 +0100 http://bitbucket.org/pypy/pypy/changeset/bf785d8257b2/ Log: update to stmgc/de27e8f11e38 (branch hashtable-iter) diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -e7a6ff9e9da3 +de27e8f11e38 diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c --- a/rpython/translator/stm/src_stm/stm/core.c +++ b/rpython/translator/stm/src_stm/stm/core.c @@ -432,13 +432,12 @@ continue; /* no need to check: is pending immediate abort */ char *remote_base = get_segment_base(i); - uint8_t remote_version = get_segment(i)->transaction_read_version; LIST_FOREACH_R( STM_PSEGMENT->modified_old_objects, object_t * /*item*/, ({ - if (was_read_remote(remote_base, item, remote_version)) { + if (was_read_remote(remote_base, item)) { /* A write-read conflict! */ dprintf(("write-read conflict on %p, our seg: %d, other: %ld\n", item, STM_SEGMENT->segment_num, i)); diff --git a/rpython/translator/stm/src_stm/stm/core.h b/rpython/translator/stm/src_stm/stm/core.h --- a/rpython/translator/stm/src_stm/stm/core.h +++ b/rpython/translator/stm/src_stm/stm/core.h @@ -282,11 +282,17 @@ static stm_thread_local_t *abort_with_mutex_no_longjmp(void); static void abort_data_structures_from_segment_num(int segment_num); -static inline bool was_read_remote(char *base, object_t *obj, - uint8_t other_transaction_read_version) +static inline struct stm_read_marker_s *get_read_marker(char *base, + object_t *obj) { + return (struct stm_read_marker_s *)(base + (((uintptr_t)obj) >> 4)); +} + +static inline bool was_read_remote(char *base, object_t *obj) { - uint8_t rm = ((struct stm_read_marker_s *) - (base + (((uintptr_t)obj) >> 4)))->rm; + uint8_t other_transaction_read_version = + ((struct stm_segment_info_s *)REAL_ADDRESS(base, STM_PSEGMENT)) + ->transaction_read_version; + uint8_t rm = get_read_marker(base, obj)->rm; assert(rm <= other_transaction_read_version); return rm == other_transaction_read_version; } diff --git a/rpython/translator/stm/src_stm/stm/hashtable.c b/rpython/translator/stm/src_stm/stm/hashtable.c --- a/rpython/translator/stm/src_stm/stm/hashtable.c +++ b/rpython/translator/stm/src_stm/stm/hashtable.c @@ -13,22 +13,28 @@ collision). The main operations on a hashtable are reading or writing an object at a -given index. It might support in the future enumerating the indexes of -non-NULL objects. +given index. It also supports fetching the list of non-NULL entries. There are two markers for every index (a read and a write marker). This is unlike regular arrays, which have only two markers in total. +Additionally, we use the read marker for the hashtable object itself +to mean "we have read the complete list of keys". This plays the role +of a "global" read marker: when any thread adds a new key/value object +to the hashtable, this new object's read marker is initialized with a +copy of the "global" read marker --- in all segments. + Implementation -------------- First idea: have the hashtable in raw memory, pointing to "entry" -objects. The entry objects themselves point to the user-specified -objects. The entry objects have the read/write markers. Every entry -object, once created, stays around. It is only removed by the next -major GC if it points to NULL and its read/write markers are not set -in any currently-running transaction. +objects (which are regular, GC- and STM-managed objects). The entry +objects themselves point to the user-specified objects. The entry +objects hold the read/write markers. Every entry object, once +created, stays around. It is only removed by the next major GC if it +points to NULL and its read/write markers are not set in any +currently-running transaction. References ---------- @@ -104,9 +110,7 @@ { long i; for (i = 1; i <= NB_SEGMENTS; i++) { - char *remote_base = get_segment_base(i); - uint8_t remote_version = get_segment(i)->transaction_read_version; - if (was_read_remote(remote_base, obj, remote_version)) + if (was_read_remote(get_segment_base(i), obj)) return true; } return false; @@ -155,6 +159,7 @@ /* ^^^ this unlocks the table by writing a non-zero value to table->resize_counter, but the new value is a pointer to the new bigger table, so IS_EVEN() is still true */ + assert(IS_EVEN(table->resize_counter)); init_table(biggertable, biggercount); @@ -174,6 +179,7 @@ } } _insert_clean(biggertable, entry); + assert(rc > 6); rc -= 6; } biggertable->resize_counter = rc; @@ -217,7 +223,16 @@ perturb >>= PERTURB_SHIFT; } } - /* here, we didn't find the 'entry' with the correct index. */ + /* here, we didn't find the 'entry' with the correct index. Note + that even if the same 'table' is modified or resized by other + threads concurrently, any new item found from a race condition + would anyway contain NULL in the present segment (ensured by + the first write_fence() below). If the 'table' grows an entry + just after we checked above, then we go ahead and lock the + table; but after we get the lock, we will notice the new entry + (ensured by the second write_fence() below) and restart the + whole process. + */ uintptr_t rc = VOLATILE_TABLE(table)->resize_counter; @@ -283,12 +298,22 @@ synchronization with other pieces of the code that may change. */ + + /* First fetch the read marker of 'hashtableobj' in all + segments, before allocate_outside_nursery_large() + which might trigger a GC */ + long j; + uint8_t readmarkers[NB_SEGMENTS]; + for (j = 1; j <= NB_SEGMENTS; j++) { + readmarkers[j - 1] = get_read_marker(get_segment_base(j), + hashtableobj)->rm; + } + acquire_privatization_lock(); char *p = allocate_outside_nursery_large( sizeof(stm_hashtable_entry_t)); entry = (stm_hashtable_entry_t *)(p - stm_object_pages); - long j; for (j = 0; j <= NB_SEGMENTS; j++) { struct stm_hashtable_entry_s *e; e = (struct stm_hashtable_entry_s *) @@ -300,6 +325,11 @@ } hashtable->additions += 0x100; release_privatization_lock(); + + for (j = 1; j <= NB_SEGMENTS; j++) { + get_read_marker(get_segment_base(j), (object_t *)entry)->rm = + readmarkers[j - 1]; + } } write_fence(); /* make sure 'entry' is fully initialized here */ table->items[i] = entry; @@ -339,17 +369,94 @@ e->object = nvalue; } +long stm_hashtable_length_upper_bound(stm_hashtable_t *hashtable) +{ + stm_hashtable_table_t *table; + uintptr_t rc; + + restart: + table = VOLATILE_HASHTABLE(hashtable)->table; + rc = VOLATILE_TABLE(table)->resize_counter; + if (IS_EVEN(rc)) { + spin_loop(); + goto restart; + } + + uintptr_t initial_rc = (table->mask + 1) * 4 + 1; + uintptr_t num_entries_times_6 = initial_rc - rc; + return num_entries_times_6 / 6; +} + +long stm_hashtable_list(object_t *hobj, stm_hashtable_t *hashtable, + stm_hashtable_entry_t **results) +{ + stm_hashtable_table_t *table; + intptr_t rc; + + /* Set the read marker. It will be left as long as we're running + the same transaction. + */ + stm_read(hobj); + + /* Acquire and immediately release the lock. We don't actually + need to do anything while we hold the lock, but the point is to + wait until the lock is available, and to synchronize other + threads with the stm_read() done above. + */ + restart: + table = VOLATILE_HASHTABLE(hashtable)->table; + rc = VOLATILE_TABLE(table)->resize_counter; + if (IS_EVEN(rc)) { + spin_loop(); + goto restart; + } + if (!__sync_bool_compare_and_swap(&table->resize_counter, rc, rc)) + goto restart; + + /* Read all entries, check which ones are not NULL, count them, + and optionally list them in 'results'. + */ + uintptr_t i, mask = table->mask; + stm_hashtable_entry_t *entry; + long nresult = 0; + + if (results != NULL) { + /* collect the results in the provided list */ + for (i = 0; i <= mask; i++) { + entry = VOLATILE_TABLE(table)->items[i]; + if (entry != NULL) { + stm_read((object_t *)entry); + if (entry->object != NULL) + results[nresult++] = entry; + } + } + } + else { + /* don't collect, just get the exact number of results */ + for (i = 0; i <= mask; i++) { + entry = VOLATILE_TABLE(table)->items[i]; + if (entry != NULL) { + stm_read((object_t *)entry); + if (entry->object != NULL) + nresult++; + } + } + } + return nresult; +} + static void _stm_compact_hashtable(stm_hashtable_t *hashtable) { stm_hashtable_table_t *table = hashtable->table; - assert(!IS_EVEN(table->resize_counter)); + uintptr_t rc = table->resize_counter; + assert(!IS_EVEN(rc)); if ((hashtable->additions >> 8) * 4 > table->mask) { int segment_num = (hashtable->additions & 0xFF); if (!segment_num) segment_num = 1; hashtable->additions = segment_num; uintptr_t initial_rc = (table->mask + 1) * 4 + 1; - uintptr_t num_entries_times_6 = initial_rc - table->resize_counter; + uintptr_t num_entries_times_6 = initial_rc - rc; uintptr_t count = INITIAL_HASHTABLE_SIZE; while (count * 4 < num_entries_times_6) count *= 2; @@ -377,6 +484,7 @@ free(old_table); } hashtable->initial_table.resize_counter = (uintptr_t)table; + assert(IS_EVEN(hashtable->initial_table.resize_counter)); } } diff --git a/rpython/translator/stm/src_stm/stm/misc.c b/rpython/translator/stm/src_stm/stm/misc.c --- a/rpython/translator/stm/src_stm/stm/misc.c +++ b/rpython/translator/stm/src_stm/stm/misc.c @@ -32,8 +32,7 @@ bool _stm_was_read(object_t *obj) { - return was_read_remote(STM_SEGMENT->segment_base, obj, - STM_SEGMENT->transaction_read_version); + return was_read_remote(STM_SEGMENT->segment_base, obj); } bool _stm_was_written(object_t *obj) diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h --- a/rpython/translator/stm/src_stm/stmgc.h +++ b/rpython/translator/stm/src_stm/stmgc.h @@ -546,6 +546,9 @@ object_t *stm_hashtable_read(object_t *, stm_hashtable_t *, uintptr_t key); void stm_hashtable_write(object_t *, stm_hashtable_t *, uintptr_t key, object_t *nvalue, stm_thread_local_t *); +long stm_hashtable_length_upper_bound(stm_hashtable_t *); +long stm_hashtable_list(object_t *, stm_hashtable_t *, + stm_hashtable_entry_t **results); extern uint32_t stm_hashtable_entry_userdata; void stm_hashtable_tracefn(stm_hashtable_t *, void (object_t **)); From noreply at buildbot.pypy.org Sun Feb 1 11:21:27 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 1 Feb 2015 11:21:27 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: _ll_hashtable_len Message-ID: <20150201102127.771C51C024F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r75619:054d5d592ec4 Date: 2015-02-01 11:10 +0100 http://bitbucket.org/pypy/pypy/changeset/054d5d592ec4/ Log: _ll_hashtable_len diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py --- a/rpython/rlib/rstm.py +++ b/rpython/rlib/rstm.py @@ -182,6 +182,7 @@ ('index', lltype.Unsigned), ('object', llmemory.GCREF)) _STM_HASHTABLE_ENTRY_P = lltype.Ptr(_STM_HASHTABLE_ENTRY) +_STM_HASHTABLE_ENTRY_ARRAY = rffi.CArray(_STM_HASHTABLE_ENTRY_P) @dont_look_inside def _ll_hashtable_get(h, key): @@ -193,6 +194,11 @@ llop.stm_hashtable_write(lltype.Void, h, h.ll_raw_hashtable, key, value) @dont_look_inside +def _ll_hashtable_len(h): + return llop.stm_hashtable_list(lltype.Signed, h, h.ll_raw_hashtable, + lltype.nullptr(_STM_HASHTABLE_ENTRY_ARRAY)) + + at dont_look_inside def _ll_hashtable_lookup(h, key): return llop.stm_hashtable_lookup(_STM_HASHTABLE_ENTRY_P, h, h.ll_raw_hashtable, key) @@ -202,6 +208,7 @@ rtti=True, adtmeths={'get': _ll_hashtable_get, 'set': _ll_hashtable_set, + 'len': _ll_hashtable_len, 'lookup': _ll_hashtable_lookup}) NULL_HASHTABLE = lltype.nullptr(_HASHTABLE_OBJ) @@ -262,6 +269,9 @@ except KeyError: pass + def len(self): + return len(self._content) + def lookup(self, key): assert type(key) is int return EntryObjectForTest(self, key) diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -465,6 +465,7 @@ 'stm_hashtable_read': LLOp(), 'stm_hashtable_write': LLOp(), 'stm_hashtable_lookup': LLOp(), + 'stm_hashtable_list' : LLOp(), 'stm_hashtable_tracefn': LLOp(), # __________ address operations __________ diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py --- a/rpython/translator/stm/funcgen.py +++ b/rpython/translator/stm/funcgen.py @@ -320,6 +320,14 @@ return '%s = stm_hashtable_lookup((object_t *)%s, %s, %s);' % ( result, arg0, arg1, arg2) +def stm_hashtable_list(funcgen, op): + arg0 = funcgen.expr(op.args[0]) + arg1 = funcgen.expr(op.args[1]) + arg2 = funcgen.expr(op.args[2]) + result = funcgen.expr(op.result) + return '%s = stm_hashtable_list((object_t *)%s, %s, %s);' % ( + result, arg0, arg1, arg2) + def stm_hashtable_tracefn(funcgen, op): arg0 = funcgen.expr(op.args[0]) arg1 = funcgen.expr(op.args[1]) diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py --- a/rpython/translator/stm/test/test_ztranslated.py +++ b/rpython/translator/stm/test/test_ztranslated.py @@ -575,6 +575,14 @@ x2 = cast_gcref_to_instance(X, p2) assert x2 is x1 # + entry = h.lookup(-1234) + assert cast_gcref_to_instance(X, entry.object) is x1 + assert h.len() == 1 + # + entry = h.lookup(4242) + assert cast_gcref_to_instance(X, entry.object) is None + assert h.len() == 1 + # print "ok!" return 0 From noreply at buildbot.pypy.org Sun Feb 1 11:21:28 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 1 Feb 2015 11:21:28 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: _ll_hashtable_list Message-ID: <20150201102128.F26AF1C024F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r75620:dce2ea23c327 Date: 2015-02-01 11:21 +0100 http://bitbucket.org/pypy/pypy/changeset/dce2ea23c327/ Log: _ll_hashtable_list diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py --- a/rpython/rlib/rstm.py +++ b/rpython/rlib/rstm.py @@ -1,5 +1,6 @@ from rpython.rlib.objectmodel import we_are_translated, specialize from rpython.rlib.objectmodel import CDefinedIntSymbolic, stm_ignored +from rpython.rlib.rarithmetic import r_uint from rpython.rlib.nonconst import NonConstant from rpython.rlib import rgc from rpython.rtyper.lltypesystem import lltype, rffi, rstr, llmemory @@ -199,6 +200,20 @@ lltype.nullptr(_STM_HASHTABLE_ENTRY_ARRAY)) @dont_look_inside +def _ll_hashtable_list(h): + upper_bound = llop.stm_hashtable_length_upper_bound(lltype.Signed, + h.ll_raw_hashtable) + array = lltype.malloc(_STM_HASHTABLE_ENTRY_ARRAY, upper_bound, + flavor='raw') + count = llop.stm_hashtable_list(lltype.Signed, h, h.ll_raw_hashtable, + array) + return (array, count) + + at dont_look_inside +def _ll_hashtable_freelist(h, array): + lltype.free(array, flavor='raw') + + at dont_look_inside def _ll_hashtable_lookup(h, key): return llop.stm_hashtable_lookup(_STM_HASHTABLE_ENTRY_P, h, h.ll_raw_hashtable, key) @@ -209,6 +224,8 @@ adtmeths={'get': _ll_hashtable_get, 'set': _ll_hashtable_set, 'len': _ll_hashtable_len, + 'list': _ll_hashtable_list, + 'freelist': _ll_hashtable_freelist, 'lookup': _ll_hashtable_lookup}) NULL_HASHTABLE = lltype.nullptr(_HASHTABLE_OBJ) @@ -272,6 +289,16 @@ def len(self): return len(self._content) + def list(self): + items = [self.lookup(key) for key in self._content] + count = len(items) + for i in range(3): + items.append("additional garbage for testing") + return items, count + + def freelist(self, array): + pass + def lookup(self, key): assert type(key) is int return EntryObjectForTest(self, key) @@ -280,6 +307,7 @@ def __init__(self, hashtable, key): self.hashtable = hashtable self.key = key + self.index = r_uint(key) def _getobj(self): return self.hashtable.get(self.key) diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -465,6 +465,7 @@ 'stm_hashtable_read': LLOp(), 'stm_hashtable_write': LLOp(), 'stm_hashtable_lookup': LLOp(), + 'stm_hashtable_length_upper_bound': LLOp(), 'stm_hashtable_list' : LLOp(), 'stm_hashtable_tracefn': LLOp(), diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py --- a/rpython/translator/stm/funcgen.py +++ b/rpython/translator/stm/funcgen.py @@ -320,6 +320,12 @@ return '%s = stm_hashtable_lookup((object_t *)%s, %s, %s);' % ( result, arg0, arg1, arg2) +def stm_hashtable_length_upper_bound(funcgen, op): + arg0 = funcgen.expr(op.args[0]) + result = funcgen.expr(op.result) + return '%s = stm_hashtable_length_upper_bound(%s);' % ( + result, arg0) + def stm_hashtable_list(funcgen, op): arg0 = funcgen.expr(op.args[0]) arg1 = funcgen.expr(op.args[1]) diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py --- a/rpython/translator/stm/test/test_ztranslated.py +++ b/rpython/translator/stm/test/test_ztranslated.py @@ -1,6 +1,7 @@ import py from rpython.rlib import rstm, rgc, objectmodel from rpython.rlib.debug import debug_print +from rpython.rlib.rarithmetic import intmask from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.rclass import OBJECTPTR @@ -583,6 +584,12 @@ assert cast_gcref_to_instance(X, entry.object) is None assert h.len() == 1 # + array, count = h.list() + assert count == 1 + assert intmask(array[0].index) == -1234 + assert cast_gcref_to_instance(X, array[0].object) is x1 + h.freelist(array) + # print "ok!" return 0 From noreply at buildbot.pypy.org Sun Feb 1 11:39:29 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 1 Feb 2015 11:39:29 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: hashtable.keys(), values(), items() Message-ID: <20150201103929.0181B1C039A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r75624:d7857d4bcf41 Date: 2015-02-01 11:39 +0100 http://bitbucket.org/pypy/pypy/changeset/d7857d4bcf41/ Log: hashtable.keys(), values(), items() diff --git a/pypy/module/pypystm/hashtable.py b/pypy/module/pypystm/hashtable.py --- a/pypy/module/pypystm/hashtable.py +++ b/pypy/module/pypystm/hashtable.py @@ -7,6 +7,7 @@ from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from rpython.rlib import rstm +from rpython.rlib.rarithmetic import intmask from rpython.rtyper.annlowlevel import cast_gcref_to_instance from rpython.rtyper.annlowlevel import cast_instance_to_gcref @@ -59,6 +60,34 @@ def len_w(self, space): return space.wrap(self.h.len()) + def keys_w(self, space): + array, count = self.h.list() + try: + lst = [intmask(array[i].index) for i in range(count)] + finally: + self.h.freelist(array) + return space.newlist_int(lst) + + def values_w(self, space): + array, count = self.h.list() + try: + lst_w = [cast_gcref_to_instance(W_Root, array[i].object) + for i in range(count)] + finally: + self.h.freelist(array) + return space.newlist(lst_w) + + def items_w(self, space): + array, count = self.h.list() + try: + lst_w = [space.newtuple([ + space.wrap(intmask(array[i].index)), + cast_gcref_to_instance(W_Root, array[i].object)]) + for i in range(count)] + finally: + self.h.freelist(array) + return space.newlist(lst_w) + def W_Hashtable___new__(space, w_subtype): r = space.allocate_instance(W_Hashtable, w_subtype) @@ -76,4 +105,7 @@ setdefault = interp2app(W_Hashtable.setdefault_w), __len__ = interp2app(W_Hashtable.len_w), + keys = interp2app(W_Hashtable.keys_w), + values = interp2app(W_Hashtable.values_w), + items = interp2app(W_Hashtable.items_w), ) diff --git a/pypy/module/pypystm/test/test_hashtable.py b/pypy/module/pypystm/test/test_hashtable.py --- a/pypy/module/pypystm/test/test_hashtable.py +++ b/pypy/module/pypystm/test/test_hashtable.py @@ -46,3 +46,12 @@ assert len(h) == 2 del h[42] assert len(h) == 1 + + def test_keys_values_items(self): + import pypystm + h = pypystm.hashtable() + h[42] = "foo" + h[43] = "bar" + assert sorted(h.keys()) == [42, 43] + assert sorted(h.values()) == ["bar", "foo"] + assert sorted(h.items()) == [(42, "foo"), (43, "bar")] From noreply at buildbot.pypy.org Sun Feb 1 12:03:41 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 1 Feb 2015 12:03:41 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: stmdict.keys() and similar methods Message-ID: <20150201110341.B44FE1C0507@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r75625:707dcfba2317 Date: 2015-02-01 12:03 +0100 http://bitbucket.org/pypy/pypy/changeset/707dcfba2317/ Log: stmdict.keys() and similar methods diff --git a/pypy/module/pypystm/stmdict.py b/pypy/module/pypystm/stmdict.py --- a/pypy/module/pypystm/stmdict.py +++ b/pypy/module/pypystm/stmdict.py @@ -138,6 +138,69 @@ entry.object = lltype.cast_opaque_ptr(llmemory.GCREF, narray) return w_default + def get_length(self): + array, count = self.h.list() + try: + total_length_times_two = 0 + for i in range(count): + subarray = lltype.cast_opaque_ptr(PARRAY, array[i].object) + assert subarray + total_length_times_two += len(subarray) + finally: + self.h.freelist(array) + return total_length_times_two >> 1 + + def get_keys_values_w(self, offset): + array, count = self.h.list() + try: + result_list_w = [] + for i in range(count): + subarray = lltype.cast_opaque_ptr(PARRAY, array[i].object) + assert subarray + j = offset + limit = len(subarray) + while j < limit: + w_item = cast_gcref_to_instance(W_Root, subarray[j]) + result_list_w.append(w_item) + j += 2 + finally: + self.h.freelist(array) + return result_list_w + + def get_items_w(self, space): + array, count = self.h.list() + try: + result_list_w = [] + for i in range(count): + subarray = lltype.cast_opaque_ptr(PARRAY, array[i].object) + assert subarray + j = 0 + limit = len(subarray) + while j < limit: + w_key = cast_gcref_to_instance(W_Root, subarray[j]) + w_value = cast_gcref_to_instance(W_Root, subarray[j + 1]) + result_list_w.append(space.newtuple([w_key, w_value])) + j += 2 + finally: + self.h.freelist(array) + return result_list_w + + def len_w(self, space): + return space.wrap(self.get_length()) + + def iter_w(self, space): + # not a real lazy iterator! + return space.iter(self.keys_w(space)) + + def keys_w(self, space): + return space.newlist(self.get_keys_values_w(offset=0)) + + def values_w(self, space): + return space.newlist(self.get_keys_values_w(offset=1)) + + def items_w(self, space): + return space.newlist(self.get_items_w(space)) + def W_STMDict___new__(space, w_subtype): r = space.allocate_instance(W_STMDict, w_subtype) @@ -153,4 +216,10 @@ __contains__ = interp2app(W_STMDict.contains_w), get = interp2app(W_STMDict.get_w), setdefault = interp2app(W_STMDict.setdefault_w), + + __len__ = interp2app(W_STMDict.len_w), + __iter__ = interp2app(W_STMDict.iter_w), + keys = interp2app(W_STMDict.keys_w), + values = interp2app(W_STMDict.values_w), + items = interp2app(W_STMDict.items_w), ) diff --git a/pypy/module/pypystm/stmset.py b/pypy/module/pypystm/stmset.py --- a/pypy/module/pypystm/stmset.py +++ b/pypy/module/pypystm/stmset.py @@ -94,6 +94,39 @@ def discard_w(self, space, w_key): self.try_remove(space, w_key) + def get_length(self): + array, count = self.h.list() + try: + total_length = 0 + for i in range(count): + subarray = lltype.cast_opaque_ptr(PARRAY, array[i].object) + assert subarray + total_length += len(subarray) + finally: + self.h.freelist(array) + return total_length + + def get_items_w(self): + array, count = self.h.list() + try: + result_list_w = [] + for i in range(count): + subarray = lltype.cast_opaque_ptr(PARRAY, array[i].object) + assert subarray + for j in range(len(subarray)): + w_item = cast_gcref_to_instance(W_Root, subarray[j]) + result_list_w.append(w_item) + finally: + self.h.freelist(array) + return result_list_w + + def len_w(self, space): + return space.wrap(self.get_length()) + + def iter_w(self, space): + # not a real lazy iterator! + return space.iter(space.newlist(self.get_items_w())) + def W_STMSet___new__(space, w_subtype): r = space.allocate_instance(W_STMSet, w_subtype) @@ -107,4 +140,7 @@ add = interp2app(W_STMSet.add_w), remove = interp2app(W_STMSet.remove_w), discard = interp2app(W_STMSet.discard_w), + + __len__ = interp2app(W_STMSet.len_w), + __iter__ = interp2app(W_STMSet.iter_w), ) diff --git a/pypy/module/pypystm/test/test_stmdict.py b/pypy/module/pypystm/test/test_stmdict.py --- a/pypy/module/pypystm/test/test_stmdict.py +++ b/pypy/module/pypystm/test/test_stmdict.py @@ -70,3 +70,34 @@ assert d[42] == "hello" assert d.get(42L) == "hello" assert d.get(42.001) is None + + def test_list_from_dict(self): + import pypystm + d = pypystm.stmdict() + assert len(d) == 0 + assert tuple(d) == () + d[42.5] = "foo" + d[42.0] = ["bar"] + assert sorted(d) == [42.0, 42.5] + assert len(d) == 2 + del d[42] + assert len(d) == 1 + assert list(d) == [42.5] + # + class Key(object): + def __hash__(self): + return hash(42.5) + key3 = Key() + d[key3] = "other" + assert len(d) == 2 + items = list(d) + assert items == [42.5, key3] or items == [key3, 42.5] + + def test_keys_values_items(self): + import pypystm + d = pypystm.stmdict() + d[42.5] = "bar" + d[42.0] = "foo" + assert sorted(d.keys()) == [42.0, 42.5] + assert sorted(d.values()) == ["bar", "foo"] + assert sorted(d.items()) == [(42.0, "foo"), (42.5, "bar")] diff --git a/pypy/module/pypystm/test/test_stmset.py b/pypy/module/pypystm/test/test_stmset.py --- a/pypy/module/pypystm/test/test_stmset.py +++ b/pypy/module/pypystm/test/test_stmset.py @@ -62,3 +62,24 @@ assert 42 in s assert 42L in s assert 42.001 not in s + + def test_list_from_set(self): + import pypystm + s = pypystm.stmset() + assert len(s) == 0 + assert tuple(s) == () + s.add(42.5) + s.add(42.0) + assert sorted(s) == [42.0, 42.5] + assert len(s) == 2 + s.remove(42.0) + assert list(s) == [42.5] + # + class Key(object): + def __hash__(self): + return hash(42.5) + key3 = Key() + s.add(key3) + assert len(s) == 2 + items = list(s) + assert items == [42.5, key3] or items == [key3, 42.5] From noreply at buildbot.pypy.org Sun Feb 1 13:44:52 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 1 Feb 2015 13:44:52 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: stmdict.pop() Message-ID: <20150201124452.19D571C02CD@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r75626:b4c5f43782f1 Date: 2015-02-01 13:44 +0100 http://bitbucket.org/pypy/pypy/changeset/b4c5f43782f1/ Log: stmdict.pop() diff --git a/pypy/module/pypystm/stmdict.py b/pypy/module/pypystm/stmdict.py --- a/pypy/module/pypystm/stmdict.py +++ b/pypy/module/pypystm/stmdict.py @@ -43,6 +43,25 @@ for i in range(length): dest[dest_start + i] = source[source_start + i] +def pop_from_entry(entry, space, w_key): + array = lltype.cast_opaque_ptr(PARRAY, entry.object) + if not array: + return None + i = find_equal_item(space, array, w_key) + if i < 0: + return None + # found + w_value = cast_gcref_to_instance(W_Root, array[i + 1]) + L = len(array) - 2 + if L == 0: + narray = lltype.nullptr(ARRAY) + else: + narray = lltype.malloc(ARRAY, L) + ll_arraycopy(array, narray, 0, 0, i) + ll_arraycopy(array, narray, i + 2, i, L - i) + entry.object = lltype.cast_opaque_ptr(llmemory.GCREF, narray) + return w_value + class W_STMDict(W_Root): @@ -82,21 +101,8 @@ def delitem_w(self, space, w_key): hkey = space.hash_w(w_key) entry = self.h.lookup(hkey) - array = lltype.cast_opaque_ptr(PARRAY, entry.object) - if array: - i = find_equal_item(space, array, w_key) - if i >= 0: - # found - L = len(array) - 2 - if L == 0: - narray = lltype.nullptr(ARRAY) - else: - narray = lltype.malloc(ARRAY, L) - ll_arraycopy(array, narray, 0, 0, i) - ll_arraycopy(array, narray, i + 2, i, L - i) - entry.object = lltype.cast_opaque_ptr(llmemory.GCREF, narray) - return - space.raise_key_error(w_key) + if pop_from_entry(entry, space, w_key) is None: + space.raise_key_error(w_key) def contains_w(self, space, w_key): hkey = space.hash_w(w_key) @@ -117,6 +123,17 @@ return cast_gcref_to_instance(W_Root, array[i + 1]) return w_default + def pop_w(self, space, w_key, w_default=None): + hkey = space.hash_w(w_key) + entry = self.h.lookup(hkey) + w_value = pop_from_entry(entry, space, w_key) + if w_value is not None: + return w_value + elif w_default is not None: + return w_default + else: + space.raise_key_error(w_key) + @unwrap_spec(w_default=WrappedDefault(None)) def setdefault_w(self, space, w_key, w_default): hkey = space.hash_w(w_key) @@ -215,6 +232,7 @@ __delitem__ = interp2app(W_STMDict.delitem_w), __contains__ = interp2app(W_STMDict.contains_w), get = interp2app(W_STMDict.get_w), + pop = interp2app(W_STMDict.pop_w), setdefault = interp2app(W_STMDict.setdefault_w), __len__ = interp2app(W_STMDict.len_w), diff --git a/pypy/module/pypystm/test/test_stmdict.py b/pypy/module/pypystm/test/test_stmdict.py --- a/pypy/module/pypystm/test/test_stmdict.py +++ b/pypy/module/pypystm/test/test_stmdict.py @@ -101,3 +101,18 @@ assert sorted(d.keys()) == [42.0, 42.5] assert sorted(d.values()) == ["bar", "foo"] assert sorted(d.items()) == [(42.0, "foo"), (42.5, "bar")] + + def test_pop(self): + import pypystm + d = pypystm.stmdict() + raises(KeyError, d.pop, 42.0) + assert d.pop(42.0, "foo") == "foo" + raises(KeyError, "d[42.0]") + d[42.0] = "bar" + res = d.pop(42.0) + assert res == "bar" + raises(KeyError, "d[42.0]") + d[42.0] = "bar" + res = d.pop(42.0, "foo") + assert res == "bar" + raises(KeyError, "d[42.0]") From noreply at buildbot.pypy.org Sun Feb 1 19:21:08 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 1 Feb 2015 19:21:08 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: fix Message-ID: <20150201182108.0FEA81C0507@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r75627:4aff38646069 Date: 2015-02-01 19:21 +0100 http://bitbucket.org/pypy/pypy/changeset/4aff38646069/ Log: fix diff --git a/pypy/module/pypystm/stmdict.py b/pypy/module/pypystm/stmdict.py --- a/pypy/module/pypystm/stmdict.py +++ b/pypy/module/pypystm/stmdict.py @@ -225,7 +225,7 @@ return space.wrap(r) W_STMDict.typedef = TypeDef( - 'pypystm.stmset', + 'pypystm.stmdict', __new__ = interp2app(W_STMDict___new__), __getitem__ = interp2app(W_STMDict.getitem_w), __setitem__ = interp2app(W_STMDict.setitem_w), From noreply at buildbot.pypy.org Sun Feb 1 19:34:02 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 1 Feb 2015 19:34:02 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.9: More fields in decode_certificate. Message-ID: <20150201183402.D01671C0FDE@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: stdlib-2.7.9 Changeset: r75628:60293e8655ee Date: 2015-01-31 19:08 +0100 http://bitbucket.org/pypy/pypy/changeset/60293e8655ee/ Log: More fields in decode_certificate. diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -636,7 +636,7 @@ space.setitem(w_retval, space.wrap("issuer"), w_issuer) space.setitem(w_retval, space.wrap("version"), - space.wrap(libssl_X509_get_version(certificate))) + space.wrap(libssl_X509_get_version(certificate) + 1)) biobuf = libssl_BIO_new(libssl_BIO_s_mem()) try: @@ -680,6 +680,11 @@ if w_alt_names is not space.w_None: space.setitem(w_retval, space.wrap("subjectAltName"), w_alt_names) + # CDP (CRL distribution points) + w_cdp = _get_crl_dp(space, certificate) + if not space.is_none(w_cdp): + space.setitem(w_retval, space.wrap("crlDistributionPoints"), w_cdp) + return w_retval @@ -828,6 +833,29 @@ return space.newtuple([w_name, w_value]) +def _get_crl_dp(space, certificate): + # Calls x509v3_cache_extensions and sets up crldp + libssl_X509_check_ca(certificate) + dps = certificate[0].c_crldp + if not dps: + return None + + cdp_w = [] + for i in range(libssl_sk_DIST_POINT_num(dps)): + dp = libssl_sk_DIST_POINT_value(dps, i) + gns = libssl_pypy_DIST_POINT_fullname(dp) + + for j in range(libssl_sk_GENERAL_NAME_num(gns)): + name = libssl_sk_GENERAL_NAME_value(gns, j) + gntype = intmask(name[0].c_type) + if gntype != GEN_URI: + continue + uri = libssl_pypy_GENERAL_NAME_uri(name) + length = intmask(uri.c_length) + s_uri = rffi.charpsize2str(uri.c_data, length) + cdp_w.append(space.wrap(s_uri)) + return space.newtuple(cdp_w[:]) + def checkwait(space, w_sock, writing): """If the socket has a timeout, do a select()/poll() on the socket. The argument writing indicates the direction. diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py --- a/pypy/module/_ssl/test/test_ssl.py +++ b/pypy/module/_ssl/test/test_ssl.py @@ -340,6 +340,9 @@ assert len(certs) == 1 print(certs) assert len(certs[0]['issuer']) == 4 + assert certs[0]['version'] == 3 + assert certs[0]['crlDistributionPoints'] == ( + 'https://www.cacert.org/revoke.crl',) def test_load_dh_params(self): import _ssl diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py --- a/rpython/rlib/ropenssl.py +++ b/rpython/rlib/ropenssl.py @@ -38,7 +38,9 @@ # Unnamed structures are not supported by rffi_platform. # So we replace an attribute access with a macro call. '#define pypy_GENERAL_NAME_dirn(name) (name->d.dirn)', + '#define pypy_GENERAL_NAME_uri(name) (name->d.uniformResourceIdentifier)', '#define pypy_X509_OBJECT_data_x509(obj) (obj->data.x509)', + '#define pypy_DIST_POINT_fullname(obj) (obj->distpoint->name.fullname)', ], ) @@ -48,12 +50,14 @@ include_dir='inc32', library_dir='out32'), ]) -X509 = rffi.COpaquePtr('X509') ASN1_STRING = lltype.Ptr(lltype.ForwardReference()) +ASN1_IA5STRING = ASN1_STRING ASN1_ITEM = rffi.COpaquePtr('ASN1_ITEM') X509_NAME = rffi.COpaquePtr('X509_NAME') X509_VERIFY_PARAM = rffi.COpaquePtr('X509_VERIFY_PARAM') stack_st_X509_OBJECT = rffi.COpaquePtr('struct stack_st_X509_OBJECT') +DIST_POINT = rffi.COpaquePtr('DIST_POINT') +stack_st_DIST_POINT = rffi.COpaquePtr('struct stack_st_DIST_POINT') DH = rffi.COpaquePtr('DH') class CConfigBootstrap: @@ -138,6 +142,10 @@ OBJ_NAME_TYPE_MD_METH = rffi_platform.ConstantInteger( "OBJ_NAME_TYPE_MD_METH") + X509_st = rffi_platform.Struct( + 'struct x509_st', + [('crldp', stack_st_DIST_POINT)]) + # Some structures, with only the fields used in the _ssl module X509_name_entry_st = rffi_platform.Struct('struct X509_name_entry_st', [('set', rffi.INT)]) @@ -193,6 +201,7 @@ SSL_CIPHER = rffi.COpaquePtr('SSL_CIPHER') SSL = rffi.COpaquePtr('SSL') BIO = rffi.COpaquePtr('BIO') +X509 = rffi.CArrayPtr(X509_st) X509_NAME_ENTRY = rffi.CArrayPtr(X509_name_entry_st) X509_EXTENSION = rffi.CArrayPtr(X509_extension_st) X509_STORE = rffi.CArrayPtr(x509_store_st) @@ -341,10 +350,18 @@ X509_OBJECT, macro=True) ssl_external('pypy_X509_OBJECT_data_x509', [X509_OBJECT], X509, macro=True) +ssl_external('sk_DIST_POINT_num', [stack_st_DIST_POINT], rffi.INT, + macro=True) +ssl_external('sk_DIST_POINT_value', [stack_st_DIST_POINT, rffi.INT], DIST_POINT, + macro=True) +ssl_external('pypy_DIST_POINT_fullname', [DIST_POINT], GENERAL_NAMES, + macro=True) ssl_external('GENERAL_NAME_print', [BIO, GENERAL_NAME], rffi.INT) ssl_external('pypy_GENERAL_NAME_dirn', [GENERAL_NAME], X509_NAME, macro=True) +ssl_external('pypy_GENERAL_NAME_uri', [GENERAL_NAME], ASN1_IA5STRING, + macro=True) ssl_external('SSL_get_current_cipher', [SSL], SSL_CIPHER) ssl_external('SSL_CIPHER_get_name', [SSL_CIPHER], rffi.CCHARP) From noreply at buildbot.pypy.org Sun Feb 1 19:34:03 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 1 Feb 2015 19:34:03 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.9: Fix an issue in cert_store_stats() Message-ID: <20150201183403.F3E4B1C0FDE@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: stdlib-2.7.9 Changeset: r75629:1c34bb5d8133 Date: 2015-01-31 23:02 +0100 http://bitbucket.org/pypy/pypy/changeset/1c34bb5d8133/ Log: Fix an issue in cert_store_stats() diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -1353,7 +1353,8 @@ obj = libssl_sk_X509_OBJECT_value(store[0].c_objs, i) if intmask(obj.c_type) == X509_LU_X509: counters['x509'] += 1 - if libssl_pypy_X509_OBJECT_data_x509(obj): + if libssl_X509_check_ca( + libssl_pypy_X509_OBJECT_data_x509(obj)): counters['x509_ca'] += 1 elif intmask(obj.c_type) == X509_LU_CRL: counters['crl'] += 1 diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py --- a/pypy/module/_ssl/test/test_ssl.py +++ b/pypy/module/_ssl/test/test_ssl.py @@ -344,6 +344,15 @@ assert certs[0]['crlDistributionPoints'] == ( 'https://www.cacert.org/revoke.crl',) + def test_cert_store_stats(self): + import _ssl + ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1) + assert ctx.cert_store_stats() == {'x509_ca': 0, 'crl': 0, 'x509': 0} + ctx.load_cert_chain(self.keycert) + assert ctx.cert_store_stats() == {'x509_ca': 0, 'crl': 0, 'x509': 0} + ctx.load_verify_locations(self.keycert) + assert ctx.cert_store_stats() == {'x509_ca': 0, 'crl': 0, 'x509': 1} + def test_load_dh_params(self): import _ssl ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1) From noreply at buildbot.pypy.org Sun Feb 1 19:34:05 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 1 Feb 2015 19:34:05 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.9: Implement SSLContext.set_servername_callback. Message-ID: <20150201183405.23CEF1C0FDE@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: stdlib-2.7.9 Changeset: r75630:1db91f1d2dbe Date: 2015-02-01 19:04 +0100 http://bitbucket.org/pypy/pypy/changeset/1db91f1d2dbe/ Log: Implement SSLContext.set_servername_callback. No test so far. diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -1,9 +1,9 @@ -from rpython.rlib import rpoll, rsocket, rthread +from rpython.rlib import rpoll, rsocket, rthread, rweakref from rpython.rlib.rarithmetic import intmask, widen, r_uint from rpython.rlib.ropenssl import * from rpython.rlib.rposix import get_errno, set_errno from rpython.rlib.rweakref import RWeakValueDictionary -from rpython.rlib.objectmodel import specialize +from rpython.rlib.objectmodel import specialize, compute_unique_id from rpython.rtyper.lltypesystem import lltype, rffi from pypy.interpreter.baseobjspace import W_Root @@ -244,7 +244,7 @@ self.socket_type = socket_type self.w_socket = w_sock - self.w_ssl_sock = None + self.ssl_sock_weakref_w = rweakref.ref(w_ssl_sock) return self def __del__(self): @@ -1059,6 +1059,64 @@ buf[i] = c return rffi.cast(rffi.INT, len(password)) +class ServernameCallback(object): + w_ctx = None + space = None +SERVERNAME_CALLBACKS = RWeakValueDictionary(int, ServernameCallback) + +def _servername_callback(ssl, ad, arg): + struct = SERVERNAME_CALLBACKS.get(rffi.cast(lltype.Signed, arg)) + w_ctx = struct.w_ctx + space = struct.space + servername = libssl_SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name) + if not w_ctx.w_set_hostname: + # remove race condition. + return SSL_TLSEXT_ERR_OK + # The high-level ssl.SSLSocket object + w_ssl = libssl_SSL_get_app_data(ssl) + assert isinstance(w_ssl, _SSLSocket) + if w_ssl.ssl_sock_weakref_w: + w_ssl_socket = w_ssl.ssl_sock_weakref_w.get() + else: + w_ssl_socket = space.w_None + if space.is_none(w_ssl_socket): + ad[0] = SSL_AD_INTERNAL_ERROR + return SSL_TLSEXT_ERR_ALERT_FATAL + + try: + if not servername: + w_result = space.call_function(w_ctx.w_set_hostname, + w_ssl_socket, space.w_None, w_ctx) + + else: + try: + w_servername = space.wrap_bytes(rffi.charp2str(servername)) + w_servername_idna = space.call_method( + w_servername, 'decode', space.wrap('idna')) + except OperationError as e: + space.write_unraisable(e, "w_servername") + ad[0] = SSL_AD_INTERNAL_ERROR; + return SSL_TLSEXT_ERR_ALERT_FATAL + + w_result = space.call_function(w_ctx.w_set_hostname, + w_ssl_socket, + w_servername_idna, w_ctx) + except OperationError as e: + space.write_unraisable(e, "ssl_ctx->set_hostname") + ad[0] = SSL_AD_HANDSHAKE_FAILURE + return SSL_TLSEXT_ERR_ALERT_FATAL + + if space.is_none(w_result): + return SSL_TLSEXT_ERR_OK + else: + try: + ad[0] = space.int_w(w_result) + except OperationError as e: + space.write_unraisable(e, "w_result") + ad[0] = SSL_AD_INTERNAL_ERROR + return SSL_TLSEXT_ERR_ALERT_FATAL + + class _SSLContext(W_Root): @staticmethod @unwrap_spec(protocol=int) @@ -1402,6 +1460,24 @@ rlist.append(_decode_certificate(space, cert)) return space.newlist(rlist) + def set_servername_callback_w(self, space, w_callback): + if space.is_none(w_callback): + libssl_SSL_CTX_set_tlsext_servername_callback( + self.ctx, lltype.nullptr(servername_cb.TO)) + return + if not space.is_true(space.callable(w_callback)): + raise oefmt(space.w_TypeError, "not a callable object") + self.w_set_hostname = w_callback + struct = ServernameCallback() + struct.space = space + struct.w_ctx = self + index = compute_unique_id(self) + SERVERNAME_CALLBACKS.set(index, struct) + libssl_SSL_CTX_set_tlsext_servername_callback( + self.ctx, _servername_callback) + libssl_SSL_CTX_set_tlsext_servername_arg(self.ctx, + rffi.cast(rffi.VOIDP, index)) + _SSLContext.typedef = TypeDef( "_ssl._SSLContext", __new__=interp2app(_SSLContext.descr_new), @@ -1414,6 +1490,7 @@ set_default_verify_paths=interp2app(_SSLContext.descr_set_default_verify_paths), _set_npn_protocols=interp2app(_SSLContext.set_npn_protocols_w), get_ca_certs=interp2app(_SSLContext.get_ca_certs_w), + set_servername_callback=interp2app(_SSLContext.set_servername_callback_w), options=GetSetProperty(_SSLContext.descr_get_options, _SSLContext.descr_set_options), diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py --- a/rpython/rlib/ropenssl.py +++ b/rpython/rlib/ropenssl.py @@ -115,6 +115,12 @@ SSL_MODE_AUTO_RETRY = rffi_platform.ConstantInteger("SSL_MODE_AUTO_RETRY") SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER = rffi_platform.ConstantInteger("SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER") SSL_TLSEXT_ERR_OK = rffi_platform.ConstantInteger("SSL_TLSEXT_ERR_OK") + SSL_TLSEXT_ERR_ALERT_FATAL = rffi_platform.ConstantInteger("SSL_TLSEXT_ERR_ALERT_FATAL") + + SSL_AD_INTERNAL_ERROR = rffi_platform.ConstantInteger("SSL_AD_INTERNAL_ERROR") + SSL_AD_HANDSHAKE_FAILURE = rffi_platform.ConstantInteger("SSL_AD_HANDSHAKE_FAILURE") + + TLSEXT_NAMETYPE_host_name = rffi_platform.ConstantInteger("TLSEXT_NAMETYPE_host_name") ERR_LIB_X509 = rffi_platform.ConstantInteger("ERR_LIB_X509") ERR_LIB_PEM = rffi_platform.ConstantInteger("ERR_LIB_PEM") @@ -273,6 +279,11 @@ pem_password_cb = lltype.Ptr(lltype.FuncType([rffi.CCHARP, rffi.INT, rffi.INT, rffi.VOIDP], rffi.INT)) ssl_external('SSL_CTX_set_default_passwd_cb', [SSL_CTX, pem_password_cb], lltype.Void) ssl_external('SSL_CTX_set_default_passwd_cb_userdata', [SSL_CTX, rffi.VOIDP], lltype.Void) +servername_cb = lltype.Ptr(lltype.FuncType([SSL, rffi.INTP, rffi.VOIDP], rffi.INT)) +ssl_external('SSL_CTX_set_tlsext_servername_callback', [SSL_CTX, servername_cb], + lltype.Void, macro=True) +ssl_external('SSL_CTX_set_tlsext_servername_arg', [SSL_CTX, rffi.VOIDP], lltype.Void) + SSL_CTX_STATS_NAMES = """ number connect connect_good connect_renegotiate accept accept_good accept_renegotiate hits misses timeouts cache_full""".split() @@ -303,6 +314,8 @@ ssl_external('SSL_get_version', [SSL], rffi.CCHARP) ssl_external('SSL_get_peer_certificate', [SSL], X509) +ssl_external('SSL_get_servername', [SSL, rffi.INT], rffi.CCHARP) +ssl_external('SSL_get_app_data', [SSL], rffi.VOIDP, macro=True) ssl_external('X509_get_subject_name', [X509], X509_NAME) ssl_external('X509_get_issuer_name', [X509], X509_NAME) ssl_external('X509_NAME_oneline', [X509_NAME, rffi.CCHARP, rffi.INT], rffi.CCHARP) From noreply at buildbot.pypy.org Sun Feb 1 19:34:06 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 1 Feb 2015 19:34:06 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.9: Fixes Message-ID: <20150201183406.4E3A21C0FDE@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: stdlib-2.7.9 Changeset: r75631:adaa2c6b7b13 Date: 2015-02-01 19:32 +0100 http://bitbucket.org/pypy/pypy/changeset/adaa2c6b7b13/ Log: Fixes diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -170,6 +170,8 @@ NPN_STORAGE = RWeakValueDictionary(r_uint, SSLNpnProtocols) +SOCKET_STORAGE = RWeakValueDictionary(int, W_Root) + if HAVE_OPENSSL_RAND: # helper routines for seeding the SSL PRNG @@ -222,6 +224,11 @@ sock_fd = space.int_w(space.call_method(w_sock, "fileno")) self.ssl = libssl_SSL_new(w_ctx.ctx) # new ssl struct + + index = compute_unique_id(self) + libssl_SSL_set_app_data(self.ssl, rffi.cast(rffi.VOIDP, index)) + SOCKET_STORAGE.set(index, self) + libssl_SSL_set_fd(self.ssl, sock_fd) # set the socket for SSL # The ACCEPT_MOVING_WRITE_BUFFER flag is necessary because the address # of a str object may be changed by the garbage collector. @@ -1068,21 +1075,22 @@ struct = SERVERNAME_CALLBACKS.get(rffi.cast(lltype.Signed, arg)) w_ctx = struct.w_ctx space = struct.space - servername = libssl_SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name) if not w_ctx.w_set_hostname: - # remove race condition. + # Possible race condition. return SSL_TLSEXT_ERR_OK # The high-level ssl.SSLSocket object - w_ssl = libssl_SSL_get_app_data(ssl) + index = rffi.cast(lltype.Signed, libssl_SSL_get_app_data(ssl)) + w_ssl = SOCKET_STORAGE.get(index) assert isinstance(w_ssl, _SSLSocket) - if w_ssl.ssl_sock_weakref_w: - w_ssl_socket = w_ssl.ssl_sock_weakref_w.get() + if w_ssl.ssl_sock_weakref_w is not None: + w_ssl_socket = w_ssl.ssl_sock_weakref_w() else: w_ssl_socket = space.w_None if space.is_none(w_ssl_socket): - ad[0] = SSL_AD_INTERNAL_ERROR + ad[0] = rffi.cast(rffi.INT, SSL_AD_INTERNAL_ERROR) return SSL_TLSEXT_ERR_ALERT_FATAL + servername = libssl_SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name) try: if not servername: w_result = space.call_function(w_ctx.w_set_hostname, @@ -1090,30 +1098,30 @@ else: try: - w_servername = space.wrap_bytes(rffi.charp2str(servername)) + w_servername = space.wrapbytes(rffi.charp2str(servername)) w_servername_idna = space.call_method( w_servername, 'decode', space.wrap('idna')) except OperationError as e: - space.write_unraisable(e, "w_servername") - ad[0] = SSL_AD_INTERNAL_ERROR; + e.write_unraisable(space, w_ctx) + ad[0] = rffi.cast(rffi.INT, SSL_AD_INTERNAL_ERROR) return SSL_TLSEXT_ERR_ALERT_FATAL w_result = space.call_function(w_ctx.w_set_hostname, w_ssl_socket, w_servername_idna, w_ctx) except OperationError as e: - space.write_unraisable(e, "ssl_ctx->set_hostname") - ad[0] = SSL_AD_HANDSHAKE_FAILURE + e.write_unraisable(space, w_ctx.w_set_hostname) + ad[0] = rffi.cast(rffi.INT, SSL_AD_HANDSHAKE_FAILURE) return SSL_TLSEXT_ERR_ALERT_FATAL if space.is_none(w_result): return SSL_TLSEXT_ERR_OK else: try: - ad[0] = space.int_w(w_result) + ad[0] = rffi.cast(rffi.INT, space.int_w(w_result)) except OperationError as e: - space.write_unraisable(e, "w_result") - ad[0] = SSL_AD_INTERNAL_ERROR + e.write_unraisable(space, w_result) + ad[0] = rffi.cast(rffi.INT, SSL_AD_INTERNAL_ERROR) return SSL_TLSEXT_ERR_ALERT_FATAL diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py --- a/pypy/objspace/fake/objspace.py +++ b/pypy/objspace/fake/objspace.py @@ -182,6 +182,9 @@ "NOT_RPYTHON" raise NotImplementedError + def wrapbytes(self, x): + return w_some_obj() + def wrap(self, x): if not we_are_translated(): if isinstance(x, gateway.interp2app): diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py --- a/pypy/objspace/std/objspace.py +++ b/pypy/objspace/std/objspace.py @@ -127,6 +127,9 @@ assert typedef is not None return self.fromcache(TypeCache).getorbuild(typedef) + def wrapbytes(self, x): + return wrapstr(self, x) + def wrap(self, x): "Wraps the Python value 'x' into one of the wrapper classes." # You might notice that this function is rather conspicuously diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py --- a/rpython/rlib/ropenssl.py +++ b/rpython/rlib/ropenssl.py @@ -316,6 +316,7 @@ ssl_external('SSL_get_peer_certificate', [SSL], X509) ssl_external('SSL_get_servername', [SSL, rffi.INT], rffi.CCHARP) ssl_external('SSL_get_app_data', [SSL], rffi.VOIDP, macro=True) +ssl_external('SSL_set_app_data', [SSL, rffi.VOIDP], lltype.Void, macro=True) ssl_external('X509_get_subject_name', [X509], X509_NAME) ssl_external('X509_get_issuer_name', [X509], X509_NAME) ssl_external('X509_NAME_oneline', [X509_NAME, rffi.CCHARP, rffi.INT], rffi.CCHARP) From noreply at buildbot.pypy.org Sun Feb 1 20:38:04 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 1 Feb 2015 20:38:04 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.9: Translation fix Message-ID: <20150201193804.945591C0FDD@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: stdlib-2.7.9 Changeset: r75632:4c44fb30552f Date: 2015-02-01 20:37 +0100 http://bitbucket.org/pypy/pypy/changeset/4c44fb30552f/ Log: Translation fix diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -251,6 +251,7 @@ self.socket_type = socket_type self.w_socket = w_sock + assert w_ssl_sock is not None self.ssl_sock_weakref_w = rweakref.ref(w_ssl_sock) return self From noreply at buildbot.pypy.org Sun Feb 1 21:43:57 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 1 Feb 2015 21:43:57 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.9: Note to self: always run tests. Message-ID: <20150201204357.45E2C1C02CD@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: stdlib-2.7.9 Changeset: r75634:6a56bba1899b Date: 2015-02-01 21:43 +0100 http://bitbucket.org/pypy/pypy/changeset/6a56bba1899b/ Log: Note to self: always run tests. diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -382,5 +382,8 @@ return "" % ( self.co_name, self.co_filename, self.co_firstlineno) + def __repr__(self): + return self.get_repr() + def repr(self, space): return space.wrap(self.get_repr()) diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -80,6 +80,12 @@ self.initialize_frame_scopes(outer_func, code) self.f_lineno = code.co_firstlineno + def __repr__(self): + # NOT_RPYTHON: useful in tracebacks + return "<%s.%s executing %s at line %s" % ( + self.__class__.__module__, self.__class__.__name__, + self.pycode, self.get_last_lineno()) + def mark_as_escaped(self): """ Must be called on frames that are exposed to applevel, e.g. by diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -251,8 +251,10 @@ self.socket_type = socket_type self.w_socket = w_sock - assert w_ssl_sock is not None - self.ssl_sock_weakref_w = rweakref.ref(w_ssl_sock) + if w_ssl_sock: + self.ssl_sock_weakref_w = rweakref.ref(w_ssl_sock) + else: + self.ssl_sock_weakref_w = None return self def __del__(self): From noreply at buildbot.pypy.org Sun Feb 1 23:44:30 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 1 Feb 2015 23:44:30 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.9: Fixes for test_sni_callback Message-ID: <20150201224430.10E111C0507@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: stdlib-2.7.9 Changeset: r75635:7834e24c3b41 Date: 2015-02-01 23:44 +0100 http://bitbucket.org/pypy/pypy/changeset/7834e24c3b41/ Log: Fixes for test_sni_callback diff --git a/pypy/bin/pyinteractive.py b/pypy/bin/pyinteractive.py --- a/pypy/bin/pyinteractive.py +++ b/pypy/bin/pyinteractive.py @@ -10,7 +10,8 @@ import sys import time -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) +pypy_path = os.path.join(os.path.dirname(__file__), '..', '..') +sys.path.insert(0, os.path.abspath(pypy_path)) from pypy.tool import option from pypy.interpreter import main, interactive, error, gateway diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1506,6 +1506,8 @@ def str_w(self, w_obj): return w_obj.str_w(self) + bytes_w = str_w # Python2 + def str0_w(self, w_obj): "Like str_w, but rejects strings with NUL bytes." from rpython.rlib import rstring diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -213,7 +213,8 @@ class _SSLSocket(W_Root): @staticmethod - def descr_new(space, w_ctx, w_sock, socket_type, hostname, w_ssl_sock): + def descr_new(space, w_ctx, w_sock, socket_type, + w_hostname=None, w_ssl_sock=None): self = _SSLSocket() self.space = space @@ -235,6 +236,13 @@ libssl_SSL_set_mode( self.ssl, SSL_MODE_AUTO_RETRY | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER) + if not space.is_none(w_hostname): + if space.isinstance_w(w_hostname, space.w_unicode): + w_hostname = space.call_method(w_hostname, "encode", + space.wrap("idna")) + libssl_SSL_set_tlsext_host_name( + self.ssl, space.bytes_w(w_hostname)) + # If the socket is in non-blocking mode or timeout mode, set the BIO # to non-blocking mode (blocking is the default) w_timeout = space.call_method(w_sock, "gettimeout") @@ -1078,9 +1086,10 @@ struct = SERVERNAME_CALLBACKS.get(rffi.cast(lltype.Signed, arg)) w_ctx = struct.w_ctx space = struct.space - if not w_ctx.w_set_hostname: + w_callback = struct.w_set_hostname + if not w_ctx.servername_callback: # Possible race condition. - return SSL_TLSEXT_ERR_OK + return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_OK) # The high-level ssl.SSLSocket object index = rffi.cast(lltype.Signed, libssl_SSL_get_app_data(ssl)) w_ssl = SOCKET_STORAGE.get(index) @@ -1091,12 +1100,12 @@ w_ssl_socket = space.w_None if space.is_none(w_ssl_socket): ad[0] = rffi.cast(rffi.INT, SSL_AD_INTERNAL_ERROR) - return SSL_TLSEXT_ERR_ALERT_FATAL + return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_ALERT_FATAL) servername = libssl_SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name) try: if not servername: - w_result = space.call_function(w_ctx.w_set_hostname, + w_result = space.call_function(w_callback, w_ssl_socket, space.w_None, w_ctx) else: @@ -1107,25 +1116,25 @@ except OperationError as e: e.write_unraisable(space, "undecodable server name") ad[0] = rffi.cast(rffi.INT, SSL_AD_INTERNAL_ERROR) - return SSL_TLSEXT_ERR_ALERT_FATAL + return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_ALERT_FATAL) - w_result = space.call_function(w_ctx.w_set_hostname, + w_result = space.call_function(w_callback, w_ssl_socket, w_servername_idna, w_ctx) except OperationError as e: e.write_unraisable(space, "in servername callback") ad[0] = rffi.cast(rffi.INT, SSL_AD_HANDSHAKE_FAILURE) - return SSL_TLSEXT_ERR_ALERT_FATAL + return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_ALERT_FATAL) if space.is_none(w_result): - return SSL_TLSEXT_ERR_OK + return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_OK) else: try: ad[0] = rffi.cast(rffi.INT, space.int_w(w_result)) except OperationError as e: - e.write_unraisable(space, "bad result in servername callback") + e.write_unraisable(space, "servername callback result") ad[0] = rffi.cast(rffi.INT, SSL_AD_INTERNAL_ERROR) - return SSL_TLSEXT_ERR_ALERT_FATAL + return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_ALERT_FATAL) class _SSLContext(W_Root): @@ -1475,15 +1484,17 @@ if space.is_none(w_callback): libssl_SSL_CTX_set_tlsext_servername_callback( self.ctx, lltype.nullptr(servername_cb.TO)) + self.servername_callback = None return if not space.is_true(space.callable(w_callback)): raise oefmt(space.w_TypeError, "not a callable object") - self.w_set_hostname = w_callback - struct = ServernameCallback() - struct.space = space - struct.w_ctx = self + callback_struct = ServernameCallback() + callback_struct.space = space + callback_struct.w_ctx = self + callback_struct.w_set_hostname = w_callback + self.servername_callback = callback_struct index = compute_unique_id(self) - SERVERNAME_CALLBACKS.set(index, struct) + SERVERNAME_CALLBACKS.set(index, callback_struct) libssl_SSL_CTX_set_tlsext_servername_callback( self.ctx, _servername_callback) libssl_SSL_CTX_set_tlsext_servername_arg(self.ctx, diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py --- a/rpython/rlib/ropenssl.py +++ b/rpython/rlib/ropenssl.py @@ -282,7 +282,7 @@ servername_cb = lltype.Ptr(lltype.FuncType([SSL, rffi.INTP, rffi.VOIDP], rffi.INT)) ssl_external('SSL_CTX_set_tlsext_servername_callback', [SSL_CTX, servername_cb], lltype.Void, macro=True) -ssl_external('SSL_CTX_set_tlsext_servername_arg', [SSL_CTX, rffi.VOIDP], lltype.Void) +ssl_external('SSL_CTX_set_tlsext_servername_arg', [SSL_CTX, rffi.VOIDP], lltype.Void, macro=True) SSL_CTX_STATS_NAMES = """ number connect connect_good connect_renegotiate accept accept_good From noreply at buildbot.pypy.org Mon Feb 2 08:22:56 2015 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 2 Feb 2015 08:22:56 +0100 (CET) Subject: [pypy-commit] pypy default: handle the easiest ravel('K') case - is_c_contiguous and no negative strides Message-ID: <20150202072256.D87E51C102E@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r75636:4d999c098b39 Date: 2015-02-02 00:33 +0200 http://bitbucket.org/pypy/pypy/changeset/4d999c098b39/ Log: handle the easiest ravel('K') case - is_c_contiguous and no negative strides diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -430,9 +430,15 @@ order = 'C' else: order = space.str_w(w_order) + if order == 'K' and is_c_contiguous(self.implementation): + for s in self.implementation.get_strides(): + if s < 0: + break + else: + order = 'C' if order != 'C': raise OperationError(space.w_NotImplementedError, space.wrap( - "order not implemented")) + "order != 'C' only partially implemented")) return self.reshape(space, space.wrap(-1)) @unwrap_spec(w_axis=WrappedDefault(None), diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -2994,6 +2994,7 @@ assert (arange(3).ravel() == arange(3)).all() assert (arange(6).reshape(2, 3).ravel() == arange(6)).all() assert (arange(6).reshape(2, 3).T.ravel() == [0, 3, 1, 4, 2, 5]).all() + assert (arange(3).ravel('K') == arange(3)).all() def test_nonzero(self): from numpy import array From noreply at buildbot.pypy.org Mon Feb 2 08:22:58 2015 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 2 Feb 2015 08:22:58 +0100 (CET) Subject: [pypy-commit] pypy default: imporove error message for missing dtype casting Message-ID: <20150202072258.1515B1C102E@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r75637:f7072390ff5d Date: 2015-02-02 00:51 +0200 http://bitbucket.org/pypy/pypy/changeset/f7072390ff5d/ Log: imporove error message for missing dtype casting diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py --- a/pypy/module/micronumpy/test/test_ufuncs.py +++ b/pypy/module/micronumpy/test/test_ufuncs.py @@ -7,6 +7,7 @@ from pypy.module.micronumpy.concrete import VoidBoxStorage from pypy.interpreter.gateway import interp2app from pypy.conftest import option +from pypy.interpreter.error import OperationError class TestUfuncCoercion(object): @@ -129,7 +130,10 @@ '', ufunc.dtypes) assert index == 0 assert dtypes == [f32_dtype, c64_dtype] - + raises(OperationError, ufunc.type_resolver, space, [f32_array], [None], + 'u->u', ufunc.dtypes) + exc = raises(OperationError, ufunc.type_resolver, space, [f32_array], [None], + 'i->i', ufunc.dtypes) class AppTestUfuncs(BaseNumpyAppTest): def test_constants(self): diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py --- a/pypy/module/micronumpy/ufuncs.py +++ b/pypy/module/micronumpy/ufuncs.py @@ -733,20 +733,24 @@ nop = len(inargs) + len(outargs) dtypes = [] if isinstance(type_tup, str) and len(type_tup) > 0: - if len(type_tup) == 1: - dtypes = [get_dtype_cache(space).dtypes_by_name[type_tup]] * self.nargs - elif len(type_tup) == self.nargs + 2: - for i in range(self.nin): - dtypes.append(get_dtype_cache(space).dtypes_by_name[type_tup[i]]) - #skip the '->' in the signature - for i in range(self.nout): - j = i + self.nin + 2 - dtypes.append(get_dtype_cache(space).dtypes_by_name[type_tup[j]]) - else: - raise oefmt(space.w_TypeError, "a type-string for %s " \ - "requires 1 typecode or %d typecode(s) before and %d" \ - " after the -> sign, not '%s'", self.name, self.nin, - self.nout, type_tup) + try: + if len(type_tup) == 1: + dtypes = [get_dtype_cache(space).dtypes_by_name[type_tup]] * self.nargs + elif len(type_tup) == self.nargs + 2: + for i in range(self.nin): + dtypes.append(get_dtype_cache(space).dtypes_by_name[type_tup[i]]) + #skip the '->' in the signature + for i in range(self.nout): + j = i + self.nin + 2 + dtypes.append(get_dtype_cache(space).dtypes_by_name[type_tup[j]]) + else: + raise oefmt(space.w_TypeError, "a type-string for %s " \ + "requires 1 typecode or %d typecode(s) before and %d" \ + " after the -> sign, not '%s'", self.name, self.nin, + self.nout, type_tup) + except KeyError: + raise oefmt(space.w_ValueError, "unknown typecode in" \ + " call to %s with type-string '%s'", self.name, type_tup) else: # XXX why does the next line not pass translation? # dtypes = [i.get_dtype() for i in inargs] @@ -770,9 +774,13 @@ break else: if len(self.funcs) > 1: + dtypesstr = ','.join(['%s%s%s' % (d.byteorder, d.kind, d.elsize) \ + for d in dtypes]) + _dtypesstr = ','.join(['%s%s%s' % (d.byteorder, d.kind, d.elsize) \ + for d in _dtypes]) raise oefmt(space.w_TypeError, - 'input dtype did not match any known dtypes', - ) + "input dtype [%s] did not match any known dtypes [%s] ", + dtypesstr,_dtypesstr) i = 0 # Fill in empty dtypes for j in range(self.nargs): From noreply at buildbot.pypy.org Mon Feb 2 08:22:59 2015 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 2 Feb 2015 08:22:59 +0100 (CET) Subject: [pypy-commit] pypy default: test, fix and start to optimize a=np.array(b) where b is a ndarray subtype Message-ID: <20150202072259.4493F1C102E@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r75638:a0d3e64efd8b Date: 2015-02-02 09:23 +0200 http://bitbucket.org/pypy/pypy/changeset/a0d3e64efd8b/ Log: test, fix and start to optimize a=np.array(b) where b is a ndarray subtype diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py --- a/pypy/module/micronumpy/ctors.py +++ b/pypy/module/micronumpy/ctors.py @@ -82,9 +82,18 @@ return w_object.descr_copy(space, w_order) elif not copy and (subok or type(w_object) is W_NDimArray): return w_object - - # not an array or incorrect dtype - shape, elems_w = strides.find_shape_and_elems(space, w_object, dtype) + if isinstance(w_object, W_NDimArray) and copy and not subok: + # TODO do the loop.assign without copying elems_w + shape = w_object.get_shape() + _elems_w = w_object.reshape(space, space.wrap(-1)) + elems_w = [None] * w_object.get_size() + for i in range(len(elems_w)): + elems_w[i] = _elems_w.descr_getitem(space, space.wrap(i)) + if space.is_none(w_dtype): + dtype = w_object.get_dtype() + else: + # not an array + shape, elems_w = strides.find_shape_and_elems(space, w_object, dtype) if dtype is None or (dtype.is_str_or_unicode() and dtype.elsize < 1): dtype = strides.find_dtype_for_seq(space, elems_w, dtype) if dtype is None: diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py --- a/pypy/module/micronumpy/test/test_subtype.py +++ b/pypy/module/micronumpy/test/test_subtype.py @@ -304,10 +304,13 @@ out.shape = (sh, 1) else: out.shape = (1, sh) - print 'out, shape was',old_shape,'now',out.shape + #print 'out, shape was',old_shape,'now',out.shape,'out',out return out - a = matrix([[1., 2.]]) + a = matrix([[1., 2.], [3., 4.]]) b = N.array([a]) + assert (b == a).all() + b = N.array(a) + assert len(b.shape) == 2 def test_setstate_no_version(self): # Some subclasses of ndarray, like MaskedArray, do not use From noreply at buildbot.pypy.org Mon Feb 2 08:30:01 2015 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 2 Feb 2015 08:30:01 +0100 (CET) Subject: [pypy-commit] pypy release-2.5.x: merge default into branch Message-ID: <20150202073001.147481C102E@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: release-2.5.x Changeset: r75639:7abc5b13f7f0 Date: 2015-02-02 09:26 +0200 http://bitbucket.org/pypy/pypy/changeset/7abc5b13f7f0/ Log: merge default into branch diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst --- a/pypy/doc/faq.rst +++ b/pypy/doc/faq.rst @@ -197,6 +197,55 @@ (now-dead) object are still true about the new object. + +Would type annotations help PyPy's performance? +----------------------------------------------- + +Two examples of type annotations that are being proposed for improved +performance are `Cython types`__ and `PEP 484 - Type Hints`__. + +.. __: http://docs.cython.org/src/reference/language_basics.html#declaring-data-types +.. __: https://www.python.org/dev/peps/pep-0484/ + +**Cython types** are, by construction, similar to C declarations. For +example, a local variable or an instance attribute can be declared +``"cdef int"`` to force a machine word to be used. This changes the +usual Python semantics (e.g. no overflow checks, and errors when +trying to write other types of objects there). It gives some extra +performance, but the exact benefits are unclear: right now +(January 2015) for example we are investigating a technique that would +store machine-word integers directly on instances, giving part of the +benefits without the user-supplied ``"cdef int"``. + +**PEP 484 - Type Hints,** on the other hand, is almost entirely +useless if you're looking at performance. First, as the name implies, +they are *hints:* they must still be checked at runtime, like PEP 484 +says. Or maybe you're fine with a mode in which you get very obscure +crashes when the type annotations are wrong; but even in that case the +speed benefits would be extremely minor. + +There are several reasons for why. One of them is that annotations +are at the wrong level (e.g. a PEP 484 "int" corresponds to Python 3's +int type, which does not necessarily fits inside one machine word; +even worse, an "int" annotation allows arbitrary int subclasses). +Another is that a lot more information is needed to produce good code +(e.g. "this ``f()`` called here really means this function there, and +will never be monkey-patched" -- same with ``len()`` or ``list()``, +btw). The third reason is that some "guards" in PyPy's JIT traces +don't really have an obvious corresponding type (e.g. "this dict is so +far using keys which don't override ``__hash__`` so a more efficient +implementation was used"). Many guards don't even have any correspondence +with types at all ("this class attribute was not modified"; "the loop +counter did not reach zero so we don't need to release the GIL"; and +so on). + +As PyPy works right now, it is able to derive far more useful +information than can ever be given by PEP 484, and it works +automatically. As far as we know, this is true even if we would add +other techniques to PyPy, like a fast first-pass JIT. + + + .. _`prolog and javascript`: Can I use PyPy's translation toolchain for other languages besides Python? diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1017,6 +1017,9 @@ def newlist_unicode(self, list_u): return self.newlist([self.wrap(u) for u in list_u]) + def newlist_int(self, list_i): + return self.newlist([self.wrap(i) for i in list_i]) + def newlist_hint(self, sizehint): from pypy.objspace.std.listobject import make_empty_list_with_size return make_empty_list_with_size(self, sizehint) diff --git a/pypy/module/cpyext/ndarrayobject.py b/pypy/module/cpyext/ndarrayobject.py --- a/pypy/module/cpyext/ndarrayobject.py +++ b/pypy/module/cpyext/ndarrayobject.py @@ -291,6 +291,6 @@ Py_ssize_t, Py_ssize_t, rffi.CCHARP, rffi.CCHARP, Py_ssize_t], PyObject) def PyUFunc_FromFuncAndData(space, funcs, data, types, ntypes, nin, nout, identity, name, doc, check_return): - w_signature = "" + w_signature = ','.join(['()'] * nin) + '->' + ','.join(['()'] * nout) return do_ufunc(space, funcs, data, types, ntypes, nin, nout, identity, name, doc, check_return, w_signature) diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py --- a/pypy/module/micronumpy/ctors.py +++ b/pypy/module/micronumpy/ctors.py @@ -82,9 +82,18 @@ return w_object.descr_copy(space, w_order) elif not copy and (subok or type(w_object) is W_NDimArray): return w_object - - # not an array or incorrect dtype - shape, elems_w = strides.find_shape_and_elems(space, w_object, dtype) + if isinstance(w_object, W_NDimArray) and copy and not subok: + # TODO do the loop.assign without copying elems_w + shape = w_object.get_shape() + _elems_w = w_object.reshape(space, space.wrap(-1)) + elems_w = [None] * w_object.get_size() + for i in range(len(elems_w)): + elems_w[i] = _elems_w.descr_getitem(space, space.wrap(i)) + if space.is_none(w_dtype): + dtype = w_object.get_dtype() + else: + # not an array + shape, elems_w = strides.find_shape_and_elems(space, w_object, dtype) if dtype is None or (dtype.is_str_or_unicode() and dtype.elsize < 1): dtype = strides.find_dtype_for_seq(space, elems_w, dtype) if dtype is None: diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -430,9 +430,15 @@ order = 'C' else: order = space.str_w(w_order) + if order == 'K' and is_c_contiguous(self.implementation): + for s in self.implementation.get_strides(): + if s < 0: + break + else: + order = 'C' if order != 'C': raise OperationError(space.w_NotImplementedError, space.wrap( - "order not implemented")) + "order != 'C' only partially implemented")) return self.reshape(space, space.wrap(-1)) @unwrap_spec(w_axis=WrappedDefault(None), diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -2994,6 +2994,7 @@ assert (arange(3).ravel() == arange(3)).all() assert (arange(6).reshape(2, 3).ravel() == arange(6)).all() assert (arange(6).reshape(2, 3).T.ravel() == [0, 3, 1, 4, 2, 5]).all() + assert (arange(3).ravel('K') == arange(3)).all() def test_nonzero(self): from numpy import array diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py --- a/pypy/module/micronumpy/test/test_subtype.py +++ b/pypy/module/micronumpy/test/test_subtype.py @@ -304,10 +304,13 @@ out.shape = (sh, 1) else: out.shape = (1, sh) - print 'out, shape was',old_shape,'now',out.shape + #print 'out, shape was',old_shape,'now',out.shape,'out',out return out - a = matrix([[1., 2.]]) + a = matrix([[1., 2.], [3., 4.]]) b = N.array([a]) + assert (b == a).all() + b = N.array(a) + assert len(b.shape) == 2 def test_setstate_no_version(self): # Some subclasses of ndarray, like MaskedArray, do not use diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py --- a/pypy/module/micronumpy/test/test_ufuncs.py +++ b/pypy/module/micronumpy/test/test_ufuncs.py @@ -7,6 +7,7 @@ from pypy.module.micronumpy.concrete import VoidBoxStorage from pypy.interpreter.gateway import interp2app from pypy.conftest import option +from pypy.interpreter.error import OperationError class TestUfuncCoercion(object): @@ -129,7 +130,10 @@ '', ufunc.dtypes) assert index == 0 assert dtypes == [f32_dtype, c64_dtype] - + raises(OperationError, ufunc.type_resolver, space, [f32_array], [None], + 'u->u', ufunc.dtypes) + exc = raises(OperationError, ufunc.type_resolver, space, [f32_array], [None], + 'i->i', ufunc.dtypes) class AppTestUfuncs(BaseNumpyAppTest): def test_constants(self): @@ -169,8 +173,7 @@ dtypes=[int, int, int, float, float, float]) int_func22 = frompyfunc([int, int], 2, 2, signature='(i),(i)->(i),(i)', dtypes=['match']) - int_func12 = frompyfunc([int], 1, 2, signature='(i)->(i),(i)', - dtypes=['match']) + int_func12 = frompyfunc([int], 1, 2, dtypes=['match']) retype = dtype(int) a = arange(10) assert isinstance(adder_ufunc1, ufunc) @@ -223,6 +226,7 @@ assert len(in_array.shape) == 2 assert in_array.shape == out_array.shape out_array[:] = in_array * 2 + from numpy import frompyfunc, dtype, arange ufunc = frompyfunc([times_2], 1, 1, signature='(m,n)->(n,m)', @@ -233,6 +237,7 @@ ai3 = ufunc(ai[0,:,:]) ai2 = ufunc(ai) assert (ai2 == ai * 2).all() + ufunc = frompyfunc([times_2], 1, 1, signature='(m,m)->(m,m)', dtypes=[dtype(int), dtype(int)], @@ -245,6 +250,21 @@ ai2 = ufunc(ai) assert (ai2 == ai * 2).all() + def test_frompyfunc_needs_nditer(self): + def summer(in0): + print 'in summer, in0=',in0,'in0.shape=',in0.shape + return in0.sum() + + from numpy import frompyfunc, dtype, arange + ufunc = frompyfunc([summer], 1, 1, + signature='(m,m)->()', + dtypes=[dtype(int), dtype(int)], + stack_inputs=False, + ) + ai = arange(12, dtype=int).reshape(3, 2, 2) + ao = ufunc(ai) + assert ao.size == 3 + def test_frompyfunc_sig_broadcast(self): def sum_along_0(in_array, out_array): out_array[...] = in_array.sum(axis=0) diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py --- a/pypy/module/micronumpy/ufuncs.py +++ b/pypy/module/micronumpy/ufuncs.py @@ -520,7 +520,9 @@ ''' _immutable_fields_ = ["funcs", "dtypes", "data", "match_dtypes"] - def __init__(self, space, funcs, name, identity, nin, nout, dtypes, signature, match_dtypes=False, stack_inputs=False): + def __init__(self, space, funcs, name, identity, nin, nout, dtypes, + signature, match_dtypes=False, stack_inputs=False, + external_loop=False): # XXX make sure funcs, signature, dtypes, nin, nout are consistent # These don't matter, we use the signature and dtypes for determining @@ -549,6 +551,7 @@ self.core_num_dims = [0] * self.nargs # number of core dimensions of each nargs self.core_offsets = [0] * self.nargs self.core_dim_ixs = [] # indices into unique shapes for each arg + self.external_loop = external_loop def reduce(self, space, w_obj, w_axis, keepdims=False, out=None, dtype=None, cumulative=False): @@ -586,29 +589,21 @@ _dtypes.append(_dtypes[0]) index, dtypes = self.type_resolver(space, inargs, outargs, sig, _dtypes) func = self.funcs[index] - if not self.core_enabled: - # func is going to do all the work, it must accept W_NDimArray args - inargs0 = inargs[0] - assert isinstance(inargs0, W_NDimArray) - arg_shapes = [inargs0.get_shape()] * self.nargs - inargs, outargs, need_to_cast = self.alloc_args(space, inargs, outargs, - dtypes, arg_shapes) - for tf in need_to_cast: - if tf: - raise oefmt(space.w_NotImplementedError, "casting not supported yet") - if self.stack_inputs: - arglist = space.newlist(list(inargs + outargs)) - space.call_args(func, Arguments.frompacked(space, arglist)) - else: - arglist = space.newlist(inargs) - outargs = space.call_args(func, Arguments.frompacked(space, arglist)) - return outargs - if len(outargs) < 2: - return outargs[0] - return space.newtuple(outargs) iter_shape, arg_shapes, matched_dims = self.verify_args(space, inargs, outargs) inargs, outargs, need_to_cast = self.alloc_args(space, inargs, outargs, dtypes, arg_shapes) + if not self.external_loop: + inargs0 = inargs[0] + outargs0 = outargs[0] + assert isinstance(inargs0, W_NDimArray) + assert isinstance(outargs0, W_NDimArray) + res_dtype = outargs0.get_dtype() + new_shape = inargs0.get_shape() + if len(outargs) < 2: + return loop.call_many_to_one(space, new_shape, func, + res_dtype, inargs, outargs[0]) + return loop.call_many_to_many(space, new_shape, func, + res_dtype, inargs, outargs) for tf in need_to_cast: if tf: raise oefmt(space.w_NotImplementedError, "casting not supported yet") @@ -619,44 +614,44 @@ w_casting = space.w_None w_op_axes = space.w_None + #print '\nsignature', sig + #print [(d, getattr(self,d)) for d in dir(self) if 'core' in d or 'broad' in d] + #print [(d, locals()[d]) for d in locals() if 'core' in d or 'broad' in d] + #print 'shapes',[d.get_shape() for d in inargs + outargs] + #print 'steps',[d.implementation.strides for d in inargs + outargs] + if isinstance(func, W_GenericUFuncCaller): + # Use GeneralizeUfunc interface with signature + # Unlike numpy, we will not broadcast dims before + # the core_ndims rather we use nditer iteration + # so dims[0] == 1 + dims = [1] + matched_dims + steps = [] + allargs = inargs + outargs + for i in range(len(allargs)): + steps.append(0) + for i in range(len(allargs)): + _arg = allargs[i] + assert isinstance(_arg, W_NDimArray) + start_dim = len(iter_shape) + steps += _arg.implementation.strides[start_dim:] + func.set_dims_and_steps(space, dims, steps) + else: + # it is a function, ready to be called by the iterator, + # from frompyfunc + pass + # mimic NpyIter_AdvancedNew with a nditer + w_itershape = space.newlist([space.wrap(i) for i in iter_shape]) + nd_it = W_NDIter(space, space.newlist(inargs + outargs), w_flags, + w_op_flags, w_op_dtypes, w_casting, w_op_axes, + w_itershape) + # coalesce each iterators, according to inner_dimensions + for i in range(len(inargs) + len(outargs)): + for j in range(self.core_num_dims[i]): + new_iter = coalesce_iter(nd_it.iters[i][0], nd_it.op_flags[i], + nd_it, nd_it.order, flat=False) + nd_it.iters[i] = (new_iter, new_iter.reset()) + # do the iteration if self.stack_inputs: - #print '\nsignature', sig - #print [(d, getattr(self,d)) for d in dir(self) if 'core' in d or 'broad' in d] - #print [(d, locals()[d]) for d in locals() if 'core' in d or 'broad' in d] - #print 'shapes',[d.get_shape() for d in inargs + outargs] - #print 'steps',[d.implementation.strides for d in inargs + outargs] - if isinstance(func, W_GenericUFuncCaller): - # Use GeneralizeUfunc interface with signature - # Unlike numpy, we will not broadcast dims before - # the core_ndims rather we use nditer iteration - # so dims[0] == 1 - dims = [1] + matched_dims - steps = [] - allargs = inargs + outargs - for i in range(len(allargs)): - steps.append(0) - for i in range(len(allargs)): - _arg = allargs[i] - assert isinstance(_arg, W_NDimArray) - start_dim = len(iter_shape) - steps += _arg.implementation.strides[start_dim:] - func.set_dims_and_steps(space, dims, steps) - else: - # it is a function, ready to be called by the iterator, - # from frompyfunc - pass - # mimic NpyIter_AdvancedNew with a nditer - w_itershape = space.newlist([space.wrap(i) for i in iter_shape]) - nd_it = W_NDIter(space, space.newlist(inargs + outargs), w_flags, - w_op_flags, w_op_dtypes, w_casting, w_op_axes, - w_itershape) - # coalesce each iterators, according to inner_dimensions - for i in range(len(inargs) + len(outargs)): - for j in range(self.core_num_dims[i]): - new_iter = coalesce_iter(nd_it.iters[i][0], nd_it.op_flags[i], - nd_it, nd_it.order, flat=False) - nd_it.iters[i] = (new_iter, new_iter.reset()) - # do the iteration while not nd_it.done: # XXX jit me for it, st in nd_it.iters: @@ -670,20 +665,35 @@ args.append(nd_it.getitem(it, st)) nd_it.iters[i] = (it, it.next(st)) space.call_args(func, Arguments.frompacked(space, space.newlist(args))) - if len(outargs) > 1: - return space.newtuple([convert_to_array(space, o) for o in outargs]) - return outargs[0] - inargs0 = inargs[0] - outargs0 = outargs[0] - assert isinstance(inargs0, W_NDimArray) - assert isinstance(outargs0, W_NDimArray) - res_dtype = outargs0.get_dtype() - new_shape = inargs0.get_shape() - if len(outargs) < 2: - return loop.call_many_to_one(space, new_shape, func, - res_dtype, inargs, outargs[0]) - return loop.call_many_to_many(space, new_shape, func, - res_dtype, inargs, outargs) + else: + # do the iteration + while not nd_it.done: + # XXX jit me + for it, st in nd_it.iters: + if not it.done(st): + break + else: + nd_it.done = True + break + initers = [] + outiters = [] + nin = len(inargs) + for i, (it, st) in enumerate(nd_it.iters[:nin]): + initers.append(nd_it.getitem(it, st)) + nd_it.iters[i] = (it, it.next(st)) + for i, (it, st) in enumerate(nd_it.iters[nin:]): + outiters.append(nd_it.getitem(it, st)) + nd_it.iters[i + nin] = (it, it.next(st)) + outs = space.call_args(func, Arguments.frompacked(space, space.newlist(initers))) + if len(outiters) < 2: + outiters[0].descr_setitem(space, space.w_Ellipsis, outs) + else: + for i in range(self.nout): + w_val = space.getitem(outs, space.wrap(i)) + outiters[i].descr_setitem(space, space.w_Ellipsis, w_val) + if len(outargs) > 1: + return space.newtuple([convert_to_array(space, o) for o in outargs]) + return outargs[0] def parse_kwargs(self, space, kwargs_w): w_subok, w_out, casting, sig, extobj = \ @@ -723,20 +733,24 @@ nop = len(inargs) + len(outargs) dtypes = [] if isinstance(type_tup, str) and len(type_tup) > 0: - if len(type_tup) == 1: - dtypes = [get_dtype_cache(space).dtypes_by_name[type_tup]] * self.nargs - elif len(type_tup) == self.nargs + 2: - for i in range(self.nin): - dtypes.append(get_dtype_cache(space).dtypes_by_name[type_tup[i]]) - #skip the '->' in the signature - for i in range(self.nout): - j = i + self.nin + 2 - dtypes.append(get_dtype_cache(space).dtypes_by_name[type_tup[j]]) - else: - raise oefmt(space.w_TypeError, "a type-string for %s " \ - "requires 1 typecode or %d typecode(s) before and %d" \ - " after the -> sign, not '%s'", self.name, self.nin, - self.nout, type_tup) + try: + if len(type_tup) == 1: + dtypes = [get_dtype_cache(space).dtypes_by_name[type_tup]] * self.nargs + elif len(type_tup) == self.nargs + 2: + for i in range(self.nin): + dtypes.append(get_dtype_cache(space).dtypes_by_name[type_tup[i]]) + #skip the '->' in the signature + for i in range(self.nout): + j = i + self.nin + 2 + dtypes.append(get_dtype_cache(space).dtypes_by_name[type_tup[j]]) + else: + raise oefmt(space.w_TypeError, "a type-string for %s " \ + "requires 1 typecode or %d typecode(s) before and %d" \ + " after the -> sign, not '%s'", self.name, self.nin, + self.nout, type_tup) + except KeyError: + raise oefmt(space.w_ValueError, "unknown typecode in" \ + " call to %s with type-string '%s'", self.name, type_tup) else: # XXX why does the next line not pass translation? # dtypes = [i.get_dtype() for i in inargs] @@ -760,9 +774,13 @@ break else: if len(self.funcs) > 1: + dtypesstr = ','.join(['%s%s%s' % (d.byteorder, d.kind, d.elsize) \ + for d in dtypes]) + _dtypesstr = ','.join(['%s%s%s' % (d.byteorder, d.kind, d.elsize) \ + for d in _dtypes]) raise oefmt(space.w_TypeError, - 'input dtype did not match any known dtypes', - ) + "input dtype [%s] did not match any known dtypes [%s] ", + dtypesstr,_dtypesstr) i = 0 # Fill in empty dtypes for j in range(self.nargs): @@ -1235,7 +1253,8 @@ w_identity=None, name='', doc='', stack_inputs=False): ''' frompyfunc(func, nin, nout) #cpython numpy compatible frompyfunc(func, nin, nout, dtypes=None, signature='', - identity=None, name='', doc='', stack_inputs=False) + identity=None, name='', doc='', + stack_inputs=False) Takes an arbitrary Python function and returns a ufunc. @@ -1251,10 +1270,12 @@ nout : int The number of arrays returned by `func`. dtypes: None or [dtype, ...] of the input, output args for each function, - or 'match' to force output to exactly match input dtype + or 'match' to force output to exactly match input dtype + Note that 'match' is a pypy-only extension to allow non-object + return dtypes signature*: str, default='' The mapping of input args to output args, defining the - inner-loop indexing + inner-loop indexing. If it is empty, the func operates on scalars identity*: None (default) or int For reduce-type ufuncs, the default value name: str, default='' @@ -1273,7 +1294,7 @@ Notes ----- - If the signature and out_dtype are both missing, the returned ufunc + If the signature and dtype are both missing, the returned ufunc always returns PyObject arrays (cpython numpy compatability). Input arguments marked with a * are pypy-only extensions @@ -1329,16 +1350,15 @@ 'identity must be None or an int') if len(signature) == 0: - # cpython compatability, func is of the form (),()->() - signature = ','.join(['()'] * nin) + '->' + ','.join(['()'] * nout) + external_loop=False else: - #stack_inputs = True - pass + external_loop=True w_ret = W_UfuncGeneric(space, func, name, identity, nin, nout, dtypes, signature, match_dtypes=match_dtypes, - stack_inputs=stack_inputs) - _parse_signature(space, w_ret, w_ret.signature) + stack_inputs=stack_inputs, external_loop=external_loop) + if w_ret.external_loop: + _parse_signature(space, w_ret, w_ret.signature) if doc: w_ret.w_doc = space.wrap(doc) return w_ret diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py --- a/pypy/module/pypyjit/interp_jit.py +++ b/pypy/module/pypyjit/interp_jit.py @@ -173,7 +173,7 @@ exiting (blackhole) steps, but just not from the final assembler. Note that the return value of the callable is ignored, because -there is no reasonable way to guess what it sound be in case the +there is no reasonable way to guess what it should be in case the function is not called. This is meant to be used notably in sys.settrace() for coverage- diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py --- a/pypy/objspace/std/dictmultiobject.py +++ b/pypy/objspace/std/dictmultiobject.py @@ -1173,8 +1173,8 @@ def wrapkey(space, key): return space.wrap(key) - # XXX there is no space.newlist_int yet to implement w_keys more - # efficiently + def w_keys(self, w_dict): + return self.space.newlist_int(self.listview_int(w_dict)) create_iterator_classes(IntDictStrategy) diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -176,6 +176,12 @@ storage = strategy.erase(list_u) return W_ListObject.from_storage_and_strategy(space, storage, strategy) + @staticmethod + def newlist_int(space, list_i): + strategy = space.fromcache(IntegerListStrategy) + storage = strategy.erase(list_i) + return W_ListObject.from_storage_and_strategy(space, storage, strategy) + def __repr__(self): """ representation for debugging purposes """ return "%s(%s, %s)" % (self.__class__.__name__, self.strategy, diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py --- a/pypy/objspace/std/objspace.py +++ b/pypy/objspace/std/objspace.py @@ -288,6 +288,9 @@ def newlist_unicode(self, list_u): return W_ListObject.newlist_unicode(self, list_u) + def newlist_int(self, list_i): + return W_ListObject.newlist_int(self, list_i) + def newdict(self, module=False, instance=False, kwargs=False, strdict=False): return W_DictMultiObject.allocate_and_init_instance( diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py --- a/rpython/rtyper/rclass.py +++ b/rpython/rtyper/rclass.py @@ -711,8 +711,15 @@ continue value = self.classdef.classdesc.read_attribute(fldname, None) if value is not None: - cvalue = inputconst(r.lowleveltype, - r.convert_desc_or_const(value)) + ll_value = r.convert_desc_or_const(value) + # don't write NULL GC pointers: we know that the malloc + # done above initialized at least the GC Ptr fields to + # NULL already, and that's true for all our GCs + if (isinstance(r.lowleveltype, Ptr) and + r.lowleveltype.TO._gckind == 'gc' and + not ll_value): + continue + cvalue = inputconst(r.lowleveltype, ll_value) self.setfield(vptr, fldname, cvalue, llops, flags={'access_directly': True}) return vptr diff --git a/rpython/translator/backendopt/test/test_malloc.py b/rpython/translator/backendopt/test/test_malloc.py --- a/rpython/translator/backendopt/test/test_malloc.py +++ b/rpython/translator/backendopt/test/test_malloc.py @@ -340,3 +340,15 @@ u[0].s.x = x return u[0].s.x graph = self.check(f, [int], [42], 42) + + def test_two_paths_one_with_constant(self): + py.test.skip("XXX implement me?") + def fn(n): + if n > 100: + tup = (0,) + else: + tup = (n,) + (n,) # <- flowspace + return tup[0] + + self.check(fn, [int], [42], 42) diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -808,12 +808,7 @@ t, cbuilder = self.compile(entry_point, shared=True) assert cbuilder.shared_library_name is not None assert cbuilder.shared_library_name != cbuilder.executable_name - if os.name == 'posix': - library_path = cbuilder.shared_library_name.dirpath() - if sys.platform == 'darwin': - monkeypatch.setenv('DYLD_LIBRARY_PATH', library_path) - else: - monkeypatch.setenv('LD_LIBRARY_PATH', library_path) + #Do not set LD_LIBRARY_PATH, make sure $ORIGIN flag is working out, err = cbuilder.cmdexec("a b") assert out == "3" diff --git a/rpython/translator/platform/freebsd.py b/rpython/translator/platform/freebsd.py --- a/rpython/translator/platform/freebsd.py +++ b/rpython/translator/platform/freebsd.py @@ -12,6 +12,7 @@ cflags = tuple( ['-O3', '-pthread', '-fomit-frame-pointer'] + os.environ.get('CFLAGS', '').split()) + rpath_flags = ['-Wl,-rpath=\'$$ORIGIN/\'', '-Wl,-z,origin'] class Freebsd_64(Freebsd): shared_only = ('-fPIC',) diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py --- a/rpython/translator/platform/posix.py +++ b/rpython/translator/platform/posix.py @@ -112,9 +112,9 @@ target_name = exe_name.basename if shared: - cflags = self.cflags + self.get_shared_only_compile_flags() + cflags = tuple(self.cflags) + self.get_shared_only_compile_flags() else: - cflags = self.cflags + self.standalone_only + cflags = tuple(self.cflags) + tuple(self.standalone_only) m = GnuMakefile(path) m.exe_name = path.join(exe_name.basename) From noreply at buildbot.pypy.org Mon Feb 2 08:30:02 2015 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 2 Feb 2015 08:30:02 +0100 (CET) Subject: [pypy-commit] pypy release-2.5.x: update version info to release Message-ID: <20150202073002.685EC1C102E@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: release-2.5.x Changeset: r75640:75dd29f10ef3 Date: 2015-02-02 09:29 +0200 http://bitbucket.org/pypy/pypy/changeset/75dd29f10ef3/ Log: update version info to release diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,7 +29,7 @@ #define PY_VERSION "2.7.8" /* PyPy version as a string */ -#define PYPY_VERSION "2.5.0-beta0" +#define PYPY_VERSION "2.5.0-final0" /* Subversion Revision number of this file (not of the repository). * Empty since Mercurial migration. */ diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (2, 5, 0, "beta", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (2, 5, 0, "final", 0) #XXX # sync patchlevel.h if platform.name == 'msvc': COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600) From noreply at buildbot.pypy.org Mon Feb 2 09:26:33 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 2 Feb 2015 09:26:33 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.9: Add SSLContext.session_stats() Message-ID: <20150202082633.28A711C1032@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: stdlib-2.7.9 Changeset: r75641:d48d4991869d Date: 2015-02-02 08:43 +0100 http://bitbucket.org/pypy/pypy/changeset/d48d4991869d/ Log: Add SSLContext.session_stats() diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -1175,6 +1175,13 @@ libssl_ERR_clear_error() raise ssl_error(space, "No cipher can be selected.") + def session_stats_w(self, space): + w_stats = space.newdict() + for name, ssl_func in SSL_CTX_STATS: + w_value = space.wrap(ssl_func(self.ctx)) + space.setitem_str(w_stats, name, w_value) + return w_stats + def descr_set_default_verify_paths(self, space): if not libssl_SSL_CTX_set_default_verify_paths(self.ctx): raise ssl_error(space, "") @@ -1509,6 +1516,7 @@ load_cert_chain=interp2app(_SSLContext.load_cert_chain_w), load_dh_params=interp2app(_SSLContext.load_dh_params_w), load_verify_locations=interp2app(_SSLContext.load_verify_locations_w), + session_stats = interp2app(_SSLContext.session_stats_w), set_default_verify_paths=interp2app(_SSLContext.descr_set_default_verify_paths), _set_npn_protocols=interp2app(_SSLContext.set_npn_protocols_w), get_ca_certs=interp2app(_SSLContext.get_ca_certs_w), From noreply at buildbot.pypy.org Mon Feb 2 09:26:34 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 2 Feb 2015 09:26:34 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.9: ssl: Pass last ssl error code to the exception. Message-ID: <20150202082634.663631C1032@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: stdlib-2.7.9 Changeset: r75642:19337a71252a Date: 2015-02-02 09:17 +0100 http://bitbucket.org/pypy/pypy/changeset/19337a71252a/ Log: ssl: Pass last ssl error code to the exception. diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -932,9 +932,10 @@ def _ssl_seterror(space, ss, ret): assert ret <= 0 + errcode = libssl_ERR_peek_last_error() + if ss is None: - errval = libssl_ERR_peek_last_error() - return ssl_error(space, None, errcode=errval) + return ssl_error(space, None, errcode=errcode) elif ss.ssl: err = libssl_SSL_get_error(ss.ssl, ret) else: @@ -979,17 +980,17 @@ errstr = rffi.charp2str(libssl_ERR_error_string(e, None)) errval = PY_SSL_ERROR_SYSCALL elif err == SSL_ERROR_SSL: - e = libssl_ERR_get_error() errval = PY_SSL_ERROR_SSL - if e != 0: - errstr = rffi.charp2str(libssl_ERR_error_string(e, None)) + if errcode != 0: + errstr = rffi.charp2str(libssl_ERR_error_string(errcode, None)) else: errstr = "A failure in the SSL library occurred" else: errstr = "Invalid error code" errval = PY_SSL_ERROR_INVALID_ERROR_CODE - return ssl_error(space, errstr, errval, w_errtype=w_errtype) + return ssl_error(space, errstr, errval, w_errtype=w_errtype, + errcode=errcode) def SSLError_descr_str(space, w_exc): w_strerror = space.getattr(w_exc, space.wrap("strerror")) From noreply at buildbot.pypy.org Mon Feb 2 09:55:04 2015 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 2 Feb 2015 09:55:04 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: fix my dates and add sebastian Message-ID: <20150202085504.61D071C026B@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: extradoc Changeset: r5496:0ade475b25d2 Date: 2015-02-02 10:55 +0200 http://bitbucket.org/pypy/extradoc/changeset/0ade475b25d2/ Log: fix my dates and add sebastian diff --git a/sprintinfo/leysin-winter-2015/people.txt b/sprintinfo/leysin-winter-2015/people.txt --- a/sprintinfo/leysin-winter-2015/people.txt +++ b/sprintinfo/leysin-winter-2015/people.txt @@ -11,8 +11,9 @@ Name Arrive/Depart Accomodation ==================== ============== ======================= Armin Rigo private -Maciej Fijalkowski 20-28 Ermina +Maciej Fijalkowski 21-28 Ermina Remi Meier 21-28 Ermina +Sebastian Pawlus 21-27 Ermina ==================== ============== ======================= From noreply at buildbot.pypy.org Mon Feb 2 15:44:21 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Feb 2015 15:44:21 +0100 (CET) Subject: [pypy-commit] stmgc default: finalizer bug: found a test Message-ID: <20150202144421.1C6091C02CD@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1603:f8215ea24895 Date: 2015-02-02 15:44 +0100 http://bitbucket.org/pypy/stmgc/changeset/f8215ea24895/ Log: finalizer bug: found a test diff --git a/c7/test/test_finalizer.py b/c7/test/test_finalizer.py --- a/c7/test/test_finalizer.py +++ b/c7/test/test_finalizer.py @@ -176,12 +176,14 @@ def test_finalizer_in_major_collection(self): self.start_transaction() - lp1 = stm_allocate_with_finalizer(48) - lp2 = stm_allocate_with_finalizer(48) - lp3 = stm_allocate_with_finalizer(48) - print lp1, lp2, lp3 - stm_major_collect() - self.expect_finalized([lp1, lp2, lp3]) + for repeat in range(2): + lp1 = stm_allocate_with_finalizer(48) + lp2 = stm_allocate_with_finalizer(48) + lp3 = stm_allocate_with_finalizer(48) + print repeat, lp1, lp2, lp3 + self.expect_finalized([]) + stm_major_collect() + self.expect_finalized([lp1, lp2, lp3]) def test_finalizer_from_other_thread(self): self.start_transaction() From noreply at buildbot.pypy.org Mon Feb 2 15:48:51 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Feb 2015 15:48:51 +0100 (CET) Subject: [pypy-commit] stmgc default: fix Message-ID: <20150202144851.13C2B1C02D4@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1604:97f88816858e Date: 2015-02-02 15:49 +0100 http://bitbucket.org/pypy/stmgc/changeset/97f88816858e/ Log: fix diff --git a/c7/stm/finalizer.c b/c7/stm/finalizer.c --- a/c7/stm/finalizer.c +++ b/c7/stm/finalizer.c @@ -138,6 +138,8 @@ init_finalizers(f); STM_PSEGMENT->finalizers = f; } + assert(STM_PSEGMENT->finalizers->count_non_young + <= list_count(STM_PSEGMENT->finalizers->objects_with_finalizers)); LIST_APPEND(STM_PSEGMENT->finalizers->objects_with_finalizers, obj); return obj; } @@ -289,6 +291,8 @@ struct list_s *lst = f->objects_with_finalizers; long i, count = list_count(lst); lst->count = 0; + f->count_non_young = 0; + for (i = 0; i < count; i++) { object_t *x = (object_t *)list_item(lst, i); From noreply at buildbot.pypy.org Mon Feb 2 15:49:09 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Feb 2015 15:49:09 +0100 (CET) Subject: [pypy-commit] stmgc hashtable-iter: hg merge default Message-ID: <20150202144909.8C1611C02D4@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: hashtable-iter Changeset: r1605:88492dc56aa8 Date: 2015-02-02 15:49 +0100 http://bitbucket.org/pypy/stmgc/changeset/88492dc56aa8/ Log: hg merge default diff --git a/c7/stm/finalizer.c b/c7/stm/finalizer.c --- a/c7/stm/finalizer.c +++ b/c7/stm/finalizer.c @@ -138,6 +138,8 @@ init_finalizers(f); STM_PSEGMENT->finalizers = f; } + assert(STM_PSEGMENT->finalizers->count_non_young + <= list_count(STM_PSEGMENT->finalizers->objects_with_finalizers)); LIST_APPEND(STM_PSEGMENT->finalizers->objects_with_finalizers, obj); return obj; } @@ -289,6 +291,8 @@ struct list_s *lst = f->objects_with_finalizers; long i, count = list_count(lst); lst->count = 0; + f->count_non_young = 0; + for (i = 0; i < count; i++) { object_t *x = (object_t *)list_item(lst, i); diff --git a/c7/test/test_finalizer.py b/c7/test/test_finalizer.py --- a/c7/test/test_finalizer.py +++ b/c7/test/test_finalizer.py @@ -176,12 +176,14 @@ def test_finalizer_in_major_collection(self): self.start_transaction() - lp1 = stm_allocate_with_finalizer(48) - lp2 = stm_allocate_with_finalizer(48) - lp3 = stm_allocate_with_finalizer(48) - print lp1, lp2, lp3 - stm_major_collect() - self.expect_finalized([lp1, lp2, lp3]) + for repeat in range(2): + lp1 = stm_allocate_with_finalizer(48) + lp2 = stm_allocate_with_finalizer(48) + lp3 = stm_allocate_with_finalizer(48) + print repeat, lp1, lp2, lp3 + self.expect_finalized([]) + stm_major_collect() + self.expect_finalized([lp1, lp2, lp3]) def test_finalizer_from_other_thread(self): self.start_transaction() From noreply at buildbot.pypy.org Mon Feb 2 15:53:28 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Feb 2015 15:53:28 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: update to stmgc/88492dc56aa8 (branch hashtable-iter) Message-ID: <20150202145328.0529D1C02CD@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r75644:55be055e388a Date: 2015-02-02 15:51 +0100 http://bitbucket.org/pypy/pypy/changeset/55be055e388a/ Log: update to stmgc/88492dc56aa8 (branch hashtable-iter) diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -de27e8f11e38 +88492dc56aa8 diff --git a/rpython/translator/stm/src_stm/stm/finalizer.c b/rpython/translator/stm/src_stm/stm/finalizer.c --- a/rpython/translator/stm/src_stm/stm/finalizer.c +++ b/rpython/translator/stm/src_stm/stm/finalizer.c @@ -139,6 +139,8 @@ init_finalizers(f); STM_PSEGMENT->finalizers = f; } + assert(STM_PSEGMENT->finalizers->count_non_young + <= list_count(STM_PSEGMENT->finalizers->objects_with_finalizers)); LIST_APPEND(STM_PSEGMENT->finalizers->objects_with_finalizers, obj); return obj; } @@ -290,6 +292,8 @@ struct list_s *lst = f->objects_with_finalizers; long i, count = list_count(lst); lst->count = 0; + f->count_non_young = 0; + for (i = 0; i < count; i++) { object_t *x = (object_t *)list_item(lst, i); From noreply at buildbot.pypy.org Mon Feb 2 16:15:14 2015 From: noreply at buildbot.pypy.org (mjacob) Date: Mon, 2 Feb 2015 16:15:14 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: Add myself to the Leysin sprint. Message-ID: <20150202151514.DD5F71C02CD@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: extradoc Changeset: r5498:ac5668253cbc Date: 2015-02-02 16:14 +0100 http://bitbucket.org/pypy/extradoc/changeset/ac5668253cbc/ Log: Add myself to the Leysin sprint. diff --git a/sprintinfo/leysin-winter-2015/people.txt b/sprintinfo/leysin-winter-2015/people.txt --- a/sprintinfo/leysin-winter-2015/people.txt +++ b/sprintinfo/leysin-winter-2015/people.txt @@ -14,6 +14,7 @@ Maciej Fijalkowski 21-28 Ermina Remi Meier 21-28 Ermina Sebastian Pawlus 21-27 Ermina +Manuel Jacob 21-28 Ermina ==================== ============== ======================= From noreply at buildbot.pypy.org Mon Feb 2 17:08:00 2015 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Mon, 2 Feb 2015 17:08:00 +0100 (CET) Subject: [pypy-commit] pypy default: Fixed #1968 -- bump the version for greenlet.py to the latest Message-ID: <20150202160800.AA14A1C02CD@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r75647:0def3207e879 Date: 2015-02-02 08:07 -0800 http://bitbucket.org/pypy/pypy/changeset/0def3207e879/ Log: Fixed #1968 -- bump the version for greenlet.py to the latest diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info --- a/lib_pypy/greenlet.egg-info +++ b/lib_pypy/greenlet.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: greenlet -Version: 0.4.0 +Version: 0.4.5 Summary: Lightweight in-process concurrent programming Home-page: https://github.com/python-greenlet/greenlet Author: Ralf Schmitt (for CPython), PyPy team diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py --- a/lib_pypy/greenlet.py +++ b/lib_pypy/greenlet.py @@ -1,7 +1,7 @@ import sys import _continuation -__version__ = "0.4.0" +__version__ = "0.4.5" # ____________________________________________________________ # Exceptions From noreply at buildbot.pypy.org Mon Feb 2 18:16:34 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Feb 2015 18:16:34 +0100 (CET) Subject: [pypy-commit] pypy default: Support for "with open(...) as f" in RPython. Message-ID: <20150202171634.959091C02D4@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75649:6657cb5a838d Date: 2015-02-02 18:15 +0100 http://bitbucket.org/pypy/pypy/changeset/6657cb5a838d/ Log: Support for "with open(...) as f" in RPython. diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -535,3 +535,9 @@ def isatty(self): self._check_closed() return os.isatty(c_fileno(self._ll_file)) + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py --- a/rpython/rlib/test/test_rfile.py +++ b/rpython/rlib/test/test_rfile.py @@ -395,6 +395,25 @@ os.unlink(fname) self.interpret(f, []) + def test_with_statement(self): + fname = str(self.tmpdir.join('file_6')) + + def f(): + with open(fname, "w") as f: + f.write("dupa") + try: + f.write("dupb") + except ValueError: + pass + else: + assert False + + f() + assert open(fname, "r").read() == "dupa" + os.unlink(fname) + self.interpret(f, []) + assert open(fname, "r").read() == "dupa" + class TestDirect: def setup_class(cls): From noreply at buildbot.pypy.org Mon Feb 2 18:16:33 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Feb 2015 18:16:33 +0100 (CET) Subject: [pypy-commit] pypy default: Improve the tests to actually check that the file is created by the Message-ID: <20150202171633.77D321C02D4@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75648:aaf45cb3d1a5 Date: 2015-02-02 18:14 +0100 http://bitbucket.org/pypy/pypy/changeset/aaf45cb3d1a5/ Log: Improve the tests to actually check that the file is created by the interpret() version. They could pass by simply having the same file already created by the direct f() version first. diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py --- a/rpython/rlib/test/test_rfile.py +++ b/rpython/rlib/test/test_rfile.py @@ -26,6 +26,7 @@ f() assert open(fname, "r").read() == "dupa" + os.unlink(fname) self.interpret(f, []) assert open(fname, "r").read() == "dupa" @@ -102,6 +103,7 @@ f2.close() f() + os.unlink(fname) self.interpret(f, []) @py.test.mark.skipif("sys.platform == 'win32'") @@ -121,6 +123,7 @@ f2.close() f() + os.unlink(fname) self.interpret(f, []) def test_open_buffering_full(self): @@ -138,6 +141,7 @@ f2.close() f() + os.unlink(fname) self.interpret(f, []) def test_fdopen_buffering_full(self): @@ -157,6 +161,7 @@ f2.close() f() + os.unlink(fname) self.interpret(f, []) def test_read_write(self): @@ -203,6 +208,7 @@ f2.close() f() + os.unlink(fname) self.interpret(f, []) def test_read_sequentially(self): @@ -277,6 +283,7 @@ f.close() f() + os.unlink(fname) self.interpret(f, []) def test_tempfile(self): @@ -309,6 +316,7 @@ f() assert open(fname).read() == "xxx" + os.unlink(fname) self.interpret(f, []) assert open(fname).read() == "xxx" @@ -325,6 +333,7 @@ res = f() assert res > 2 + os.unlink(fname) res = self.interpret(f, []) assert res > 2 @@ -341,6 +350,7 @@ res = f() assert res == 3 + os.unlink(fname) res = self.interpret(f, []) assert res == 3 @@ -357,6 +367,7 @@ f.close() f() + os.unlink(fname) self.interpret(f, []) def test_truncate(self): @@ -381,6 +392,7 @@ f.close() f() + os.unlink(fname) self.interpret(f, []) From noreply at buildbot.pypy.org Mon Feb 2 18:51:31 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Mon, 2 Feb 2015 18:51:31 +0100 (CET) Subject: [pypy-commit] pypy typed-cells: merge default Message-ID: <20150202175131.16C8F1C026B@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: typed-cells Changeset: r75650:2ac6556197f9 Date: 2015-02-02 16:30 +0100 http://bitbucket.org/pypy/pypy/changeset/2ac6556197f9/ Log: merge default diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py --- a/lib-python/2.7/ctypes/test/test_frombuffer.py +++ b/lib-python/2.7/ctypes/test/test_frombuffer.py @@ -2,7 +2,6 @@ import array import gc import unittest -from ctypes.test import xfail class X(Structure): _fields_ = [("c_int", c_int)] @@ -11,7 +10,6 @@ self._init_called = True class Test(unittest.TestCase): - @xfail def test_fom_buffer(self): a = array.array("i", range(16)) x = (c_int * 16).from_buffer(a) @@ -34,10 +32,9 @@ del a; gc.collect(); gc.collect(); gc.collect() self.assertEqual(x[:], expected) - self.assertRaises(TypeError, + self.assertRaises((TypeError, ValueError), (c_char * 16).from_buffer, "a" * 16) - @xfail def test_fom_buffer_with_offset(self): a = array.array("i", range(16)) x = (c_int * 15).from_buffer(a, sizeof(c_int)) @@ -46,7 +43,6 @@ self.assertRaises(ValueError, lambda: (c_int * 16).from_buffer(a, sizeof(c_int))) self.assertRaises(ValueError, lambda: (c_int * 1).from_buffer(a, 16 * sizeof(c_int))) - @xfail def test_from_buffer_copy(self): a = array.array("i", range(16)) x = (c_int * 16).from_buffer_copy(a) @@ -71,7 +67,6 @@ x = (c_char * 16).from_buffer_copy("a" * 16) self.assertEqual(x[:], "a" * 16) - @xfail def test_fom_buffer_copy_with_offset(self): a = array.array("i", range(16)) x = (c_int * 15).from_buffer_copy(a, sizeof(c_int)) diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py --- a/lib_pypy/_ctypes/basics.py +++ b/lib_pypy/_ctypes/basics.py @@ -83,6 +83,37 @@ def in_dll(self, dll, name): return self.from_address(dll._handle.getaddressindll(name)) + def from_buffer(self, obj, offset=0): + size = self._sizeofinstances() + buf = buffer(obj, offset, size) + if len(buf) < size: + raise ValueError( + "Buffer size too small (%d instead of at least %d bytes)" + % (len(buf) + offset, size + offset)) + raw_addr = buf._pypy_raw_address() + result = self.from_address(raw_addr) + result._ensure_objects()['ffffffff'] = obj + return result + + def from_buffer_copy(self, obj, offset=0): + size = self._sizeofinstances() + buf = buffer(obj, offset, size) + if len(buf) < size: + raise ValueError( + "Buffer size too small (%d instead of at least %d bytes)" + % (len(buf) + offset, size + offset)) + result = self() + dest = result._buffer.buffer + try: + raw_addr = buf._pypy_raw_address() + except ValueError: + _rawffi.rawstring2charp(dest, buf) + else: + from ctypes import memmove + memmove(dest, raw_addr, size) + return result + + class CArgObject(object): """ simple wrapper around buffer, just for the case of freeing it afterwards diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -1,3 +1,4 @@ +import sys import _rawffi from _ctypes.basics import _CData, _CDataMeta, keepalive_key,\ store_reference, ensure_objects, CArgObject @@ -178,6 +179,8 @@ instance = StructOrUnion.__new__(self) if isinstance(address, _rawffi.StructureInstance): address = address.buffer + # fix the address: turn it into as unsigned, in case it is negative + address = address & (sys.maxint * 2 + 1) instance.__dict__['_buffer'] = self._ffistruct.fromaddress(address) return instance diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst --- a/pypy/doc/faq.rst +++ b/pypy/doc/faq.rst @@ -3,6 +3,13 @@ .. contents:: +See also: `Frequently ask questions about RPython.`__ + +.. __: http://rpython.readthedocs.org/en/latest/faq.html + +--------------------------- + + What is PyPy? ------------- diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -1,14 +1,23 @@ Potential project list ====================== -=========================== -Simple tasks for newcomers: -=========================== +========================== +Simple tasks for newcomers +========================== -Optimize random ---------------- +* Tkinter module missing support for threads: + https://bitbucket.org/pypy/pypy/issue/1929/tkinter-broken-for-threaded-python-on-both -https://bitbucket.org/pypy/pypy/issue/1901/try-using-a-different-implementation-of +* Optimize random: + https://bitbucket.org/pypy/pypy/issue/1901/try-using-a-different-implementation-of + +* Implement AF_XXX packet types of sockets: + https://bitbucket.org/pypy/pypy/issue/1942/support-for-af_xxx-sockets + + +================== +Mid-to-large tasks +================== Below is a list of projects that are interesting for potential contributors who are seriously interested in the PyPy project. They mostly share common @@ -33,15 +42,8 @@ ------------------------ PyPy's bytearray type is very inefficient. It would be an interesting -task to look into possible optimizations on this. - -Implement AF_XXX packet types for PyPy --------------------------------------- - -PyPy is missing AF_XXX types of sockets. Implementing it is easy-to-medium -task. `bug report`_ - -.. _`bug report`: https://bitbucket.org/pypy/pypy/issue/1942/support-for-af_xxx-sockets#more +task to look into possible optimizations on this. (XXX current status +unknown; ask on #pypy for updates on this.) Implement copy-on-write list slicing ------------------------------------ @@ -106,6 +108,8 @@ Translation Toolchain --------------------- +(XXX this is unlikely to be feasible.) + * Incremental or distributed translation. * Allow separate compilation of extension modules. diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -439,7 +439,10 @@ f_back = space.interp_w(PyFrame, w_f_back, can_be_None=True) new_frame.f_backref = jit.non_virtual_ref(f_back) - new_frame.builtin = space.interp_w(Module, w_builtin) + if space.config.objspace.honor__builtins__: + new_frame.builtin = space.interp_w(Module, w_builtin) + else: + assert space.interp_w(Module, w_builtin) is space.builtin new_frame.set_blocklist([unpickle_block(space, w_blk) for w_blk in space.unpackiterable(w_blockstack)]) values_w = maker.slp_from_tuple_with_nulls(space, w_valuestack) @@ -524,9 +527,10 @@ # cellvars are values exported to inner scopes # freevars are values coming from outer scopes - freevarnames = list(self.pycode.co_cellvars) + # (see locals2fast for why CO_OPTIMIZED) + freevarnames = self.pycode.co_cellvars if self.pycode.co_flags & consts.CO_OPTIMIZED: - freevarnames.extend(self.pycode.co_freevars) + freevarnames = freevarnames + self.pycode.co_freevars for i in range(len(freevarnames)): name = freevarnames[i] cell = self.cells[i] @@ -555,7 +559,16 @@ self.setfastscope(new_fastlocals_w) - freevarnames = self.pycode.co_cellvars + self.pycode.co_freevars + freevarnames = self.pycode.co_cellvars + if self.pycode.co_flags & consts.CO_OPTIMIZED: + freevarnames = freevarnames + self.pycode.co_freevars + # If the namespace is unoptimized, then one of the + # following cases applies: + # 1. It does not contain free variables, because it + # uses import * or is a top-level namespace. + # 2. It is a class namespace. + # We don't want to accidentally copy free variables + # into the locals dict used by the class. for i in range(len(freevarnames)): name = freevarnames[i] cell = self.cells[i] diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py --- a/pypy/interpreter/test/test_pyframe.py +++ b/pypy/interpreter/test/test_pyframe.py @@ -516,3 +516,21 @@ assert seen == [(1, f, firstline + 6, 'line', None), (1, f, firstline + 7, 'line', None), (1, f, firstline + 8, 'line', None)] + + def test_locals2fast_freevar_bug(self): + import sys + def f(n): + class A(object): + def g(self): + return n + n = 42 + return A() + res = f(10).g() + assert res == 10 + # + def trace(*args): + return trace + sys.settrace(trace) + res = f(10).g() + sys.settrace(None) + assert res == 10 diff --git a/rpython/doc/faq.rst b/rpython/doc/faq.rst --- a/rpython/doc/faq.rst +++ b/rpython/doc/faq.rst @@ -3,6 +3,12 @@ .. contents:: +See also: `Frequently ask questions about PyPy.`__ + +.. __: http://pypy.readthedocs.org/en/latest/faq.html + +-------------------------- + What is RPython? ---------------- diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -1298,5 +1298,6 @@ self.load_reg(self.mc, res_loc, r.sp, ofs) scale = get_scale(size_loc.value) signed = (sign_loc.value != 0) - self._load_from_mem(res_loc, res_loc, ofs_loc, scale, signed, fcond) + self._load_from_mem(res_loc, res_loc, ofs_loc, imm(scale), signed, + fcond) return fcond diff --git a/rpython/jit/backend/x86/valgrind.py b/rpython/jit/backend/x86/valgrind.py --- a/rpython/jit/backend/x86/valgrind.py +++ b/rpython/jit/backend/x86/valgrind.py @@ -6,6 +6,7 @@ from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.rlib.objectmodel import we_are_translated +from rpython.jit.backend.x86.arch import WORD eci = ExternalCompilationInfo(includes = ['valgrind/valgrind.h']) @@ -13,9 +14,91 @@ try: rffi_platform.verify_eci(eci) except rffi_platform.CompilationError: - VALGRIND_DISCARD_TRANSLATIONS = None -else: - VALGRIND_DISCARD_TRANSLATIONS = rffi.llexternal( + # Can't open 'valgrind/valgrind.h'. It is a bad idea to just go + # ahead and not compile the valgrind-specific hacks. Instead, + # we'll include manually the few needed macros from a hopefully + # standard valgrind.h file. + eci = ExternalCompilationInfo(post_include_bits = [r""" +/************ Valgrind support: only with GCC/clang for now ***********/ +/** This code is inserted only if valgrind/valgrind.h is not found **/ +/**********************************************************************/ +#ifdef __GNUC__ + +#if ${WORD} == 4 /* if 32-bit x86 */ + +#define VG__SPECIAL_INSTRUCTION_PREAMBLE \ + "roll $3, %%edi ; roll $13, %%edi\n\t" \ + "roll $29, %%edi ; roll $19, %%edi\n\t" +#define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ + _zzq_default, _zzq_request, \ + _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ + __extension__ \ + ({volatile unsigned int _zzq_args[6]; \ + volatile unsigned int _zzq_result; \ + _zzq_args[0] = (unsigned int)(_zzq_request); \ + _zzq_args[1] = (unsigned int)(_zzq_arg1); \ + _zzq_args[2] = (unsigned int)(_zzq_arg2); \ + _zzq_args[3] = (unsigned int)(_zzq_arg3); \ + _zzq_args[4] = (unsigned int)(_zzq_arg4); \ + _zzq_args[5] = (unsigned int)(_zzq_arg5); \ + __asm__ volatile(VG__SPECIAL_INSTRUCTION_PREAMBLE \ + /* %EDX = client_request ( %EAX ) */ \ + "xchgl %%ebx,%%ebx" \ + : "=d" (_zzq_result) \ + : "a" (&_zzq_args[0]), "0" (_zzq_default) \ + : "cc", "memory" \ + ); \ + _zzq_result; \ + }) + +#else /* 64-bit x86-64 */ + +#define VG__SPECIAL_INSTRUCTION_PREAMBLE \ + "rolq $3, %%rdi ; rolq $13, %%rdi\n\t" \ + "rolq $61, %%rdi ; rolq $51, %%rdi\n\t" +#define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ + _zzq_default, _zzq_request, \ + _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ + __extension__ \ + ({ volatile unsigned long long int _zzq_args[6]; \ + volatile unsigned long long int _zzq_result; \ + _zzq_args[0] = (unsigned long long int)(_zzq_request); \ + _zzq_args[1] = (unsigned long long int)(_zzq_arg1); \ + _zzq_args[2] = (unsigned long long int)(_zzq_arg2); \ + _zzq_args[3] = (unsigned long long int)(_zzq_arg3); \ + _zzq_args[4] = (unsigned long long int)(_zzq_arg4); \ + _zzq_args[5] = (unsigned long long int)(_zzq_arg5); \ + __asm__ volatile(VG__SPECIAL_INSTRUCTION_PREAMBLE \ + /* %RDX = client_request ( %RAX ) */ \ + "xchgq %%rbx,%%rbx" \ + : "=d" (_zzq_result) \ + : "a" (&_zzq_args[0]), "0" (_zzq_default) \ + : "cc", "memory" \ + ); \ + _zzq_result; \ + }) +#endif + +#define VALGRIND_DO_CLIENT_REQUEST_STMT(_zzq_request, _zzq_arg1, \ + _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ + do { (void) VALGRIND_DO_CLIENT_REQUEST_EXPR(0, \ + (_zzq_request), (_zzq_arg1), (_zzq_arg2), \ + (_zzq_arg3), (_zzq_arg4), (_zzq_arg5)); } while (0) + +#define VG_USERREQ__DISCARD_TRANSLATIONS 0x1002 +#define VALGRIND_DISCARD_TRANSLATIONS(_qzz_addr,_qzz_len) \ + VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__DISCARD_TRANSLATIONS, \ + _qzz_addr, _qzz_len, 0, 0, 0) + +/**********************************************************************/ +#else /* if !__GNUC__ */ +#define VALGRIND_DISCARD_TRANSLATIONS(_qzz_addr,_qzz_len) do { } while(0) +#endif +/**********************************************************************/ +""".replace("${WORD}", str(WORD))]) + + +VALGRIND_DISCARD_TRANSLATIONS = rffi.llexternal( "VALGRIND_DISCARD_TRANSLATIONS", [llmemory.Address, lltype.Signed], lltype.Void, @@ -26,5 +109,5 @@ # ____________________________________________________________ def discard_translations(data, size): - if we_are_translated() and VALGRIND_DISCARD_TRANSLATIONS is not None: + if we_are_translated(): VALGRIND_DISCARD_TRANSLATIONS(llmemory.cast_int_to_adr(data), size) diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -477,8 +477,7 @@ @enforceargs(None, str) def write(self, value): self._check_closed() - ll_value, is_pinned, is_raw = rffi.get_nonmovingbuffer(value) - try: + with rffi.scoped_nonmovingbuffer(value) as ll_value: # note that since we got a nonmoving buffer, it is either raw # or already cannot move, so the arithmetics below are fine length = len(value) @@ -487,8 +486,6 @@ errno = rposix.get_saved_errno() c_clearerr(self._ll_file) raise IOError(errno, os.strerror(errno)) - finally: - rffi.free_nonmovingbuffer(value, ll_value, is_pinned, is_raw) def flush(self): self._check_closed() diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -684,7 +684,7 @@ class RegisterGcTraceEntry(ExtRegistryEntry): _about_ = register_custom_trace_hook - def compute_result_annotation(self, *args_s): + def compute_result_annotation(self, s_tp, s_lambda_func): pass def specialize_call(self, hop): @@ -692,3 +692,26 @@ lambda_func = hop.args_s[1].const hop.exception_cannot_occur() hop.rtyper.custom_trace_funcs.append((TP, lambda_func())) + +def register_custom_light_finalizer(TP, lambda_func): + """ This function does not do anything, but called from any annotated + place, will tell that "func" is used as a lightweight finalizer for TP. + The func must be specified as "lambda: func" in this call, for internal + reasons. + """ + +class RegisterCustomLightFinalizer(ExtRegistryEntry): + _about_ = register_custom_light_finalizer + + def compute_result_annotation(self, s_tp, s_lambda_func): + pass + + def specialize_call(self, hop): + from rpython.rtyper.llannotation import SomePtr + TP = hop.args_s[0].const + lambda_func = hop.args_s[1].const + ll_func = lambda_func() + args_s = [SomePtr(lltype.Ptr(TP))] + funcptr = hop.rtyper.annotate_helper_fn(ll_func, args_s) + hop.exception_cannot_occur() + lltype.attachRuntimeTypeInfo(TP, destrptr=funcptr) diff --git a/rpython/translator/c/src/asm_msvc.h b/rpython/translator/c/src/asm_msvc.h --- a/rpython/translator/c/src/asm_msvc.h +++ b/rpython/translator/c/src/asm_msvc.h @@ -2,3 +2,14 @@ #define PYPY_X86_CHECK_SSE2_DEFINED RPY_EXTERN void pypy_x86_check_sse2(void); #endif + + +/* Provides the same access to RDTSC as used by the JIT backend. This + is needed (at least if the JIT is enabled) because otherwise the + JIT-produced assembler would use RDTSC while the non-jitted code + would use QueryPerformanceCounter(), giving different incompatible + results. See issue #900. +*/ +#include +#pragma intrinsic(__rdtsc) +#define READ_TIMESTAMP(val) do { val = (long long)__rdtsc(); } while (0) diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -473,6 +473,31 @@ res = self.run('custom_trace', 0) assert res == 10000 + def define_custom_light_finalizer(cls): + from rpython.rtyper.annlowlevel import llhelper + # + T = lltype.Struct('T', ('count', lltype.Signed)) + t = lltype.malloc(T, zero=True, immortal=True, flavor='raw') + # + S = lltype.GcStruct('S', rtti=True) + def customlightfinlz(addr): + t.count += 1 + lambda_customlightfinlz = lambda: customlightfinlz + # + def setup(): + rgc.register_custom_light_finalizer(S, lambda_customlightfinlz) + for i in range(10000): + lltype.malloc(S) + def f(n): + setup() + llop.gc__collect(lltype.Void) + return t.count + return f + + def test_custom_light_finalizer(self): + res = self.run('custom_light_finalizer', 0) + assert res == 10000 + def define_weakref(cls): import weakref diff --git a/rpython/translator/tool/cbuild.py b/rpython/translator/tool/cbuild.py --- a/rpython/translator/tool/cbuild.py +++ b/rpython/translator/tool/cbuild.py @@ -59,7 +59,11 @@ separately and linked later on. (If an .h file is needed for other .c files to access this, it can be put in includes.) - (export_symbols: killed, replaced by @rlib.entrypoint.export_symbol) + (export_symbols: killed; you need, depending on the case, to + add the RPY_EXTERN or RPY_EXPORTED macro just before the + declaration of each function in the C header file, as explained + in translator/c/src/precommondefs.h; or you need the decorator + @rlib.entrypoint.export_symbol) compile_extra: list of parameters which will be directly passed to the compiler From noreply at buildbot.pypy.org Mon Feb 2 18:51:33 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Mon, 2 Feb 2015 18:51:33 +0100 (CET) Subject: [pypy-commit] pypy typed-cells: merge default Message-ID: <20150202175133.1F23E1C026B@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: typed-cells Changeset: r75651:153478e63e23 Date: 2015-02-02 16:31 +0100 http://bitbucket.org/pypy/pypy/changeset/153478e63e23/ Log: merge default diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst --- a/pypy/doc/faq.rst +++ b/pypy/doc/faq.rst @@ -197,6 +197,55 @@ (now-dead) object are still true about the new object. + +Would type annotations help PyPy's performance? +----------------------------------------------- + +Two examples of type annotations that are being proposed for improved +performance are `Cython types`__ and `PEP 484 - Type Hints`__. + +.. __: http://docs.cython.org/src/reference/language_basics.html#declaring-data-types +.. __: https://www.python.org/dev/peps/pep-0484/ + +**Cython types** are, by construction, similar to C declarations. For +example, a local variable or an instance attribute can be declared +``"cdef int"`` to force a machine word to be used. This changes the +usual Python semantics (e.g. no overflow checks, and errors when +trying to write other types of objects there). It gives some extra +performance, but the exact benefits are unclear: right now +(January 2015) for example we are investigating a technique that would +store machine-word integers directly on instances, giving part of the +benefits without the user-supplied ``"cdef int"``. + +**PEP 484 - Type Hints,** on the other hand, is almost entirely +useless if you're looking at performance. First, as the name implies, +they are *hints:* they must still be checked at runtime, like PEP 484 +says. Or maybe you're fine with a mode in which you get very obscure +crashes when the type annotations are wrong; but even in that case the +speed benefits would be extremely minor. + +There are several reasons for why. One of them is that annotations +are at the wrong level (e.g. a PEP 484 "int" corresponds to Python 3's +int type, which does not necessarily fits inside one machine word; +even worse, an "int" annotation allows arbitrary int subclasses). +Another is that a lot more information is needed to produce good code +(e.g. "this ``f()`` called here really means this function there, and +will never be monkey-patched" -- same with ``len()`` or ``list()``, +btw). The third reason is that some "guards" in PyPy's JIT traces +don't really have an obvious corresponding type (e.g. "this dict is so +far using keys which don't override ``__hash__`` so a more efficient +implementation was used"). Many guards don't even have any correspondence +with types at all ("this class attribute was not modified"; "the loop +counter did not reach zero so we don't need to release the GIL"; and +so on). + +As PyPy works right now, it is able to derive far more useful +information than can ever be given by PEP 484, and it works +automatically. As far as we know, this is true even if we would add +other techniques to PyPy, like a fast first-pass JIT. + + + .. _`prolog and javascript`: Can I use PyPy's translation toolchain for other languages besides Python? diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1017,6 +1017,9 @@ def newlist_unicode(self, list_u): return self.newlist([self.wrap(u) for u in list_u]) + def newlist_int(self, list_i): + return self.newlist([self.wrap(i) for i in list_i]) + def newlist_hint(self, sizehint): from pypy.objspace.std.listobject import make_empty_list_with_size return make_empty_list_with_size(self, sizehint) diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,7 +29,7 @@ #define PY_VERSION "2.7.8" /* PyPy version as a string */ -#define PYPY_VERSION "2.5.0-alpha0" +#define PYPY_VERSION "2.6.0-alpha0" /* Subversion Revision number of this file (not of the repository). * Empty since Mercurial migration. */ diff --git a/pypy/module/cpyext/ndarrayobject.py b/pypy/module/cpyext/ndarrayobject.py --- a/pypy/module/cpyext/ndarrayobject.py +++ b/pypy/module/cpyext/ndarrayobject.py @@ -291,6 +291,6 @@ Py_ssize_t, Py_ssize_t, rffi.CCHARP, rffi.CCHARP, Py_ssize_t], PyObject) def PyUFunc_FromFuncAndData(space, funcs, data, types, ntypes, nin, nout, identity, name, doc, check_return): - w_signature = "" + w_signature = ','.join(['()'] * nin) + '->' + ','.join(['()'] * nout) return do_ufunc(space, funcs, data, types, ntypes, nin, nout, identity, name, doc, check_return, w_signature) diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py --- a/pypy/module/micronumpy/ctors.py +++ b/pypy/module/micronumpy/ctors.py @@ -82,9 +82,18 @@ return w_object.descr_copy(space, w_order) elif not copy and (subok or type(w_object) is W_NDimArray): return w_object - - # not an array or incorrect dtype - shape, elems_w = strides.find_shape_and_elems(space, w_object, dtype) + if isinstance(w_object, W_NDimArray) and copy and not subok: + # TODO do the loop.assign without copying elems_w + shape = w_object.get_shape() + _elems_w = w_object.reshape(space, space.wrap(-1)) + elems_w = [None] * w_object.get_size() + for i in range(len(elems_w)): + elems_w[i] = _elems_w.descr_getitem(space, space.wrap(i)) + if space.is_none(w_dtype): + dtype = w_object.get_dtype() + else: + # not an array + shape, elems_w = strides.find_shape_and_elems(space, w_object, dtype) if dtype is None or (dtype.is_str_or_unicode() and dtype.elsize < 1): dtype = strides.find_dtype_for_seq(space, elems_w, dtype) if dtype is None: diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -430,9 +430,15 @@ order = 'C' else: order = space.str_w(w_order) + if order == 'K' and is_c_contiguous(self.implementation): + for s in self.implementation.get_strides(): + if s < 0: + break + else: + order = 'C' if order != 'C': raise OperationError(space.w_NotImplementedError, space.wrap( - "order not implemented")) + "order != 'C' only partially implemented")) return self.reshape(space, space.wrap(-1)) @unwrap_spec(w_axis=WrappedDefault(None), diff --git a/pypy/module/micronumpy/nditer.py b/pypy/module/micronumpy/nditer.py --- a/pypy/module/micronumpy/nditer.py +++ b/pypy/module/micronumpy/nditer.py @@ -252,10 +252,6 @@ # Copy logic from npyiter_coalesce_axes, used in ufunc iterators # and in nditer's with 'external_loop' flag can_coalesce = True - if it.order == 'F': - fastest = 0 - else: - fastest = -1 for idim in range(it.ndim - 1): for op_it, _ in it.iters: if op_it is None: @@ -275,7 +271,7 @@ if can_coalesce: for i in range(len(it.iters)): new_iter = coalesce_iter(it.iters[i][0], it.op_flags[i], it, - it.order, fastest) + it.order) it.iters[i] = (new_iter, new_iter.reset()) if len(it.shape) > 1: if it.order == 'F': @@ -289,7 +285,7 @@ break # Always coalesce at least one for i in range(len(it.iters)): - new_iter = coalesce_iter(it.iters[i][0], it.op_flags[i], it, 'C', -1) + new_iter = coalesce_iter(it.iters[i][0], it.op_flags[i], it, 'C') it.iters[i] = (new_iter, new_iter.reset()) if len(it.shape) > 1: if it.order == 'F': @@ -300,12 +296,11 @@ it.shape = [1] -def coalesce_iter(old_iter, op_flags, it, order, fastest=-1, flat=True): +def coalesce_iter(old_iter, op_flags, it, order, flat=True): ''' We usually iterate through an array one value at a time. But after coalesce(), getoperand() will return a slice by removing - the fastest varying dimension from the beginning or end of the shape. - XXX - what happens on swapaxis arrays? + the fastest varying dimension(s) from the beginning or end of the shape. If flat is true, then the slice will be 1d, otherwise stack up the shape of the fastest varying dimension in the slice, so an iterator of a 'C' array of shape (2,4,3) after two calls to coalesce will iterate 2 times over a slice @@ -319,6 +314,9 @@ new_strides = strides[1:] new_backstrides = backstrides[1:] _stride = old_iter.slice_stride + [strides[0]] + _shape = old_iter.slice_shape + [shape[0]] + _backstride = old_iter.slice_backstride + [strides[0] * (shape[0] - 1)] + fastest = shape[0] else: new_shape = shape[:-1] new_strides = strides[:-1] @@ -326,14 +324,15 @@ # use the operand's iterator's rightmost stride, # even if it is not the fastest (for 'F' or swapped axis) _stride = [strides[-1]] + old_iter.slice_stride - _shape = [shape[fastest]] + old_iter.slice_shape - _backstride = [(_shape[fastest] - 1) * _stride[0]] + old_iter.slice_backstride + _shape = [shape[-1]] + old_iter.slice_shape + _backstride = [(shape[-1] - 1) * strides[-1]] + old_iter.slice_backstride + fastest = shape[-1] if flat: _shape = [support.product(_shape)] if len(_stride) > 1: _stride = [min(_stride[0], _stride[1])] _backstride = [(shape[0] - 1) * _stride[0]] - return SliceIter(old_iter.array, old_iter.size / shape[fastest], + return SliceIter(old_iter.array, old_iter.size / fastest, new_shape, new_strides, new_backstrides, _shape, _stride, _backstride, op_flags, it) diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -2994,6 +2994,7 @@ assert (arange(3).ravel() == arange(3)).all() assert (arange(6).reshape(2, 3).ravel() == arange(6)).all() assert (arange(6).reshape(2, 3).T.ravel() == [0, 3, 1, 4, 2, 5]).all() + assert (arange(3).ravel('K') == arange(3)).all() def test_nonzero(self): from numpy import array diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py --- a/pypy/module/micronumpy/test/test_subtype.py +++ b/pypy/module/micronumpy/test/test_subtype.py @@ -304,10 +304,13 @@ out.shape = (sh, 1) else: out.shape = (1, sh) - print 'out, shape was',old_shape,'now',out.shape + #print 'out, shape was',old_shape,'now',out.shape,'out',out return out - a = matrix([[1., 2.]]) + a = matrix([[1., 2.], [3., 4.]]) b = N.array([a]) + assert (b == a).all() + b = N.array(a) + assert len(b.shape) == 2 def test_setstate_no_version(self): # Some subclasses of ndarray, like MaskedArray, do not use diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py --- a/pypy/module/micronumpy/test/test_ufuncs.py +++ b/pypy/module/micronumpy/test/test_ufuncs.py @@ -7,6 +7,7 @@ from pypy.module.micronumpy.concrete import VoidBoxStorage from pypy.interpreter.gateway import interp2app from pypy.conftest import option +from pypy.interpreter.error import OperationError class TestUfuncCoercion(object): @@ -129,7 +130,10 @@ '', ufunc.dtypes) assert index == 0 assert dtypes == [f32_dtype, c64_dtype] - + raises(OperationError, ufunc.type_resolver, space, [f32_array], [None], + 'u->u', ufunc.dtypes) + exc = raises(OperationError, ufunc.type_resolver, space, [f32_array], [None], + 'i->i', ufunc.dtypes) class AppTestUfuncs(BaseNumpyAppTest): def test_constants(self): @@ -169,8 +173,7 @@ dtypes=[int, int, int, float, float, float]) int_func22 = frompyfunc([int, int], 2, 2, signature='(i),(i)->(i),(i)', dtypes=['match']) - int_func12 = frompyfunc([int], 1, 2, signature='(i)->(i),(i)', - dtypes=['match']) + int_func12 = frompyfunc([int], 1, 2, dtypes=['match']) retype = dtype(int) a = arange(10) assert isinstance(adder_ufunc1, ufunc) @@ -223,6 +226,7 @@ assert len(in_array.shape) == 2 assert in_array.shape == out_array.shape out_array[:] = in_array * 2 + from numpy import frompyfunc, dtype, arange ufunc = frompyfunc([times_2], 1, 1, signature='(m,n)->(n,m)', @@ -233,6 +237,7 @@ ai3 = ufunc(ai[0,:,:]) ai2 = ufunc(ai) assert (ai2 == ai * 2).all() + ufunc = frompyfunc([times_2], 1, 1, signature='(m,m)->(m,m)', dtypes=[dtype(int), dtype(int)], @@ -245,6 +250,21 @@ ai2 = ufunc(ai) assert (ai2 == ai * 2).all() + def test_frompyfunc_needs_nditer(self): + def summer(in0): + print 'in summer, in0=',in0,'in0.shape=',in0.shape + return in0.sum() + + from numpy import frompyfunc, dtype, arange + ufunc = frompyfunc([summer], 1, 1, + signature='(m,m)->()', + dtypes=[dtype(int), dtype(int)], + stack_inputs=False, + ) + ai = arange(12, dtype=int).reshape(3, 2, 2) + ao = ufunc(ai) + assert ao.size == 3 + def test_frompyfunc_sig_broadcast(self): def sum_along_0(in_array, out_array): out_array[...] = in_array.sum(axis=0) @@ -269,6 +289,26 @@ aout = ufunc_sum(ai) assert aout.shape == (3, 3) + def test_frompyfunc_fortran(self): + import numpy as np + def tofrom_fortran(in0, out0): + out0[:] = in0.T + + def lapack_like_times2(in0, out0): + a = np.empty(in0.T.shape, in0.dtype) + tofrom_fortran(in0, a) + a *= 2 + tofrom_fortran(a, out0) + + times2 = np.frompyfunc([lapack_like_times2], 1, 1, + signature='(m,n)->(m,n)', + dtypes=[np.dtype(float), np.dtype(float)], + stack_inputs=True, + ) + in0 = np.arange(3300, dtype=float).reshape(100, 33) + out0 = times2(in0) + assert out0.shape == in0.shape + assert (out0 == in0 * 2).all() def test_ufunc_kwargs(self): from numpy import ufunc, frompyfunc, arange, dtype diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py --- a/pypy/module/micronumpy/ufuncs.py +++ b/pypy/module/micronumpy/ufuncs.py @@ -520,7 +520,9 @@ ''' _immutable_fields_ = ["funcs", "dtypes", "data", "match_dtypes"] - def __init__(self, space, funcs, name, identity, nin, nout, dtypes, signature, match_dtypes=False, stack_inputs=False): + def __init__(self, space, funcs, name, identity, nin, nout, dtypes, + signature, match_dtypes=False, stack_inputs=False, + external_loop=False): # XXX make sure funcs, signature, dtypes, nin, nout are consistent # These don't matter, we use the signature and dtypes for determining @@ -549,6 +551,7 @@ self.core_num_dims = [0] * self.nargs # number of core dimensions of each nargs self.core_offsets = [0] * self.nargs self.core_dim_ixs = [] # indices into unique shapes for each arg + self.external_loop = external_loop def reduce(self, space, w_obj, w_axis, keepdims=False, out=None, dtype=None, cumulative=False): @@ -586,29 +589,21 @@ _dtypes.append(_dtypes[0]) index, dtypes = self.type_resolver(space, inargs, outargs, sig, _dtypes) func = self.funcs[index] - if not self.core_enabled: - # func is going to do all the work, it must accept W_NDimArray args - inargs0 = inargs[0] - assert isinstance(inargs0, W_NDimArray) - arg_shapes = [inargs0.get_shape()] * self.nargs - inargs, outargs, need_to_cast = self.alloc_args(space, inargs, outargs, - dtypes, arg_shapes) - for tf in need_to_cast: - if tf: - raise oefmt(space.w_NotImplementedError, "casting not supported yet") - if self.stack_inputs: - arglist = space.newlist(list(inargs + outargs)) - space.call_args(func, Arguments.frompacked(space, arglist)) - else: - arglist = space.newlist(inargs) - outargs = space.call_args(func, Arguments.frompacked(space, arglist)) - return outargs - if len(outargs) < 2: - return outargs[0] - return space.newtuple(outargs) iter_shape, arg_shapes, matched_dims = self.verify_args(space, inargs, outargs) inargs, outargs, need_to_cast = self.alloc_args(space, inargs, outargs, dtypes, arg_shapes) + if not self.external_loop: + inargs0 = inargs[0] + outargs0 = outargs[0] + assert isinstance(inargs0, W_NDimArray) + assert isinstance(outargs0, W_NDimArray) + res_dtype = outargs0.get_dtype() + new_shape = inargs0.get_shape() + if len(outargs) < 2: + return loop.call_many_to_one(space, new_shape, func, + res_dtype, inargs, outargs[0]) + return loop.call_many_to_many(space, new_shape, func, + res_dtype, inargs, outargs) for tf in need_to_cast: if tf: raise oefmt(space.w_NotImplementedError, "casting not supported yet") @@ -619,48 +614,44 @@ w_casting = space.w_None w_op_axes = space.w_None + #print '\nsignature', sig + #print [(d, getattr(self,d)) for d in dir(self) if 'core' in d or 'broad' in d] + #print [(d, locals()[d]) for d in locals() if 'core' in d or 'broad' in d] + #print 'shapes',[d.get_shape() for d in inargs + outargs] + #print 'steps',[d.implementation.strides for d in inargs + outargs] + if isinstance(func, W_GenericUFuncCaller): + # Use GeneralizeUfunc interface with signature + # Unlike numpy, we will not broadcast dims before + # the core_ndims rather we use nditer iteration + # so dims[0] == 1 + dims = [1] + matched_dims + steps = [] + allargs = inargs + outargs + for i in range(len(allargs)): + steps.append(0) + for i in range(len(allargs)): + _arg = allargs[i] + assert isinstance(_arg, W_NDimArray) + start_dim = len(iter_shape) + steps += _arg.implementation.strides[start_dim:] + func.set_dims_and_steps(space, dims, steps) + else: + # it is a function, ready to be called by the iterator, + # from frompyfunc + pass + # mimic NpyIter_AdvancedNew with a nditer + w_itershape = space.newlist([space.wrap(i) for i in iter_shape]) + nd_it = W_NDIter(space, space.newlist(inargs + outargs), w_flags, + w_op_flags, w_op_dtypes, w_casting, w_op_axes, + w_itershape) + # coalesce each iterators, according to inner_dimensions + for i in range(len(inargs) + len(outargs)): + for j in range(self.core_num_dims[i]): + new_iter = coalesce_iter(nd_it.iters[i][0], nd_it.op_flags[i], + nd_it, nd_it.order, flat=False) + nd_it.iters[i] = (new_iter, new_iter.reset()) + # do the iteration if self.stack_inputs: - #print '\nsignature', sig - #print [(d, getattr(self,d)) for d in dir(self) if 'core' in d or 'broad' in d] - #print [(d, locals()[d]) for d in locals() if 'core' in d or 'broad' in d] - #print 'shapes',[d.get_shape() for d in inargs + outargs] - #print 'steps',[d.implementation.strides for d in inargs + outargs] - if isinstance(func, W_GenericUFuncCaller): - # Use GeneralizeUfunc interface with signature - # Unlike numpy, we will not broadcast dims before - # the core_ndims rather we use nditer iteration - # so dims[0] == 1 - dims = [1] + matched_dims - steps = [] - allargs = inargs + outargs - for i in range(len(allargs)): - steps.append(0) - for i in range(len(allargs)): - _arg = allargs[i] - assert isinstance(_arg, W_NDimArray) - start_dim = len(iter_shape) - steps += _arg.implementation.strides[start_dim:] - func.set_dims_and_steps(space, dims, steps) - else: - # it is a function, ready to be called by the iterator, - # from frompyfunc - pass - # mimic NpyIter_AdvancedNew with a nditer - w_itershape = space.newlist([space.wrap(i) for i in iter_shape]) - nd_it = W_NDIter(space, space.newlist(inargs + outargs), w_flags, - w_op_flags, w_op_dtypes, w_casting, w_op_axes, - w_itershape) - # coalesce each iterators, according to inner_dimensions - if nd_it.order == 'F': - fastest = 0 - else: - fastest = -1 - for i in range(len(inargs) + len(outargs)): - for j in range(self.core_num_dims[i]): - new_iter = coalesce_iter(nd_it.iters[i][0], nd_it.op_flags[i], - nd_it, nd_it.order, fastest, flat=False) - nd_it.iters[i] = (new_iter, new_iter.reset()) - # do the iteration while not nd_it.done: # XXX jit me for it, st in nd_it.iters: @@ -674,20 +665,35 @@ args.append(nd_it.getitem(it, st)) nd_it.iters[i] = (it, it.next(st)) space.call_args(func, Arguments.frompacked(space, space.newlist(args))) - if len(outargs) > 1: - return space.newtuple([convert_to_array(space, o) for o in outargs]) - return outargs[0] - inargs0 = inargs[0] - outargs0 = outargs[0] - assert isinstance(inargs0, W_NDimArray) - assert isinstance(outargs0, W_NDimArray) - res_dtype = outargs0.get_dtype() - new_shape = inargs0.get_shape() - if len(outargs) < 2: - return loop.call_many_to_one(space, new_shape, func, - res_dtype, inargs, outargs[0]) - return loop.call_many_to_many(space, new_shape, func, - res_dtype, inargs, outargs) + else: + # do the iteration + while not nd_it.done: + # XXX jit me + for it, st in nd_it.iters: + if not it.done(st): + break + else: + nd_it.done = True + break + initers = [] + outiters = [] + nin = len(inargs) + for i, (it, st) in enumerate(nd_it.iters[:nin]): + initers.append(nd_it.getitem(it, st)) + nd_it.iters[i] = (it, it.next(st)) + for i, (it, st) in enumerate(nd_it.iters[nin:]): + outiters.append(nd_it.getitem(it, st)) + nd_it.iters[i + nin] = (it, it.next(st)) + outs = space.call_args(func, Arguments.frompacked(space, space.newlist(initers))) + if len(outiters) < 2: + outiters[0].descr_setitem(space, space.w_Ellipsis, outs) + else: + for i in range(self.nout): + w_val = space.getitem(outs, space.wrap(i)) + outiters[i].descr_setitem(space, space.w_Ellipsis, w_val) + if len(outargs) > 1: + return space.newtuple([convert_to_array(space, o) for o in outargs]) + return outargs[0] def parse_kwargs(self, space, kwargs_w): w_subok, w_out, casting, sig, extobj = \ @@ -727,20 +733,24 @@ nop = len(inargs) + len(outargs) dtypes = [] if isinstance(type_tup, str) and len(type_tup) > 0: - if len(type_tup) == 1: - dtypes = [get_dtype_cache(space).dtypes_by_name[type_tup]] * self.nargs - elif len(type_tup) == self.nargs + 2: - for i in range(self.nin): - dtypes.append(get_dtype_cache(space).dtypes_by_name[type_tup[i]]) - #skip the '->' in the signature - for i in range(self.nout): - j = i + self.nin + 2 - dtypes.append(get_dtype_cache(space).dtypes_by_name[type_tup[j]]) - else: - raise oefmt(space.w_TypeError, "a type-string for %s " \ - "requires 1 typecode or %d typecode(s) before and %d" \ - " after the -> sign, not '%s'", self.name, self.nin, - self.nout, type_tup) + try: + if len(type_tup) == 1: + dtypes = [get_dtype_cache(space).dtypes_by_name[type_tup]] * self.nargs + elif len(type_tup) == self.nargs + 2: + for i in range(self.nin): + dtypes.append(get_dtype_cache(space).dtypes_by_name[type_tup[i]]) + #skip the '->' in the signature + for i in range(self.nout): + j = i + self.nin + 2 + dtypes.append(get_dtype_cache(space).dtypes_by_name[type_tup[j]]) + else: + raise oefmt(space.w_TypeError, "a type-string for %s " \ + "requires 1 typecode or %d typecode(s) before and %d" \ + " after the -> sign, not '%s'", self.name, self.nin, + self.nout, type_tup) + except KeyError: + raise oefmt(space.w_ValueError, "unknown typecode in" \ + " call to %s with type-string '%s'", self.name, type_tup) else: # XXX why does the next line not pass translation? # dtypes = [i.get_dtype() for i in inargs] @@ -764,9 +774,13 @@ break else: if len(self.funcs) > 1: + dtypesstr = ','.join(['%s%s%s' % (d.byteorder, d.kind, d.elsize) \ + for d in dtypes]) + _dtypesstr = ','.join(['%s%s%s' % (d.byteorder, d.kind, d.elsize) \ + for d in _dtypes]) raise oefmt(space.w_TypeError, - 'input dtype did not match any known dtypes', - ) + "input dtype [%s] did not match any known dtypes [%s] ", + dtypesstr,_dtypesstr) i = 0 # Fill in empty dtypes for j in range(self.nargs): @@ -1239,7 +1253,8 @@ w_identity=None, name='', doc='', stack_inputs=False): ''' frompyfunc(func, nin, nout) #cpython numpy compatible frompyfunc(func, nin, nout, dtypes=None, signature='', - identity=None, name='', doc='', stack_inputs=False) + identity=None, name='', doc='', + stack_inputs=False) Takes an arbitrary Python function and returns a ufunc. @@ -1255,10 +1270,12 @@ nout : int The number of arrays returned by `func`. dtypes: None or [dtype, ...] of the input, output args for each function, - or 'match' to force output to exactly match input dtype + or 'match' to force output to exactly match input dtype + Note that 'match' is a pypy-only extension to allow non-object + return dtypes signature*: str, default='' The mapping of input args to output args, defining the - inner-loop indexing + inner-loop indexing. If it is empty, the func operates on scalars identity*: None (default) or int For reduce-type ufuncs, the default value name: str, default='' @@ -1277,7 +1294,7 @@ Notes ----- - If the signature and out_dtype are both missing, the returned ufunc + If the signature and dtype are both missing, the returned ufunc always returns PyObject arrays (cpython numpy compatability). Input arguments marked with a * are pypy-only extensions @@ -1333,16 +1350,15 @@ 'identity must be None or an int') if len(signature) == 0: - # cpython compatability, func is of the form (),()->() - signature = ','.join(['()'] * nin) + '->' + ','.join(['()'] * nout) + external_loop=False else: - #stack_inputs = True - pass + external_loop=True w_ret = W_UfuncGeneric(space, func, name, identity, nin, nout, dtypes, signature, match_dtypes=match_dtypes, - stack_inputs=stack_inputs) - _parse_signature(space, w_ret, w_ret.signature) + stack_inputs=stack_inputs, external_loop=external_loop) + if w_ret.external_loop: + _parse_signature(space, w_ret, w_ret.signature) if doc: w_ret.w_doc = space.wrap(doc) return w_ret diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py --- a/pypy/module/pypyjit/interp_jit.py +++ b/pypy/module/pypyjit/interp_jit.py @@ -173,7 +173,7 @@ exiting (blackhole) steps, but just not from the final assembler. Note that the return value of the callable is ignored, because -there is no reasonable way to guess what it sound be in case the +there is no reasonable way to guess what it should be in case the function is not called. This is meant to be used notably in sys.settrace() for coverage- diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (2, 5, 0, "alpha", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (2, 6, 0, "alpha", 0) #XXX # sync patchlevel.h if platform.name == 'msvc': COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600) diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py --- a/pypy/objspace/std/dictmultiobject.py +++ b/pypy/objspace/std/dictmultiobject.py @@ -1173,8 +1173,8 @@ def wrapkey(space, key): return space.wrap(key) - # XXX there is no space.newlist_int yet to implement w_keys more - # efficiently + def w_keys(self, w_dict): + return self.space.newlist_int(self.listview_int(w_dict)) create_iterator_classes(IntDictStrategy) diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -176,6 +176,12 @@ storage = strategy.erase(list_u) return W_ListObject.from_storage_and_strategy(space, storage, strategy) + @staticmethod + def newlist_int(space, list_i): + strategy = space.fromcache(IntegerListStrategy) + storage = strategy.erase(list_i) + return W_ListObject.from_storage_and_strategy(space, storage, strategy) + def __repr__(self): """ representation for debugging purposes """ return "%s(%s, %s)" % (self.__class__.__name__, self.strategy, diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py --- a/pypy/objspace/std/objspace.py +++ b/pypy/objspace/std/objspace.py @@ -288,6 +288,9 @@ def newlist_unicode(self, list_u): return W_ListObject.newlist_unicode(self, list_u) + def newlist_int(self, list_i): + return W_ListObject.newlist_int(self, list_i) + def newdict(self, module=False, instance=False, kwargs=False, strdict=False): return W_DictMultiObject.allocate_and_init_instance( diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py --- a/rpython/rtyper/rclass.py +++ b/rpython/rtyper/rclass.py @@ -711,8 +711,15 @@ continue value = self.classdef.classdesc.read_attribute(fldname, None) if value is not None: - cvalue = inputconst(r.lowleveltype, - r.convert_desc_or_const(value)) + ll_value = r.convert_desc_or_const(value) + # don't write NULL GC pointers: we know that the malloc + # done above initialized at least the GC Ptr fields to + # NULL already, and that's true for all our GCs + if (isinstance(r.lowleveltype, Ptr) and + r.lowleveltype.TO._gckind == 'gc' and + not ll_value): + continue + cvalue = inputconst(r.lowleveltype, ll_value) self.setfield(vptr, fldname, cvalue, llops, flags={'access_directly': True}) return vptr diff --git a/rpython/translator/backendopt/test/test_malloc.py b/rpython/translator/backendopt/test/test_malloc.py --- a/rpython/translator/backendopt/test/test_malloc.py +++ b/rpython/translator/backendopt/test/test_malloc.py @@ -340,3 +340,15 @@ u[0].s.x = x return u[0].s.x graph = self.check(f, [int], [42], 42) + + def test_two_paths_one_with_constant(self): + py.test.skip("XXX implement me?") + def fn(n): + if n > 100: + tup = (0,) + else: + tup = (n,) + (n,) # <- flowspace + return tup[0] + + self.check(fn, [int], [42], 42) diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -808,12 +808,7 @@ t, cbuilder = self.compile(entry_point, shared=True) assert cbuilder.shared_library_name is not None assert cbuilder.shared_library_name != cbuilder.executable_name - if os.name == 'posix': - library_path = cbuilder.shared_library_name.dirpath() - if sys.platform == 'darwin': - monkeypatch.setenv('DYLD_LIBRARY_PATH', library_path) - else: - monkeypatch.setenv('LD_LIBRARY_PATH', library_path) + #Do not set LD_LIBRARY_PATH, make sure $ORIGIN flag is working out, err = cbuilder.cmdexec("a b") assert out == "3" diff --git a/rpython/translator/platform/freebsd.py b/rpython/translator/platform/freebsd.py --- a/rpython/translator/platform/freebsd.py +++ b/rpython/translator/platform/freebsd.py @@ -12,6 +12,7 @@ cflags = tuple( ['-O3', '-pthread', '-fomit-frame-pointer'] + os.environ.get('CFLAGS', '').split()) + rpath_flags = ['-Wl,-rpath=\'$$ORIGIN/\'', '-Wl,-z,origin'] class Freebsd_64(Freebsd): shared_only = ('-fPIC',) diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py --- a/rpython/translator/platform/posix.py +++ b/rpython/translator/platform/posix.py @@ -112,9 +112,9 @@ target_name = exe_name.basename if shared: - cflags = self.cflags + self.get_shared_only_compile_flags() + cflags = tuple(self.cflags) + self.get_shared_only_compile_flags() else: - cflags = self.cflags + self.standalone_only + cflags = tuple(self.cflags) + tuple(self.standalone_only) m = GnuMakefile(path) m.exe_name = path.join(exe_name.basename) From noreply at buildbot.pypy.org Mon Feb 2 18:51:34 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Mon, 2 Feb 2015 18:51:34 +0100 (CET) Subject: [pypy-commit] pypy typed-cells: also unwrap floats Message-ID: <20150202175134.5F63E1C026B@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: typed-cells Changeset: r75652:b193ed03acc0 Date: 2015-02-02 18:51 +0100 http://bitbucket.org/pypy/pypy/changeset/b193ed03acc0/ Log: also unwrap floats diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -9,7 +9,8 @@ BaseValueIterator, BaseItemIterator, _never_equal_to_string ) from pypy.objspace.std.typeobject import ( - MutableCell, IntMutableCell, ObjectMutableCell, unwrap_cell) + MutableCell, IntMutableCell, FloatMutableCell, ObjectMutableCell, + unwrap_cell) # ____________________________________________________________ # attribute shapes @@ -303,15 +304,24 @@ def _write_cell(self, w_cell, w_value): from pypy.objspace.std.intobject import W_IntObject + from pypy.objspace.std.floatobject import W_FloatObject assert not isinstance(w_cell, ObjectMutableCell) if isinstance(w_cell, IntMutableCell) and type(w_value) is W_IntObject: w_cell.intvalue = w_value.intval return None + if isinstance(w_cell, FloatMutableCell) and type(w_value) is W_FloatObject: + w_cell.floatvalue = w_value.floatval + return None if type(w_value) is W_IntObject: if not self.can_contain_mutable_cell: self.can_contain_mutable_cell = True if self.ever_mutated: return IntMutableCell(w_value.intval) + if type(w_value) is W_FloatObject: + if not self.can_contain_mutable_cell: + self.can_contain_mutable_cell = True + if self.ever_mutated: + return FloatMutableCell(w_value.floatval) return w_value def _copy_attr(self, obj, new_obj): diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -182,6 +182,35 @@ assert mutcell2.intvalue == 7 assert mutcell2 is mutcell1 + +def test_mutcell_not_immutable_float(): + from pypy.objspace.std.floatobject import W_FloatObject + cls = Class() + obj = cls.instantiate() + # make sure the attribute counts as mutable + obj.setdictvalue(space, "a", W_FloatObject(4.43)) + obj.setdictvalue(space, "a", W_FloatObject(5.43)) + assert obj.map.ever_mutated + + obj = cls.instantiate() + obj.setdictvalue(space, "a", W_FloatObject(5.43)) + assert obj.getdictvalue(space, "a") == 5.43 + mutcell = obj._mapdict_read_storage(0) + assert mutcell.floatvalue == 5.43 + + obj.setdictvalue(space, "a", W_FloatObject(6.43)) + assert obj.getdictvalue(space, "a") == 6.43 + mutcell1 = obj._mapdict_read_storage(0) + assert mutcell1.floatvalue == 6.43 + assert mutcell is mutcell1 + + obj.setdictvalue(space, "a", W_FloatObject(7.43)) + assert obj.getdictvalue(space, "a") == 7.43 + mutcell2 = obj._mapdict_read_storage(0) + assert mutcell2.floatvalue == 7.43 + assert mutcell2 is mutcell1 + + def test_no_mutcell_if_immutable(): # don't introduce an immutable cell if the attribute seems immutable from pypy.objspace.std.intobject import W_IntObject diff --git a/pypy/objspace/std/test/test_versionedtype.py b/pypy/objspace/std/test/test_versionedtype.py --- a/pypy/objspace/std/test/test_versionedtype.py +++ b/pypy/objspace/std/test/test_versionedtype.py @@ -259,6 +259,43 @@ cell = w_A._getdictvalue_no_unwrapping(space, "x") assert space.float_w(cell.w_value) == 2.2 + def test_float_cells(self): + space = self.space + w_x = space.wrap("x") + w_A, w_B, w_C = self.get_three_classes() + atag = w_A.version_tag() + space.setattr(w_A, w_x, space.newfloat(1.1)) + assert w_A.version_tag() is not atag + assert space.float_w(space.getattr(w_A, w_x)) == 1.1 + + atag = w_A.version_tag() + space.setattr(w_A, w_x, space.newfloat(2.1)) + assert w_A.version_tag() is not atag + assert space.float_w(space.getattr(w_A, w_x)) == 2.1 + cell = w_A._getdictvalue_no_unwrapping(space, "x") + assert cell.floatvalue == 2.1 + + atag = w_A.version_tag() + space.setattr(w_A, w_x, space.newfloat(3.1)) + assert w_A.version_tag() is atag + assert space.float_w(space.getattr(w_A, w_x)) == 3.1 + assert cell.floatvalue == 3.1 + + space.setattr(w_A, w_x, space.newfloat(4.1)) + assert w_A.version_tag() is atag + assert space.float_w(space.getattr(w_A, w_x)) == 4.1 + assert cell.floatvalue == 4.1 + + def test_float_cell_turns_into_cell(self): + space = self.space + w_x = space.wrap("x") + w_A, w_B, w_C = self.get_three_classes() + atag = w_A.version_tag() + space.setattr(w_A, w_x, space.newfloat(1.1)) + space.setattr(w_A, w_x, space.newfloat(2.1)) + space.setattr(w_A, w_x, space.wrap("abc")) + cell = w_A._getdictvalue_no_unwrapping(space, "x") + assert space.str_w(cell.w_value) == "abc" class AppTestVersionedType(test_typeobject.AppTestTypeObject): diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -36,6 +36,17 @@ def __repr__(self): return "" % (self.intvalue, ) +class FloatMutableCell(MutableCell): + def __init__(self, floatvalue): + self.floatvalue = floatvalue + + def unwrap_cell(self, space): + return space.wrap(self.floatvalue) + + def __repr__(self): + return "" % (self.floatvalue, ) + + def unwrap_cell(space, w_value): if isinstance(w_value, MutableCell): return w_value.unwrap_cell(space) @@ -49,6 +60,7 @@ def write_cell(space, w_cell, w_value): from pypy.objspace.std.intobject import W_IntObject + from pypy.objspace.std.floatobject import W_FloatObject if w_cell is None: # attribute does not exist at all, write it without a cell first return w_value @@ -58,14 +70,19 @@ elif isinstance(w_cell, IntMutableCell) and type(w_value) is W_IntObject: w_cell.intvalue = w_value.intval return None + elif isinstance(w_cell, FloatMutableCell) and type(w_value) is W_FloatObject: + w_cell.floatvalue = w_value.floatval + return None elif space.is_w(w_cell, w_value): # If the new value and the current value are the same, don't # create a level of indirection, or mutate the version. return None - if type(w_value) is W_IntObject: - return IntMutableCell(w_value.intval) - else: - return ObjectMutableCell(w_value) + if not isinstance(w_cell, MutableCell): + if type(w_value) is W_IntObject: + return IntMutableCell(w_value.intval) + if type(w_value) is W_FloatObject: + return FloatMutableCell(w_value.floatval) + return ObjectMutableCell(w_value) class VersionTag(object): pass From noreply at buildbot.pypy.org Mon Feb 2 21:15:51 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Feb 2015 21:15:51 +0100 (CET) Subject: [pypy-commit] stmgc hashtable-iter: ready for merge Message-ID: <20150202201551.9971E1C026B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: hashtable-iter Changeset: r1606:ddb0b264ee12 Date: 2015-02-02 19:57 +0100 http://bitbucket.org/pypy/stmgc/changeset/ddb0b264ee12/ Log: ready for merge From noreply at buildbot.pypy.org Mon Feb 2 21:15:53 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Feb 2015 21:15:53 +0100 (CET) Subject: [pypy-commit] stmgc default: hg merge hashtable-iter Message-ID: <20150202201553.178961C026B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1607:ee68afa62c2c Date: 2015-02-02 19:58 +0100 http://bitbucket.org/pypy/stmgc/changeset/ee68afa62c2c/ Log: hg merge hashtable-iter Add methods like .items(), .keys() and so on. Unlike the name of the branch suggests, we *don't* support the lazy iterators. diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -431,13 +431,12 @@ continue; /* no need to check: is pending immediate abort */ char *remote_base = get_segment_base(i); - uint8_t remote_version = get_segment(i)->transaction_read_version; LIST_FOREACH_R( STM_PSEGMENT->modified_old_objects, object_t * /*item*/, ({ - if (was_read_remote(remote_base, item, remote_version)) { + if (was_read_remote(remote_base, item)) { /* A write-read conflict! */ dprintf(("write-read conflict on %p, our seg: %d, other: %ld\n", item, STM_SEGMENT->segment_num, i)); diff --git a/c7/stm/core.h b/c7/stm/core.h --- a/c7/stm/core.h +++ b/c7/stm/core.h @@ -281,11 +281,17 @@ static stm_thread_local_t *abort_with_mutex_no_longjmp(void); static void abort_data_structures_from_segment_num(int segment_num); -static inline bool was_read_remote(char *base, object_t *obj, - uint8_t other_transaction_read_version) +static inline struct stm_read_marker_s *get_read_marker(char *base, + object_t *obj) { + return (struct stm_read_marker_s *)(base + (((uintptr_t)obj) >> 4)); +} + +static inline bool was_read_remote(char *base, object_t *obj) { - uint8_t rm = ((struct stm_read_marker_s *) - (base + (((uintptr_t)obj) >> 4)))->rm; + uint8_t other_transaction_read_version = + ((struct stm_segment_info_s *)REAL_ADDRESS(base, STM_PSEGMENT)) + ->transaction_read_version; + uint8_t rm = get_read_marker(base, obj)->rm; assert(rm <= other_transaction_read_version); return rm == other_transaction_read_version; } diff --git a/c7/stm/hashtable.c b/c7/stm/hashtable.c --- a/c7/stm/hashtable.c +++ b/c7/stm/hashtable.c @@ -12,22 +12,28 @@ collision). The main operations on a hashtable are reading or writing an object at a -given index. It might support in the future enumerating the indexes of -non-NULL objects. +given index. It also supports fetching the list of non-NULL entries. There are two markers for every index (a read and a write marker). This is unlike regular arrays, which have only two markers in total. +Additionally, we use the read marker for the hashtable object itself +to mean "we have read the complete list of keys". This plays the role +of a "global" read marker: when any thread adds a new key/value object +to the hashtable, this new object's read marker is initialized with a +copy of the "global" read marker --- in all segments. + Implementation -------------- First idea: have the hashtable in raw memory, pointing to "entry" -objects. The entry objects themselves point to the user-specified -objects. The entry objects have the read/write markers. Every entry -object, once created, stays around. It is only removed by the next -major GC if it points to NULL and its read/write markers are not set -in any currently-running transaction. +objects (which are regular, GC- and STM-managed objects). The entry +objects themselves point to the user-specified objects. The entry +objects hold the read/write markers. Every entry object, once +created, stays around. It is only removed by the next major GC if it +points to NULL and its read/write markers are not set in any +currently-running transaction. References ---------- @@ -103,9 +109,7 @@ { long i; for (i = 1; i <= NB_SEGMENTS; i++) { - char *remote_base = get_segment_base(i); - uint8_t remote_version = get_segment(i)->transaction_read_version; - if (was_read_remote(remote_base, obj, remote_version)) + if (was_read_remote(get_segment_base(i), obj)) return true; } return false; @@ -154,6 +158,7 @@ /* ^^^ this unlocks the table by writing a non-zero value to table->resize_counter, but the new value is a pointer to the new bigger table, so IS_EVEN() is still true */ + assert(IS_EVEN(table->resize_counter)); init_table(biggertable, biggercount); @@ -173,6 +178,7 @@ } } _insert_clean(biggertable, entry); + assert(rc > 6); rc -= 6; } biggertable->resize_counter = rc; @@ -216,7 +222,16 @@ perturb >>= PERTURB_SHIFT; } } - /* here, we didn't find the 'entry' with the correct index. */ + /* here, we didn't find the 'entry' with the correct index. Note + that even if the same 'table' is modified or resized by other + threads concurrently, any new item found from a race condition + would anyway contain NULL in the present segment (ensured by + the first write_fence() below). If the 'table' grows an entry + just after we checked above, then we go ahead and lock the + table; but after we get the lock, we will notice the new entry + (ensured by the second write_fence() below) and restart the + whole process. + */ uintptr_t rc = VOLATILE_TABLE(table)->resize_counter; @@ -282,12 +297,22 @@ synchronization with other pieces of the code that may change. */ + + /* First fetch the read marker of 'hashtableobj' in all + segments, before allocate_outside_nursery_large() + which might trigger a GC */ + long j; + uint8_t readmarkers[NB_SEGMENTS]; + for (j = 1; j <= NB_SEGMENTS; j++) { + readmarkers[j - 1] = get_read_marker(get_segment_base(j), + hashtableobj)->rm; + } + acquire_privatization_lock(); char *p = allocate_outside_nursery_large( sizeof(stm_hashtable_entry_t)); entry = (stm_hashtable_entry_t *)(p - stm_object_pages); - long j; for (j = 0; j <= NB_SEGMENTS; j++) { struct stm_hashtable_entry_s *e; e = (struct stm_hashtable_entry_s *) @@ -299,6 +324,11 @@ } hashtable->additions += 0x100; release_privatization_lock(); + + for (j = 1; j <= NB_SEGMENTS; j++) { + get_read_marker(get_segment_base(j), (object_t *)entry)->rm = + readmarkers[j - 1]; + } } write_fence(); /* make sure 'entry' is fully initialized here */ table->items[i] = entry; @@ -338,17 +368,94 @@ e->object = nvalue; } +long stm_hashtable_length_upper_bound(stm_hashtable_t *hashtable) +{ + stm_hashtable_table_t *table; + uintptr_t rc; + + restart: + table = VOLATILE_HASHTABLE(hashtable)->table; + rc = VOLATILE_TABLE(table)->resize_counter; + if (IS_EVEN(rc)) { + spin_loop(); + goto restart; + } + + uintptr_t initial_rc = (table->mask + 1) * 4 + 1; + uintptr_t num_entries_times_6 = initial_rc - rc; + return num_entries_times_6 / 6; +} + +long stm_hashtable_list(object_t *hobj, stm_hashtable_t *hashtable, + stm_hashtable_entry_t **results) +{ + stm_hashtable_table_t *table; + intptr_t rc; + + /* Set the read marker. It will be left as long as we're running + the same transaction. + */ + stm_read(hobj); + + /* Acquire and immediately release the lock. We don't actually + need to do anything while we hold the lock, but the point is to + wait until the lock is available, and to synchronize other + threads with the stm_read() done above. + */ + restart: + table = VOLATILE_HASHTABLE(hashtable)->table; + rc = VOLATILE_TABLE(table)->resize_counter; + if (IS_EVEN(rc)) { + spin_loop(); + goto restart; + } + if (!__sync_bool_compare_and_swap(&table->resize_counter, rc, rc)) + goto restart; + + /* Read all entries, check which ones are not NULL, count them, + and optionally list them in 'results'. + */ + uintptr_t i, mask = table->mask; + stm_hashtable_entry_t *entry; + long nresult = 0; + + if (results != NULL) { + /* collect the results in the provided list */ + for (i = 0; i <= mask; i++) { + entry = VOLATILE_TABLE(table)->items[i]; + if (entry != NULL) { + stm_read((object_t *)entry); + if (entry->object != NULL) + results[nresult++] = entry; + } + } + } + else { + /* don't collect, just get the exact number of results */ + for (i = 0; i <= mask; i++) { + entry = VOLATILE_TABLE(table)->items[i]; + if (entry != NULL) { + stm_read((object_t *)entry); + if (entry->object != NULL) + nresult++; + } + } + } + return nresult; +} + static void _stm_compact_hashtable(stm_hashtable_t *hashtable) { stm_hashtable_table_t *table = hashtable->table; - assert(!IS_EVEN(table->resize_counter)); + uintptr_t rc = table->resize_counter; + assert(!IS_EVEN(rc)); if ((hashtable->additions >> 8) * 4 > table->mask) { int segment_num = (hashtable->additions & 0xFF); if (!segment_num) segment_num = 1; hashtable->additions = segment_num; uintptr_t initial_rc = (table->mask + 1) * 4 + 1; - uintptr_t num_entries_times_6 = initial_rc - table->resize_counter; + uintptr_t num_entries_times_6 = initial_rc - rc; uintptr_t count = INITIAL_HASHTABLE_SIZE; while (count * 4 < num_entries_times_6) count *= 2; @@ -376,6 +483,7 @@ free(old_table); } hashtable->initial_table.resize_counter = (uintptr_t)table; + assert(IS_EVEN(hashtable->initial_table.resize_counter)); } } diff --git a/c7/stm/misc.c b/c7/stm/misc.c --- a/c7/stm/misc.c +++ b/c7/stm/misc.c @@ -31,8 +31,7 @@ bool _stm_was_read(object_t *obj) { - return was_read_remote(STM_SEGMENT->segment_base, obj, - STM_SEGMENT->transaction_read_version); + return was_read_remote(STM_SEGMENT->segment_base, obj); } bool _stm_was_written(object_t *obj) diff --git a/c7/stmgc.h b/c7/stmgc.h --- a/c7/stmgc.h +++ b/c7/stmgc.h @@ -545,6 +545,9 @@ object_t *stm_hashtable_read(object_t *, stm_hashtable_t *, uintptr_t key); void stm_hashtable_write(object_t *, stm_hashtable_t *, uintptr_t key, object_t *nvalue, stm_thread_local_t *); +long stm_hashtable_length_upper_bound(stm_hashtable_t *); +long stm_hashtable_list(object_t *, stm_hashtable_t *, + stm_hashtable_entry_t **results); extern uint32_t stm_hashtable_entry_userdata; void stm_hashtable_tracefn(stm_hashtable_t *, void (object_t **)); diff --git a/c7/test/support.py b/c7/test/support.py --- a/c7/test/support.py +++ b/c7/test/support.py @@ -167,17 +167,23 @@ void (*stmcb_finalizer)(object_t *); typedef struct stm_hashtable_s stm_hashtable_t; +typedef ... stm_hashtable_entry_t; stm_hashtable_t *stm_hashtable_create(void); void stm_hashtable_free(stm_hashtable_t *); bool _check_hashtable_read(object_t *, stm_hashtable_t *, uintptr_t key); object_t *hashtable_read_result; bool _check_hashtable_write(object_t *, stm_hashtable_t *, uintptr_t key, object_t *nvalue, stm_thread_local_t *tl); +long stm_hashtable_length_upper_bound(stm_hashtable_t *); +long stm_hashtable_list(object_t *, stm_hashtable_t *, + stm_hashtable_entry_t **results); uint32_t stm_hashtable_entry_userdata; void stm_hashtable_tracefn(stm_hashtable_t *, void (object_t **)); void _set_hashtable(object_t *obj, stm_hashtable_t *h); stm_hashtable_t *_get_hashtable(object_t *obj); +uintptr_t _get_entry_index(stm_hashtable_entry_t *entry); +object_t *_get_entry_object(stm_hashtable_entry_t *entry); """) @@ -308,6 +314,18 @@ return *(stm_hashtable_t *TLPREFIX *)field_addr; } +uintptr_t _get_entry_index(stm_hashtable_entry_t *entry) +{ + stm_read((object_t *)entry); + return entry->index; +} + +object_t *_get_entry_object(stm_hashtable_entry_t *entry) +{ + stm_read((object_t *)entry); + return entry->object; +} + void _set_ptr(object_t *obj, int n, object_t *v) { long nrefs = (long)((myobj_t*)obj)->type_id - 421420; diff --git a/c7/test/test_hashtable.py b/c7/test/test_hashtable.py --- a/c7/test/test_hashtable.py +++ b/c7/test/test_hashtable.py @@ -16,6 +16,24 @@ if res: raise Conflict +def ht_length_upper_bound(o): + h = get_hashtable(o) + return lib.stm_hashtable_length_upper_bound(h) + +def htitems(o): + h = get_hashtable(o) + upper_bound = lib.stm_hashtable_length_upper_bound(h) + entries = ffi.new("stm_hashtable_entry_t *[]", upper_bound) + count = lib.stm_hashtable_list(o, h, entries) + assert count <= upper_bound + return [(lib._get_entry_index(entries[i]), + lib._get_entry_object(entries[i])) for i in range(count)] + +def htlen(o): + h = get_hashtable(o) + count = lib.stm_hashtable_list(o, h, ffi.NULL) + return count + class BaseTestHashtable(BaseTest): @@ -236,6 +254,36 @@ assert htget(h, 11) != ffi.NULL assert htget(h, 12) != ffi.NULL + def test_list_1(self): + self.start_transaction() + h = self.allocate_hashtable() + tl0 = self.tls[self.current_thread] + for i in range(32): + assert ht_length_upper_bound(h) == i + htset(h, 19 ^ i, stm_allocate(32), tl0) + assert ht_length_upper_bound(h) == 32 + + def test_list_2(self): + self.start_transaction() + h = self.allocate_hashtable() + tl0 = self.tls[self.current_thread] + expected = [] + for i in range(29): + lp = stm_allocate(32) + htset(h, 19 ^ i, lp, tl0) + expected.append((19 ^ i, lp)) + lst = htitems(h) + assert len(lst) == 29 + assert sorted(lst) == sorted(expected) + + def test_list_3(self): + self.start_transaction() + h = self.allocate_hashtable() + tl0 = self.tls[self.current_thread] + for i in range(29): + htset(h, 19 ^ i, stm_allocate(32), tl0) + assert htlen(h) == 29 + class TestRandomHashtable(BaseTestHashtable): From noreply at buildbot.pypy.org Mon Feb 2 21:15:54 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Feb 2015 21:15:54 +0100 (CET) Subject: [pypy-commit] stmgc default: One very probable bug fix, and one more theoretical Message-ID: <20150202201554.1CA931C026B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1608:10c636ae449e Date: 2015-02-02 21:15 +0100 http://bitbucket.org/pypy/stmgc/changeset/10c636ae449e/ Log: One very probable bug fix, and one more theoretical diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -1111,6 +1111,8 @@ dprintf(("become_inevitable: %s\n", msg)); timing_fetch_inev(); + write_fence(); /* make sure others see a correct 'marker_inev' + if they see TS_INEVITABLE */ wait_for_end_of_inevitable_transaction(); STM_PSEGMENT->transaction_state = TS_INEVITABLE; stm_rewind_jmp_forget(STM_SEGMENT->running_thread); diff --git a/c7/stm/prof.c b/c7/stm/prof.c --- a/c7/stm/prof.c +++ b/c7/stm/prof.c @@ -32,12 +32,27 @@ int len0 = 0; int len1 = 0; if (markers != NULL) { + /* Obscure. In rare cases we see an uninitialized object + at 'markers[1]->segment_base + object'. I *think* it + is related to the other thread busy privatizing the + page. We thus need to acquire the privatization_lock + here (any will do). + + Note that we have to read the object from the foreign + segment. We can't assume the code object to be already + committed at all; maybe it was created during the remote + transaction. + */ + acquire_privatization_lock(); + if (markers[1].tl != NULL) buf.other_thread_num = markers[1].tl->thread_local_counter; if (markers[0].odd_number != 0) len0 = profiling_expand_marker(&markers[0], buf.extra, 128); if (markers[1].odd_number != 0) len1 = profiling_expand_marker(&markers[1], buf.extra + len0, 128); + + release_privatization_lock(); } buf.marker_length[0] = len0; buf.marker_length[1] = len1; From noreply at buildbot.pypy.org Mon Feb 2 21:23:51 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Feb 2015 21:23:51 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: import stmgc/10c636ae449e Message-ID: <20150202202351.B268B1C026B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r75654:79d715e680ed Date: 2015-02-02 21:23 +0100 http://bitbucket.org/pypy/pypy/changeset/79d715e680ed/ Log: import stmgc/10c636ae449e diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -88492dc56aa8 +10c636ae449e diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c --- a/rpython/translator/stm/src_stm/stm/core.c +++ b/rpython/translator/stm/src_stm/stm/core.c @@ -1112,6 +1112,8 @@ dprintf(("become_inevitable: %s\n", msg)); timing_fetch_inev(); + write_fence(); /* make sure others see a correct 'marker_inev' + if they see TS_INEVITABLE */ wait_for_end_of_inevitable_transaction(); STM_PSEGMENT->transaction_state = TS_INEVITABLE; stm_rewind_jmp_forget(STM_SEGMENT->running_thread); diff --git a/rpython/translator/stm/src_stm/stm/prof.c b/rpython/translator/stm/src_stm/stm/prof.c --- a/rpython/translator/stm/src_stm/stm/prof.c +++ b/rpython/translator/stm/src_stm/stm/prof.c @@ -33,12 +33,27 @@ int len0 = 0; int len1 = 0; if (markers != NULL) { + /* Obscure. In rare cases we see an uninitialized object + at 'markers[1]->segment_base + object'. I *think* it + is related to the other thread busy privatizing the + page. We thus need to acquire the privatization_lock + here (any will do). + + Note that we have to read the object from the foreign + segment. We can't assume the code object to be already + committed at all; maybe it was created during the remote + transaction. + */ + acquire_privatization_lock(); + if (markers[1].tl != NULL) buf.other_thread_num = markers[1].tl->thread_local_counter; if (markers[0].odd_number != 0) len0 = profiling_expand_marker(&markers[0], buf.extra, 128); if (markers[1].odd_number != 0) len1 = profiling_expand_marker(&markers[1], buf.extra + len0, 128); + + release_privatization_lock(); } buf.marker_length[0] = len0; buf.marker_length[1] = len1; From noreply at buildbot.pypy.org Mon Feb 2 21:52:26 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 2 Feb 2015 21:52:26 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.9: ssl: Add ALERT_DESCRIPTION_* constants Message-ID: <20150202205226.EE6041C02D4@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: stdlib-2.7.9 Changeset: r75656:5be3e3aef9b7 Date: 2015-02-02 21:20 +0100 http://bitbucket.org/pypy/pypy/changeset/5be3e3aef9b7/ Log: ssl: Add ALERT_DESCRIPTION_* constants diff --git a/pypy/module/_ssl/__init__.py b/pypy/module/_ssl/__init__.py --- a/pypy/module/_ssl/__init__.py +++ b/pypy/module/_ssl/__init__.py @@ -1,4 +1,5 @@ from pypy.interpreter.mixedmodule import MixedModule +from pypy.module._ssl import ssl_data class Module(MixedModule): """Implementation module for SSL socket operations. @@ -36,6 +37,9 @@ Module.interpleveldefs['RAND_status'] = "interp_ssl.RAND_status" Module.interpleveldefs['RAND_egd'] = "interp_ssl.RAND_egd" + for name, value in ssl_data.ALERT_DESCRIPTION_CODES.items(): + Module.interpleveldefs[name] = "space.wrap(%r)" % value + super(Module, cls).buildloaders() def startup(self, space): diff --git a/pypy/module/_ssl/ssl_data.py b/pypy/module/_ssl/ssl_data.py --- a/pypy/module/_ssl/ssl_data.py +++ b/pypy/module/_ssl/ssl_data.py @@ -347,6 +347,21 @@ setattr(CConfig, code, rffi_platform.DefinedConstantInteger( '%s_R_%s' % (lib, code))) +# Constants for ALERT_DESCRIPTION_ error codes +AD_NAMES = """ + CLOSE_NOTIFY UNEXPECTED_MESSAGE BAD_RECORD_MAC RECORD_OVERFLOW + DECOMPRESSION_FAILURE HANDSHAKE_FAILURE BAD_CERTIFICATE + UNSUPPORTED_CERTIFICATE CERTIFICATE_REVOKED CERTIFICATE_EXPIRED + CERTIFICATE_UNKNOWN ILLEGAL_PARAMETER UNKNOWN_CA ACCESS_DENIED + DECODE_ERROR DECRYPT_ERROR PROTOCOL_VERSION INSUFFICIENT_SECURITY + INTERNAL_ERROR USER_CANCELLED NO_RENEGOTIATION UNSUPPORTED_EXTENSION + CERTIFICATE_UNOBTAINABLE UNRECOGNIZED_NAME BAD_CERTIFICATE_STATUS_RESPONSE + BAD_CERTIFICATE_HASH_VALUE UNKNOWN_PSK_IDENTITY + """.split() +for name in AD_NAMES: + setattr(CConfig, name, rffi_platform.DefinedConstantInteger( + 'SSL_AD_%s' % name)) + cconfig = rffi_platform.configure(CConfig) LIBRARY_CODES_TO_NAMES = {} @@ -356,3 +371,8 @@ for lib, code in error_codes: ERROR_CODES_TO_NAMES[cconfig[lib], cconfig[code]] = code +ALERT_DESCRIPTION_CODES = {} +for name in AD_NAMES: + value = cconfig[name] + if value is not None: + ALERT_DESCRIPTION_CODES['ALERT_DESCRIPTION_%s' % name] = value diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py --- a/pypy/module/_ssl/test/test_ssl.py +++ b/pypy/module/_ssl/test/test_ssl.py @@ -38,6 +38,8 @@ assert isinstance(_ssl.OPENSSL_VERSION, str) assert 'openssl' in _ssl.OPENSSL_VERSION.lower() + assert isinstance(_ssl.ALERT_DESCRIPTION_ACCESS_DENIED, int) + def test_RAND_add(self): import _ssl if not hasattr(_ssl, "RAND_add"): From noreply at buildbot.pypy.org Mon Feb 2 21:52:28 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 2 Feb 2015 21:52:28 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.9: Fix test, checked with CPython Message-ID: <20150202205228.1B3691C02D4@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: stdlib-2.7.9 Changeset: r75657:22cbcc2a0bd4 Date: 2015-02-02 21:38 +0100 http://bitbucket.org/pypy/pypy/changeset/22cbcc2a0bd4/ Log: Fix test, checked with CPython diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py --- a/pypy/module/_ssl/test/test_ssl.py +++ b/pypy/module/_ssl/test/test_ssl.py @@ -330,7 +330,7 @@ with open(self.keycert) as f: cacert_pem = f.read().decode('ascii') ctx.load_verify_locations(cadata=cacert_pem) - assert ctx.cert_store_stats()["x509_ca"] + assert ctx.cert_store_stats()["x509_ca"] == 0 def test_get_ca_certs(self): import _ssl From noreply at buildbot.pypy.org Mon Feb 2 21:52:29 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 2 Feb 2015 21:52:29 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.9: ssl.get_default_verify_paths() Message-ID: <20150202205229.46E4D1C02D4@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: stdlib-2.7.9 Changeset: r75658:447c4c3f1931 Date: 2015-02-02 21:51 +0100 http://bitbucket.org/pypy/pypy/changeset/447c4c3f1931/ Log: ssl.get_default_verify_paths() diff --git a/pypy/module/_ssl/__init__.py b/pypy/module/_ssl/__init__.py --- a/pypy/module/_ssl/__init__.py +++ b/pypy/module/_ssl/__init__.py @@ -9,6 +9,7 @@ '_test_decode_cert': 'interp_ssl._test_decode_cert', 'txt2obj': 'interp_ssl.txt2obj', 'nid2obj': 'interp_ssl.nid2obj', + 'get_default_verify_paths': 'interp_ssl.get_default_verify_paths', 'SSLError': "interp_ssl.get_exception_class(space, 'w_sslerror')", 'SSLZeroReturnError': "interp_ssl.get_exception_class(space, 'w_sslzeroreturnerror')", diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -1563,3 +1563,18 @@ @unwrap_spec(nid=int) def nid2obj(space, nid): return space.newtuple([]) + + +def w_convert_path(space, path): + if not path: + return space.w_None + else: + return space.wrapbytes(rffi.charp2str(path)) + +def get_default_verify_paths(space): + return space.newtuple([ + w_convert_path(space, libssl_X509_get_default_cert_file_env()), + w_convert_path(space, libssl_X509_get_default_cert_file()), + w_convert_path(space, libssl_X509_get_default_cert_dir_env()), + w_convert_path(space, libssl_X509_get_default_cert_dir()), + ]) diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py --- a/pypy/module/_ssl/test/test_ssl.py +++ b/pypy/module/_ssl/test/test_ssl.py @@ -363,6 +363,14 @@ raises(TypeError, ctx.load_dh_params, None) raises(_ssl.SSLError, ctx.load_dh_params, self.keycert) + def test_get_default_verify_paths(self): + import _ssl + paths = _ssl.get_default_verify_paths() + assert paths[0] == 'SSL_CERT_FILE' + assert paths[2] == 'SSL_CERT_DIR' + assert paths[1].endswith('cert.pem') + assert paths[3].endswith('certs') + class AppTestSSLError: spaceconfig = dict(usemodules=('_ssl', '_socket', 'thread')) diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py --- a/rpython/rlib/ropenssl.py +++ b/rpython/rlib/ropenssl.py @@ -338,6 +338,11 @@ ssl_external('X509_VERIFY_PARAM_get_flags', [X509_VERIFY_PARAM], rffi.ULONG) ssl_external('X509_STORE_add_cert', [X509_STORE, X509], rffi.INT) +ssl_external('X509_get_default_cert_file_env', [], rffi.CCHARP) +ssl_external('X509_get_default_cert_file', [], rffi.CCHARP) +ssl_external('X509_get_default_cert_dir_env', [], rffi.CCHARP) +ssl_external('X509_get_default_cert_dir', [], rffi.CCHARP) + ssl_external('OBJ_obj2txt', [rffi.CCHARP, rffi.INT, ASN1_OBJECT, rffi.INT], rffi.INT) ssl_external('OBJ_obj2nid', [ASN1_OBJECT], rffi.INT) From noreply at buildbot.pypy.org Mon Feb 2 22:01:59 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Feb 2015 22:01:59 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Improve the basic tests Message-ID: <20150202210159.477D51C02D4@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r75659:3ab2ca363785 Date: 2015-02-02 22:01 +0100 http://bitbucket.org/pypy/pypy/changeset/3ab2ca363785/ Log: Improve the basic tests diff --git a/pypy/module/pypystm/test_pypy_c/support.py b/pypy/module/pypystm/test_pypy_c/support.py --- a/pypy/module/pypystm/test_pypy_c/support.py +++ b/pypy/module/pypystm/test_pypy_c/support.py @@ -72,7 +72,10 @@ stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = pipe.communicate() - if getattr(pipe, 'returncode', 0) < 0: + if pipe.returncode > 0: + raise IOError("subprocess error %d:\n%s" % (pipe.returncode, + stderr)) + if pipe.returncode < 0: raise IOError("subprocess was killed by signal %d" % ( pipe.returncode,)) @@ -80,8 +83,18 @@ from pypy.stm.print_stm_log import StmLog return StmLog(str(self.logfile)) - def check_almost_no_conflict(self, func_or_src, args=[]): + def _check_count_conflicts(self, func_or_src, args=[]): self._write_source(func_or_src, args) self._execute() stmlog = self._parse_log() - assert stmlog.get_total_aborts_and_pauses() < 1000 + count = stmlog.get_total_aborts_and_pauses() + print 'stmlog.get_total_aborts_and_pauses():', count + return count + + def check_almost_no_conflict(self, *args): + count = self._check_count_conflicts(*args) + assert count < 500 + + def check_many_conflicts(self, *args): + count = self._check_count_conflicts(*args) + assert count > 20000 diff --git a/pypy/module/pypystm/test_pypy_c/test_conflict.py b/pypy/module/pypystm/test_pypy_c/test_conflict.py new file mode 100644 --- /dev/null +++ b/pypy/module/pypystm/test_pypy_c/test_conflict.py @@ -0,0 +1,16 @@ +from pypy.module.pypystm.test_pypy_c.support import BaseTestSTM + + +class TestConflict(BaseTestSTM): + + def test_obvious(self): + def f(): + class X(object): + pass + x = X() # shared + x.a = 0 + def g(): + x.a += 1 + run_in_threads(g) + # + self.check_many_conflicts(f) From noreply at buildbot.pypy.org Mon Feb 2 22:10:09 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Feb 2015 22:10:09 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: more tests Message-ID: <20150202211009.DE9F11C026B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r75662:255b484dd279 Date: 2015-02-02 22:09 +0100 http://bitbucket.org/pypy/pypy/changeset/255b484dd279/ Log: more tests diff --git a/pypy/module/pypystm/test_pypy_c/support.py b/pypy/module/pypystm/test_pypy_c/support.py --- a/pypy/module/pypystm/test_pypy_c/support.py +++ b/pypy/module/pypystm/test_pypy_c/support.py @@ -8,24 +8,27 @@ HEADER = """ import thread, pypystm -def _run(lock, result, function): +def _run(lock, result, function, args): start = pypystm.time() try: while True: - function() + function(*args) if pypystm.time() - start >= 3.0: break result.append(1) finally: lock.release() -def run_in_threads(function): +def run_in_threads(function, arg_thread_num=False): locks = [] result = [] for i in range(3): lock = thread.allocate_lock() lock.acquire() - thread.start_new_thread(_run, (lock, result, function)) + args = () + if arg_thread_num: + args += (i,) + thread.start_new_thread(_run, (lock, result, function, args)) locks.append(lock) for lock in locks: lock._py3k_acquire(timeout=30) diff --git a/pypy/module/pypystm/test_pypy_c/test_conflict.py b/pypy/module/pypystm/test_pypy_c/test_conflict.py --- a/pypy/module/pypystm/test_pypy_c/test_conflict.py +++ b/pypy/module/pypystm/test_pypy_c/test_conflict.py @@ -14,3 +14,12 @@ run_in_threads(g) # self.check_many_conflicts(f) + + def test_plain_dict_access(self): + def f(): + d = {} # shared + def g(n): + d[n] = d.get(n, 0) + 1 + run_in_threads(g, arg_thread_num=True) + # + self.check_many_conflicts(f) diff --git a/pypy/module/pypystm/test_pypy_c/test_no_conflict.py b/pypy/module/pypystm/test_pypy_c/test_no_conflict.py --- a/pypy/module/pypystm/test_pypy_c/test_no_conflict.py +++ b/pypy/module/pypystm/test_pypy_c/test_no_conflict.py @@ -17,3 +17,13 @@ run_in_threads(g) # self.check_almost_no_conflict(f) + + def test_stmdict_access(self): + def f(): + import pypystm + d = pypystm.stmdict() # shared + def g(n): + d[n] = d.get(n, 0) + 1 + run_in_threads(g, arg_thread_num=True) + # + self.check_almost_no_conflict(f) From noreply at buildbot.pypy.org Mon Feb 2 22:12:20 2015 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 2 Feb 2015 22:12:20 +0100 (CET) Subject: [pypy-commit] pypy release-2.5.x: merge default into branch Message-ID: <20150202211220.A3BF61C026B@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: release-2.5.x Changeset: r75663:10f1b29a2bd2 Date: 2015-02-02 23:12 +0200 http://bitbucket.org/pypy/pypy/changeset/10f1b29a2bd2/ Log: merge default into branch diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info --- a/lib_pypy/greenlet.egg-info +++ b/lib_pypy/greenlet.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: greenlet -Version: 0.4.0 +Version: 0.4.5 Summary: Lightweight in-process concurrent programming Home-page: https://github.com/python-greenlet/greenlet Author: Ralf Schmitt (for CPython), PyPy team diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py --- a/lib_pypy/greenlet.py +++ b/lib_pypy/greenlet.py @@ -1,7 +1,7 @@ import sys import _continuation -__version__ = "0.4.0" +__version__ = "0.4.5" # ____________________________________________________________ # Exceptions diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py --- a/pypy/module/micronumpy/ctors.py +++ b/pypy/module/micronumpy/ctors.py @@ -75,22 +75,32 @@ if order != 'C': # or order != 'F': raise oefmt(space.w_ValueError, "Unknown order: %s", order) - # arrays with correct dtype - if isinstance(w_object, W_NDimArray) and \ - (space.is_none(w_dtype) or w_object.get_dtype() is dtype): - if copy and (subok or type(w_object) is W_NDimArray): - return w_object.descr_copy(space, w_order) - elif not copy and (subok or type(w_object) is W_NDimArray): - return w_object - if isinstance(w_object, W_NDimArray) and copy and not subok: - # TODO do the loop.assign without copying elems_w - shape = w_object.get_shape() - _elems_w = w_object.reshape(space, space.wrap(-1)) - elems_w = [None] * w_object.get_size() - for i in range(len(elems_w)): - elems_w[i] = _elems_w.descr_getitem(space, space.wrap(i)) - if space.is_none(w_dtype): + if isinstance(w_object, W_NDimArray): + if (dtype is None or w_object.get_dtype() is dtype): + if copy and (subok or type(w_object) is W_NDimArray): + return w_object.descr_copy(space, w_order) + elif not copy and (subok or type(w_object) is W_NDimArray): + return w_object + # we have a ndarray, but need to copy or change dtype or create W_NDimArray + if dtype is None: dtype = w_object.get_dtype() + if dtype != w_object.get_dtype(): + # silently reject the copy value + copy = True + if copy: + shape = w_object.get_shape() + _elems_w = w_object.reshape(space, space.wrap(-1)) + elems_w = [None] * w_object.get_size() + for i in range(len(elems_w)): + elems_w[i] = _elems_w.descr_getitem(space, space.wrap(i)) + elif subok: + raise oefmt(space.w_NotImplementedError, + "array(...copy=False, subok=True) not implemented yet") + else: + sz = support.product(w_object.get_shape()) * dtype.elsize + return W_NDimArray.from_shape_and_storage(space, + w_object.get_shape(),w_object.implementation.storage, + dtype, storage_bytes=sz, w_base=w_object) else: # not an array shape, elems_w = strides.find_shape_and_elems(space, w_object, dtype) diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py --- a/pypy/module/micronumpy/test/test_subtype.py +++ b/pypy/module/micronumpy/test/test_subtype.py @@ -268,7 +268,7 @@ c = array(a, float) assert c.dtype is dtype(float) - def test__getitem_modifies_shape(self): + def test_array_of_subtype(self): import numpy as N # numpy's matrix class caused an infinite loop class matrix(N.ndarray): @@ -309,8 +309,14 @@ a = matrix([[1., 2.], [3., 4.]]) b = N.array([a]) assert (b == a).all() + b = N.array(a) assert len(b.shape) == 2 + assert (b == a).all() + + b = N.array(a, copy=False) + assert len(b.shape) == 2 + assert (b == a).all() def test_setstate_no_version(self): # Some subclasses of ndarray, like MaskedArray, do not use diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py --- a/pypy/module/micronumpy/test/test_ufuncs.py +++ b/pypy/module/micronumpy/test/test_ufuncs.py @@ -220,6 +220,9 @@ af = arange(10, dtype=float) af2 = ufunc(af) assert all(af2 == af * 2) + ac = arange(10, dtype=complex) + skip('casting not implemented yet') + ac1 = ufunc(ac) def test_frompyfunc_2d_sig(self): def times_2(in_array, out_array): diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py --- a/pypy/module/micronumpy/ufuncs.py +++ b/pypy/module/micronumpy/ufuncs.py @@ -774,8 +774,13 @@ break else: if len(self.funcs) > 1: - dtypesstr = ','.join(['%s%s%s' % (d.byteorder, d.kind, d.elsize) \ - for d in dtypes]) + + dtypesstr = '' + for d in dtypes: + if d is None: + dtypesstr += 'None,' + else: + dtypesstr += '%s%s%s,' % (d.byteorder, d.kind, d.elsize) _dtypesstr = ','.join(['%s%s%s' % (d.byteorder, d.kind, d.elsize) \ for d in _dtypes]) raise oefmt(space.w_TypeError, diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -535,3 +535,9 @@ def isatty(self): self._check_closed() return os.isatty(c_fileno(self._ll_file)) + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() diff --git a/rpython/rlib/test/test_rfile.py b/rpython/rlib/test/test_rfile.py --- a/rpython/rlib/test/test_rfile.py +++ b/rpython/rlib/test/test_rfile.py @@ -26,6 +26,7 @@ f() assert open(fname, "r").read() == "dupa" + os.unlink(fname) self.interpret(f, []) assert open(fname, "r").read() == "dupa" @@ -102,6 +103,7 @@ f2.close() f() + os.unlink(fname) self.interpret(f, []) @py.test.mark.skipif("sys.platform == 'win32'") @@ -121,6 +123,7 @@ f2.close() f() + os.unlink(fname) self.interpret(f, []) def test_open_buffering_full(self): @@ -138,6 +141,7 @@ f2.close() f() + os.unlink(fname) self.interpret(f, []) def test_fdopen_buffering_full(self): @@ -157,6 +161,7 @@ f2.close() f() + os.unlink(fname) self.interpret(f, []) def test_read_write(self): @@ -203,6 +208,7 @@ f2.close() f() + os.unlink(fname) self.interpret(f, []) def test_read_sequentially(self): @@ -277,6 +283,7 @@ f.close() f() + os.unlink(fname) self.interpret(f, []) def test_tempfile(self): @@ -309,6 +316,7 @@ f() assert open(fname).read() == "xxx" + os.unlink(fname) self.interpret(f, []) assert open(fname).read() == "xxx" @@ -325,6 +333,7 @@ res = f() assert res > 2 + os.unlink(fname) res = self.interpret(f, []) assert res > 2 @@ -341,6 +350,7 @@ res = f() assert res == 3 + os.unlink(fname) res = self.interpret(f, []) assert res == 3 @@ -357,6 +367,7 @@ f.close() f() + os.unlink(fname) self.interpret(f, []) def test_truncate(self): @@ -381,8 +392,28 @@ f.close() f() + os.unlink(fname) self.interpret(f, []) + def test_with_statement(self): + fname = str(self.tmpdir.join('file_6')) + + def f(): + with open(fname, "w") as f: + f.write("dupa") + try: + f.write("dupb") + except ValueError: + pass + else: + assert False + + f() + assert open(fname, "r").read() == "dupa" + os.unlink(fname) + self.interpret(f, []) + assert open(fname, "r").read() == "dupa" + class TestDirect: def setup_class(cls): From noreply at buildbot.pypy.org Mon Feb 2 23:38:59 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 2 Feb 2015 23:38:59 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Fight fight until I can reliably get a failure mode for weakrefs Message-ID: <20150202223859.A22271C0FDC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r75670:5b9137e56e6b Date: 2015-02-02 23:38 +0100 http://bitbucket.org/pypy/pypy/changeset/5b9137e56e6b/ Log: Fight fight until I can reliably get a failure mode for weakrefs diff --git a/pypy/module/pypystm/test_pypy_c/support.py b/pypy/module/pypystm/test_pypy_c/support.py --- a/pypy/module/pypystm/test_pypy_c/support.py +++ b/pypy/module/pypystm/test_pypy_c/support.py @@ -8,27 +8,66 @@ HEADER = """ import thread, pypystm -def _run(lock, result, function, args): +NUM_THREADS = 3 + +_b_to_go = NUM_THREADS +_b_done = False +_b_lock = thread.allocate_lock() +_b_locks = [thread.allocate_lock() for _i in range(NUM_THREADS)] +for _bl in _b_locks: + _bl.acquire() + +class BarrierThreadsDone(Exception): + pass + +def barrier(tnum, done=False): + '''Waits until NUM_THREADS call this function, and then returns + in all these threads at once.''' + global _b_to_go, _b_done + _b_lock.acquire() + if done: + _b_done = True + _b_to_go -= 1 + if _b_to_go > 0: + _b_lock.release() + _b_locks[tnum].acquire() + else: + _b_to_go = NUM_THREADS + for i in range(NUM_THREADS): + if i != tnum: + _b_locks[i].release() + _b_lock.release() + if _b_done: + raise BarrierThreadsDone + +def _run(tnum, lock, result, function, args): start = pypystm.time() try: - while True: - function(*args) - if pypystm.time() - start >= 3.0: - break + try: + while True: + function(*args) + if pypystm.time() - start >= 3.0: + break + except BarrierThreadsDone: + pass result.append(1) finally: lock.release() + while len(result) != NUM_THREADS: + barrier(tnum, done=True) -def run_in_threads(function, arg_thread_num=False): +def run_in_threads(function, arg_thread_num=False, arg_class=None): locks = [] result = [] - for i in range(3): + for i in range(NUM_THREADS): lock = thread.allocate_lock() lock.acquire() args = () if arg_thread_num: args += (i,) - thread.start_new_thread(_run, (lock, result, function, args)) + if arg_class: + args += (arg_class(),) + thread.start_new_thread(_run, (i, lock, result, function, args)) locks.append(lock) for lock in locks: lock._py3k_acquire(timeout=30) @@ -98,6 +137,10 @@ count = self._check_count_conflicts(*args) assert count < 500 - def check_many_conflicts(self, *args): + def check_MANY_conflicts(self, *args): count = self._check_count_conflicts(*args) assert count > 20000 + + def check_SOME_conflicts(self, *args): + count = self._check_count_conflicts(*args) + assert count > 1000 diff --git a/pypy/module/pypystm/test_pypy_c/test_conflict.py b/pypy/module/pypystm/test_pypy_c/test_conflict.py --- a/pypy/module/pypystm/test_pypy_c/test_conflict.py +++ b/pypy/module/pypystm/test_pypy_c/test_conflict.py @@ -13,7 +13,7 @@ x.a += 1 run_in_threads(g) # - self.check_many_conflicts(f) + self.check_MANY_conflicts(f) def test_plain_dict_access(self): def f(): @@ -22,4 +22,25 @@ d[n] = d.get(n, 0) + 1 run_in_threads(g, arg_thread_num=True) # - self.check_many_conflicts(f) + self.check_MANY_conflicts(f) + + def test_write_to_many_objects_in_order(self): + def f(): + import weakref + + class X(object): + pass + + lst = [] # shared + + def g(tnum): + if tnum == 0: + lst[:] = [X() for i in range(1000)] + barrier(tnum) + for x in lst: + x.a = 5 + barrier(tnum) + + run_in_threads(g, arg_thread_num=True) + # + self.check_SOME_conflicts(f) diff --git a/pypy/module/pypystm/test_pypy_c/test_no_conflict.py b/pypy/module/pypystm/test_pypy_c/test_no_conflict.py --- a/pypy/module/pypystm/test_pypy_c/test_no_conflict.py +++ b/pypy/module/pypystm/test_pypy_c/test_no_conflict.py @@ -1,3 +1,4 @@ +import py from pypy.module.pypystm.test_pypy_c.support import BaseTestSTM @@ -27,3 +28,25 @@ run_in_threads(g, arg_thread_num=True) # self.check_almost_no_conflict(f) + + def test_weakrefs(self): + py.test.skip("next issue") + def f(): + import weakref + + class X(object): + pass + + lst = [] # shared + + def g(tnum): + if tnum == 0: + lst[:] = [X() for i in range(1000)] + barrier(tnum) + for x in lst: + weakref.ref(x) + barrier(tnum) + + run_in_threads(g, arg_thread_num=True) + # + self.check_almost_no_conflict(f) From noreply at buildbot.pypy.org Tue Feb 3 11:36:34 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 3 Feb 2015 11:36:34 +0100 (CET) Subject: [pypy-commit] pypy default: Fix jittest.py to accept targets with no specified jitpolicy() Message-ID: <20150203103634.A81CC1C068A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75674:410c76551947 Date: 2015-02-03 11:36 +0100 http://bitbucket.org/pypy/pypy/changeset/410c76551947/ Log: Fix jittest.py to accept targets with no specified jitpolicy() diff --git a/rpython/jit/tl/jittest.py b/rpython/jit/tl/jittest.py --- a/rpython/jit/tl/jittest.py +++ b/rpython/jit/tl/jittest.py @@ -15,16 +15,23 @@ reload(runsubprocess) -ARGS = ["--jit", "threshold=100000,trace_eagerness=100000", - "-S", "/home/arigo/pypysrc/32compiled/z.py"] +#ARGS = ["--jit", "threshold=100000,trace_eagerness=100000", +# "-S", "/home/arigo/pypysrc/32compiled/z.py"] +ARGS = ["targettest_executable_name", + "-r", "13", "/home/arigo/git/pyrlang/test_beam/fact.beam", + "fact", "20000"] def jittest(driver): graph = driver.translator._graphof(driver.entry_point) interp = LLInterpreter(driver.translator.rtyper) - get_policy = driver.extra['jitpolicy'] - jitpolicy = get_policy(driver) + get_policy = driver.extra.get('jitpolicy', None) + if get_policy is None: + from rpython.jit.codewriter.policy import JitPolicy + jitpolicy = JitPolicy() + else: + jitpolicy = get_policy(driver) from rpython.jit.backend.llgraph.runner import LLGraphCPU apply_jit(jitpolicy, interp, graph, LLGraphCPU) From noreply at buildbot.pypy.org Tue Feb 3 12:02:20 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 3 Feb 2015 12:02:20 +0100 (CET) Subject: [pypy-commit] pypy default: Fix for tests Message-ID: <20150203110220.B3F301C114E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75676:fdb74eb88378 Date: 2015-02-03 12:00 +0100 http://bitbucket.org/pypy/pypy/changeset/fdb74eb88378/ Log: Fix for tests diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py --- a/rpython/jit/metainterp/blackhole.py +++ b/rpython/jit/metainterp/blackhole.py @@ -3,6 +3,7 @@ from rpython.jit.metainterp.compile import ResumeAtPositionDescr from rpython.jit.metainterp.jitexc import get_llexception, reraise from rpython.jit.metainterp import jitexc +from rpython.jit.metainterp.history import MissingValue from rpython.rlib import longlong2float from rpython.rlib.debug import ll_assert, make_sure_not_resized from rpython.rlib.objectmodel import we_are_translated @@ -29,9 +30,6 @@ class LeaveFrame(jitexc.JitException): pass -class MissingValue(object): - "NOT_RPYTHON" - def signedord(c): value = ord(c) value = intmask(value << (LONG_BIT-8)) >> (LONG_BIT-8) @@ -1595,14 +1593,20 @@ self.setposition(miframe.jitcode, miframe.pc) for i in range(self.jitcode.num_regs_i()): box = miframe.registers_i[i] + if not we_are_translated() and isinstance(box, MissingValue): + continue if box is not None: self.setarg_i(i, box.getint()) for i in range(self.jitcode.num_regs_r()): box = miframe.registers_r[i] + if not we_are_translated() and isinstance(box, MissingValue): + continue if box is not None: self.setarg_r(i, box.getref_base()) for i in range(self.jitcode.num_regs_f()): box = miframe.registers_f[i] + if not we_are_translated() and isinstance(box, MissingValue): + continue if box is not None: self.setarg_f(i, box.getfloatstorage()) diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py --- a/rpython/jit/metainterp/history.py +++ b/rpython/jit/metainterp/history.py @@ -170,6 +170,10 @@ self.identifier = identifier # for testing +class MissingValue(object): + "NOT_RPYTHON" + + class Const(AbstractValue): __slots__ = () diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -8,7 +8,7 @@ from rpython.jit.metainterp import history, compile, resume, executor, jitexc from rpython.jit.metainterp.heapcache import HeapCache from rpython.jit.metainterp.history import (Const, ConstInt, ConstPtr, - ConstFloat, Box, TargetToken) + ConstFloat, Box, TargetToken, MissingValue) from rpython.jit.metainterp.jitprof import EmptyProfiler from rpython.jit.metainterp.logger import Logger from rpython.jit.metainterp.optimizeopt.util import args_dict @@ -43,9 +43,6 @@ "ge": "history.CONST_TRUE", } -class MissingValue(object): - "NOT_RPYTHON" - class MIFrame(object): debug = False From noreply at buildbot.pypy.org Tue Feb 3 12:24:52 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 3 Feb 2015 12:24:52 +0100 (CET) Subject: [pypy-commit] pypy default: even clearer Message-ID: <20150203112452.A053D1C114F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75678:ed8716ebf22a Date: 2015-02-03 12:24 +0100 http://bitbucket.org/pypy/pypy/changeset/ed8716ebf22a/ Log: even clearer diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst --- a/pypy/doc/how-to-release.rst +++ b/pypy/doc/how-to-release.rst @@ -41,7 +41,7 @@ * repackage and upload source "-src.tar.bz2" to bitbucket and to cobra, as some packagers prefer a clearly labeled source package (download e.g. https://bitbucket.org/pypy/pypy/get/release-2.5.x.tar.bz2, unpack, - rename the top-level directory, repack, and upload) + rename the top-level directory to "pypy-2.5.0-src", repack, and upload) * write release announcement pypy/doc/release-x.y(.z).txt the release announcement should contain a direct link to the download page From noreply at buildbot.pypy.org Tue Feb 3 15:24:15 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 3 Feb 2015 15:24:15 +0100 (CET) Subject: [pypy-commit] pypy default: Document this restriction (for now). Message-ID: <20150203142415.09AAE1C0F98@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75681:b06ed3201641 Date: 2015-02-03 15:23 +0100 http://bitbucket.org/pypy/pypy/changeset/b06ed3201641/ Log: Document this restriction (for now). diff --git a/rpython/doc/jit/virtualizable.rst b/rpython/doc/jit/virtualizable.rst --- a/rpython/doc/jit/virtualizable.rst +++ b/rpython/doc/jit/virtualizable.rst @@ -66,3 +66,11 @@ virtualizable survives for longer, you want to force it before returning. It's better to do it that way than by an external call some time later. It's done using ``jit.hint(frame, force_virtualizable=True)`` + +* Your interpreter should have a local variable similar to ``frame`` + above. It must not be modified as long as it runs its + ``jit_merge_point`` loop, and in the loop it must be passed directly + to the ``jit_merge_point()`` and ``can_enter_jit()`` calls. The JIT + generator is known to produce buggy code if you fetch the + virtualizable from somewhere every iteration, instead of reusing the + same unmodified local variable. From noreply at buildbot.pypy.org Tue Feb 3 19:06:02 2015 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 3 Feb 2015 19:06:02 +0100 (CET) Subject: [pypy-commit] pypy vmprof: add some asserts Message-ID: <20150203180602.052A41C0820@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r75683:c93a10fdfeef Date: 2015-02-03 20:05 +0200 http://bitbucket.org/pypy/pypy/changeset/c93a10fdfeef/ Log: add some asserts diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py --- a/rpython/jit/backend/llsupport/codemap.py +++ b/rpython/jit/backend/llsupport/codemap.py @@ -52,5 +52,13 @@ pos = self.patch_position.pop() self.l[pos] = size self.l[pos + 1] = len(self.l) + # at the end there should be no zeros + for i in range(len(self.l) / 4): + item = self.l[i * 4] # unique_id + assert item > 0 # no zeros here + item = self.l[i * 4 + 2] # end in asm + assert item > 0 + item = self.l[i * 4 + 3] # end in l + assert item > 0 return (addr, size, self.l) # XXX compact self.l From noreply at buildbot.pypy.org Tue Feb 3 20:06:19 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 3 Feb 2015 20:06:19 +0100 (CET) Subject: [pypy-commit] pypy release-2.5.x: cleanup release notice Message-ID: <20150203190619.24A0C1C0F98@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: release-2.5.x Changeset: r75684:62a418bcbc7f Date: 2015-02-03 17:28 +0200 http://bitbucket.org/pypy/pypy/changeset/62a418bcbc7f/ Log: cleanup release notice diff --git a/pypy/doc/release-2.5.0.rst b/pypy/doc/release-2.5.0.rst --- a/pypy/doc/release-2.5.0.rst +++ b/pypy/doc/release-2.5.0.rst @@ -3,7 +3,7 @@ ================================================= We're pleased to announce PyPy 2.5, which contains significant performance -enhancements and bug fixes. +enhancements and bug fixes. You can download the PyPy 2.5.0 release here: @@ -11,7 +11,7 @@ We would like to thank our donors for the continued support of the PyPy project, and for those who donate to our three sub-projects, as well as our -volunteers and contributors (10 new commiters joined PyPy since the last +volunteers and contributors (10 new commiters joined PyPy since the last release). We've shown quite a bit of progress, but we're slowly running out of funds. Please consider donating more, or even better convince your employer to donate, @@ -23,13 +23,13 @@ * `STM`_ (software transactional memory): We have released a first working version, and continue to try out new promising paths of achieving a fast multithreaded Python -* `NumPy`_ which requires installation of our fork of upstream numpy, +* `NumPy`_ which requires installation of our fork of upstream numpy, available `on bitbucket`_ .. _`Py3k`: http://pypy.org/py3donate.html .. _`STM`: http://pypy.org/tmdonate2.html .. _`NumPy`: http://pypy.org/numpydonate.html -.. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy +.. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy What is PyPy? ============= @@ -38,9 +38,9 @@ CPython 2.7. It's fast (`pypy and cpython 2.7.x`_ performance comparison) due to its integrated tracing JIT compiler. -This release supports **x86** machines on most common operating systems +This release supports **x86** machines on most common operating systems (Linux 32/64, Mac OS X 64, Windows, and OpenBSD), -as well as newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux. +as well as newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux. While we support 32 bit python on Windows, work on the native Windows 64 bit python is still stalling, we would welcome a volunteer @@ -55,36 +55,34 @@ * The past months have seen pypy mature and grow, as rpython becomes the goto solution for writing fast dynamic language interpreters. Our separation of rpython and the python interpreter PyPy is now much clearer in the - `documentation`_ . + `PyPy documentation`_ and we now have seperate `Rpython documentation`_. * We have improved warmup time as well as jitted code performance: more than 10% - compared to pypy-2.4.0, due to internal cleanup and gc nursery improvements. + compared to pypy-2.4.0, due to internal cleanup and gc nursery improvements. * Our integrated numpy support gained much of the GenericUfunc api in order to support the lapack/blas linalg module of numpy. This dovetails with work in the pypy/numpy repository to support linalg both through the (slower) cpyext capi interface and also via (the faster) pure python cffi interface, using an - extended frompyfunc() api. + extended frompyfunc() api. We will soon post a seperate blog post specifically + about linalg and PyPy. * Dictionaries are now ordered by default, see the `blog post`_ -* Issues reported with our previous release were fixed after reports from users on - our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at - #pypy. Here is a summary of some of the user-facing changes; - for more information see `whats-new`_: - * Our nightly translations use --shared by default, including on OS/X and linux * We now more carefully handle errno (and GetLastError, WSAGetLastError) tying the handlers as close as possible to the external function call, in non-jitted as well as jitted code. - -* Many issues were resolved_ since the 2.4.0 release in September 2014 -.. _`documentation`: http://doc.pypy.org +* Issues reported with our previous release were resolved_ after reports from users on + our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at + #pypy. + +.. _`PyPy documentation`: http://doc.pypy.org +.. _`Rpython documentation`: http://rpython.readthedocs.org .. _`blog post`: http://morepypy.blogspot.com/2015/01/faster-more-memory-efficient-and-more.html .. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.5.0.html -.. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved We have further improvements on the way: rpython file handling, finishing numpy linalg compatibility, numpy object dtypes, a better profiler, @@ -96,4 +94,3 @@ Cheers The PyPy Team - From noreply at buildbot.pypy.org Tue Feb 3 20:45:36 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 3 Feb 2015 20:45:36 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: 2.5.0 release Message-ID: <20150203194536.D91981C0F98@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: extradoc Changeset: r564:5072e83d9f25 Date: 2015-02-03 21:46 +0200 http://bitbucket.org/pypy/pypy.org/changeset/5072e83d9f25/ Log: 2.5.0 release diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -9,32 +9,31 @@ .. class:: download_menu There are `nightly binary builds`_ available. Those builds are not always - as stable as the release, but they contain numerous bugfixes and + as stable as the release, but they contain numerous bugfixes and performance improvements. **Note that the OS X nightly builds (but not the release) are slightly miscompiled due to buildslave being old. Contributions are welcomed**. We provide binaries for x86 and ARM Linux, Mac OS/X and Windows for: -* the Python2.7 compatible release — **PyPy 2.4.0** — (`what's new in PyPy 2.4.0?`_ ) +* the Python2.7 compatible release — **PyPy 2.5.0** — (`what's new in PyPy 2.5.0?`_ ) * the Python3.2.5 compatible release — **PyPy3 2.4.0** — (`what's new in PyPy3 2.4.0?`_). -.. _what's new in PyPy 2.4.0?: http://doc.pypy.org/en/latest/release-2.4.0.html +.. _what's new in PyPy 2.5.0?: http://doc.pypy.org/en/latest/release-2.5.0.html .. _what's new in PyPy3 2.4.0?: http://doc.pypy.org/en/latest/release-pypy3-2.4.0.html -.. _what's new in PyPy 2.4?: http://doc.pypy.org/en/latest/release-2.4.0.html .. class:: download_menu - * Download - + * Download + * `Default (with a JIT Compiler)`_ * `Other versions`_ * `Installing`_ (optional) * `Installing more modules`_ * `Installing NumPy`_ (optional) - * `Building from source`_ + * `Building from source`_ * `Packaging`_ * `Checksums`_ @@ -72,7 +71,7 @@ .. _`portable Linux binaries`: https://github.com/squeaky-pl/portable-pypy -Python2.7 compatible PyPy 2.4.0 +Python2.7 compatible PyPy 2.5.0 ----------------------------------- * `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below) @@ -82,18 +81,22 @@ * `ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2, Ubuntu Precise)`__ (see ``[1]`` below) * `Mac OS/X binary (64bit)`__ * `Windows binary (32bit)`__ (you might need the `VS 2008 runtime library +* `Source (tar.bz2)`__ +* `Source (zip)`__ installer vcredist_x86.exe`_.) * `All our downloads,`__ including previous versions. We also have a mirror_, but please use only if you have troubles accessing the links above * See below for the sources. -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4.0-linux.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4.0-linux64.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4.0-linux-armhf-raspbian.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4.0-linux-armhf-raring.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4.0-linux-armel.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4.0-osx64.tar.bz2 -.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4.0-win32.zip +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.0-linux.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.0-linux64.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.0-linux-armhf-raspbian.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.0-linux-armhf-raring.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.0-linux-armel.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.0-osx64.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.0-win32.zip +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.0-src.tar.bz2 +.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.0-src.zip .. _`VS 2008 runtime library installer vcredist_x86.exe`: http://www.microsoft.com/en-us/download/details.aspx?id=5582 .. __: https://bitbucket.org/pypy/pypy/downloads .. _mirror: http://cobra.cs.uni-duesseldorf.de/~buildmaster/mirror/ @@ -172,7 +175,7 @@ uncompressed, they run in-place. For now you can uncompress them either somewhere in your home directory or, say, in ``/opt``, and if you want, put a symlink from somewhere like -``/usr/local/bin/pypy`` to ``/path/to/pypy-2.4.0/bin/pypy``. Do +``/usr/local/bin/pypy`` to ``/path/to/pypy-2.5.0/bin/pypy``. Do not move or copy the executable ``pypy`` outside the tree --- put a symlink to it, otherwise it will not find its libraries. @@ -221,9 +224,9 @@ 1. Get the source code. The following packages contain the source at the same revision as the above binaries: - * `pypy-2.4.0-src.tar.bz2`__ (sources) + * `pypy-2.5.0-src.tar.bz2`__ (sources) - .. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.4.0-src.tar.bz2 + .. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.0-src.tar.bz2 Or you can checkout the current trunk using Mercurial_ (the trunk usually works and is of course more up-to-date):: @@ -297,7 +300,7 @@ cd ./pypy/pypy/tool/release/ python package.py --help #for information - python package.py --archive-name pypy-my-own-package-name + python package.py --archive-name pypy-my-own-package-name It is recommended to use package.py because custom scripts will invariably become out-of-date. If you want to write custom scripts @@ -317,14 +320,15 @@ Here are the checksums for each of the downloads (md5 and sha1):: - 63bd68546f60cf5921ba7654f3fe47aa pypy-2.4.0-linux64.tar.bz2 - 6c9b444a1cd090ab7b43083a24e07734 pypy-2.4.0-linux-armel.tar.bz2 - 5ff951da5989a00e01611678c311f8af pypy-2.4.0-linux-armhf-raring.tar.bz2 - d7540883a52f91433da62b0cdfaaa30f pypy-2.4.0-linux-armhf-raspbian.tar.bz2 - 77a971f5198685ff60528de5def407dd pypy-2.4.0-linux.tar.bz2 - 907d6fbabc5bcd5bafdcf02a76a8ca33 pypy-2.4.0-win32.zip - 6a25a212e7c5121f1f3988c118d05695 pypy-2.4.0-src.tar.bz2 - 07896c0ac37f82884e021c9a4514f479 pypy-2.4.0-osx64.tar.bz2 + + 0a109c96bbce20931c17dfded57a3d46 pypy-2.5.0-linux64.tar.bz2 + fc79e0ef640371fa7bdbb040c3211bcf pypy-2.5.0-linux-armel.tar.bz2 + d8d0fe8a1503a7ed4ec5ff52d0630b88 pypy-2.5.0-linux-armhf-raring.tar.bz2 + 9a4430abd36dd619c45705c3fe70f5ca pypy-2.5.0-linux-armhf-raspbian.tar.bz2 + 1c14e954fdd1d112540790ec5b0cea26 pypy-2.5.0-linux.tar.bz2 + 9916a1485f32953a46afdd36f8f01f83 pypy-2.5.0-osx64.tar.bz2 + f4700c0af45e986178b36ce91a45136e pypy-2.5.0-src.tar.bz2 + 2461a4e4fe4ea64ce7fb1a07aa310e6a pypy-2.5.0-win32.zip eadbc9790823fc0ae40c943087cd7cb3 pypy3-2.4.0-linux64.tar.bz2 7ab84727da2d5363866907f2f7921d86 pypy3-2.4.0-linux-armel.tar.bz2 @@ -339,14 +343,14 @@ 2c9f0054f3b93a6473f10be35277825a pypy-1.8-sandbox-linux64.tar.bz2 009c970b5fa75754ae4c32a5d108a8d4 pypy-1.8-sandbox-linux.tar.bz2 - c362247226f1cde2b957ab5e885f41475381553b pypy-2.4.0-linux64.tar.bz2 - d542ee549ded9face573ac9fb49a23f5a5b4be60 pypy-2.4.0-linux-armel.tar.bz2 - b8e02dc381e5040e2bf50541e82f0148f9a46a48 pypy-2.4.0-linux-armhf-raring.tar.bz2 - ad65e7ddb1582b465a37090dc4a13bc37a8cd95d pypy-2.4.0-linux-armhf-raspbian.tar.bz2 - fd52b42069287ca11e816c8e18fc95f53542c73d pypy-2.4.0-linux.tar.bz2 - e2e0bcf8457c0ae5a24f126a60aa921dabfe60fb pypy-2.4.0-src.tar.bz2 - b72c3365c23c34ffd35a781fb72d8722e0b7517e pypy-2.4.0-win32.zip - aa7f9b41d8bfda16239b629cd1b8dc884c2ad808 pypy-2.4.0-osx64.tar.bz2 + 303bdc29e80640e337f1d65c7787870738a3ebf5 pypy-2.5.0-linux64.tar.bz2 + 1bc8de2db37233051d84c9c27e1e64608283b67c pypy-2.5.0-linux-armel.tar.bz2 + 01427d7dd52e2b679c38bfad414ebe676b019ea0 pypy-2.5.0-linux-armhf-raring.tar.bz2 + e3483d52182ab3ee02f665fda02f9fac4ab40ab1 pypy-2.5.0-linux-armhf-raspbian.tar.bz2 + 7334013255670f4ae60b316a5b35499861e0714f pypy-2.5.0-linux.tar.bz2 + ad47285526b1b3c14f4eecc874bb82a133a8e551 pypy-2.5.0-osx64.tar.bz2 + 1d215a22ea16581de338700d556b21a8c02b4eff pypy-2.5.0-src.tar.bz2 + 3492aa6fe96987537b23d09d66bb0c52f1760652 pypy-2.5.0-win32.zip 7d715742f6929351b310a2ca3b924cab35913089 pypy3-2.4.0-linux64.tar.bz2 b33e817f3557f91c434032c9f74e5220fe70036c pypy3-2.4.0-linux-armel.tar.bz2 diff --git a/source/features.txt b/source/features.txt --- a/source/features.txt +++ b/source/features.txt @@ -10,7 +10,7 @@ language that was co-developed with it. The main reason to use it instead of CPython is speed: it runs generally faster (see next section). -**PyPy 2.4** implements **Python 2.7.8** and runs on Intel +**PyPy 2.5** implements **Python 2.7.8** and runs on Intel `x86 (IA-32)`_ , `x86_64`_ and `ARM`_ platforms, with PPC being stalled. It supports all of the core language, passing the Python test suite (with minor modifications that were already accepted in the main python From noreply at buildbot.pypy.org Tue Feb 3 21:31:38 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 3 Feb 2015 21:31:38 +0100 (CET) Subject: [pypy-commit] pypy default: merge release-2.5.x into default to get documentation updates Message-ID: <20150203203138.3BFD01C114E@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r75685:22ce6f3699f0 Date: 2015-02-03 21:48 +0200 http://bitbucket.org/pypy/pypy/changeset/22ce6f3699f0/ Log: merge release-2.5.x into default to get documentation updates diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -42,19 +42,19 @@ Amaury Forgeot d'Arc Samuele Pedroni Alex Gaynor + Brian Kearns + Matti Picus + Philip Jenvey Michael Hudson David Schneider - Matti Picus - Brian Kearns - Philip Jenvey Holger Krekel Christian Tismer Hakan Ardo Benjamin Peterson Manuel Jacob + Ronan Lamy Anders Chrigstrom Eric van Riet Paap - Ronan Lamy Wim Lavrijsen Richard Emslie Alexander Schremmer @@ -68,9 +68,9 @@ Camillo Bruni Laura Creighton Toon Verwaest + Romain Guillebert Leonardo Santagada Seo Sanghyeon - Romain Guillebert Justin Peel Ronny Pfannschmidt David Edelsohn @@ -91,15 +91,16 @@ Michal Bendowski Jan de Mooij stian + Tyler Wade Michael Foord Stephan Diehl - Tyler Wade Stefan Schwarzer Valentino Volonghi Tomek Meka Patrick Maupin Bob Ippolito Bruno Gola + David Malcolm Jean-Paul Calderone Timo Paulssen Squeaky @@ -108,18 +109,19 @@ Marius Gedminas Martin Matusiak Konstantin Lopuhin + Wenzhu Man John Witulski - Wenzhu Man + Laurence Tratt + Ivan Sichmann Freitas Greg Price Dario Bertini Mark Pearse Simon Cross - Ivan Sichmann Freitas Andreas Stührk + Stefano Rivera Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov - Stefano Rivera Paweł Piotr Przeradowski Paul deGrandis Ilya Osadchiy @@ -129,7 +131,6 @@ tav Taavi Burns Georg Brandl - Laurence Tratt Bert Freudenberg Stian Andreassen Wanja Saatkamp @@ -141,13 +142,12 @@ Jeremy Thurgood Rami Chowdhury Tobias Pape - David Malcolm Eugene Oden Henry Mason Vasily Kuznetsov Preston Timmons + David Ripton Jeff Terrace - David Ripton Dusty Phillips Lukas Renggli Guenter Jantzen @@ -166,13 +166,16 @@ Gintautas Miliauskas Michael Twomey Lucian Branescu Mihaila + Yichao Yu Gabriel Lavoie Olivier Dormond Jared Grubb Karl Bartel + Wouter van Heyst Brian Dorsey Victor Stinner Andrews Medina + anatoly techtonik Stuart Williams Jasper Schulz Christian Hudon @@ -182,12 +185,11 @@ Michael Cheng Justas Sadzevicius Gasper Zejn - anatoly techtonik Neil Shepperd + Stanislaw Halik Mikael Schönenberg Elmo M?ntynen Jonathan David Riehl - Stanislaw Halik Anders Qvist Corbin Simpson Chirag Jadwani @@ -196,10 +198,13 @@ Vincent Legoll Alan McIntyre Alexander Sedov + Attila Gobi Christopher Pope Christian Tismer Marc Abramowitz Dan Stromberg + Arjun Naik + Valentina Mukhamedzhanova Stefano Parmesan Alexis Daboville Jens-Uwe Mager @@ -213,8 +218,6 @@ Sylvain Thenault Nathan Taylor Vladimir Kryachko - Arjun Naik - Attila Gobi Jacek Generowicz Alejandro J. Cura Jacob Oscarson @@ -222,22 +225,23 @@ Ryan Gonzalez Ian Foote Kristjan Valur Jonsson + David Lievens Neil Blakey-Milner Lutz Paelike Lucio Torre Lars Wassermann - Valentina Mukhamedzhanova Henrik Vendelbo Dan Buch Miguel de Val Borro Artur Lisiecki Sergey Kishchenko - Yichao Yu Ignas Mikalajunas Christoph Gerum Martin Blais Lene Wagner Tomo Cocoa + Toni Mattis + Lucas Stadler roberto at goyle Yury V. Zaytsev Anna Katrina Dominguez @@ -265,23 +269,30 @@ Stephan Busemann Rafał Gałczyński Christian Muirhead + Berker Peksag James Lan shoma hosaka - Daniel Neuh?user - Matthew Miller + Daniel Neuhäuser + Ben Mather + halgari + Boglarka Vezer + Chris Pressey Buck Golemon Konrad Delong Dinu Gherman Chris Lambacher coolbutuseless at gmail.com + Jim Baker Rodrigo Araújo - Jim Baker + Nikolaos-Digenis Karagiannis James Robert Armin Ronacher Brett Cannon + Donald Stufft yrttyr aliceinwire OlivierBlanvillain + Dan Sanders Zooko Wilcox-O Hearn Tomer Chachamu Christopher Groskopf @@ -295,6 +306,7 @@ Markus Unterwaditzer Even Wiik Thomassen jbs + squeaky soareschen Kurt Griffiths Mike Bayer @@ -306,6 +318,7 @@ Anna Ravencroft Dan Crosta Julien Phalip + Roman Podoliaka Dan Loewenherz Heinrich-Heine University, Germany diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -65,9 +65,9 @@ # built documents. # # The short X.Y version. -version = '2.4' +version = '2.5' # The full version, including alpha/beta/rc tags. -release = '2.4.0' +release = '2.5.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -12,19 +12,19 @@ Amaury Forgeot d'Arc Samuele Pedroni Alex Gaynor + Brian Kearns + Matti Picus + Philip Jenvey Michael Hudson David Schneider - Matti Picus - Brian Kearns - Philip Jenvey Holger Krekel Christian Tismer Hakan Ardo Benjamin Peterson Manuel Jacob + Ronan Lamy Anders Chrigstrom Eric van Riet Paap - Ronan Lamy Wim Lavrijsen Richard Emslie Alexander Schremmer @@ -38,9 +38,9 @@ Camillo Bruni Laura Creighton Toon Verwaest + Romain Guillebert Leonardo Santagada Seo Sanghyeon - Romain Guillebert Justin Peel Ronny Pfannschmidt David Edelsohn @@ -61,15 +61,16 @@ Michal Bendowski Jan de Mooij stian + Tyler Wade Michael Foord Stephan Diehl - Tyler Wade Stefan Schwarzer Valentino Volonghi Tomek Meka Patrick Maupin Bob Ippolito Bruno Gola + David Malcolm Jean-Paul Calderone Timo Paulssen Squeaky @@ -78,18 +79,19 @@ Marius Gedminas Martin Matusiak Konstantin Lopuhin + Wenzhu Man John Witulski - Wenzhu Man + Laurence Tratt + Ivan Sichmann Freitas Greg Price Dario Bertini Mark Pearse Simon Cross - Ivan Sichmann Freitas Andreas Stührk + Stefano Rivera Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov - Stefano Rivera Paweł Piotr Przeradowski Paul deGrandis Ilya Osadchiy @@ -99,7 +101,6 @@ tav Taavi Burns Georg Brandl - Laurence Tratt Bert Freudenberg Stian Andreassen Wanja Saatkamp @@ -111,13 +112,12 @@ Jeremy Thurgood Rami Chowdhury Tobias Pape - David Malcolm Eugene Oden Henry Mason Vasily Kuznetsov Preston Timmons + David Ripton Jeff Terrace - David Ripton Dusty Phillips Lukas Renggli Guenter Jantzen @@ -136,13 +136,16 @@ Gintautas Miliauskas Michael Twomey Lucian Branescu Mihaila + Yichao Yu Gabriel Lavoie Olivier Dormond Jared Grubb Karl Bartel + Wouter van Heyst Brian Dorsey Victor Stinner Andrews Medina + anatoly techtonik Stuart Williams Jasper Schulz Christian Hudon @@ -152,12 +155,11 @@ Michael Cheng Justas Sadzevicius Gasper Zejn - anatoly techtonik Neil Shepperd + Stanislaw Halik Mikael Schönenberg Elmo M?ntynen Jonathan David Riehl - Stanislaw Halik Anders Qvist Corbin Simpson Chirag Jadwani @@ -166,10 +168,13 @@ Vincent Legoll Alan McIntyre Alexander Sedov + Attila Gobi Christopher Pope Christian Tismer Marc Abramowitz Dan Stromberg + Arjun Naik + Valentina Mukhamedzhanova Stefano Parmesan Alexis Daboville Jens-Uwe Mager @@ -183,8 +188,6 @@ Sylvain Thenault Nathan Taylor Vladimir Kryachko - Arjun Naik - Attila Gobi Jacek Generowicz Alejandro J. Cura Jacob Oscarson @@ -192,22 +195,23 @@ Ryan Gonzalez Ian Foote Kristjan Valur Jonsson + David Lievens Neil Blakey-Milner Lutz Paelike Lucio Torre Lars Wassermann - Valentina Mukhamedzhanova Henrik Vendelbo Dan Buch Miguel de Val Borro Artur Lisiecki Sergey Kishchenko - Yichao Yu Ignas Mikalajunas Christoph Gerum Martin Blais Lene Wagner Tomo Cocoa + Toni Mattis + Lucas Stadler roberto at goyle Yury V. Zaytsev Anna Katrina Dominguez @@ -235,23 +239,30 @@ Stephan Busemann Rafał Gałczyński Christian Muirhead + Berker Peksag James Lan shoma hosaka - Daniel Neuh?user - Matthew Miller + Daniel Neuhäuser + Ben Mather + halgari + Boglarka Vezer + Chris Pressey Buck Golemon Konrad Delong Dinu Gherman Chris Lambacher coolbutuseless at gmail.com + Jim Baker Rodrigo Araújo - Jim Baker + Nikolaos-Digenis Karagiannis James Robert Armin Ronacher Brett Cannon + Donald Stufft yrttyr aliceinwire OlivierBlanvillain + Dan Sanders Zooko Wilcox-O Hearn Tomer Chachamu Christopher Groskopf @@ -265,6 +276,7 @@ Markus Unterwaditzer Even Wiik Thomassen jbs + squeaky soareschen Kurt Griffiths Mike Bayer @@ -276,5 +288,6 @@ Anna Ravencroft Dan Crosta Julien Phalip + Roman Podoliaka Dan Loewenherz diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst --- a/pypy/doc/how-to-release.rst +++ b/pypy/doc/how-to-release.rst @@ -22,12 +22,12 @@ will capture the revision number of this change for the release; some of the next updates may be done before or after branching; make sure things are ported back to the trunk and to the branch as - necessary; also update the version number in pypy/doc/conf.py, - and in pypy/doc/index.rst + necessary; also update the version number in pypy/doc/conf.py. * update pypy/doc/contributor.rst (and possibly LICENSE) pypy/doc/tool/makecontributor.py generates the list of contributors * rename pypy/doc/whatsnew_head.rst to whatsnew_VERSION.rst - and create a fresh whatsnew_head.rst after the release + create a fresh whatsnew_head.rst after the release + and add the new file to pypy/doc/index-of-whatsnew.rst * go to pypy/tool/release and run: force-builds.py * wait for builds to complete, make sure there are no failures @@ -45,6 +45,7 @@ * write release announcement pypy/doc/release-x.y(.z).txt the release announcement should contain a direct link to the download page + and add new files to pypy/doc/index-of-release-notes.rst * update pypy.org (under extradoc/pypy.org), rebuild and commit * post announcement on morepypy.blogspot.com diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ .. toctree:: + release-2.5.0.rst release-2.4.0.rst release-2.3.1.rst release-2.3.0.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-2.5.0.rst whatsnew-2.4.0.rst whatsnew-2.3.1.rst whatsnew-2.3.0.rst diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -191,3 +191,37 @@ to make them work at all if they currently don't. A part of this work would be to get cpyext into a shape where it supports running Cython generated extensions. + +====================================== +Make more python modules pypy-friendly +====================================== + +Work has been started on a few popular python packages. Here is a partial +list of good work that needs to be finished: + +**matplotlib** https://github.com/mattip/matplotlib + + Status: the repo is an older version of matplotlib adapted to pypy and cpyext + + TODO: A suggested first step would be to merge the differences into + matplotlib/HEAD. The major problem is the use of a generic view into a + numpy ndarray. The int* fields would need to be converted into int[MAX_DIMS] + c-arrays and filled in. + +**wxPython** https://bitbucket.org/waedt/wxpython_cffi + + Status: A GSOC 2013 project to adapt the Phoenix sip build system to cffi + + TODO: Merge the latest version of the wrappers and finish the sip conversion + +**pygame** https://github.com/CTPUG/pygame_cffi + + Status: see blog post + + TODO: see the end of the blog post + +**pyopengl** https://bitbucket.org/duangle/pyopengl-cffi + + Status: unknown + + diff --git a/pypy/doc/release-2.5.0.rst b/pypy/doc/release-2.5.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-2.5.0.rst @@ -0,0 +1,96 @@ +================================================= +PyPy 2.5 - XXXXXX +================================================= + +We're pleased to announce PyPy 2.5, which contains significant performance +enhancements and bug fixes. + +You can download the PyPy 2.5.0 release here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project, and for those who donate to our three sub-projects, as well as our +volunteers and contributors (10 new commiters joined PyPy since the last +release). +We've shown quite a bit of progress, but we're slowly running out of funds. +Please consider donating more, or even better convince your employer to donate, +so we can finish those projects! The three sub-projects are: + +* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatible version + we call PyPy3 2.4.0, and are working toward a Python 3.3 compatible version + +* `STM`_ (software transactional memory): We have released a first working version, + and continue to try out new promising paths of achieving a fast multithreaded Python + +* `NumPy`_ which requires installation of our fork of upstream numpy, + available `on bitbucket`_ + +.. _`Py3k`: http://pypy.org/py3donate.html +.. _`STM`: http://pypy.org/tmdonate2.html +.. _`NumPy`: http://pypy.org/numpydonate.html +.. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`pypy and cpython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +This release supports **x86** machines on most common operating systems +(Linux 32/64, Mac OS X 64, Windows, and OpenBSD), +as well as newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux. + +While we support 32 bit python on Windows, work on the native Windows 64 +bit python is still stalling, we would welcome a volunteer +to `handle that`_. + +.. _`pypy and cpython 2.7.x`: http://speed.pypy.org +.. _`handle that`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation + +Highlights +========== + +* The past months have seen pypy mature and grow, as rpython becomes the goto + solution for writing fast dynamic language interpreters. Our separation of + rpython and the python interpreter PyPy is now much clearer in the + `PyPy documentation`_ and we now have seperate `Rpython documentation`_. + +* We have improved warmup time as well as jitted code performance: more than 10% + compared to pypy-2.4.0, due to internal cleanup and gc nursery improvements. + +* Our integrated numpy support gained much of the GenericUfunc api in order to + support the lapack/blas linalg module of numpy. This dovetails with work in the + pypy/numpy repository to support linalg both through the (slower) cpyext capi + interface and also via (the faster) pure python cffi interface, using an + extended frompyfunc() api. We will soon post a seperate blog post specifically + about linalg and PyPy. + +* Dictionaries are now ordered by default, see the `blog post`_ + +* Our nightly translations use --shared by default, including on OS/X and linux + +* We now more carefully handle errno (and GetLastError, WSAGetLastError) tying + the handlers as close as possible to the external function call, in non-jitted + as well as jitted code. + +* Issues reported with our previous release were resolved_ after reports from users on + our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at + #pypy. + +.. _`PyPy documentation`: http://doc.pypy.org +.. _`Rpython documentation`: http://rpython.readthedocs.org +.. _`blog post`: http://morepypy.blogspot.com/2015/01/faster-more-memory-efficient-and-more.html +.. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.5.0.html + +We have further improvements on the way: rpython file handling, +finishing numpy linalg compatibility, numpy object dtypes, a better profiler, +as well as support for Python stdlib 2.7.9. + +Please try it out and let us know what you think. We especially welcome +success stories, we know you are using PyPy, please tell us about it! + +Cheers + +The PyPy Team diff --git a/pypy/doc/whatsnew-2.5.0.rst b/pypy/doc/whatsnew-2.5.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-2.5.0.rst @@ -0,0 +1,139 @@ +======================= +What's new in PyPy 2.5 +======================= + +.. this is a revision shortly after release-2.4.x +.. startrev: 7026746cbb1b + +.. branch: win32-fixes5 + +Fix c code generation for msvc so empty "{ }" are avoided in unions, +Avoid re-opening files created with NamedTemporaryFile, +Allocate by 4-byte chunks in rffi_platform, +Skip testing objdump if it does not exist, +and other small adjustments in own tests + +.. branch: rtyper-stuff + +Small internal refactorings in the rtyper. + +.. branch: var-in-Some + +Store annotations on the Variable objects, rather than in a big dict. +Introduce a new framework for double-dispatched annotation implementations. + +.. branch: ClassRepr + +Refactor ClassRepr and make normalizecalls independent of the rtyper. + +.. branch: remove-remaining-smm + +Remove all remaining multimethods. + +.. branch: improve-docs + +Split RPython documentation from PyPy documentation and clean up. There now is +a clearer separation between documentation for users, developers and people +interested in background information. + +.. branch: kill-multimethod + +Kill multimethod machinery, all multimethods were removed earlier. + +.. branch nditer-external_loop + +Implement `external_loop` arguement to numpy's nditer + +.. branch kill-rctime + +Rename pypy/module/rctime to pypy/module/time, since it contains the implementation of the 'time' module. + +.. branch: ssa-flow + +Use SSA form for flow graphs inside build_flow() and part of simplify_graph() + +.. branch: ufuncapi + +Implement most of the GenericUfunc api to support numpy linalg. The strategy is +to encourage use of pure python or cffi ufuncs by extending frompyfunc(). +See the docstring of frompyfunc for more details. This dovetails with a branch +of pypy/numpy - cffi-linalg which is a rewrite of the _umath_linalg module in +python, calling lapack from cffi. The branch also support traditional use of +cpyext GenericUfunc definitions in c. + +.. branch: all_ordered_dicts + +This makes ordered dicts the default dictionary implementation in +RPython and in PyPy. It polishes the basic idea of rordereddict.py +and then fixes various things, up to simplifying +collections.OrderedDict. + +Note: Python programs can rely on the guaranteed dict order in PyPy +now, but for compatibility with other Python implementations they +should still use collections.OrderedDict where that really matters. +Also, support for reversed() was *not* added to the 'dict' class; +use OrderedDict. + +Benchmark results: in the noise. A few benchmarks see good speed +improvements but the average is very close to parity. + +.. branch: berkerpeksag/fix-broken-link-in-readmerst-1415127402066 +.. branch: bigint-with-int-ops +.. branch: dstufft/update-pip-bootstrap-location-to-the-new-1420760611527 +.. branch: float-opt +.. branch: gc-incminimark-pinning + +This branch adds an interface rgc.pin which would (very temporarily) +make object non-movable. That's used by rffi.alloc_buffer and +rffi.get_nonmovable_buffer and improves performance considerably for +IO operations. + +.. branch: gc_no_cleanup_nursery + +A branch started by Wenzhu Man (SoC'14) and then done by fijal. It +removes the clearing of the nursery. The drawback is that new objects +are not automatically filled with zeros any longer, which needs some +care, mostly for GC references (which the GC tries to follow, so they +must not contain garbage). The benefit is a quite large speed-up. + +.. branch: improve-gc-tracing-hooks +.. branch: improve-ptr-conv-error +.. branch: intern-not-immortal + +Fix intern() to return mortal strings, like in CPython. + +.. branch: issue1922-take2 +.. branch: kill-exported-symbols-list +.. branch: kill-rctime +.. branch: kill_ll_termios +.. branch: look-into-all-modules +.. branch: nditer-external_loop +.. branch: numpy-generic-item +.. branch: osx-shared + +``--shared`` support on OS/X (thanks wouter) + +.. branch: portable-threadlocal +.. branch: pypy-dont-copy-ops +.. branch: recursion_and_inlining +.. branch: slim-down-resumedescr +.. branch: squeaky/use-cflags-for-compiling-asm +.. branch: unicode-fix +.. branch: zlib_zdict + +.. branch: errno-again + +Changes how errno, GetLastError, and WSAGetLastError are handled. +The idea is to tie reading the error status as close as possible to +the external function call. This fixes some bugs, both of the very +rare kind (e.g. errno on Linux might in theory be overwritten by +mmap(), called rarely during major GCs, if such a major GC occurs at +exactly the wrong time), and some of the less rare kind +(particularly on Windows tests). + +.. branch: osx-package.py +.. branch: package.py-helpful-error-message + +.. branch: typed-cells + +Improve performance of integer globals and class attributes. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -2,138 +2,6 @@ What's new in PyPy 2.5+ ======================= -.. this is a revision shortly after release-2.4.x -.. startrev: 7026746cbb1b +.. this is a revision shortly after release-2.5.x +.. startrev: 397b96217b85 -.. branch: win32-fixes5 - -Fix c code generation for msvc so empty "{ }" are avoided in unions, -Avoid re-opening files created with NamedTemporaryFile, -Allocate by 4-byte chunks in rffi_platform, -Skip testing objdump if it does not exist, -and other small adjustments in own tests - -.. branch: rtyper-stuff - -Small internal refactorings in the rtyper. - -.. branch: var-in-Some - -Store annotations on the Variable objects, rather than in a big dict. -Introduce a new framework for double-dispatched annotation implementations. - -.. branch: ClassRepr - -Refactor ClassRepr and make normalizecalls independent of the rtyper. - -.. branch: remove-remaining-smm - -Remove all remaining multimethods. - -.. branch: improve-docs - -Split RPython documentation from PyPy documentation and clean up. There now is -a clearer separation between documentation for users, developers and people -interested in background information. - -.. branch: kill-multimethod - -Kill multimethod machinery, all multimethods were removed earlier. - -.. branch nditer-external_loop - -Implement `external_loop` arguement to numpy's nditer - -.. branch kill-rctime - -Rename pypy/module/rctime to pypy/module/time, since it contains the implementation of the 'time' module. - -.. branch: ssa-flow - -Use SSA form for flow graphs inside build_flow() and part of simplify_graph() - -.. branch: ufuncapi - -Implement most of the GenericUfunc api to support numpy linalg. The strategy is -to encourage use of pure python or cffi ufuncs by extending frompyfunc(). -See the docstring of frompyfunc for more details. This dovetails with a branch -of pypy/numpy - cffi-linalg which is a rewrite of the _umath_linalg module in -python, calling lapack from cffi. The branch also support traditional use of -cpyext GenericUfunc definitions in c. - -.. branch: all_ordered_dicts - -This makes ordered dicts the default dictionary implementation in -RPython and in PyPy. It polishes the basic idea of rordereddict.py -and then fixes various things, up to simplifying -collections.OrderedDict. - -Note: Python programs can rely on the guaranteed dict order in PyPy -now, but for compatibility with other Python implementations they -should still use collections.OrderedDict where that really matters. -Also, support for reversed() was *not* added to the 'dict' class; -use OrderedDict. - -Benchmark results: in the noise. A few benchmarks see good speed -improvements but the average is very close to parity. - -.. branch: berkerpeksag/fix-broken-link-in-readmerst-1415127402066 -.. branch: bigint-with-int-ops -.. branch: dstufft/update-pip-bootstrap-location-to-the-new-1420760611527 -.. branch: float-opt -.. branch: gc-incminimark-pinning - -This branch adds an interface rgc.pin which would (very temporarily) -make object non-movable. That's used by rffi.alloc_buffer and -rffi.get_nonmovable_buffer and improves performance considerably for -IO operations. - -.. branch: gc_no_cleanup_nursery - -A branch started by Wenzhu Man (SoC'14) and then done by fijal. It -removes the clearing of the nursery. The drawback is that new objects -are not automatically filled with zeros any longer, which needs some -care, mostly for GC references (which the GC tries to follow, so they -must not contain garbage). The benefit is a quite large speed-up. - -.. branch: improve-gc-tracing-hooks -.. branch: improve-ptr-conv-error -.. branch: intern-not-immortal - -Fix intern() to return mortal strings, like in CPython. - -.. branch: issue1922-take2 -.. branch: kill-exported-symbols-list -.. branch: kill-rctime -.. branch: kill_ll_termios -.. branch: look-into-all-modules -.. branch: nditer-external_loop -.. branch: numpy-generic-item -.. branch: osx-shared - -``--shared`` support on OS/X (thanks wouter) - -.. branch: portable-threadlocal -.. branch: pypy-dont-copy-ops -.. branch: recursion_and_inlining -.. branch: slim-down-resumedescr -.. branch: squeaky/use-cflags-for-compiling-asm -.. branch: unicode-fix -.. branch: zlib_zdict - -.. branch: errno-again - -Changes how errno, GetLastError, and WSAGetLastError are handled. -The idea is to tie reading the error status as close as possible to -the external function call. This fixes some bugs, both of the very -rare kind (e.g. errno on Linux might in theory be overwritten by -mmap(), called rarely during major GCs, if such a major GC occurs at -exactly the wrong time), and some of the less rare kind -(particularly on Windows tests). - -.. branch: osx-package.py -.. branch: package.py-helpful-error-message - -.. branch: typed-cells - -Improve performance of integer globals and class attributes. diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,7 +29,7 @@ #define PY_VERSION "2.7.8" /* PyPy version as a string */ -#define PYPY_VERSION "2.6.0-alpha0" +#define PYPY_VERSION "2.5.0" /* Subversion Revision number of this file (not of the repository). * Empty since Mercurial migration. */ diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (2, 6, 0, "alpha", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (2, 5, 0, "final", 0) #XXX # sync patchlevel.h if platform.name == 'msvc': COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600) From noreply at buildbot.pypy.org Tue Feb 3 21:31:39 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 3 Feb 2015 21:31:39 +0100 (CET) Subject: [pypy-commit] pypy default: tweak 2.5.0 release notice Message-ID: <20150203203139.5BBCF1C114E@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r75686:d924ae5db23e Date: 2015-02-03 22:32 +0200 http://bitbucket.org/pypy/pypy/changeset/d924ae5db23e/ Log: tweak 2.5.0 release notice diff --git a/pypy/doc/release-2.5.0.rst b/pypy/doc/release-2.5.0.rst --- a/pypy/doc/release-2.5.0.rst +++ b/pypy/doc/release-2.5.0.rst @@ -1,6 +1,6 @@ -================================================= -PyPy 2.5 - XXXXXX -================================================= +============================== +PyPy 2.5.0 - Protea Pincushion +============================== We're pleased to announce PyPy 2.5, which contains significant performance enhancements and bug fixes. @@ -59,6 +59,10 @@ * We have improved warmup time as well as jitted code performance: more than 10% compared to pypy-2.4.0, due to internal cleanup and gc nursery improvements. + We no longer zero-out memory allocated in the gc nursery by default. + +* IO operations got a performance boost by adding a pinning interface that is + used by rffi.alloc_buffer and rff.get_nonmovablebuffer. * Our integrated numpy support gained much of the GenericUfunc api in order to support the lapack/blas linalg module of numpy. This dovetails with work in the From noreply at buildbot.pypy.org Tue Feb 3 21:41:12 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 3 Feb 2015 21:41:12 +0100 (CET) Subject: [pypy-commit] pypy default: Added tag release-2.5.0 for changeset 62a418bcbc7f Message-ID: <20150203204112.820F61C0F98@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r75688:94feb5506033 Date: 2015-02-03 22:41 +0200 http://bitbucket.org/pypy/pypy/changeset/94feb5506033/ Log: Added tag release-2.5.0 for changeset 62a418bcbc7f diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -14,3 +14,4 @@ 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 0000000000000000000000000000000000000000 release-2.2=3.1 +62a418bcbc7ffc542136569cec30373c5cbe7fb1 release-2.5.0 From noreply at buildbot.pypy.org Tue Feb 3 21:45:55 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 3 Feb 2015 21:45:55 +0100 (CET) Subject: [pypy-commit] pypy default: spelling (camara) Message-ID: <20150203204555.3C7A51C114E@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r75689:84a627b39a2c Date: 2015-02-03 22:46 +0200 http://bitbucket.org/pypy/pypy/changeset/84a627b39a2c/ Log: spelling (camara) diff --git a/pypy/doc/release-2.5.0.rst b/pypy/doc/release-2.5.0.rst --- a/pypy/doc/release-2.5.0.rst +++ b/pypy/doc/release-2.5.0.rst @@ -63,7 +63,7 @@ was started during a GSoC. * IO operations got a performance boost by adding a **pinning** interface that is - used by rffi.alloc_buffer and rff.get_nonmovablebuffer. + used by rffi.alloc_buffer and rffi.get_nonmovablebuffer. * Our integrated numpy support gained much of the GenericUfunc api in order to support the lapack/blas linalg module of numpy. This dovetails with work in the From noreply at buildbot.pypy.org Tue Feb 3 21:58:27 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 3 Feb 2015 21:58:27 +0100 (CET) Subject: [pypy-commit] pypy default: rework pinning description Message-ID: <20150203205827.7FA1E1C114F@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r75690:e781f33da96b Date: 2015-02-03 22:59 +0200 http://bitbucket.org/pypy/pypy/changeset/e781f33da96b/ Log: rework pinning description diff --git a/pypy/doc/release-2.5.0.rst b/pypy/doc/release-2.5.0.rst --- a/pypy/doc/release-2.5.0.rst +++ b/pypy/doc/release-2.5.0.rst @@ -62,8 +62,11 @@ We no longer zero-out memory allocated in the gc nursery by default, work that was started during a GSoC. -* IO operations got a performance boost by adding a **pinning** interface that is - used by rffi.alloc_buffer and rffi.get_nonmovablebuffer. +* Passing objects between C and PyPy has been improved. We are now able to pass + raw pointers to C (without copying) using **pinning**. This improves I/O; + benchmarks that use networking intensively improved by about 50%. File() + operations still need some refactoring but are already showing a 20% + improvement on our benchmarks. Let us know if you see similar improvements. * Our integrated numpy support gained much of the GenericUfunc api in order to support the lapack/blas linalg module of numpy. This dovetails with work in the From noreply at buildbot.pypy.org Tue Feb 3 22:01:00 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 3 Feb 2015 22:01:00 +0100 (CET) Subject: [pypy-commit] pypy default: fix capitalization Message-ID: <20150203210100.7CAB81C114F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75691:6f31bff804a9 Date: 2015-02-03 22:00 +0100 http://bitbucket.org/pypy/pypy/changeset/6f31bff804a9/ Log: fix capitalization diff --git a/pypy/doc/release-2.5.0.rst b/pypy/doc/release-2.5.0.rst --- a/pypy/doc/release-2.5.0.rst +++ b/pypy/doc/release-2.5.0.rst @@ -55,7 +55,7 @@ * The past months have seen pypy mature and grow, as rpython becomes the goto solution for writing fast dynamic language interpreters. Our separation of rpython and the python interpreter PyPy is now much clearer in the - `PyPy documentation`_ and we now have seperate `Rpython documentation`_. + `PyPy documentation`_ and we now have seperate `RPython documentation`_. * We have improved warmup time as well as jitted code performance: more than 10% compared to pypy-2.4.0, due to internal cleanup and gc nursery improvements. @@ -88,7 +88,7 @@ #pypy. .. _`PyPy documentation`: http://doc.pypy.org -.. _`Rpython documentation`: http://rpython.readthedocs.org +.. _`RPython documentation`: http://rpython.readthedocs.org .. _`blog post`: http://morepypy.blogspot.com/2015/01/faster-more-memory-efficient-and-more.html .. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.5.0.html From noreply at buildbot.pypy.org Tue Feb 3 22:09:03 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 3 Feb 2015 22:09:03 +0100 (CET) Subject: [pypy-commit] pypy default: Removed tag release-2.5.0 Message-ID: <20150203210903.7732A1C0820@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r75692:55d23c7d8a2c Date: 2015-02-03 23:03 +0200 http://bitbucket.org/pypy/pypy/changeset/55d23c7d8a2c/ Log: Removed tag release-2.5.0 diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -15,3 +15,5 @@ 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 0000000000000000000000000000000000000000 release-2.2=3.1 62a418bcbc7ffc542136569cec30373c5cbe7fb1 release-2.5.0 +62a418bcbc7ffc542136569cec30373c5cbe7fb1 release-2.5.0 +0000000000000000000000000000000000000000 release-2.5.0 From noreply at buildbot.pypy.org Tue Feb 3 22:09:04 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 3 Feb 2015 22:09:04 +0100 (CET) Subject: [pypy-commit] pypy default: Added tag release-2.5.0 for changeset 10f1b29a2bd2 Message-ID: <20150203210904.945201C0820@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r75693:063a74d884c9 Date: 2015-02-03 23:03 +0200 http://bitbucket.org/pypy/pypy/changeset/063a74d884c9/ Log: Added tag release-2.5.0 for changeset 10f1b29a2bd2 diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -17,3 +17,5 @@ 62a418bcbc7ffc542136569cec30373c5cbe7fb1 release-2.5.0 62a418bcbc7ffc542136569cec30373c5cbe7fb1 release-2.5.0 0000000000000000000000000000000000000000 release-2.5.0 +0000000000000000000000000000000000000000 release-2.5.0 +10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0 From noreply at buildbot.pypy.org Tue Feb 3 22:09:05 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 3 Feb 2015 22:09:05 +0100 (CET) Subject: [pypy-commit] pypy default: clean up strangeness in hgtags file Message-ID: <20150203210905.DF8721C0820@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r75694:2ef6e6fedee6 Date: 2015-02-03 23:09 +0200 http://bitbucket.org/pypy/pypy/changeset/2ef6e6fedee6/ Log: clean up strangeness in hgtags file diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -7,15 +7,7 @@ 9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm 20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0 -20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0 -0000000000000000000000000000000000000000 release-2.3.0 394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1 -32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 -0000000000000000000000000000000000000000 release-2.2=3.1 -62a418bcbc7ffc542136569cec30373c5cbe7fb1 release-2.5.0 -62a418bcbc7ffc542136569cec30373c5cbe7fb1 release-2.5.0 -0000000000000000000000000000000000000000 release-2.5.0 -0000000000000000000000000000000000000000 release-2.5.0 10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0 From noreply at buildbot.pypy.org Tue Feb 3 22:34:03 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 3 Feb 2015 22:34:03 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: create html Message-ID: <20150203213403.304BD1C0FF4@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: extradoc Changeset: r565:7935f316ec12 Date: 2015-02-03 23:35 +0200 http://bitbucket.org/pypy/pypy.org/changeset/7935f316ec12/ Log: create html diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -76,7 +76,7 @@ miscompiled due to buildslave being old. Contributions are welcomed.

We provide binaries for x86 and ARM Linux, Mac OS/X and Windows for:

    @@ -112,20 +112,37 @@ degrees of being up-to-date. You may have more luck trying out Squeaky's portable Linux binaries.

    -
    -

    Python2.7 compatible PyPy 2.4.0

    -
      -
    • Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04) (see [1] below)
      • -
    • Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04) (see [1] below)
      • -
    • ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Raspbian) (see [1] below)
      • -
    • ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Ubuntu Raring) (see [1] below)
      • -
    • ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2, Ubuntu Precise) (see [1] below)
      • -
    • Mac OS/X binary (64bit)
      • -
    • Windows binary (32bit) (you might need the VS 2008 runtime library -installer vcredist_x86.exe.)
      • -
    • All our downloads, including previous versions. We also have a -mirror, but please use only if you have troubles accessing the links above
      • -
    • See below for the sources.
      • +
      +

      Python2.7 compatible PyPy 2.5.0

      +
      @@ -178,7 +195,7 @@ uncompressed, they run in-place. For now you can uncompress them either somewhere in your home directory or, say, in /opt, and if you want, put a symlink from somewhere like -/usr/local/bin/pypy to /path/to/pypy-2.4.0/bin/pypy. Do +/usr/local/bin/pypy to /path/to/pypy-2.5.0/bin/pypy. Do not move or copy the executable pypy outside the tree – put a symlink to it, otherwise it will not find its libraries.

      @@ -216,7 +233,7 @@

    • Get the source code. The following packages contain the source at the same revision as the above binaries:

      Or you can checkout the current trunk using Mercurial (the trunk usually works and is of course more up-to-date):

      @@ -298,14 +315,14 @@

      Checksums

      Here are the checksums for each of the downloads (md5 and sha1):

      -63bd68546f60cf5921ba7654f3fe47aa  pypy-2.4.0-linux64.tar.bz2
-6c9b444a1cd090ab7b43083a24e07734  pypy-2.4.0-linux-armel.tar.bz2
-5ff951da5989a00e01611678c311f8af  pypy-2.4.0-linux-armhf-raring.tar.bz2
-d7540883a52f91433da62b0cdfaaa30f  pypy-2.4.0-linux-armhf-raspbian.tar.bz2
-77a971f5198685ff60528de5def407dd  pypy-2.4.0-linux.tar.bz2
-907d6fbabc5bcd5bafdcf02a76a8ca33  pypy-2.4.0-win32.zip
-6a25a212e7c5121f1f3988c118d05695  pypy-2.4.0-src.tar.bz2
-07896c0ac37f82884e021c9a4514f479  pypy-2.4.0-osx64.tar.bz2
+0a109c96bbce20931c17dfded57a3d46  pypy-2.5.0-linux64.tar.bz2
+fc79e0ef640371fa7bdbb040c3211bcf  pypy-2.5.0-linux-armel.tar.bz2
+d8d0fe8a1503a7ed4ec5ff52d0630b88  pypy-2.5.0-linux-armhf-raring.tar.bz2
+9a4430abd36dd619c45705c3fe70f5ca  pypy-2.5.0-linux-armhf-raspbian.tar.bz2
+1c14e954fdd1d112540790ec5b0cea26  pypy-2.5.0-linux.tar.bz2
+9916a1485f32953a46afdd36f8f01f83  pypy-2.5.0-osx64.tar.bz2
+f4700c0af45e986178b36ce91a45136e  pypy-2.5.0-src.tar.bz2
+2461a4e4fe4ea64ce7fb1a07aa310e6a  pypy-2.5.0-win32.zip
 eadbc9790823fc0ae40c943087cd7cb3  pypy3-2.4.0-linux64.tar.bz2
 7ab84727da2d5363866907f2f7921d86  pypy3-2.4.0-linux-armel.tar.bz2
 83158d3a55ca134b179ef01dc2bb6a30  pypy3-2.4.0-linux-armhf-raring.tar.bz2
@@ -317,14 +334,14 @@
 c58015d0d3e08a9f24b93b8edca26d4d  pypy3-2.4.0-src.zip
 2c9f0054f3b93a6473f10be35277825a  pypy-1.8-sandbox-linux64.tar.bz2
 009c970b5fa75754ae4c32a5d108a8d4  pypy-1.8-sandbox-linux.tar.bz2
-c362247226f1cde2b957ab5e885f41475381553b  pypy-2.4.0-linux64.tar.bz2
-d542ee549ded9face573ac9fb49a23f5a5b4be60  pypy-2.4.0-linux-armel.tar.bz2
-b8e02dc381e5040e2bf50541e82f0148f9a46a48  pypy-2.4.0-linux-armhf-raring.tar.bz2
-ad65e7ddb1582b465a37090dc4a13bc37a8cd95d  pypy-2.4.0-linux-armhf-raspbian.tar.bz2
-fd52b42069287ca11e816c8e18fc95f53542c73d  pypy-2.4.0-linux.tar.bz2
-e2e0bcf8457c0ae5a24f126a60aa921dabfe60fb  pypy-2.4.0-src.tar.bz2
-b72c3365c23c34ffd35a781fb72d8722e0b7517e  pypy-2.4.0-win32.zip
-aa7f9b41d8bfda16239b629cd1b8dc884c2ad808  pypy-2.4.0-osx64.tar.bz2
+303bdc29e80640e337f1d65c7787870738a3ebf5  pypy-2.5.0-linux64.tar.bz2
+1bc8de2db37233051d84c9c27e1e64608283b67c  pypy-2.5.0-linux-armel.tar.bz2
+01427d7dd52e2b679c38bfad414ebe676b019ea0  pypy-2.5.0-linux-armhf-raring.tar.bz2
+e3483d52182ab3ee02f665fda02f9fac4ab40ab1  pypy-2.5.0-linux-armhf-raspbian.tar.bz2
+7334013255670f4ae60b316a5b35499861e0714f  pypy-2.5.0-linux.tar.bz2
+ad47285526b1b3c14f4eecc874bb82a133a8e551  pypy-2.5.0-osx64.tar.bz2
+1d215a22ea16581de338700d556b21a8c02b4eff  pypy-2.5.0-src.tar.bz2
+3492aa6fe96987537b23d09d66bb0c52f1760652  pypy-2.5.0-win32.zip
 7d715742f6929351b310a2ca3b924cab35913089  pypy3-2.4.0-linux64.tar.bz2
 b33e817f3557f91c434032c9f74e5220fe70036c  pypy3-2.4.0-linux-armel.tar.bz2
 bb098b72ecc83a0e73c426f364bb6a0974fb9360  pypy3-2.4.0-linux-armhf-raring.tar.bz2
diff --git a/features.html b/features.html
--- a/features.html
+++ b/features.html
@@ -72,7 +72,7 @@
 
      PyPy is a replacement for CPython.  It is built using the RPython
 language that was co-developed with it.  The main reason to use it
 instead of CPython is speed: it runs generally faster (see next section).
      
-
      PyPy 2.4 implements Python 2.7.8 and runs on Intel
+
      PyPy 2.5 implements Python 2.7.8 and runs on Intel
 x86 (IA-32) , x86_64 and ARM platforms, with PPC being
 stalled. It supports all of the core language, passing the Python test suite
 (with minor modifications that were already accepted in the main python

From noreply at buildbot.pypy.org  Tue Feb  3 23:17:05 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Tue,  3 Feb 2015 23:17:05 +0100 (CET)
Subject: [pypy-commit] jitviewer default: Added tag pypy-2.5 for changeset
	ec561fb900e0
Message-ID: <20150203221705.A57F61C1150@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r267:43103ce6ee31
Date: 2015-02-04 00:18 +0200
http://bitbucket.org/pypy/jitviewer/changeset/43103ce6ee31/

Log:	Added tag pypy-2.5 for changeset ec561fb900e0

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -3,3 +3,4 @@
 15e03325a227c4c7145a56e841b6a8a3c59730ed pypy-2.2
 62ad3e746dacc21c8e5dff2a37738659e1b61b7a pypy-2.4
 62ad3e746dacc21c8e5dff2a37738659e1b61b7a pypy-2.3
+ec561fb900e02df04e47b11c413f4a8449cbbb3a pypy-2.5

From noreply at buildbot.pypy.org  Wed Feb  4 09:53:19 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  4 Feb 2015 09:53:19 +0100 (CET)
Subject: [pypy-commit] pypy default: fix link
Message-ID: <20150204085319.C3AF01C01A7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r75696:5593c8e14454
Date: 2015-02-04 09:53 +0100
http://bitbucket.org/pypy/pypy/changeset/5593c8e14454/

Log:	fix link

diff --git a/pypy/doc/release-2.5.0.rst b/pypy/doc/release-2.5.0.rst
--- a/pypy/doc/release-2.5.0.rst
+++ b/pypy/doc/release-2.5.0.rst
@@ -90,7 +90,7 @@
 .. _`PyPy documentation`: http://doc.pypy.org
 .. _`RPython documentation`: http://rpython.readthedocs.org
 .. _`blog post`: http://morepypy.blogspot.com/2015/01/faster-more-memory-efficient-and-more.html
-.. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.5.0.html
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-2.5.0.html
 
 We have further improvements on the way: rpython file handling,
 finishing numpy linalg compatibility, numpy object dtypes, a better profiler,

From noreply at buildbot.pypy.org  Wed Feb  4 10:50:04 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  4 Feb 2015 10:50:04 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: actually define the unique id
Message-ID: <20150204095004.692671C11A4@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r75697:90e71bdbaf0c
Date: 2015-02-04 11:49 +0200
http://bitbucket.org/pypy/pypy/changeset/90e71bdbaf0c/

Log:	actually define the unique id

diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -35,6 +35,9 @@
     name = opcode_method_names[ord(bytecode.co_code[next_instr])]
     return '%s #%d %s' % (bytecode.get_repr(), next_instr, name)
 
+def get_unique_id(next_instr, is_being_profiled, bytecode):
+    return bytecode._unique_id
+
 
 def should_unroll_one_iteration(next_instr, is_being_profiled, bytecode):
     return (bytecode.co_flags & CO_GENERATOR) != 0
@@ -45,6 +48,7 @@
     virtualizables = ['frame']
 
 pypyjitdriver = PyPyJitDriver(get_printable_location = get_printable_location,
+                              get_unique_id = get_unique_id,
                               should_unroll_one_iteration =
                               should_unroll_one_iteration,
                               name='pypyjit')

From noreply at buildbot.pypy.org  Wed Feb  4 12:18:31 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  4 Feb 2015 12:18:31 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: Add vmprof only on 64bit linux and blind
 fixes for ARM (which is a bit
Message-ID: <20150204111831.2E5571C080A@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r75698:aee43a736a67
Date: 2015-02-04 13:18 +0200
http://bitbucket.org/pypy/pypy/changeset/aee43a736a67/

Log:	Add vmprof only on 64bit linux and blind fixes for ARM (which is a
	bit future-only since we don't have 64bit ARM)

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -35,9 +35,12 @@
     "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "micronumpy", "_continuation", "_cffi_backend",
-    "_csv", "cppyy", "_pypyjson", "_vmprof"
+    "_csv", "cppyy", "_pypyjson"
 ])
 
+if sys.platform.startswith('linux') and sys.maxint > 2147483647:
+    working_modules.add('_vmprof')
+
 translation_modules = default_modules.copy()
 translation_modules.update([
     "fcntl", "time", "select", "signal", "_rawffi", "zlib", "struct", "_md5",
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -663,6 +663,7 @@
             assert len(set(inputargs)) == len(inputargs)
 
         self.setup(original_loop_token)
+        self.codemap.inherit_code_from_position(faildescr.adr_jump_offset)
         descr_number = compute_unique_id(faildescr)
         if log:
             operations = self._inject_debugging_code(faildescr, operations,
@@ -883,8 +884,12 @@
         self.datablockwrapper.done()      # finish using cpu.asmmemmgr
         self.datablockwrapper = None
         allblocks = self.get_asmmemmgr_blocks(looptoken)
-        return self.mc.materialize(self.cpu.asmmemmgr, allblocks,
+        size = self.mc.get_relative_pos() 
+        res = self.mc.materialize(self.cpu.asmmemmgr, allblocks,
                                    self.cpu.gc_ll_descr.gcrootmap)
+        self.cpu.asmmemmgr.register_codemap(
+            self.codemap.get_final_bytecode(res, size))
+        return res
 
     def update_frame_depth(self, frame_depth):
         baseofs = self.cpu.get_baseofs_of_frame_field()

From noreply at buildbot.pypy.org  Wed Feb  4 16:27:04 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  4 Feb 2015 16:27:04 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: try returning const
Message-ID: <20150204152704.B33681C105C@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r75705:f984a5235681
Date: 2015-02-04 17:26 +0200
http://bitbucket.org/pypy/pypy/changeset/f984a5235681/

Log:	try returning const

diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -306,7 +306,7 @@
         while not hasattr(T, 'vable_token'):
             if not hasattr(T, 'super'):
                 # we're not really in a jitted build
-                return hop.inputconst(lltype.Void,
+                return hop.inputconst(llmemory.GCREF,
                                       lltype.nullptr(llmemory.GCREF.TO))
             v = hop.genop('getfield', [v, c_super], resulttype=T.super)
             T = T.super

From noreply at buildbot.pypy.org  Wed Feb  4 16:38:46 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  4 Feb 2015 16:38:46 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: fix
Message-ID: <20150204153846.D28C31C1152@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: vmprof
Changeset: r75706:ded7218f97ef
Date: 2015-02-04 16:38 +0100
http://bitbucket.org/pypy/pypy/changeset/ded7218f97ef/

Log:	fix

diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -302,14 +302,13 @@
         hop.exception_cannot_occur()
         T = hop.args_r[0].lowleveltype.TO
         v = hop.inputarg(hop.args_r[0], arg=0)
-        c_super = hop.inputconst(lltype.Void, 'super')
         while not hasattr(T, 'vable_token'):
             if not hasattr(T, 'super'):
                 # we're not really in a jitted build
                 return hop.inputconst(llmemory.GCREF,
                                       lltype.nullptr(llmemory.GCREF.TO))
-            v = hop.genop('getfield', [v, c_super], resulttype=T.super)
             T = T.super
+        v = hop.genop('cast_pointer', [v], resulttype=lltype.Ptr(T))
         c_vable_token = hop.inputconst(lltype.Void, 'vable_token')
         return hop.genop('getfield', [v, c_vable_token],
                          resulttype=llmemory.GCREF)

From noreply at buildbot.pypy.org  Wed Feb  4 16:50:18 2015
From: noreply at buildbot.pypy.org (krono)
Date: Wed,  4 Feb 2015 16:50:18 +0100 (CET)
Subject: [pypy-commit] pypy default: Fix FAQ reference
Message-ID: <20150204155018.3D2931C11B2@cobra.cs.uni-duesseldorf.de>

Author: Tobias Pape 
Branch: 
Changeset: r75707:c0eb18eeee51
Date: 2015-02-04 16:41 +0100
http://bitbucket.org/pypy/pypy/changeset/c0eb18eeee51/

Log:	Fix FAQ reference

diff --git a/rpython/doc/getting-started.rst b/rpython/doc/getting-started.rst
--- a/rpython/doc/getting-started.rst
+++ b/rpython/doc/getting-started.rst
@@ -7,7 +7,7 @@
 
     Please `read this FAQ entry`_ first!
 
-.. _`read this FAQ entry`: http://doc.pypy.org/en/latest/faq.html#do-i-have-to-rewrite-my-programs-in-rpython
+.. _`read this FAQ entry`: http://rpython.readthedocs.org/en/latest/faq.html#do-i-have-to-rewrite-my-programs-in-rpython
 
 RPython is a subset of Python that can be statically compiled. The PyPy
 interpreter is written mostly in RPython (with pieces in Python), while

From noreply at buildbot.pypy.org  Wed Feb  4 19:43:26 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  4 Feb 2015 19:43:26 +0100 (CET)
Subject: [pypy-commit] pypy stackroot-speedup-2: Pass the optional
 "is_minor" argument to walk_roots(), 
Message-ID: <20150204184326.BAFC11C105F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stackroot-speedup-2
Changeset: r75709:99fb7bf8cb49
Date: 2015-02-04 18:08 +0100
http://bitbucket.org/pypy/pypy/changeset/99fb7bf8cb49/

Log:	Pass the optional "is_minor" argument to walk_roots(), and implement
	it in the llinterp

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1662,7 +1662,8 @@
         self.root_walker.walk_roots(
             callback,     # stack roots
             callback,     # static in prebuilt non-gc
-            None)         # static in prebuilt gc
+            None,         # static in prebuilt gc
+            is_minor=True)
         debug_stop("gc-minor-walkroots")
 
     def collect_cardrefs_to_nursery(self):
diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -1322,7 +1322,8 @@
         self.root_walker.walk_roots(
             MiniMarkGC._trace_drag_out1,  # stack roots
             MiniMarkGC._trace_drag_out1,  # static in prebuilt non-gc
-            None)                         # static in prebuilt gc
+            None,                         # static in prebuilt gc
+            is_minor=True)
         debug_stop("gc-minor-walkroots")
 
     def collect_cardrefs_to_nursery(self):
diff --git a/rpython/memory/gctransform/asmgcroot.py b/rpython/memory/gctransform/asmgcroot.py
--- a/rpython/memory/gctransform/asmgcroot.py
+++ b/rpython/memory/gctransform/asmgcroot.py
@@ -340,7 +340,7 @@
         # called first, to initialize self.belongs_to_current_thread.
         assert not hasattr(self, 'gc_detach_callback_pieces_ptr')
 
-    def walk_stack_roots(self, collect_stack_root):
+    def walk_stack_roots(self, collect_stack_root, is_minor=False):
         gcdata = self.gcdata
         gcdata._gc_collect_stack_root = collect_stack_root
         pypy_asm_stackwalk(llhelper(ASM_CALLBACK_PTR, self._asm_callback),
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -1462,7 +1462,8 @@
 
     def walk_roots(self, collect_stack_root,
                    collect_static_in_prebuilt_nongc,
-                   collect_static_in_prebuilt_gc):
+                   collect_static_in_prebuilt_gc,
+                   is_minor=False):
         gcdata = self.gcdata
         gc = self.gc
         if collect_static_in_prebuilt_nongc:
@@ -1482,7 +1483,7 @@
                     collect_static_in_prebuilt_gc(gc, result)
                 addr += sizeofaddr
         if collect_stack_root:
-            self.walk_stack_roots(collect_stack_root)     # abstract
+            self.walk_stack_roots(collect_stack_root, is_minor)     # abstract
 
     def finished_minor_collection(self):
         func = self.finished_minor_collection_func
diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py
--- a/rpython/memory/gctransform/shadowstack.py
+++ b/rpython/memory/gctransform/shadowstack.py
@@ -99,7 +99,7 @@
         self.shadow_stack_pool.initial_setup()
         BaseRootWalker.setup_root_walker(self)
 
-    def walk_stack_roots(self, collect_stack_root):
+    def walk_stack_roots(self, collect_stack_root, is_minor=False):
         gcdata = self.gcdata
         self.rootstackhook(collect_stack_root,
                            gcdata.root_stack_base, gcdata.root_stack_top)
diff --git a/rpython/memory/gcwrapper.py b/rpython/memory/gcwrapper.py
--- a/rpython/memory/gcwrapper.py
+++ b/rpython/memory/gcwrapper.py
@@ -191,7 +191,8 @@
 
     def walk_roots(self, collect_stack_root,
                    collect_static_in_prebuilt_nongc,
-                   collect_static_in_prebuilt_gc):
+                   collect_static_in_prebuilt_gc,
+                   is_minor=False):
         gcheap = self.gcheap
         gc = gcheap.gc
         if collect_static_in_prebuilt_gc:
@@ -203,7 +204,7 @@
                 if self.gcheap.gc.points_to_valid_gc_object(addrofaddr):
                     collect_static_in_prebuilt_nongc(gc, addrofaddr)
         if collect_stack_root:
-            for addrofaddr in gcheap.llinterp.find_roots():
+            for addrofaddr in gcheap.llinterp.find_roots(is_minor):
                 if self.gcheap.gc.points_to_valid_gc_object(addrofaddr):
                     collect_stack_root(gc, addrofaddr)
 
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -146,13 +146,22 @@
                            }
             return self._tlobj
 
-    def find_roots(self):
+    def find_roots(self, is_minor=False):
         """Return a list of the addresses of the roots."""
         #log.findroots("starting")
         roots = []
-        for frame in self.frame_stack:
+        for frame in reversed(self.frame_stack):
             #log.findroots("graph", frame.graph.name)
             frame.find_roots(roots)
+            # If two consecutive calls are both done with 'is_minor=True',
+            # we can stop after the first already-seen frame in the stack
+            # (which we still need to trace, but not its callers)
+            if is_minor:
+                if getattr(frame, '_find_roots_already_seen', False):
+                    break
+                frame._find_roots_already_seen = True
+            else:
+                frame._find_roots_already_seen = False
         return roots
 
     def find_exception(self, exc):

From noreply at buildbot.pypy.org  Wed Feb  4 19:43:25 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  4 Feb 2015 19:43:25 +0100 (CET)
Subject: [pypy-commit] pypy stackroot-speedup-2: Another try (now focusing
 more on asmgcc than shadowstack):
Message-ID: <20150204184325.62EDF1C105F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stackroot-speedup-2
Changeset: r75708:298e98aff859
Date: 2015-02-04 18:04 +0100
http://bitbucket.org/pypy/pypy/changeset/298e98aff859/

Log:	Another try (now focusing more on asmgcc than shadowstack): avoid
	tracing all stack roots during repeated minor collections, by
	ignoring the part of the stack that didn't change


From noreply at buildbot.pypy.org  Wed Feb  4 19:43:27 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  4 Feb 2015 19:43:27 +0100 (CET)
Subject: [pypy-commit] pypy stackroot-speedup-2: Untested code
Message-ID: <20150204184327.E7EF71C105F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stackroot-speedup-2
Changeset: r75710:1c9c0cc23c80
Date: 2015-02-04 19:01 +0100
http://bitbucket.org/pypy/pypy/changeset/1c9c0cc23c80/

Log:	Untested code

diff --git a/rpython/jit/backend/x86/arch.py b/rpython/jit/backend/x86/arch.py
--- a/rpython/jit/backend/x86/arch.py
+++ b/rpython/jit/backend/x86/arch.py
@@ -35,7 +35,9 @@
     PASS_ON_MY_FRAME = 15
     JITFRAME_FIXED_SIZE = 6 + 8 * 2 # 6 GPR + 8 XMM * 2 WORDS/float
     # 'threadlocal_addr' is passed as 2nd argument on the stack,
-    # and it can be left here for when it is needed
+    # and it can be left here for when it is needed.  As an additional hack,
+    # with asmgcc, it is made odd-valued to mean "already seen this frame
+    # during the previous minor collection".
     THREADLOCAL_OFS = (FRAME_FIXED_SIZE + 2) * WORD
 else:
     # rbp + rbx + r12 + r13 + r14 + r15 + threadlocal + 12 extra words = 19
@@ -43,7 +45,9 @@
     PASS_ON_MY_FRAME = 12
     JITFRAME_FIXED_SIZE = 28 # 13 GPR + 15 XMM
     # 'threadlocal_addr' is passed as 2nd argument in %esi,
-    # and is moved into this frame location
+    # and is moved into this frame location.  As an additional hack,
+    # with asmgcc, it is made odd-valued to mean "already seen this frame
+    # during the previous minor collection".
     THREADLOCAL_OFS = (FRAME_FIXED_SIZE - 1) * WORD
 
 assert PASS_ON_MY_FRAME >= 12       # asmgcc needs at least JIT_USE_WORDS + 3
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1980,6 +1980,23 @@
 
     def _call_assembler_emit_call(self, addr, argloc, _):
         threadlocal_loc = RawEspLoc(THREADLOCAL_OFS, INT)
+        if self._is_asmgcc():
+            # We need to remove the bit "already seen during the
+            # previous minor collection" instead of passing this
+            # value directly.
+            if IS_X86_64:
+                tmploc = esi    # already the correct place
+                if argloc is tmploc:
+                    self.mc.MOV_rr(esi.value, edi.value)
+                    argloc = edi
+            else:
+                tmploc = eax
+                if tmploc is argloc:
+                    tmploc = edx
+            self.mc.MOV(tmploc, threadlocal_ofs)
+            self.mc.AND_ri(tmploc.value, ~1)
+            threadlocal_ofs = tmploc
+        #
         self.simple_call(addr, [argloc, threadlocal_loc])
 
     def _call_assembler_emit_helper_call(self, addr, arglocs, result_loc):
@@ -2355,6 +2372,8 @@
         assert self.cpu.translate_support_code
         assert isinstance(resloc, RegLoc)
         self.mc.MOV_rs(resloc.value, THREADLOCAL_OFS)
+        if self._is_asmgcc():
+            self.mc.AND_ri(resloc.value, ~1)
         self.load_from_mem(resloc, addr_add_const(resloc, offset),
                            imm(size), imm(sign))
 
diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py
--- a/rpython/jit/backend/x86/callbuilder.py
+++ b/rpython/jit/backend/x86/callbuilder.py
@@ -167,6 +167,8 @@
                 self.tlofs_reg = r12
             self.mc.MOV_rs(self.tlofs_reg.value,
                            THREADLOCAL_OFS - self.current_esp)
+            if self.asm._is_asmgcc():
+                self.mc.AND_ri(self.tlofs_reg.value, ~1)
         return self.tlofs_reg
 
     def save_stack_position(self):
diff --git a/rpython/memory/gctransform/asmgcroot.py b/rpython/memory/gctransform/asmgcroot.py
--- a/rpython/memory/gctransform/asmgcroot.py
+++ b/rpython/memory/gctransform/asmgcroot.py
@@ -343,6 +343,7 @@
     def walk_stack_roots(self, collect_stack_root, is_minor=False):
         gcdata = self.gcdata
         gcdata._gc_collect_stack_root = collect_stack_root
+        gcdata._gc_collect_is_minor = is_minor
         pypy_asm_stackwalk(llhelper(ASM_CALLBACK_PTR, self._asm_callback),
                            gcrootanchor)
 
@@ -477,6 +478,14 @@
             addr = self.getlocation(callee, ebp_in_caller, location)
             caller.regs_stored_at[reg] = addr
             reg -= 1
+        #
+        # small hack: the JIT reserves THREADLOCAL_OFS's last bit for
+        # us.  We use it to store an "already traced past this frame"
+        # flag.
+        if self._with_jit:
+            is_minor = self.gcdata._gc_collect_is_minor
+            if self.mark_jit_frame_can_stop(callee, is_minor):
+                return False
 
         location = self._shape_decompressor.next()
         caller.frame_address = self.getlocation(callee, ebp_in_caller,
@@ -548,6 +557,23 @@
         else:  # kind == LOC_EBP_MINUS:   at -N(%ebp)
             return ebp_in_caller - offset
 
+    def mark_jit_frame_can_stop(self, callee, is_minor):
+        location = self._shape_decompressor.get_threadlocal_loc()
+        if location == LOC_NOWHERE:
+            return False
+        addr = self.getlocation(callee, llmemory.NULL, location)
+        #
+        x = addr.signed[0]
+        if is_minor:
+            if x & 1:
+                return True            # this JIT stack frame is already marked!
+            else:
+                addr.signed[0] = x | 1    # otherwise, mark it but don't stop
+                return False
+        else:
+            addr.signed[0] = x & ~1    # 'is_minor' is False, remove the marks
+            return False
+
 
 LOC_REG       = 0
 LOC_ESP_PLUS  = 1
@@ -729,6 +755,19 @@
             llop.debug_fatalerror(lltype.Void, "asmgcroot: invalid index")
             return 0   # annotator fix
 
+    def get_threadlocal_loc(self):
+        index = self.jit_index
+        if index < 0:
+            return LOC_NOWHERE     # case "outside the jit"
+        else:
+            # case "in the jit"
+            from rpython.jit.backend.x86.arch import THREADLOCAL_OFS
+            from rpython.jit.backend.x86.arch import PASS_ON_MY_FRAME
+            stack_depth = PASS_ON_MY_FRAME + self.extra_stack_depth
+            return (LOC_ESP_PLUS |
+                    ((THREADLOCAL_OFS // WORD + self.extra_stack_depth) << 2))
+
+
 # ____________________________________________________________
 
 #

From noreply at buildbot.pypy.org  Wed Feb  4 19:43:29 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  4 Feb 2015 19:43:29 +0100 (CET)
Subject: [pypy-commit] pypy stackroot-speedup-2: fixes
Message-ID: <20150204184329.1020F1C105F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stackroot-speedup-2
Changeset: r75711:526dbd4749a9
Date: 2015-02-04 19:23 +0100
http://bitbucket.org/pypy/pypy/changeset/526dbd4749a9/

Log:	fixes

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1993,7 +1993,7 @@
                 tmploc = eax
                 if tmploc is argloc:
                     tmploc = edx
-            self.mc.MOV(tmploc, threadlocal_ofs)
+            self.mc.MOV(tmploc, threadlocal_loc)
             self.mc.AND_ri(tmploc.value, ~1)
             threadlocal_ofs = tmploc
         #
diff --git a/rpython/memory/gctransform/asmgcroot.py b/rpython/memory/gctransform/asmgcroot.py
--- a/rpython/memory/gctransform/asmgcroot.py
+++ b/rpython/memory/gctransform/asmgcroot.py
@@ -761,9 +761,7 @@
             return LOC_NOWHERE     # case "outside the jit"
         else:
             # case "in the jit"
-            from rpython.jit.backend.x86.arch import THREADLOCAL_OFS
-            from rpython.jit.backend.x86.arch import PASS_ON_MY_FRAME
-            stack_depth = PASS_ON_MY_FRAME + self.extra_stack_depth
+            from rpython.jit.backend.x86.arch import THREADLOCAL_OFS, WORD
             return (LOC_ESP_PLUS |
                     ((THREADLOCAL_OFS // WORD + self.extra_stack_depth) << 2))
 

From noreply at buildbot.pypy.org  Wed Feb  4 19:43:30 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  4 Feb 2015 19:43:30 +0100 (CET)
Subject: [pypy-commit] pypy stackroot-speedup-2: apply this test to
 Incremental MiniMark at least
Message-ID: <20150204184330.285701C105F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stackroot-speedup-2
Changeset: r75712:8d63051f794e
Date: 2015-02-04 19:23 +0100
http://bitbucket.org/pypy/pypy/changeset/8d63051f794e/

Log:	apply this test to Incremental MiniMark at least

diff --git a/rpython/translator/c/gcc/test/test_asmgcroot.py b/rpython/translator/c/gcc/test/test_asmgcroot.py
--- a/rpython/translator/c/gcc/test/test_asmgcroot.py
+++ b/rpython/translator/c/gcc/test/test_asmgcroot.py
@@ -251,13 +251,17 @@
     def define_callback_with_collect(cls):
         return lambda: 0
 
-class TestAsmGCRootWithSemiSpaceGC_Shared(TestAsmGCRootWithSemiSpaceGC):
-    @classmethod
-    def make_config(cls):
-        config = TestAsmGCRootWithSemiSpaceGC.make_config()
-        config.translation.shared = True
-        return config
+#class TestAsmGCRootWithSemiSpaceGC_Shared(TestAsmGCRootWithSemiSpaceGC):
+#    @classmethod
+#    def make_config(cls):
+#        config = TestAsmGCRootWithSemiSpaceGC.make_config()
+#        config.translation.shared = True
+#        return config
 
 class TestAsmGCRootWithHybridTagged(AbstractTestAsmGCRoot,
                                     test_newgc.TestHybridTaggedPointers):
     pass
+
+class TestAsmGCRootWithIncrementalMinimark(AbstractTestAsmGCRoot,
+                                    test_newgc.TestIncrementalMiniMarkGC):
+    pass

From noreply at buildbot.pypy.org  Wed Feb  4 19:53:31 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  4 Feb 2015 19:53:31 +0100 (CET)
Subject: [pypy-commit] pypy stackroot-speedup-2: Simplify the condition: we
 can just ignore 'walk_roots(is_minor=False)'
Message-ID: <20150204185331.A65A01C114E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stackroot-speedup-2
Changeset: r75713:3dc3cccf1b6f
Date: 2015-02-04 19:53 +0100
http://bitbucket.org/pypy/pypy/changeset/3dc3cccf1b6f/

Log:	Simplify the condition: we can just ignore
	'walk_roots(is_minor=False)' for the purpose of knowing how far the
	next 'is_minor=True' needs to go until for sure it doesn't see any
	young object

diff --git a/rpython/memory/gctransform/asmgcroot.py b/rpython/memory/gctransform/asmgcroot.py
--- a/rpython/memory/gctransform/asmgcroot.py
+++ b/rpython/memory/gctransform/asmgcroot.py
@@ -482,9 +482,8 @@
         # small hack: the JIT reserves THREADLOCAL_OFS's last bit for
         # us.  We use it to store an "already traced past this frame"
         # flag.
-        if self._with_jit:
-            is_minor = self.gcdata._gc_collect_is_minor
-            if self.mark_jit_frame_can_stop(callee, is_minor):
+        if self._with_jit and self.gcdata._gc_collect_is_minor:
+            if self.mark_jit_frame_can_stop(callee):
                 return False
 
         location = self._shape_decompressor.next()
@@ -557,21 +556,17 @@
         else:  # kind == LOC_EBP_MINUS:   at -N(%ebp)
             return ebp_in_caller - offset
 
-    def mark_jit_frame_can_stop(self, callee, is_minor):
+    def mark_jit_frame_can_stop(self, callee):
         location = self._shape_decompressor.get_threadlocal_loc()
         if location == LOC_NOWHERE:
             return False
         addr = self.getlocation(callee, llmemory.NULL, location)
         #
         x = addr.signed[0]
-        if is_minor:
-            if x & 1:
-                return True            # this JIT stack frame is already marked!
-            else:
-                addr.signed[0] = x | 1    # otherwise, mark it but don't stop
-                return False
+        if x & 1:
+            return True            # this JIT stack frame is already marked!
         else:
-            addr.signed[0] = x & ~1    # 'is_minor' is False, remove the marks
+            addr.signed[0] = x | 1    # otherwise, mark it but don't stop
             return False
 
 
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -153,15 +153,14 @@
         for frame in reversed(self.frame_stack):
             #log.findroots("graph", frame.graph.name)
             frame.find_roots(roots)
-            # If two consecutive calls are both done with 'is_minor=True',
-            # we can stop after the first already-seen frame in the stack
-            # (which we still need to trace, but not its callers)
+            # If a call is done with 'is_minor=True', we can stop after the
+            # first frame in the stack that was already seen by the previous
+            # call with 'is_minor=True'.  (We still need to trace that frame,
+            # but not its callers.)
             if is_minor:
                 if getattr(frame, '_find_roots_already_seen', False):
                     break
                 frame._find_roots_already_seen = True
-            else:
-                frame._find_roots_already_seen = False
         return roots
 
     def find_exception(self, exc):

From noreply at buildbot.pypy.org  Wed Feb  4 20:10:58 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  4 Feb 2015 20:10:58 +0100 (CET)
Subject: [pypy-commit] pypy stackroot-speedup-2: Disable the optimization in
 case there are pinned objects still
Message-ID: <20150204191058.96C201C0013@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stackroot-speedup-2
Changeset: r75714:77a2c41cc7ac
Date: 2015-02-04 20:10 +0100
http://bitbucket.org/pypy/pypy/changeset/77a2c41cc7ac/

Log:	Disable the optimization in case there are pinned objects still
	waiting in the nursery.

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1489,6 +1489,7 @@
         # The following counter keeps track of alive and pinned young objects
         # inside the nursery. We reset it here and increace it in
         # '_trace_drag_out()'.
+        any_pinned_objects_in_nursery = (self.pinned_objects_in_nursery > 0)
         self.pinned_objects_in_nursery = 0
         #
         # Before everything else, remove from 'old_objects_pointing_to_young'
@@ -1513,7 +1514,7 @@
         # are copied out or flagged.  They are also added to the list
         # 'old_objects_pointing_to_young'.
         self.nursery_surviving_size = 0
-        self.collect_roots_in_nursery()
+        self.collect_roots_in_nursery(any_pinned_objects_in_nursery)
         #
         # visit all objects that are known for pointing to pinned
         # objects. This way we populate 'surviving_pinned_objects'
@@ -1649,7 +1650,7 @@
     def _visit_old_objects_pointing_to_pinned(self, obj, ignore):
         self.trace(obj, self._trace_drag_out, obj)
 
-    def collect_roots_in_nursery(self):
+    def collect_roots_in_nursery(self, any_pinned_objects_in_nursery):
         # we don't need to trace prebuilt GcStructs during a minor collect:
         # if a prebuilt GcStruct contains a pointer to a young object,
         # then the write_barrier must have ensured that the prebuilt
@@ -1659,11 +1660,17 @@
             callback = IncrementalMiniMarkGC._trace_drag_out1_marking_phase
         else:
             callback = IncrementalMiniMarkGC._trace_drag_out1
+        #
+        # Note a subtlety: if the nursery contains pinned objects right
+        # now, we can't use the "is_minor=True" optimization.  We really
+        # need to walk the complete stack to be sure we still see them.
+        use_jit_frame_stoppers = not any_pinned_objects_in_nursery
+        #
         self.root_walker.walk_roots(
             callback,     # stack roots
             callback,     # static in prebuilt non-gc
             None,         # static in prebuilt gc
-            is_minor=True)
+            is_minor=use_jit_frame_stoppers)
         debug_stop("gc-minor-walkroots")
 
     def collect_cardrefs_to_nursery(self):

From noreply at buildbot.pypy.org  Wed Feb  4 20:43:00 2015
From: noreply at buildbot.pypy.org (rguillebert)
Date: Wed,  4 Feb 2015 20:43:00 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: Fix typo
Message-ID: <20150204194300.B9B601C02D4@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: extradoc
Changeset: r5501:74934f466440
Date: 2015-02-04 20:42 +0100
http://bitbucket.org/pypy/extradoc/changeset/74934f466440/

Log:	Fix typo

diff --git a/talk/fosdem2015/talk.pdf b/talk/fosdem2015/talk.pdf
index e1d849071a9ecfed8044921f817cc8476db0d0a6..a64540c5df6d1e7d57d78b1feda51dc7eeb26e53
GIT binary patch

[cut]

diff --git a/talk/fosdem2015/talk.rst b/talk/fosdem2015/talk.rst
--- a/talk/fosdem2015/talk.rst
+++ b/talk/fosdem2015/talk.rst
@@ -54,7 +54,7 @@
 Current situation (3/3)
 -----------------------
 
-* It's pretty hard to switch between implementations because of C extensions ()
+* It's pretty hard to switch between implementations because of C extensions
 
 * C extensions are very useful but CPython can't evolve because of them
 

From noreply at buildbot.pypy.org  Wed Feb  4 21:39:18 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  4 Feb 2015 21:39:18 +0100 (CET)
Subject: [pypy-commit] pypy stackroot-speedup-2: I think this was a bug,
 and this is a fix.  Theoretical thinking though.
Message-ID: <20150204203918.BA4BB1C02A4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stackroot-speedup-2
Changeset: r75715:41114edfebad
Date: 2015-02-04 21:39 +0100
http://bitbucket.org/pypy/pypy/changeset/41114edfebad/

Log:	I think this was a bug, and this is a fix. Theoretical thinking
	though.

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -375,6 +375,10 @@
         # the nursery.
         self.pinned_objects_in_nursery = 0
         #
+        # This flag is set if the previous minor collection found at least
+        # one pinned object alive.
+        self.any_pinned_object_kept = False
+        #
         # Keeps track of old objects pointing to pinned objects. These objects
         # must be traced every minor collection. Without tracing them the
         # referenced pinned object wouldn't be visited and therefore collected.
@@ -1489,8 +1493,9 @@
         # The following counter keeps track of alive and pinned young objects
         # inside the nursery. We reset it here and increace it in
         # '_trace_drag_out()'.
-        any_pinned_objects_in_nursery = (self.pinned_objects_in_nursery > 0)
+        any_pinned_object_from_earlier = self.any_pinned_object_kept
         self.pinned_objects_in_nursery = 0
+        self.any_pinned_object_kept = False
         #
         # Before everything else, remove from 'old_objects_pointing_to_young'
         # the young arrays.
@@ -1514,7 +1519,7 @@
         # are copied out or flagged.  They are also added to the list
         # 'old_objects_pointing_to_young'.
         self.nursery_surviving_size = 0
-        self.collect_roots_in_nursery(any_pinned_objects_in_nursery)
+        self.collect_roots_in_nursery(any_pinned_object_from_earlier)
         #
         # visit all objects that are known for pointing to pinned
         # objects. This way we populate 'surviving_pinned_objects'
@@ -1650,7 +1655,7 @@
     def _visit_old_objects_pointing_to_pinned(self, obj, ignore):
         self.trace(obj, self._trace_drag_out, obj)
 
-    def collect_roots_in_nursery(self, any_pinned_objects_in_nursery):
+    def collect_roots_in_nursery(self, any_pinned_object_from_earlier):
         # we don't need to trace prebuilt GcStructs during a minor collect:
         # if a prebuilt GcStruct contains a pointer to a young object,
         # then the write_barrier must have ensured that the prebuilt
@@ -1661,10 +1666,12 @@
         else:
             callback = IncrementalMiniMarkGC._trace_drag_out1
         #
-        # Note a subtlety: if the nursery contains pinned objects right
-        # now, we can't use the "is_minor=True" optimization.  We really
-        # need to walk the complete stack to be sure we still see them.
-        use_jit_frame_stoppers = not any_pinned_objects_in_nursery
+        # Note a subtlety: if the nursery contains pinned objects "from
+        # earlier", i.e. created earlier than the previous minor
+        # collection, then we can't use the "is_minor=True" optimization.
+        # We really need to walk the complete stack to be sure we still
+        # see them.
+        use_jit_frame_stoppers = not any_pinned_object_from_earlier
         #
         self.root_walker.walk_roots(
             callback,     # stack roots
@@ -1852,6 +1859,7 @@
             self.surviving_pinned_objects.append(
                 llarena.getfakearenaaddress(obj - size_gc_header))
             self.pinned_objects_in_nursery += 1
+            self.any_pinned_object_kept = True
             return
         else:
             # First visit to an object that has already a shadow.

From noreply at buildbot.pypy.org  Thu Feb  5 00:58:15 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  5 Feb 2015 00:58:15 +0100 (CET)
Subject: [pypy-commit] pypy stackroot-speedup-2:
	aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
Message-ID: <20150204235815.1DF951C114E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stackroot-speedup-2
Changeset: r75717:16afee7b24da
Date: 2015-02-05 00:58 +0100
http://bitbucket.org/pypy/pypy/changeset/16afee7b24da/

Log:	aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1995,7 +1995,7 @@
                     tmploc = edx
             self.mc.MOV(tmploc, threadlocal_loc)
             self.mc.AND_ri(tmploc.value, ~1)
-            threadlocal_ofs = tmploc
+            threadlocal_loc = tmploc
         #
         self.simple_call(addr, [argloc, threadlocal_loc])
 

From noreply at buildbot.pypy.org  Thu Feb  5 00:58:13 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  5 Feb 2015 00:58:13 +0100 (CET)
Subject: [pypy-commit] pypy stackroot-speedup-2: Move this piece of code up.
	Minor benefit.
Message-ID: <20150204235813.C89991C114E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stackroot-speedup-2
Changeset: r75716:42a6cd5c1785
Date: 2015-02-04 23:54 +0100
http://bitbucket.org/pypy/pypy/changeset/42a6cd5c1785/

Log:	Move this piece of code up. Minor benefit.

diff --git a/rpython/memory/gctransform/asmgcroot.py b/rpython/memory/gctransform/asmgcroot.py
--- a/rpython/memory/gctransform/asmgcroot.py
+++ b/rpython/memory/gctransform/asmgcroot.py
@@ -469,6 +469,13 @@
             if gc.points_to_valid_gc_object(addr):
                 collect_stack_root(gc, addr)
         #
+        # small hack: the JIT reserves THREADLOCAL_OFS's last bit for
+        # us.  We use it to store an "already traced past this frame"
+        # flag.
+        if self._with_jit and self.gcdata._gc_collect_is_minor:
+            if self.mark_jit_frame_can_stop(callee):
+                return False
+        #
         # track where the caller_frame saved the registers from its own
         # caller
         #
@@ -478,13 +485,6 @@
             addr = self.getlocation(callee, ebp_in_caller, location)
             caller.regs_stored_at[reg] = addr
             reg -= 1
-        #
-        # small hack: the JIT reserves THREADLOCAL_OFS's last bit for
-        # us.  We use it to store an "already traced past this frame"
-        # flag.
-        if self._with_jit and self.gcdata._gc_collect_is_minor:
-            if self.mark_jit_frame_can_stop(callee):
-                return False
 
         location = self._shape_decompressor.next()
         caller.frame_address = self.getlocation(callee, ebp_in_caller,

From noreply at buildbot.pypy.org  Thu Feb  5 10:45:09 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  5 Feb 2015 10:45:09 +0100 (CET)
Subject: [pypy-commit] pypy stackroot-speedup-2: fix
Message-ID: <20150205094509.D05921C0103@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stackroot-speedup-2
Changeset: r75718:4a2be540c12e
Date: 2015-02-05 10:44 +0100
http://bitbucket.org/pypy/pypy/changeset/4a2be540c12e/

Log:	fix

diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py
--- a/rpython/memory/gc/test/test_direct.py
+++ b/rpython/memory/gc/test/test_direct.py
@@ -34,7 +34,8 @@
 
     def walk_roots(self, collect_stack_root,
                    collect_static_in_prebuilt_nongc,
-                   collect_static_in_prebuilt_gc):
+                   collect_static_in_prebuilt_gc,
+                   is_minor=False):
         gc = self.tester.gc
         layoutbuilder = self.tester.layoutbuilder
         if collect_static_in_prebuilt_gc:

From noreply at buildbot.pypy.org  Thu Feb  5 12:18:42 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  5 Feb 2015 12:18:42 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: a bit blind fix
Message-ID: <20150205111842.B03E11C115A@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r75719:f2c96b90445b
Date: 2015-02-05 12:58 +0200
http://bitbucket.org/pypy/pypy/changeset/f2c96b90445b/

Log:	a bit blind fix

diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -392,6 +392,9 @@
         else:
             self.rm._sync_var(v)
 
+    def prepare_op_debug_merge_point(self, op, fcond):
+        self.assembler.codemap.debug_merge_point(op)
+
     def _prepare_op_int_add(self, op, fcond):
         boxes = op.getarglist()
         a0, a1 = boxes

From noreply at buildbot.pypy.org  Thu Feb  5 12:18:44 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  5 Feb 2015 12:18:44 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: fixes for 32bit
Message-ID: <20150205111844.03E2F1C115A@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r75720:0fcac08c3273
Date: 2015-02-05 13:10 +0200
http://bitbucket.org/pypy/pypy/changeset/0fcac08c3273/

Log:	fixes for 32bit

diff --git a/rpython/jit/backend/x86/test/test_recursive.py b/rpython/jit/backend/x86/test/test_recursive.py
--- a/rpython/jit/backend/x86/test/test_recursive.py
+++ b/rpython/jit/backend/x86/test/test_recursive.py
@@ -1,15 +1,16 @@
 
-from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.jit.metainterp.test.test_recursive import RecursiveTests
 from rpython.jit.backend.x86.test.test_basic import Jit386Mixin
 from rpython.jit.backend.llsupport import asmmemmgr
 from rpython.jit.backend.llsupport.asmmemmgr import unpack_traceback
-from rpython.jit.metainterp.test.support import get_stats
+from rpython.jit.backend.x86.arch import WORD
 
 class TestRecursive(Jit386Mixin, RecursiveTests):
     # for the individual tests see
     # ====> ../../../metainterp/test/test_recursive.py
-    def check_get_unique_id(self):            
+    def check_get_unique_id(self):
+        if WORD == 4:
+            return # this is 64 bit only check
         codemaps = asmmemmgr._memmngr.jit_codemap[:] # ups, sorting later
         assert len(codemaps) == 3
         codemaps.sort(lambda arg0, arg1: cmp(arg0[1], arg1[1]))

From noreply at buildbot.pypy.org  Thu Feb  5 12:18:45 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  5 Feb 2015 12:18:45 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: merge
Message-ID: <20150205111845.352F41C115A@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r75721:f9100bc57aa6
Date: 2015-02-05 13:18 +0200
http://bitbucket.org/pypy/pypy/changeset/f9100bc57aa6/

Log:	merge

diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -302,14 +302,13 @@
         hop.exception_cannot_occur()
         T = hop.args_r[0].lowleveltype.TO
         v = hop.inputarg(hop.args_r[0], arg=0)
-        c_super = hop.inputconst(lltype.Void, 'super')
         while not hasattr(T, 'vable_token'):
             if not hasattr(T, 'super'):
                 # we're not really in a jitted build
                 return hop.inputconst(llmemory.GCREF,
                                       lltype.nullptr(llmemory.GCREF.TO))
-            v = hop.genop('getfield', [v, c_super], resulttype=T.super)
             T = T.super
+        v = hop.genop('cast_pointer', [v], resulttype=lltype.Ptr(T))
         c_vable_token = hop.inputconst(lltype.Void, 'vable_token')
         return hop.genop('getfield', [v, c_vable_token],
                          resulttype=llmemory.GCREF)

From noreply at buildbot.pypy.org  Thu Feb  5 12:22:15 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  5 Feb 2015 12:22:15 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: shuffle the imports around
Message-ID: <20150205112215.DE34F1C115A@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r75722:51a273f1d8e1
Date: 2015-02-05 13:21 +0200
http://bitbucket.org/pypy/pypy/changeset/51a273f1d8e1/

Log:	shuffle the imports around

diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py
--- a/rpython/jit/backend/llsupport/asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/asmmemmgr.py
@@ -4,86 +4,11 @@
 from rpython.rlib import rmmap
 from rpython.rlib.debug import debug_start, debug_print, debug_stop
 from rpython.rlib.debug import have_debug_prints
-from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.rbisect import bisect, bisect_tuple
-from rpython.rlib import rgc
-from rpython.rlib.entrypoint import jit_entrypoint
 
 _memmngr = None # global reference so we can use @entrypoint :/
 
- at jit_entrypoint([lltype.Signed], lltype.Signed,
-                c_name='pypy_jit_stack_depth_at_loc')
- at rgc.no_collect
-def stack_depth_at_loc(loc):
-    global _memmngr
-
-    pos = bisect(_memmngr.jit_addr_map, loc)
-    if pos == 0 or pos == len(_memmngr.jit_addr_map):
-        return -1
-    return _memmngr.jit_frame_depth_map[pos-1]
-
- at jit_entrypoint([], lltype.Signed, c_name='pypy_jit_start_addr')
-def jit_start_addr():
-    global _memmngr
-
-    return _memmngr.jit_addr_map[0]
-
- at jit_entrypoint([], lltype.Signed, c_name='pypy_jit_end_addr')
-def jit_end_addr():
-    global _memmngr
-
-    return _memmngr.jit_addr_map[-1]
-
- at jit_entrypoint([lltype.Signed], lltype.Signed,
-                c_name='pypy_find_codemap_at_addr')
-def find_codemap_at_addr(addr):
-    global _memmngr
-
-    res = bisect_tuple(_memmngr.jit_codemap, addr) - 1
-    if res == len(_memmngr.jit_codemap):
-        return -1
-    return res
-
- at jit_entrypoint([lltype.Signed, lltype.Signed,
-                 rffi.CArrayPtr(lltype.Signed)], lltype.Signed,
-                 c_name='pypy_yield_codemap_at_addr')
-def yield_bytecode_at_addr(codemap_no, addr, current_pos_addr):
-    """ will return consecutive unique_ids from codemap, starting from position
-    `pos` until addr
-    """
-    global _memmngr
-
-    codemap = _memmngr.jit_codemap[codemap_no]
-    current_pos = current_pos_addr[0]
-    start_addr = codemap[0]
-    rel_addr = addr - start_addr
-    while True:
-        if current_pos >= len(codemap[2]):
-            return 0
-        next_start = codemap[2][current_pos + 1]
-        if next_start > rel_addr:
-            return 0
-        next_stop = codemap[2][current_pos + 2]
-        if next_stop > rel_addr:
-            current_pos_addr[0] = current_pos + 4
-            return codemap[2][current_pos]
-        # we need to skip potentially more than one
-        current_pos = codemap[2][current_pos + 3]
-
-def unpack_traceback(addr):
-    codemap_pos = find_codemap_at_addr(addr)
-    assert codemap_pos >= 0
-    storage = lltype.malloc(rffi.CArray(lltype.Signed), 1, flavor='raw')
-    storage[0] = 0
-    res = []
-    while True:
-        item = yield_bytecode_at_addr(codemap_pos, addr, storage)
-        if item == 0:
-            break
-        res.append(item)
-    lltype.free(storage, flavor='raw')
-    return res
-
 
 class AsmMemoryManager(object):
     LARGE_ALLOC_SIZE = 1024 * 1024   # 1MB
diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py
--- a/rpython/jit/backend/llsupport/codemap.py
+++ b/rpython/jit/backend/llsupport/codemap.py
@@ -9,7 +9,85 @@
 
 """
 
-from rpython.jit.backend.llsupport.asmmemmgr import unpack_traceback
+from rpython.rlib import rgc
+from rpython.rlib.entrypoint import jit_entrypoint
+from rpython.jit.backend.llsupport import asmmemmgr
+from rpython.rlib.rbisect import bisect, bisect_tuple
+from rpython.rtyper.lltypesystem import lltype, rffi
+
+ at jit_entrypoint([lltype.Signed], lltype.Signed,
+                c_name='pypy_jit_stack_depth_at_loc')
+ at rgc.no_collect
+def stack_depth_at_loc(loc):
+    _memmngr = asmmemmgr._memmngr
+
+    pos = bisect(_memmngr.jit_addr_map, loc)
+    if pos == 0 or pos == len(_memmngr.jit_addr_map):
+        return -1
+    return _memmngr.jit_frame_depth_map[pos-1]
+
+ at jit_entrypoint([], lltype.Signed, c_name='pypy_jit_start_addr')
+def jit_start_addr():
+    _memmngr = asmmemmgr._memmngr
+
+    return _memmngr.jit_addr_map[0]
+
+ at jit_entrypoint([], lltype.Signed, c_name='pypy_jit_end_addr')
+def jit_end_addr():
+    _memmngr = asmmemmgr._memmngr
+
+    return _memmngr.jit_addr_map[-1]
+
+ at jit_entrypoint([lltype.Signed], lltype.Signed,
+                c_name='pypy_find_codemap_at_addr')
+def find_codemap_at_addr(addr):
+    _memmngr = asmmemmgr._memmngr
+
+    res = bisect_tuple(_memmngr.jit_codemap, addr) - 1
+    if res == len(_memmngr.jit_codemap):
+        return -1
+    return res
+
+ at jit_entrypoint([lltype.Signed, lltype.Signed,
+                 rffi.CArrayPtr(lltype.Signed)], lltype.Signed,
+                 c_name='pypy_yield_codemap_at_addr')
+def yield_bytecode_at_addr(codemap_no, addr, current_pos_addr):
+    """ will return consecutive unique_ids from codemap, starting from position
+    `pos` until addr
+    """
+    _memmngr = asmmemmgr._memmngr
+
+    codemap = _memmngr.jit_codemap[codemap_no]
+    current_pos = current_pos_addr[0]
+    start_addr = codemap[0]
+    rel_addr = addr - start_addr
+    while True:
+        if current_pos >= len(codemap[2]):
+            return 0
+        next_start = codemap[2][current_pos + 1]
+        if next_start > rel_addr:
+            return 0
+        next_stop = codemap[2][current_pos + 2]
+        if next_stop > rel_addr:
+            current_pos_addr[0] = current_pos + 4
+            return codemap[2][current_pos]
+        # we need to skip potentially more than one
+        current_pos = codemap[2][current_pos + 3]
+
+def unpack_traceback(addr):
+    codemap_pos = find_codemap_at_addr(addr)
+    assert codemap_pos >= 0
+    storage = lltype.malloc(rffi.CArray(lltype.Signed), 1, flavor='raw')
+    storage[0] = 0
+    res = []
+    while True:
+        item = yield_bytecode_at_addr(codemap_pos, addr, storage)
+        if item == 0:
+            break
+        res.append(item)
+    lltype.free(storage, flavor='raw')
+    return res
+
 
 class CodemapBuilder(object):
     def __init__(self):
diff --git a/rpython/jit/backend/x86/test/test_recursive.py b/rpython/jit/backend/x86/test/test_recursive.py
--- a/rpython/jit/backend/x86/test/test_recursive.py
+++ b/rpython/jit/backend/x86/test/test_recursive.py
@@ -2,7 +2,7 @@
 from rpython.jit.metainterp.test.test_recursive import RecursiveTests
 from rpython.jit.backend.x86.test.test_basic import Jit386Mixin
 from rpython.jit.backend.llsupport import asmmemmgr
-from rpython.jit.backend.llsupport.asmmemmgr import unpack_traceback
+from rpython.jit.backend.llsupport.codemap import unpack_traceback
 from rpython.jit.backend.x86.arch import WORD
 
 class TestRecursive(Jit386Mixin, RecursiveTests):

From noreply at buildbot.pypy.org  Thu Feb  5 12:40:32 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  5 Feb 2015 12:40:32 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: add a check to the module
Message-ID: <20150205114032.A9F361C0013@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r75723:7afaf0219041
Date: 2015-02-05 13:40 +0200
http://bitbucket.org/pypy/pypy/changeset/7afaf0219041/

Log:	add a check to the module

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -102,6 +102,7 @@
     "_hashlib"  : ["pypy.module._ssl.interp_ssl"],
     "_minimal_curses": ["pypy.module._minimal_curses.fficurses"],
     "_continuation": ["rpython.rlib.rstacklet"],
+    "_vmprof" : ["pypy.module._vmprof.interp_vmprof"],
     }
 
 def get_module_validator(modname):
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -3,7 +3,8 @@
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rtyper.annlowlevel import cast_instance_to_gcref, cast_base_ptr_to_instance
 from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib import jit, rposix, entrypoint, rstruct
+from rpython.rlib import jit, rposix, entrypoint
+from rpython.rtyper.tool import rffi_platform as platform
 from rpython.rlib.rstring import StringBuilder
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import oefmt, wrap_oserror, OperationError
@@ -48,6 +49,10 @@
 
 eci = ExternalCompilationInfo(**eci_kwds)
 
+check_eci = eci.merge(ExternalCompilationInfo(separate_module_files=[
+    SRC.join('fake_pypy_api.c')]))
+
+platform.verify_eci(check_eci)
 
 pypy_execute_frame_trampoline = rffi.llexternal(
     "pypy_execute_frame_trampoline",
diff --git a/pypy/module/_vmprof/src/fake_pypy_api.c b/pypy/module/_vmprof/src/fake_pypy_api.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/src/fake_pypy_api.c
@@ -0,0 +1,29 @@
+
+long pypy_jit_start_addr(void)
+{
+	return 3;
+}
+
+long pypy_jit_end_addr(void)
+{
+	return 3;
+}
+
+long pypy_jit_stack_depth_at_loc(long x)
+{
+	return 0;
+}
+
+long pypy_find_codemap_at_addr(long x)
+{
+	return 0;
+}
+
+long pypy_yield_codemap_at_addr(long x, long y, long *a)
+{
+	return 0;
+}
+
+void pypy_pyframe_execute_frame(void)
+{
+}

From noreply at buildbot.pypy.org  Thu Feb  5 14:29:57 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  5 Feb 2015 14:29:57 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: merge vmprof
Message-ID: <20150205132957.A0B4E1C02A4@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r75724:59ba42dac21a
Date: 2015-02-05 15:29 +0200
http://bitbucket.org/pypy/pypy/changeset/59ba42dac21a/

Log:	merge vmprof


From noreply at buildbot.pypy.org  Thu Feb  5 14:29:59 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  5 Feb 2015 14:29:59 +0100 (CET)
Subject: [pypy-commit] pypy default: merge vmprof - add profiling
	capabilities
Message-ID: <20150205132959.F35931C02A4@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r75725:72433b63468f
Date: 2015-02-05 15:29 +0200
http://bitbucket.org/pypy/pypy/changeset/72433b63468f/

Log:	merge vmprof - add profiling capabilities

diff too long, truncating to 2000 out of 2680 lines

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -420,3 +420,10 @@
 the terms of the GPL license version 2 or any later version.  Thus the
 gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed
 under the terms of the GPL license as well.
+
+License for 'pypy/module/_vmprof/src'
+--------------------------------------
+
+The code is based on gperftools. You may see a copy of the License for it at
+
+    https://code.google.com/p/gperftools/source/browse/COPYING
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -38,6 +38,9 @@
     "_csv", "cppyy", "_pypyjson"
 ])
 
+if sys.platform.startswith('linux') and sys.maxint > 2147483647:
+    working_modules.add('_vmprof')
+
 translation_modules = default_modules.copy()
 translation_modules.update([
     "fcntl", "time", "select", "signal", "_rawffi", "zlib", "struct", "_md5",
@@ -99,6 +102,7 @@
     "_hashlib"  : ["pypy.module._ssl.interp_ssl"],
     "_minimal_curses": ["pypy.module._minimal_curses.fficurses"],
     "_continuation": ["rpython.rlib.rstacklet"],
+    "_vmprof" : ["pypy.module._vmprof.interp_vmprof"],
     }
 
 def get_module_validator(modname):
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -9,6 +9,7 @@
 from rpython.rlib.signature import signature
 from rpython.rlib.rarithmetic import r_uint, SHRT_MIN, SHRT_MAX, \
     INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX
+from rpython.rlib.rweaklist import RWeakListMixin
 
 from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag,
     UserDelAction)
@@ -366,6 +367,10 @@
 
 # ____________________________________________________________
 
+class CodeObjWeakList(RWeakListMixin):
+    def __init__(self):
+        self.initialize()
+
 class ObjSpace(object):
     """Base class for the interpreter-level implementations of object spaces.
     http://pypy.readthedocs.org/en/latest/objspace.html"""
@@ -389,6 +394,7 @@
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
         self._code_of_sys_exc_info = None
+        self.all_code_objs = CodeObjWeakList()
 
         # can be overridden to a subclass
         self.initialize()
@@ -666,6 +672,16 @@
             assert ec is not None
             return ec
 
+    def register_code_object(self, pycode):
+        callback = self.getexecutioncontext().register_code_callback
+        if callback is not None:
+            callback(self, pycode)
+        self.all_code_objs.add_handle(pycode)
+
+    def set_code_callback(self, callback):
+        ec = self.getexecutioncontext()
+        ec.register_code_callback = callback
+        
     def _freeze_(self):
         return True
 
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -33,6 +33,11 @@
         self.profilefunc = None
         self.w_profilefuncarg = None
         self.thread_disappeared = False   # might be set to True after os.fork()
+        self.register_code_callback = None
+        if sys.maxint == 2147483647:
+            self._code_unique_id = 0 # XXX this is wrong, it won't work on 32bit
+        else:
+            self._code_unique_id = 0x7000000000000000
 
     @staticmethod
     def _mark_thread_disappeared(space):
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -14,9 +14,10 @@
     CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED,
     CO_GENERATOR, CO_KILL_DOCSTRING, CO_YIELD_INSIDE_TRY)
 from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT
-from rpython.rlib.rarithmetic import intmask
+from rpython.rlib.rarithmetic import intmask, r_longlong
 from rpython.rlib.objectmodel import compute_hash
 from rpython.rlib import jit
+from rpython.rlib.debug import debug_start, debug_stop, debug_print
 
 
 class BytecodeCorruption(Exception):
@@ -54,8 +55,9 @@
     "CPython-style code objects."
     _immutable_ = True
     _immutable_fields_ = ["co_consts_w[*]", "co_names_w[*]", "co_varnames[*]",
-                          "co_freevars[*]", "co_cellvars[*]", "_args_as_cellvars[*]"]
-
+                          "co_freevars[*]", "co_cellvars[*]",
+                          "_args_as_cellvars[*]"]
+    
     def __init__(self, space,  argcount, nlocals, stacksize, flags,
                      code, consts, names, varnames, filename,
                      name, firstlineno, lnotab, freevars, cellvars,
@@ -83,6 +85,7 @@
         self.magic = magic
         self._signature = cpython_code_signature(self)
         self._initialize()
+        space.register_code_object(self)
 
     def _initialize(self):
         if self.co_cellvars:
@@ -124,6 +127,15 @@
             from pypy.objspace.std.mapdict import init_mapdict_cache
             init_mapdict_cache(self)
 
+        ec = self.space.getexecutioncontext()
+        self._unique_id = ec._code_unique_id
+        ec._code_unique_id += 2 # so we have one bit that we can mark stuff
+        # with
+
+    def _get_full_name(self):
+        return "py:%s:%d:%s" % (self.co_name, self.co_firstlineno,
+                                self.co_filename)
+
     def _cleanup_(self):
         if (self.magic == cpython_magic and
             '__pypy__' not in sys.builtin_module_names):
diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/__init__.py
@@ -0,0 +1,18 @@
+from pypy.interpreter.mixedmodule import MixedModule
+
+class Module(MixedModule):
+    """
+    Write me :)
+    """
+    appleveldefs = {
+    }
+
+    interpleveldefs = {
+        'enable': 'interp_vmprof.enable',
+        'disable': 'interp_vmprof.disable',
+    }
+
+    def setup_after_space_initialization(self):
+        # force the __extend__ hacks to occur early
+        from pypy.module._vmprof.interp_vmprof import VMProf
+        self.vmprof = VMProf()
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -0,0 +1,223 @@
+import py, os, sys
+from rpython.rtyper.lltypesystem import lltype, rffi, llmemory
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from rpython.rtyper.annlowlevel import cast_instance_to_gcref, cast_base_ptr_to_instance
+from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib import jit, rposix, entrypoint
+from rpython.rtyper.tool import rffi_platform as platform
+from rpython.rlib.rstring import StringBuilder
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.error import oefmt, wrap_oserror, OperationError
+from pypy.interpreter.gateway import unwrap_spec
+from pypy.interpreter.pyframe import PyFrame
+
+ROOT = py.path.local(__file__).join('..')
+SRC = ROOT.join('src')
+
+# by default, we statically link vmprof.c into pypy; however, if you set
+# DYNAMIC_VMPROF to True, it will be dynamically linked to the libvmprof.so
+# which is expected to be inside pypy/module/_vmprof/src: this is very useful
+# during development. Note that you have to manually build libvmprof by
+# running make inside the src dir
+DYNAMIC_VMPROF = False
+
+eci_kwds = dict(
+    include_dirs = [SRC],
+    includes = ['vmprof.h', 'trampoline.h'],
+    separate_module_files = [SRC.join('trampoline.asmgcc.s')],
+    libraries = ['unwind'],
+    
+    post_include_bits=["""
+        void* pypy_vmprof_get_virtual_ip(void*);
+        void pypy_vmprof_init(void);
+    """],
+    
+    separate_module_sources=["""
+        void pypy_vmprof_init(void) {
+            vmprof_set_mainloop(pypy_execute_frame_trampoline, 0,
+                                pypy_vmprof_get_virtual_ip);
+        }
+    """],
+    )
+
+
+if DYNAMIC_VMPROF:
+    eci_kwds['libraries'] += ['vmprof']
+    eci_kwds['link_extra'] = ['-Wl,-rpath,%s' % SRC, '-L%s' % SRC]
+else:
+    eci_kwds['separate_module_files'] += [SRC.join('vmprof.c')]
+
+eci = ExternalCompilationInfo(**eci_kwds)
+
+check_eci = eci.merge(ExternalCompilationInfo(separate_module_files=[
+    SRC.join('fake_pypy_api.c')]))
+
+platform.verify_eci(check_eci)
+
+pypy_execute_frame_trampoline = rffi.llexternal(
+    "pypy_execute_frame_trampoline",
+    [llmemory.GCREF, llmemory.GCREF, llmemory.GCREF],
+    llmemory.GCREF,
+    compilation_info=eci,
+    _nowrapper=True, sandboxsafe=True,
+    random_effects_on_gcobjs=True)
+
+pypy_vmprof_init = rffi.llexternal("pypy_vmprof_init", [], lltype.Void,
+                                   compilation_info=eci)
+vmprof_enable = rffi.llexternal("vmprof_enable",
+                                [rffi.INT, rffi.LONG, rffi.INT,
+                                 rffi.CCHARP, rffi.INT],
+                                rffi.INT, compilation_info=eci,
+                                save_err=rffi.RFFI_SAVE_ERRNO)
+vmprof_disable = rffi.llexternal("vmprof_disable", [], rffi.INT,
+                                 compilation_info=eci,
+                                save_err=rffi.RFFI_SAVE_ERRNO)
+
+vmprof_register_virtual_function = rffi.llexternal(
+    "vmprof_register_virtual_function",
+    [rffi.CCHARP, rffi.VOIDP, rffi.VOIDP], lltype.Void,
+    compilation_info=eci, _nowrapper=True)
+
+original_execute_frame = PyFrame.execute_frame.im_func
+original_execute_frame.c_name = 'pypy_pyframe_execute_frame'
+original_execute_frame._dont_inline_ = True
+
+class __extend__(PyFrame):
+    def execute_frame(frame, w_inputvalue=None, operr=None):
+        # go through the asm trampoline ONLY if we are translated but not being JITted.
+        #
+        # If we are not translated, we obviously don't want to go through the
+        # trampoline because there is no C function it can call.
+        #
+        # If we are being JITted, we want to skip the trampoline, else the JIT
+        # cannot see throug it
+        if we_are_translated() and not jit.we_are_jitted():
+            # if we are translated, call the trampoline
+            gc_frame = cast_instance_to_gcref(frame)
+            gc_inputvalue = cast_instance_to_gcref(w_inputvalue)
+            gc_operr = cast_instance_to_gcref(operr)
+            gc_result = pypy_execute_frame_trampoline(gc_frame, gc_inputvalue, gc_operr)
+            return cast_base_ptr_to_instance(W_Root, gc_result)
+        else:
+            return original_execute_frame(frame, w_inputvalue, operr)
+
+
+ at entrypoint.entrypoint_lowlevel('main', [llmemory.GCREF],
+                                'pypy_vmprof_get_virtual_ip', True)
+def get_virtual_ip(gc_frame):
+    frame = cast_base_ptr_to_instance(PyFrame, gc_frame)
+    if jit._get_virtualizable_token(frame):
+        return rffi.cast(rffi.VOIDP, 0)
+    virtual_ip = do_get_virtual_ip(frame)
+    return rffi.cast(rffi.VOIDP, virtual_ip)
+
+def do_get_virtual_ip(frame):
+    return frame.pycode._unique_id
+
+def write_long_to_string_builder(l, b):
+    if sys.maxint == 2147483647:
+        b.append(chr(l & 0xff))
+        b.append(chr((l >> 8) & 0xff))
+        b.append(chr((l >> 16) & 0xff))
+        b.append(chr((l >> 24) & 0xff))
+    else:
+        b.append(chr(l & 0xff))
+        b.append(chr((l >> 8) & 0xff))
+        b.append(chr((l >> 16) & 0xff))
+        b.append(chr((l >> 24) & 0xff))
+        b.append(chr((l >> 32) & 0xff))
+        b.append(chr((l >> 40) & 0xff))
+        b.append(chr((l >> 48) & 0xff))
+        b.append(chr((l >> 56) & 0xff))
+
+class VMProf(object):
+    def __init__(self):
+        self.is_enabled = False
+        self.ever_enabled = False
+        self.mapping_so_far = [] # stored mapping in between runs
+        self.fileno = -1
+
+    def enable(self, space, fileno, period):
+        if self.is_enabled:
+            raise oefmt(space.w_ValueError, "_vmprof already enabled")
+        self.fileno = fileno
+        self.is_enabled = True
+        self.write_header(fileno, period)
+        if not self.ever_enabled:
+            if we_are_translated():
+                pypy_vmprof_init()
+            self.ever_enabled = True
+        for weakcode in space.all_code_objs.get_all_handles():
+            code = weakcode()
+            if code:
+                self.register_code(space, code)
+        space.set_code_callback(vmprof_register_code)
+        if we_are_translated():
+            # does not work untranslated
+            res = vmprof_enable(fileno, period, 0,
+                                lltype.nullptr(rffi.CCHARP.TO), 0)
+        else:
+            res = 0
+        if res == -1:
+            raise wrap_oserror(space, OSError(rposix.get_saved_errno(),
+                                              "_vmprof.enable"))
+
+    def write_header(self, fileno, period):
+        if period == -1:
+            period_usec = 1000000 / 100 #  100hz
+        else:
+            period_usec = period
+        b = StringBuilder()
+        write_long_to_string_builder(0, b)
+        write_long_to_string_builder(3, b)
+        write_long_to_string_builder(0, b)
+        write_long_to_string_builder(period_usec, b)
+        write_long_to_string_builder(0, b)
+        os.write(fileno, b.build())
+
+    def register_code(self, space, code):
+        if self.fileno == -1:
+            raise OperationError(space.w_RuntimeError,
+                                 space.wrap("vmprof not running"))
+        name = code._get_full_name()
+        b = StringBuilder()
+        b.append('\x02')
+        write_long_to_string_builder(code._unique_id, b)
+        write_long_to_string_builder(len(name), b)
+        b.append(name)
+        os.write(self.fileno, b.build())
+
+    def disable(self, space):
+        if not self.is_enabled:
+            raise oefmt(space.w_ValueError, "_vmprof not enabled")
+        self.is_enabled = False
+        self.fileno = -1
+        if we_are_translated():
+           # does not work untranslated
+            res = vmprof_disable()
+        else:
+            res = 0
+        space.set_code_callback(None)
+        if res == -1:
+            raise wrap_oserror(space, OSError(rposix.get_saved_errno(),
+                                              "_vmprof.disable"))
+
+def vmprof_register_code(space, code):
+    from pypy.module._vmprof import Module
+    mod_vmprof = space.getbuiltinmodule('_vmprof')
+    assert isinstance(mod_vmprof, Module)
+    mod_vmprof.vmprof.register_code(space, code)
+        
+ at unwrap_spec(fileno=int, period=int)
+def enable(space, fileno, period=-1):
+    from pypy.module._vmprof import Module
+    mod_vmprof = space.getbuiltinmodule('_vmprof')
+    assert isinstance(mod_vmprof, Module)
+    mod_vmprof.vmprof.enable(space, fileno, period)
+
+def disable(space):
+    from pypy.module._vmprof import Module
+    mod_vmprof = space.getbuiltinmodule('_vmprof')
+    assert isinstance(mod_vmprof, Module)
+    mod_vmprof.vmprof.disable(space)
+
diff --git a/pypy/module/_vmprof/src/config.h b/pypy/module/_vmprof/src/config.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/src/config.h
@@ -0,0 +1,2 @@
+#define HAVE_SYS_UCONTEXT_H
+#define PC_FROM_UCONTEXT uc_mcontext.gregs[REG_RIP]
diff --git a/pypy/module/_vmprof/src/fake_pypy_api.c b/pypy/module/_vmprof/src/fake_pypy_api.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/src/fake_pypy_api.c
@@ -0,0 +1,29 @@
+
+long pypy_jit_start_addr(void)
+{
+	return 3;
+}
+
+long pypy_jit_end_addr(void)
+{
+	return 3;
+}
+
+long pypy_jit_stack_depth_at_loc(long x)
+{
+	return 0;
+}
+
+long pypy_find_codemap_at_addr(long x)
+{
+	return 0;
+}
+
+long pypy_yield_codemap_at_addr(long x, long y, long *a)
+{
+	return 0;
+}
+
+void pypy_pyframe_execute_frame(void)
+{
+}
diff --git a/pypy/module/_vmprof/src/get_custom_offset.c b/pypy/module/_vmprof/src/get_custom_offset.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/src/get_custom_offset.c
@@ -0,0 +1,46 @@
+
+long pypy_jit_start_addr();
+long pypy_jit_end_addr();
+long pypy_jit_stack_depth_at_loc(long);
+long pypy_find_codemap_at_addr(long);
+long pypy_yield_codemap_at_addr(long, long, long*);
+
+void vmprof_set_tramp_range(void* start, void* end)
+{
+}
+
+static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, unw_cursor_t *cp) {
+	intptr_t ip_l = (intptr_t)ip;
+
+	if (ip_l < pypy_jit_start_addr() || ip_l > pypy_jit_end_addr()) {
+		return -1;
+	}
+	return (void*)pypy_jit_stack_depth_at_loc(ip_l);
+}
+
+static long vmprof_write_header_for_jit_addr(void **result, long n,
+											 void *ip, int max_depth)
+{
+	long codemap_pos;
+	long current_pos = 0;
+	intptr_t id;
+	intptr_t addr = (intptr_t)ip;
+
+	if (addr < pypy_jit_start_addr() || addr > pypy_jit_end_addr()) {
+		return n;
+	}
+	codemap_pos = pypy_find_codemap_at_addr(addr);
+	if (codemap_pos == -1) {
+		return n;
+	}
+	while (1) {
+		id = pypy_yield_codemap_at_addr(codemap_pos, addr, ¤t_pos);
+		if (id == 0) {
+			return n;
+		}
+		result[n++] = id;
+		if (n >= max_depth) {
+			return n;
+		}
+	}
+}
diff --git a/pypy/module/_vmprof/src/getpc.h b/pypy/module/_vmprof/src/getpc.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/src/getpc.h
@@ -0,0 +1,187 @@
+// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*-
+// Copyright (c) 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// ---
+// Author: Craig Silverstein
+//
+// This is an internal header file used by profiler.cc.  It defines
+// the single (inline) function GetPC.  GetPC is used in a signal
+// handler to figure out the instruction that was being executed when
+// the signal-handler was triggered.
+//
+// To get this, we use the ucontext_t argument to the signal-handler
+// callback, which holds the full context of what was going on when
+// the signal triggered.  How to get from a ucontext_t to a Program
+// Counter is OS-dependent.
+
+#ifndef BASE_GETPC_H_
+#define BASE_GETPC_H_
+
+#include "config.h"
+
+// On many linux systems, we may need _GNU_SOURCE to get access to
+// the defined constants that define the register we want to see (eg
+// REG_EIP).  Note this #define must come first!
+#define _GNU_SOURCE 1
+// If #define _GNU_SOURCE causes problems, this might work instead.
+// It will cause problems for FreeBSD though!, because it turns off
+// the needed __BSD_VISIBLE.
+//#define _XOPEN_SOURCE 500
+
+#include          // for memcmp
+#if defined(HAVE_SYS_UCONTEXT_H)
+#include 
+#elif defined(HAVE_UCONTEXT_H)
+#include        // for ucontext_t (and also mcontext_t)
+#elif defined(HAVE_CYGWIN_SIGNAL_H)
+#include 
+typedef ucontext ucontext_t;
+#endif
+
+
+// Take the example where function Foo() calls function Bar().  For
+// many architectures, Bar() is responsible for setting up and tearing
+// down its own stack frame.  In that case, it's possible for the
+// interrupt to happen when execution is in Bar(), but the stack frame
+// is not properly set up (either before it's done being set up, or
+// after it's been torn down but before Bar() returns).  In those
+// cases, the stack trace cannot see the caller function anymore.
+//
+// GetPC can try to identify this situation, on architectures where it
+// might occur, and unwind the current function call in that case to
+// avoid false edges in the profile graph (that is, edges that appear
+// to show a call skipping over a function).  To do this, we hard-code
+// in the asm instructions we might see when setting up or tearing
+// down a stack frame.
+//
+// This is difficult to get right: the instructions depend on the
+// processor, the compiler ABI, and even the optimization level.  This
+// is a best effort patch -- if we fail to detect such a situation, or
+// mess up the PC, nothing happens; the returned PC is not used for
+// any further processing.
+struct CallUnrollInfo {
+  // Offset from (e)ip register where this instruction sequence
+  // should be matched. Interpreted as bytes. Offset 0 is the next
+  // instruction to execute. Be extra careful with negative offsets in
+  // architectures of variable instruction length (like x86) - it is
+  // not that easy as taking an offset to step one instruction back!
+  int pc_offset;
+  // The actual instruction bytes. Feel free to make it larger if you
+  // need a longer sequence.
+  unsigned char ins[16];
+  // How many bytes to match from ins array?
+  int ins_size;
+  // The offset from the stack pointer (e)sp where to look for the
+  // call return address. Interpreted as bytes.
+  int return_sp_offset;
+};
+
+
+// The dereferences needed to get the PC from a struct ucontext were
+// determined at configure time, and stored in the macro
+// PC_FROM_UCONTEXT in config.h.  The only thing we need to do here,
+// then, is to do the magic call-unrolling for systems that support it.
+
+// -- Special case 1: linux x86, for which we have CallUnrollInfo
+#if defined(__linux) && defined(__i386) && defined(__GNUC__)
+static const CallUnrollInfo callunrollinfo[] = {
+  // Entry to a function:  push %ebp;  mov  %esp,%ebp
+  // Top-of-stack contains the caller IP.
+  { 0,
+    {0x55, 0x89, 0xe5}, 3,
+    0
+  },
+  // Entry to a function, second instruction:  push %ebp;  mov  %esp,%ebp
+  // Top-of-stack contains the old frame, caller IP is +4.
+  { -1,
+    {0x55, 0x89, 0xe5}, 3,
+    4
+  },
+  // Return from a function: RET.
+  // Top-of-stack contains the caller IP.
+  { 0,
+    {0xc3}, 1,
+    0
+  }
+};
+
+inline void* GetPC(ucontext_t *signal_ucontext) {
+  // See comment above struct CallUnrollInfo.  Only try instruction
+  // flow matching if both eip and esp looks reasonable.
+  const int eip = signal_ucontext->uc_mcontext.gregs[REG_EIP];
+  const int esp = signal_ucontext->uc_mcontext.gregs[REG_ESP];
+  if ((eip & 0xffff0000) != 0 && (~eip & 0xffff0000) != 0 &&
+      (esp & 0xffff0000) != 0) {
+    char* eip_char = reinterpret_cast(eip);
+    for (int i = 0; i < sizeof(callunrollinfo)/sizeof(*callunrollinfo); ++i) {
+      if (!memcmp(eip_char + callunrollinfo[i].pc_offset,
+                  callunrollinfo[i].ins, callunrollinfo[i].ins_size)) {
+        // We have a match.
+        void **retaddr = (void**)(esp + callunrollinfo[i].return_sp_offset);
+        return *retaddr;
+      }
+    }
+  }
+  return (void*)eip;
+}
+
+// Special case #2: Windows, which has to do something totally different.
+#elif defined(_WIN32) || defined(__CYGWIN__) || defined(__CYGWIN32__) || defined(__MINGW32__)
+// If this is ever implemented, probably the way to do it is to have
+// profiler.cc use a high-precision timer via timeSetEvent:
+//    http://msdn2.microsoft.com/en-us/library/ms712713.aspx
+// We'd use it in mode TIME_CALLBACK_FUNCTION/TIME_PERIODIC.
+// The callback function would be something like prof_handler, but
+// alas the arguments are different: no ucontext_t!  I don't know
+// how we'd get the PC (using StackWalk64?)
+//    http://msdn2.microsoft.com/en-us/library/ms680650.aspx
+
+#include "base/logging.h"   // for RAW_LOG
+#ifndef HAVE_CYGWIN_SIGNAL_H
+typedef int ucontext_t;
+#endif
+
+inline void* GetPC(ucontext_t *signal_ucontext) {
+  RAW_LOG(ERROR, "GetPC is not yet implemented on Windows\n");
+  return NULL;
+}
+
+// Normal cases.  If this doesn't compile, it's probably because
+// PC_FROM_UCONTEXT is the empty string.  You need to figure out
+// the right value for your system, and add it to the list in
+// configure.ac (or set it manually in your config.h).
+#else
+inline void* GetPC(ucontext_t *signal_ucontext) {
+  return (void*)signal_ucontext->PC_FROM_UCONTEXT;   // defined in config.h
+}
+
+#endif
+
+#endif  // BASE_GETPC_H_
diff --git a/pypy/module/_vmprof/src/trampoline.asmgcc.s b/pypy/module/_vmprof/src/trampoline.asmgcc.s
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/src/trampoline.asmgcc.s
@@ -0,0 +1,17 @@
+// NOTE: you need to use TABs, not spaces!
+        
+	.text
+	.p2align 4,,-1
+	.globl	pypy_execute_frame_trampoline
+	.type	pypy_execute_frame_trampoline, @function
+pypy_execute_frame_trampoline:
+	.cfi_startproc
+	pushq	%rdi
+	.cfi_def_cfa_offset 16
+	call pypy_pyframe_execute_frame at PLT
+	/* GCROOT 0(%rsp) */
+	popq	%rdi
+	.cfi_def_cfa_offset 8
+	ret
+	.cfi_endproc
+	.size	pypy_execute_frame_trampoline, .-pypy_execute_frame_trampoline
diff --git a/pypy/module/_vmprof/src/trampoline.h b/pypy/module/_vmprof/src/trampoline.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/src/trampoline.h
@@ -0,0 +1,1 @@
+void* pypy_execute_frame_trampoline(void*, void*, void*);
diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/src/vmprof.c
@@ -0,0 +1,340 @@
+/* VMPROF
+ *
+ * statistical sampling profiler specifically designed to profile programs
+ * which run on a Virtual Machine and/or bytecode interpreter, such as Python,
+ * etc.
+ *
+ * The logic to dump the C stack traces is partly stolen from the code in gperftools.
+ * The file "getpc.h" has been entirely copied from gperftools.
+ *
+ * Tested only on gcc, linux, x86_64.
+ *
+ * Copyright (C) 2014-2015
+ *   Antonio Cuni - anto.cuni at gmail.com
+ *   Maciej Fijalkowski - fijall at gmail.com
+ *
+ */
+
+
+#include "getpc.h"      // should be first to get the _GNU_SOURCE dfn
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#define UNW_LOCAL_ONLY
+#include 
+
+#include "vmprof.h"
+
+#define _unused(x) ((void)x)
+
+#define MAX_FUNC_NAME 128
+#define MAX_STACK_DEPTH 64
+
+static FILE* profile_file = NULL;
+void* vmprof_mainloop_func;
+static ptrdiff_t mainloop_sp_offset;
+static vmprof_get_virtual_ip_t mainloop_get_virtual_ip;
+
+
+/* *************************************************************
+ * functions to write a profile file compatible with gperftools
+ * *************************************************************
+ */
+
+#define MARKER_STACKTRACE '\x01'
+#define MARKER_VIRTUAL_IP '\x02'
+#define MARKER_TRAILER '\x03'
+
+static void prof_word(FILE* f, long x) {
+    fwrite(&x, sizeof(x), 1, f);
+}
+
+static void prof_header(FILE* f, long period_usec) {
+    prof_word(f, 0);
+    prof_word(f, 3);
+    prof_word(f, 0);
+    prof_word(f, period_usec);
+    prof_word(f, 0);
+}
+
+static void prof_write_stacktrace(FILE* f, void** stack, int depth, int count) {
+    int i;
+	char marker = MARKER_STACKTRACE;
+
+	fwrite(&marker, 1, 1, f);
+    prof_word(f, count);
+    prof_word(f, depth);
+    for(i=0; isp = bp;
+		bp -= sizeof(void*);
+        cp2->ip = ((void**)bp)[0];
+        // the ret is on the top of the stack minus WORD
+        return 1;
+    }
+}
+
+
+/* *************************************************************
+ * functions to dump the stack trace
+ * *************************************************************
+ */
+
+// stolen from pprof:
+// Sometimes, we can try to get a stack trace from within a stack
+// trace, because libunwind can call mmap (maybe indirectly via an
+// internal mmap based memory allocator), and that mmap gets trapped
+// and causes a stack-trace request.  If were to try to honor that
+// recursive request, we'd end up with infinite recursion or deadlock.
+// Luckily, it's safe to ignore those subsequent traces.  In such
+// cases, we return 0 to indicate the situation.
+//static __thread int recursive;
+static int recursive; // XXX antocuni: removed __thread
+
+int get_stack_trace(void** result, int max_depth, ucontext_t *ucontext) {
+    void *ip;
+    int n = 0;
+    unw_cursor_t cursor;
+    unw_context_t uc = *ucontext;
+    if (recursive) {
+        return 0;
+    }
+    ++recursive;
+
+    int ret = unw_init_local(&cursor, &uc);
+    assert(ret >= 0);
+    _unused(ret);
+
+    while (n < max_depth) {
+        if (unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *) &ip) < 0) {
+            break;
+        }
+
+        unw_proc_info_t pip;
+        unw_get_proc_info(&cursor, &pip);
+
+        /* char funcname[4096]; */
+        /* unw_word_t offset; */
+        /* unw_get_proc_name(&cursor, funcname, 4096, &offset); */
+        /* printf("%s+%#lx <%p>\n", funcname, offset, ip); */
+
+        /* if n==0, it means that the signal handler interrupted us while we
+           were in the trampoline, so we are not executing (yet) the real main
+           loop function; just skip it */
+        if (vmprof_mainloop_func && 
+            (void*)pip.start_ip == (void*)vmprof_mainloop_func &&
+            n > 0) {
+          // found main loop stack frame
+          void* sp;
+          unw_get_reg(&cursor, UNW_REG_SP, (unw_word_t *) &sp);
+          void *arg_addr = (char*)sp + mainloop_sp_offset;
+          void **arg_ptr = (void**)arg_addr;
+          // fprintf(stderr, "stacktrace mainloop: rsp %p   &f2 %p   offset %ld\n", 
+          //         sp, arg_addr, mainloop_sp_offset);
+          ip = mainloop_get_virtual_ip(*arg_ptr);
+        }
+
+        result[n++] = ip;
+		n = vmprof_write_header_for_jit_addr(result, n, ip, max_depth);
+        if (vmprof_unw_step(&cursor) <= 0) {
+            break;
+        }
+    }
+    --recursive;
+    return n;
+}
+
+
+static int __attribute__((noinline)) frame_forcer(int rv) {
+    return rv;
+}
+
+static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext) {
+    void* stack[MAX_STACK_DEPTH];
+    stack[0] = GetPC((ucontext_t*)ucontext);
+    int depth = frame_forcer(get_stack_trace(stack+1, MAX_STACK_DEPTH-1, ucontext));
+    depth++;  // To account for pc value in stack[0];
+    prof_write_stacktrace(profile_file, stack, depth, 1);
+}
+
+/* *************************************************************
+ * functions to enable/disable the profiler
+ * *************************************************************
+ */
+
+static int open_profile(int fd, long period_usec, int write_header, char *s,
+						int slen) {
+	if ((fd = dup(fd)) == -1) {
+		return -1;
+	}
+    profile_file = fdopen(fd, "wb");
+	if (!profile_file) {
+		return -1;
+	}
+	if (write_header)
+		prof_header(profile_file, period_usec);
+	if (s)
+		fwrite(s, slen, 1, profile_file);
+	return 0;
+}
+
+static int close_profile(void) {
+	// XXX all of this can happily fail
+    FILE* src;
+    char buf[BUFSIZ];
+    size_t size;
+	int marker = MARKER_TRAILER;
+	fwrite(&marker, 1, 1, profile_file);
+
+    // copy /proc/PID/maps to the end of the profile file
+    sprintf(buf, "/proc/%d/maps", getpid());
+    src = fopen(buf, "r");    
+    while ((size = fread(buf, 1, BUFSIZ, src))) {
+        fwrite(buf, 1, size, profile_file);
+    }
+    fclose(src);
+    fclose(profile_file);
+	return 0;
+}
+
+
+static int install_sigprof_handler(void) {
+    struct sigaction sa;
+    memset(&sa, 0, sizeof(sa));
+    sa.sa_sigaction = sigprof_handler;
+    sa.sa_flags = SA_RESTART | SA_SIGINFO;
+    if (sigemptyset(&sa.sa_mask) == -1 ||
+		sigaction(SIGPROF, &sa, NULL) == -1) {
+		return -1;
+	}
+	return 0;
+}
+
+static int remove_sigprof_handler(void) {
+    //sighandler_t res = signal(SIGPROF, SIG_DFL);
+	//if (res == SIG_ERR) {
+	//	return -1;
+	//}
+	return 0;
+};
+
+static int install_sigprof_timer(long period_usec) {
+    static struct itimerval timer;
+    timer.it_interval.tv_sec = 0;
+    timer.it_interval.tv_usec = period_usec;
+    timer.it_value = timer.it_interval;
+    if (setitimer(ITIMER_PROF, &timer, NULL) != 0) {
+		return -1;
+    }
+	return 0;
+}
+
+static int remove_sigprof_timer(void) {
+    static struct itimerval timer;
+    timer.it_interval.tv_sec = 0;
+    timer.it_interval.tv_usec = 0;
+    timer.it_value = timer.it_interval;
+    if (setitimer(ITIMER_PROF, &timer, NULL) != 0) {
+		return -1;
+    }
+	return 0;
+}
+
+/* *************************************************************
+ * public API
+ * *************************************************************
+ */
+
+void vmprof_set_mainloop(void* func, ptrdiff_t sp_offset, 
+                         vmprof_get_virtual_ip_t get_virtual_ip) {
+    vmprof_mainloop_func = func;
+    mainloop_sp_offset = sp_offset;
+    mainloop_get_virtual_ip = get_virtual_ip;
+}
+
+int vmprof_enable(int fd, long period_usec, int write_header, char *s,
+				  int slen)
+{
+    if (period_usec == -1)
+        period_usec = 1000000 / 100; /* 100hz */
+    if (open_profile(fd, period_usec, write_header, s, slen) == -1) {
+		return -1;
+	}
+    if (install_sigprof_handler() == -1) {
+		return -1;
+	}
+    if (install_sigprof_timer(period_usec) == -1) {
+		return -1;
+	}
+	return 0;
+}
+
+int vmprof_disable(void) {
+    if (remove_sigprof_timer() == -1) {
+		return -1;
+	}
+    if (remove_sigprof_handler() == -1) {
+		return -1;
+	}
+    if (close_profile() == -1) {
+		return -1;
+	}
+	return 0;
+}
+
+void vmprof_register_virtual_function(const char* name, void* start, void* end) {
+    // for now *end is simply ignored
+	char buf[1024];
+	int lgt = strlen(name) + 2 * sizeof(long) + 1;
+
+	if (lgt > 1024) {
+		lgt = 1024;
+	}
+	buf[0] = MARKER_VIRTUAL_IP;
+	((void **)(((void*)buf) + 1))[0] = start;
+	((long *)(((void*)buf) + 1 + sizeof(long)))[0] = lgt - 2 * sizeof(long) - 1;
+	strncpy(buf + 2 * sizeof(long) + 1, name, 1024 - 2 * sizeof(long) - 1);
+	fwrite(buf, lgt, 1, profile_file);
+}
diff --git a/pypy/module/_vmprof/src/vmprof.h b/pypy/module/_vmprof/src/vmprof.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/src/vmprof.h
@@ -0,0 +1,22 @@
+#ifndef VMPROF_VMPROF_H_
+#define VMPROF_VMPROF_H_
+
+#include 
+
+typedef void* (*vmprof_get_virtual_ip_t)(void*);
+
+extern void* vmprof_mainloop_func;
+void vmprof_set_mainloop(void* func, ptrdiff_t sp_offset, 
+                         vmprof_get_virtual_ip_t get_virtual_ip);
+
+void vmprof_register_virtual_function(const char* name, void* start, void* end);
+
+
+int vmprof_enable(int fd, long period_usec, int write_header, char* vips,
+				  int vips_len);
+int vmprof_disable(void);
+
+// XXX: this should be part of _vmprof (the CPython extension), not vmprof (the library)
+void vmprof_set_tramp_range(void* start, void* end);
+
+#endif
diff --git a/pypy/module/_vmprof/test/__init__.py b/pypy/module/_vmprof/test/__init__.py
new file mode 100644
diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -0,0 +1,55 @@
+
+import tempfile
+from pypy.tool.pytest.objspace import gettestobjspace
+
+class AppTestVMProf(object):
+    def setup_class(cls):
+        cls.space = gettestobjspace(usemodules=['_vmprof', 'struct'])
+        cls.tmpfile = tempfile.NamedTemporaryFile()
+        cls.w_tmpfileno = cls.space.wrap(cls.tmpfile.fileno())
+        cls.w_tmpfilename = cls.space.wrap(cls.tmpfile.name)
+        cls.tmpfile2 = tempfile.NamedTemporaryFile()
+        cls.w_tmpfileno2 = cls.space.wrap(cls.tmpfile2.fileno())
+        cls.w_tmpfilename2 = cls.space.wrap(cls.tmpfile2.name)
+
+    def test_import_vmprof(self):
+        import struct, sys
+
+        WORD = struct.calcsize('l')
+        
+        def count(s):
+            i = 0
+            count = 0
+            i += 5 * WORD # header
+            while i < len(s):
+                assert s[i] == '\x02'
+                i += 1
+                _, size = struct.unpack("ll", s[i:i + 2 * WORD])
+                count += 1
+                i += 2 * WORD + size
+            return count
+        
+        import _vmprof
+        _vmprof.enable(self.tmpfileno)
+        _vmprof.disable()
+        s = open(self.tmpfilename).read()
+        no_of_codes = count(s)
+        assert no_of_codes > 10
+        d = {}
+
+        exec """def foo():
+            pass
+        """ in d
+
+        _vmprof.enable(self.tmpfileno2)
+
+        exec """def foo2():
+            pass
+        """ in d
+
+        _vmprof.disable()
+        s = open(self.tmpfilename2).read()
+        no_of_codes2 = count(s)
+        assert "py:foo:" in s
+        assert "py:foo2:" in s
+        assert no_of_codes2 >= no_of_codes + 2 # some extra codes from tests
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -35,6 +35,9 @@
     name = opcode_method_names[ord(bytecode.co_code[next_instr])]
     return '%s #%d %s' % (bytecode.get_repr(), next_instr, name)
 
+def get_unique_id(next_instr, is_being_profiled, bytecode):
+    return bytecode._unique_id
+
 
 def should_unroll_one_iteration(next_instr, is_being_profiled, bytecode):
     return (bytecode.co_flags & CO_GENERATOR) != 0
@@ -45,6 +48,7 @@
     virtualizables = ['frame']
 
 pypyjitdriver = PyPyJitDriver(get_printable_location = get_printable_location,
+                              get_unique_id = get_unique_id,
                               should_unroll_one_iteration =
                               should_unroll_one_iteration,
                               name='pypyjit')
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -57,6 +57,7 @@
         BaseAssembler.setup_once(self)
 
     def setup(self, looptoken):
+        BaseAssembler.setup(self, looptoken)
         assert self.memcpy_addr != 0, 'setup_once() not called?'
         if we_are_translated():
             self.debug = False
@@ -71,7 +72,6 @@
         self.mc.datablockwrapper = self.datablockwrapper
         self.target_tokens_currently_compiling = {}
         self.frame_depth_to_patch = []
-        self._finish_gcmap = lltype.nullptr(jitframe.GCMAP)
 
     def teardown(self):
         self.current_clt = None
@@ -663,6 +663,7 @@
             assert len(set(inputargs)) == len(inputargs)
 
         self.setup(original_loop_token)
+        self.codemap.inherit_code_from_position(faildescr.adr_jump_offset)
         descr_number = compute_unique_id(faildescr)
         if log:
             operations = self._inject_debugging_code(faildescr, operations,
@@ -883,8 +884,12 @@
         self.datablockwrapper.done()      # finish using cpu.asmmemmgr
         self.datablockwrapper = None
         allblocks = self.get_asmmemmgr_blocks(looptoken)
-        return self.mc.materialize(self.cpu.asmmemmgr, allblocks,
+        size = self.mc.get_relative_pos() 
+        res = self.mc.materialize(self.cpu.asmmemmgr, allblocks,
                                    self.cpu.gc_ll_descr.gcrootmap)
+        self.cpu.asmmemmgr.register_codemap(
+            self.codemap.get_final_bytecode(res, size))
+        return res
 
     def update_frame_depth(self, frame_depth):
         baseofs = self.cpu.get_baseofs_of_frame_field()
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -392,6 +392,9 @@
         else:
             self.rm._sync_var(v)
 
+    def prepare_op_debug_merge_point(self, op, fcond):
+        self.assembler.codemap.debug_merge_point(op)
+
     def _prepare_op_int_add(self, op, fcond):
         boxes = op.getarglist()
         a0, a1 = boxes
diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py
--- a/rpython/jit/backend/llsupport/asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/asmmemmgr.py
@@ -4,7 +4,10 @@
 from rpython.rlib import rmmap
 from rpython.rlib.debug import debug_start, debug_print, debug_stop
 from rpython.rlib.debug import have_debug_prints
-from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.rbisect import bisect, bisect_tuple
+
+_memmngr = None # global reference so we can use @entrypoint :/
 
 
 class AsmMemoryManager(object):
@@ -24,6 +27,12 @@
         self.free_blocks = {}      # map {start: stop}
         self.free_blocks_end = {}  # map {stop: start}
         self.blocks_by_size = [[] for i in range(self.num_indices)]
+        # two lists of jit addresses (sorted) and the corresponding stack
+        # depths
+        self.jit_addr_map = []
+        self.jit_frame_depth_map = []
+        self.jit_codemap = []
+        # see codemap.py
 
     def malloc(self, minsize, maxsize):
         """Allocate executable memory, between minsize and maxsize bytes,
@@ -45,6 +54,13 @@
         if r_uint is not None:
             self.total_mallocs -= r_uint(stop - start)
         self._add_free_block(start, stop)
+        # fix up jit_addr_map
+        jit_adr_start = bisect(self.jit_addr_map, start)
+        jit_adr_stop = bisect(self.jit_addr_map, stop)
+        self.jit_addr_map = (self.jit_addr_map[:jit_adr_start] +
+                             self.jit_addr_map[jit_adr_stop:])
+        self.jit_frame_depth_map = (self.jit_frame_depth_map[:jit_adr_start] +
+                                    self.jit_frame_depth_map[jit_adr_stop:])
 
     def open_malloc(self, minsize):
         """Allocate at least minsize bytes.  Returns (start, stop)."""
@@ -151,6 +167,35 @@
         del self.free_blocks_end[stop]
         return (start, stop)
 
+    def register_frame_depth_map(self, rawstart, frame_positions,
+                                 frame_assignments):
+        if not frame_positions:
+            return
+        if not self.jit_addr_map or rawstart > self.jit_addr_map[-1]:
+            start = len(self.jit_addr_map)
+            self.jit_addr_map += [0] * len(frame_positions)
+            self.jit_frame_depth_map += [0] * len(frame_positions)
+        else:
+            start = bisect(self.jit_addr_map, rawstart)
+            self.jit_addr_map = (self.jit_addr_map[:start] +
+                                 [0] * len(frame_positions) +
+                                 self.jit_addr_map[start:])
+            self.jit_frame_depth_map = (self.jit_frame_depth_map[:start] +
+                                 [0] * len(frame_positions) +
+                                 self.jit_frame_depth_map[start:])
+        for i, pos in enumerate(frame_positions):
+            self.jit_addr_map[i + start] = pos + rawstart
+            self.jit_frame_depth_map[i + start] = frame_assignments[i]
+
+    def register_codemap(self, codemap):
+        start = codemap[0]
+        pos = bisect_tuple(self.jit_codemap, start)
+        if pos == len(self.jit_codemap): # common case
+            self.jit_codemap.append(codemap)
+        else:
+            self.jit_codemap = (self.jit_codemap[:pos] + [codemap] +
+                                self.jit_codemap[pos:])
+
     def _delete(self):
         "NOT_RPYTHON"
         if self._allocated:
@@ -210,6 +255,9 @@
 
     gcroot_markers = None
 
+    frame_positions = None
+    frame_assignments = None
+
     def __init__(self, translated=None):
         if translated is None:
             translated = we_are_translated()
@@ -311,6 +359,10 @@
             assert gcrootmap is not None
             for pos, mark in self.gcroot_markers:
                 gcrootmap.register_asm_addr(rawstart + pos, mark)
+        asmmemmgr.register_frame_depth_map(rawstart, self.frame_positions,
+                                           self.frame_assignments)
+        self.frame_positions = None
+        self.frame_assignments = None
         return rawstart
 
     def _become_a_plain_block_builder(self):
diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -1,6 +1,7 @@
 from rpython.jit.backend.llsupport import jitframe
 from rpython.jit.backend.llsupport.memcpy import memcpy_fn, memset_fn
 from rpython.jit.backend.llsupport.symbolic import WORD
+from rpython.jit.backend.llsupport.codemap import CodemapBuilder
 from rpython.jit.metainterp.history import (INT, REF, FLOAT, JitCellToken,
     ConstInt, BoxInt, AbstractFailDescr)
 from rpython.jit.metainterp.resoperation import ResOperation, rop
@@ -128,6 +129,10 @@
                                               track_allocation=False)
         self.gcmap_for_finish[0] = r_uint(1)
 
+    def setup(self, looptoken):
+        self.codemap = CodemapBuilder()
+        self._finish_gcmap = lltype.nullptr(jitframe.GCMAP)
+
     def set_debug(self, v):
         r = self._debug
         self._debug = v
@@ -194,6 +199,9 @@
         guardtok.faildescr.rd_locs = positions
         return fail_descr, target
 
+    def debug_merge_point(self, op):
+        self.codemap.debug_merge_point(op, self.mc.get_relative_pos())
+
     def call_assembler(self, op, guard_op, argloc, vloc, result_loc, tmploc):
         self._store_force_index(guard_op)
         descr = op.getdescr()
@@ -276,6 +284,9 @@
         # YYY very minor leak -- we need the counters to stay alive
         # forever, just because we want to report them at the end
         # of the process
+
+        # XXX the numbers here are ALMOST unique, but not quite, use a counter
+        #     or something
         struct = lltype.malloc(DEBUG_COUNTER, flavor='raw',
                                track_allocation=False)
         struct.i = 0
diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/llsupport/codemap.py
@@ -0,0 +1,142 @@
+
+""" Bytecode for storage in asmmemmgr.jit_codemap. Format is as follows:
+
+ list of tuples of shape (addr, machine code size, bytecode info)
+ where bytecode info is a string made up of:
+    8 bytes unique_id, 4 bytes start_addr (relative), 4 bytes size (relative),
+    2 bytes how many items to skip to go to the next on similar level
+    [so far represented by a list of integers for simplicity]
+
+"""
+
+from rpython.rlib import rgc
+from rpython.rlib.entrypoint import jit_entrypoint
+from rpython.jit.backend.llsupport import asmmemmgr
+from rpython.rlib.rbisect import bisect, bisect_tuple
+from rpython.rtyper.lltypesystem import lltype, rffi
+
+ at jit_entrypoint([lltype.Signed], lltype.Signed,
+                c_name='pypy_jit_stack_depth_at_loc')
+ at rgc.no_collect
+def stack_depth_at_loc(loc):
+    _memmngr = asmmemmgr._memmngr
+
+    pos = bisect(_memmngr.jit_addr_map, loc)
+    if pos == 0 or pos == len(_memmngr.jit_addr_map):
+        return -1
+    return _memmngr.jit_frame_depth_map[pos-1]
+
+ at jit_entrypoint([], lltype.Signed, c_name='pypy_jit_start_addr')
+def jit_start_addr():
+    _memmngr = asmmemmgr._memmngr
+
+    return _memmngr.jit_addr_map[0]
+
+ at jit_entrypoint([], lltype.Signed, c_name='pypy_jit_end_addr')
+def jit_end_addr():
+    _memmngr = asmmemmgr._memmngr
+
+    return _memmngr.jit_addr_map[-1]
+
+ at jit_entrypoint([lltype.Signed], lltype.Signed,
+                c_name='pypy_find_codemap_at_addr')
+def find_codemap_at_addr(addr):
+    _memmngr = asmmemmgr._memmngr
+
+    res = bisect_tuple(_memmngr.jit_codemap, addr) - 1
+    if res == len(_memmngr.jit_codemap):
+        return -1
+    return res
+
+ at jit_entrypoint([lltype.Signed, lltype.Signed,
+                 rffi.CArrayPtr(lltype.Signed)], lltype.Signed,
+                 c_name='pypy_yield_codemap_at_addr')
+def yield_bytecode_at_addr(codemap_no, addr, current_pos_addr):
+    """ will return consecutive unique_ids from codemap, starting from position
+    `pos` until addr
+    """
+    _memmngr = asmmemmgr._memmngr
+
+    codemap = _memmngr.jit_codemap[codemap_no]
+    current_pos = current_pos_addr[0]
+    start_addr = codemap[0]
+    rel_addr = addr - start_addr
+    while True:
+        if current_pos >= len(codemap[2]):
+            return 0
+        next_start = codemap[2][current_pos + 1]
+        if next_start > rel_addr:
+            return 0
+        next_stop = codemap[2][current_pos + 2]
+        if next_stop > rel_addr:
+            current_pos_addr[0] = current_pos + 4
+            return codemap[2][current_pos]
+        # we need to skip potentially more than one
+        current_pos = codemap[2][current_pos + 3]
+
+def unpack_traceback(addr):
+    codemap_pos = find_codemap_at_addr(addr)
+    assert codemap_pos >= 0
+    storage = lltype.malloc(rffi.CArray(lltype.Signed), 1, flavor='raw')
+    storage[0] = 0
+    res = []
+    while True:
+        item = yield_bytecode_at_addr(codemap_pos, addr, storage)
+        if item == 0:
+            break
+        res.append(item)
+    lltype.free(storage, flavor='raw')
+    return res
+
+
+class CodemapBuilder(object):
+    def __init__(self):
+        self.l = []
+        self.patch_position = []
+        self.last_call_depth = -1
+
+    def debug_merge_point(self, op, pos):
+        call_depth = op.getarg(1).getint()
+        if call_depth != self.last_call_depth:
+            unique_id = op.getarg(3).getint()
+            if unique_id == 0: # uninteresting case
+                return
+            assert unique_id & 1 == 0
+            if call_depth > self.last_call_depth:
+                self.l.append(unique_id)
+                self.l.append(pos) # <- this is a relative pos
+                self.patch_position.append(len(self.l))
+                self.l.append(0) # marker
+                self.l.append(0) # second marker
+            else:
+                for i in range(self.last_call_depth - call_depth):
+                    to_patch = self.patch_position.pop()
+                    self.l[to_patch] = pos
+                    self.l[to_patch + 1] = len(self.l)
+            self.last_call_depth = call_depth
+
+    def inherit_code_from_position(self, pos):
+        lst = unpack_traceback(pos)
+        self.last_call_depth = len(lst) - 1
+        for item in lst:
+            self.l.append(item)
+            self.l.append(0)
+            self.patch_position.append(len(self.l))
+            self.l.append(0) # marker
+            self.l.append(0) # second marker
+
+    def get_final_bytecode(self, addr, size):
+        while self.patch_position:
+            pos = self.patch_position.pop()
+            self.l[pos] = size
+            self.l[pos + 1] = len(self.l)
+        # at the end there should be no zeros
+        for i in range(len(self.l) / 4):
+            item = self.l[i * 4] # unique_id
+            assert item > 0 # no zeros here
+            item = self.l[i * 4 + 2] # end in asm
+            assert item > 0
+            item = self.l[i * 4 + 3] # end in l
+            assert item > 0
+        return (addr, size, self.l) # XXX compact self.l
+
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -16,7 +16,7 @@
     FieldDescr, ArrayDescr, CallDescr, InteriorFieldDescr,
     FLAG_POINTER, FLAG_FLOAT)
 from rpython.jit.backend.llsupport.memcpy import memset_fn
-from rpython.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
+from rpython.jit.backend.llsupport import asmmemmgr
 from rpython.rlib.unroll import unrolling_iterable
 
 
@@ -48,7 +48,8 @@
             self._setup_exception_handling_translated()
         else:
             self._setup_exception_handling_untranslated()
-        self.asmmemmgr = AsmMemoryManager()
+        self.asmmemmgr = asmmemmgr.AsmMemoryManager()
+        asmmemmgr._memmngr = self.asmmemmgr
         self._setup_frame_realloc(translate_support_code)
         ad = self.gc_ll_descr.getframedescrs(self).arraydescr
         self.signedarraydescr = ad
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -17,8 +17,6 @@
 class GcRewriterAssembler(object):
     """ This class performs the following rewrites on the list of operations:
 
-     - Remove the DEBUG_MERGE_POINTs.
-
      - Turn all NEW_xxx to either a CALL_MALLOC_GC, or a CALL_MALLOC_NURSERY
        followed by SETFIELDs in order to initialize their GC fields.  The
        two advantages of CALL_MALLOC_NURSERY is that it inlines the common
@@ -60,8 +58,6 @@
         #
         for i in range(len(operations)):
             op = operations[i]
-            if op.getopnum() == rop.DEBUG_MERGE_POINT:
-                continue
             # ---------- turn NEWxxx into CALL_MALLOC_xxx ----------
             if op.is_malloc():
                 self.handle_malloc_operation(op)
diff --git a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -2,6 +2,7 @@
 from rpython.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
 from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
 from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin
+from rpython.jit.backend.llsupport import asmmemmgr
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import debug
 
@@ -157,6 +158,7 @@
         class FakeGcRootMap:
             def register_asm_addr(self, retaddr, mark):
                 puts.append((retaddr, mark))
+
         #
         mc = BlockBuilderMixin()
         mc.writechar('X')
@@ -262,3 +264,16 @@
     md.done()
     assert allblocks == [(1597, 1697), (1797, 1835)]
     assert ops == [('free', 1835, 1897)]
+
+def test_find_jit_frame_depth():
+    mgr = AsmMemoryManager()
+    mgr.register_frame_depth_map(11, [0, 5, 10], [1, 2, 3])
+    mgr.register_frame_depth_map(30, [0, 5, 10], [4, 5, 6])
+    mgr.register_frame_depth_map(0, [0, 5, 10], [7, 8, 9])
+    asmmemmgr._memmngr = mgr
+    assert asmmemmgr.stack_depth_at_loc(13) == 1
+    assert asmmemmgr.stack_depth_at_loc(-3) == -1
+    assert asmmemmgr.stack_depth_at_loc(41) == -1
+    assert asmmemmgr.stack_depth_at_loc(5) == 8
+    assert asmmemmgr.stack_depth_at_loc(17) == 2
+    assert asmmemmgr.stack_depth_at_loc(38) == 5
diff --git a/rpython/jit/backend/llsupport/test/ztranslation_test.py b/rpython/jit/backend/llsupport/test/ztranslation_test.py
--- a/rpython/jit/backend/llsupport/test/ztranslation_test.py
+++ b/rpython/jit/backend/llsupport/test/ztranslation_test.py
@@ -2,7 +2,7 @@
 from rpython.tool.udir import udir
 from rpython.rlib.jit import JitDriver, unroll_parameters, set_param
 from rpython.rlib.jit import PARAMETERS, dont_look_inside
-from rpython.rlib.jit import promote
+from rpython.rlib.jit import promote, _get_virtualizable_token
 from rpython.rlib import jit_hooks, rposix
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rthread import ThreadLocalReference, ThreadLocalField
@@ -28,12 +28,15 @@
         # - floats neg and abs
         # - llexternal with macro=True
 
-        class Frame(object):
+        class BasicFrame(object):
             _virtualizable_ = ['i']
 
             def __init__(self, i):
                 self.i = i
 
+        class Frame(BasicFrame):
+            pass
+
         eci = ExternalCompilationInfo(post_include_bits=['''
 #define pypy_my_fabs(x)  fabs(x)
 '''])
@@ -59,6 +62,7 @@
             while frame.i > 3:
                 jitdriver.can_enter_jit(frame=frame, total=total, j=j)
                 jitdriver.jit_merge_point(frame=frame, total=total, j=j)
+                _get_virtualizable_token(frame)
                 total += frame.i
                 if frame.i >= 20:
                     frame.i -= 2
diff --git a/rpython/jit/backend/x86/arch.py b/rpython/jit/backend/x86/arch.py
--- a/rpython/jit/backend/x86/arch.py
+++ b/rpython/jit/backend/x86/arch.py
@@ -47,3 +47,6 @@
     THREADLOCAL_OFS = (FRAME_FIXED_SIZE - 1) * WORD
 
 assert PASS_ON_MY_FRAME >= 12       # asmgcc needs at least JIT_USE_WORDS + 3
+
+# return address, followed by FRAME_FIXED_SIZE words
+DEFAULT_FRAME_BYTES = (1 + FRAME_FIXED_SIZE) * WORD
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -18,7 +18,8 @@
 from rpython.jit.backend.llsupport.regalloc import (get_scale, valid_addressing_size)
 from rpython.jit.backend.x86.arch import (FRAME_FIXED_SIZE, WORD, IS_X86_64,
                                        JITFRAME_FIXED_SIZE, IS_X86_32,
-                                       PASS_ON_MY_FRAME, THREADLOCAL_OFS)
+                                       PASS_ON_MY_FRAME, THREADLOCAL_OFS,
+                                       DEFAULT_FRAME_BYTES)
 from rpython.jit.backend.x86.regloc import (eax, ecx, edx, ebx, esp, ebp, esi,
     xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, r8, r9, r10, r11, edi,
     r12, r13, r14, r15, X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG,
@@ -65,6 +66,7 @@
             self._build_float_constants()
 
     def setup(self, looptoken):
+        BaseAssembler.setup(self, looptoken)
         assert self.memcpy_addr != 0, "setup_once() not called?"
         self.current_clt = looptoken.compiled_loop_token
         self.pending_guard_tokens = []
@@ -79,7 +81,6 @@
                                                         allblocks)
         self.target_tokens_currently_compiling = {}
         self.frame_depth_to_patch = []
-        self._finish_gcmap = lltype.nullptr(jitframe.GCMAP)
 
     def teardown(self):
         self.pending_guard_tokens = None
@@ -266,6 +267,10 @@
         # the correct "ret" arg
         offset = mc.get_relative_pos() - jz_location
         mc.overwrite32(jz_location-4, offset)
+        # From now on this function is basically "merged" with
+        # its caller and so contains DEFAULT_FRAME_BYTES bytes
+        # plus my own return address, which we'll ignore next
+        mc.force_frame_size(DEFAULT_FRAME_BYTES + WORD)
         mc.ADD_ri(esp.value, WORD)
         mc.JMP(imm(self.propagate_exception_path))
         #
@@ -277,6 +282,7 @@
             return      # not supported (for tests, or non-translated)
         #
         self.mc = codebuf.MachineCodeBlockWrapper()
+        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         #
         # read and reset the current exception
 
@@ -330,7 +336,10 @@
         offset = mc.get_relative_pos() - jnz_location
         assert 0 < offset <= 127
         mc.overwrite(jnz_location-1, chr(offset))
-        # adjust the esp to point back to the previous return
+        # From now on this function is basically "merged" with
+        # its caller and so contains DEFAULT_FRAME_BYTES bytes
+        # plus my own return address, which we'll ignore next
+        mc.force_frame_size(DEFAULT_FRAME_BYTES + WORD)
         mc.ADD_ri(esp.value, WORD)
         mc.JMP(imm(self.propagate_exception_path))
         #
@@ -408,6 +417,8 @@
                 mc.LEA_rs(esp.value, 2 * WORD)
             self._pop_all_regs_from_frame(mc, [], withfloats, callee_only=True)
             mc.RET16_i(WORD)
+            # Note that wb_slowpath[0..3] end with a RET16_i, which must be
+            # taken care of in the caller by stack_frame_size_delta(-WORD)
         else:
             if IS_X86_32:
                 mc.MOV_rs(edx.value, 4 * WORD)
@@ -513,6 +524,8 @@
             assert len(set(inputargs)) == len(inputargs)
 
         self.setup(original_loop_token)
+        self.codemap.inherit_code_from_position(faildescr.adr_jump_offset)
+        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         descr_number = compute_unique_id(faildescr)
         if log:
             operations = self._inject_debugging_code(faildescr, operations,
@@ -673,8 +686,12 @@
         self.datablockwrapper.done()      # finish using cpu.asmmemmgr
         self.datablockwrapper = None
         allblocks = self.get_asmmemmgr_blocks(looptoken)
-        return self.mc.materialize(self.cpu.asmmemmgr, allblocks,
-                                   self.cpu.gc_ll_descr.gcrootmap)
+        size = self.mc.get_relative_pos()
+        res = self.mc.materialize(self.cpu.asmmemmgr, allblocks,
+                                  self.cpu.gc_ll_descr.gcrootmap)
+        self.cpu.asmmemmgr.register_codemap(
+            self.codemap.get_final_bytecode(res, size))
+        return res
 
     def patch_jump_for_descr(self, faildescr, adr_new_target):
         adr_jump_offset = faildescr.adr_jump_offset
@@ -685,6 +702,7 @@
         # place, but clobber the recovery stub with a jump to the real
         # target.
         mc = codebuf.MachineCodeBlockWrapper()
+        mc.force_frame_size(DEFAULT_FRAME_BYTES)
         if rx86.fits_in_32bits(offset):
             mc.writeimm32(offset)
             mc.copy_to_raw_memory(adr_jump_offset)
@@ -1755,6 +1773,7 @@
 
     def generate_propagate_error_64(self):
         assert WORD == 8
+        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         startpos = self.mc.get_relative_pos()
         self.mc.JMP(imm(self.propagate_exception_path))
         return startpos
@@ -1762,6 +1781,7 @@
     def generate_quick_failure(self, guardtok):
         """ Gather information about failure
         """
+        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         startpos = self.mc.get_relative_pos()
         fail_descr, target = self.store_info_on_descr(startpos, guardtok)
         self.mc.PUSH(imm(fail_descr))
@@ -1837,6 +1857,9 @@
 
     def _build_failure_recovery(self, exc, withfloats=False):
         mc = codebuf.MachineCodeBlockWrapper()
+        # this is jumped to, from a stack that has DEFAULT_FRAME_BYTES
+        # followed by 2 extra words just pushed
+        mc.force_frame_size(DEFAULT_FRAME_BYTES + 2 * WORD)
         self.mc = mc
 
         self._push_all_regs_to_frame(mc, [], withfloats)
@@ -1908,6 +1931,7 @@
             self.mc.J_il(rx86.Conditions[condition], 0)
         else:
             self.mc.JMP_l(0)
+            self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         guard_token.pos_jump_offset = self.mc.get_relative_pos() - 4
         self.pending_guard_tokens.append(guard_token)
 
@@ -2002,6 +2026,7 @@
         offset = jmp_location - je_location
         assert 0 < offset <= 127
         self.mc.overwrite(je_location - 1, chr(offset))
+        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         #
         return jmp_location
 
@@ -2086,6 +2111,8 @@
         if is_frame and align_stack:
             mc.SUB_ri(esp.value, 16 - WORD) # erase the return address
         mc.CALL(imm(self.wb_slowpath[helper_num]))
+        if not is_frame:
+            mc.stack_frame_size_delta(-WORD)
         if is_frame and align_stack:
             mc.ADD_ri(esp.value, 16 - WORD) # erase the return address
 
@@ -2322,6 +2349,7 @@
         offset = self.mc.get_relative_pos() - jmp_adr1
         assert 0 < offset <= 127
         self.mc.overwrite(jmp_adr1-1, chr(offset))
+        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         # write down the tid, but not if it's the result of the CALL
         self.mc.MOV(mem(eax, 0), imm(arraydescr.tid))
         # while we're at it, this line is not needed if we've done the CALL
diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py
--- a/rpython/jit/backend/x86/callbuilder.py
+++ b/rpython/jit/backend/x86/callbuilder.py
@@ -46,7 +46,7 @@
         if not self.fnloc_is_immediate:
             self.fnloc = None
             self.arglocs = arglocs + [fnloc]
-        self.current_esp = 0     # 0 or (usually) negative, counted in bytes
+        self.start_frame_size = self.mc._frame_size
 
     def select_call_release_gil_mode(self):
         AbstractCallBuilder.select_call_release_gil_mode(self)
@@ -58,13 +58,15 @@
     def subtract_esp_aligned(self, count):
         if count > 0:
             align = align_stack_words(count)
-            self.current_esp -= align * WORD
             self.mc.SUB_ri(esp.value, align * WORD)
 
+    def get_current_esp(self):
+        return self.start_frame_size - self.mc._frame_size
+
     def restore_stack_pointer(self, target_esp=0):
-        if self.current_esp != target_esp:
-            self.mc.ADD_ri(esp.value, target_esp - self.current_esp)
-            self.current_esp = target_esp
+        current_esp = self.get_current_esp()
+        if current_esp != target_esp:
+            self.mc.ADD_ri(esp.value, target_esp - current_esp)
 
     def load_result(self):
         """Overridden in CallBuilder32 and CallBuilder64"""
@@ -87,9 +89,10 @@
         # after the rearrangements done just before, ignoring the return
         # value eax, if necessary
         assert not self.is_call_release_gil
-        self.change_extra_stack_depth = (self.current_esp != 0)
+        current_esp = self.get_current_esp()
+        self.change_extra_stack_depth = (current_esp != 0)
         if self.change_extra_stack_depth:
-            self.asm.set_extra_stack_depth(self.mc, -self.current_esp)
+            self.asm.set_extra_stack_depth(self.mc, -current_esp)
         noregs = self.asm.cpu.gc_ll_descr.is_shadow_stack()
         gcmap = self.asm._regalloc.get_gcmap([eax], noregs=noregs)
         self.asm.push_gcmap(self.mc, gcmap, store=True)
@@ -130,7 +133,7 @@
             # and 5/7 words as described for asmgcroot.ASM_FRAMEDATA, for a
             # total size of JIT_USE_WORDS.  This structure is found at
             # [ESP+css].
-            css = -self.current_esp + (
+            css = -self.get_current_esp() + (
                 WORD * (PASS_ON_MY_FRAME - asmgcroot.JIT_USE_WORDS))
             assert css >= 2 * WORD
             # Save ebp
@@ -166,7 +169,7 @@
             else:
                 self.tlofs_reg = r12
             self.mc.MOV_rs(self.tlofs_reg.value,
-                           THREADLOCAL_OFS - self.current_esp)
+                           THREADLOCAL_OFS - self.get_current_esp())
         return self.tlofs_reg
 
     def save_stack_position(self):
@@ -425,7 +428,10 @@
         else:
             self.mc.CALL(self.fnloc)
             if self.callconv != FFI_DEFAULT_ABI:
-                self.current_esp += self._fix_stdcall(self.callconv)
+                # in the STDCALL ABI, the CALL above has an effect on
+                # the stack depth.  Adjust 'mc._frame_size'.
+                delta = self._fix_stdcall(self.callconv)
+                self.mc.stack_frame_size_delta(-delta)
 
     def _fix_stdcall(self, callconv):
         from rpython.rlib.clibffi import FFI_STDCALL
diff --git a/rpython/jit/backend/x86/codebuf.py b/rpython/jit/backend/x86/codebuf.py
--- a/rpython/jit/backend/x86/codebuf.py
+++ b/rpython/jit/backend/x86/codebuf.py
@@ -23,6 +23,7 @@
                               codebuilder_cls):
     def __init__(self):
         self.init_block_builder()
+        codebuilder_cls.__init__(self)
         # a list of relative positions; for each position p, the bytes
         # at [p-4:p] encode an absolute address that will need to be
         # made relative.  Only works on 32-bit!
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -12,7 +12,7 @@
      valid_addressing_size)
 from rpython.jit.backend.x86 import rx86
 from rpython.jit.backend.x86.arch import (WORD, JITFRAME_FIXED_SIZE, IS_X86_32,
-    IS_X86_64)
+    IS_X86_64, DEFAULT_FRAME_BYTES)
 from rpython.jit.backend.x86.jump import remap_frame_layout_mixed
 from rpython.jit.backend.x86.regloc import (FrameLoc, RegLoc, ConstFloatLoc,
     FloatImmedLoc, ImmedLoc, imm, imm0, imm1, ecx, eax, edx, ebx, esi, edi,
@@ -314,6 +314,7 @@
         while i < len(operations):
             op = operations[i]
             self.assembler.mc.mark_op(op)
+            assert self.assembler.mc._frame_size == DEFAULT_FRAME_BYTES
             self.rm.position = i
             self.xrm.position = i
             if op.has_no_side_effect() and op.result not in self.longevity:
@@ -1297,7 +1298,7 @@
         assembler.closing_jump(self.jump_target_descr)
 
     def consider_debug_merge_point(self, op):
-        pass
+        self.assembler.debug_merge_point(op)
 
     def consider_jit_debug(self, op):
         pass
diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -447,6 +447,11 @@
 class AbstractX86CodeBuilder(object):
     """Abstract base class."""
 
+    def __init__(self):
+        self.frame_positions = []
+        self.frame_assignments = []
+        self.force_frame_size(self.WORD)
+
     def writechar(self, char):
         raise NotImplementedError
 
@@ -464,6 +469,23 @@
         self.writechar(chr((imm >> 16) & 0xFF))
         self.writechar(chr((imm >> 24) & 0xFF))
 
+    def force_frame_size(self, frame_size):
+        self.frame_positions.append(self.get_relative_pos())
+        self.frame_assignments.append(frame_size)
+        self._frame_size = frame_size
+
+    def stack_frame_size_delta(self, delta):
+        "Called when we generate an instruction that changes the value of ESP"
+        self._frame_size += delta
+        self.frame_positions.append(self.get_relative_pos()) 
+        self.frame_assignments.append(self._frame_size)
+        assert self._frame_size >= self.WORD
+
+    def check_stack_size_at_ret(self):
+        assert self._frame_size == self.WORD
+        if not we_are_translated():
+            self._frame_size = None
+
     # ------------------------------ MOV ------------------------------
 
     MOV_ri = insn(register(1), '\xB8', immediate(2))
@@ -474,14 +496,24 @@
     INC_m = insn(rex_w, '\xFF', orbyte(0), mem_reg_plus_const(1))
     INC_j = insn(rex_w, '\xFF', orbyte(0), abs_(1))
 
-    ADD_ri,ADD_rr,ADD_rb,_,_,ADD_rm,ADD_rj,_,_ = common_modes(0)
+    AD1_ri,ADD_rr,ADD_rb,_,_,ADD_rm,ADD_rj,_,_ = common_modes(0)
     OR_ri, OR_rr, OR_rb, _,_,OR_rm, OR_rj, _,_ = common_modes(1)
     AND_ri,AND_rr,AND_rb,_,_,AND_rm,AND_rj,_,_ = common_modes(4)
-    SUB_ri,SUB_rr,SUB_rb,_,_,SUB_rm,SUB_rj,SUB_ji8,SUB_mi8 = common_modes(5)
+    SU1_ri,SUB_rr,SUB_rb,_,_,SUB_rm,SUB_rj,SUB_ji8,SUB_mi8 = common_modes(5)
     SBB_ri,SBB_rr,SBB_rb,_,_,SBB_rm,SBB_rj,_,_ = common_modes(3)
     XOR_ri,XOR_rr,XOR_rb,_,_,XOR_rm,XOR_rj,_,_ = common_modes(6)
     CMP_ri,CMP_rr,CMP_rb,CMP_bi,CMP_br,CMP_rm,CMP_rj,_,_ = common_modes(7)
 
+    def ADD_ri(self, reg, immed):
+        self.AD1_ri(reg, immed)
+        if reg == R.esp:
+            self.stack_frame_size_delta(-immed)
+
+    def SUB_ri(self, reg, immed):
+        self.SU1_ri(reg, immed)
+        if reg == R.esp:
+            self.stack_frame_size_delta(+immed)
+
     CMP_mi8 = insn(rex_w, '\x83', orbyte(7<<3), mem_reg_plus_const(1), immediate(2, 'b'))
     CMP_mi32 = insn(rex_w, '\x81', orbyte(7<<3), mem_reg_plus_const(1), immediate(2))
     CMP_mi = select_8_or_32_bit_immed(CMP_mi8, CMP_mi32)
@@ -531,30 +563,65 @@
     # ------------------------------ Misc stuff ------------------------------
 
     NOP = insn('\x90')
-    RET = insn('\xC3')
-    RET16_i = insn('\xC2', immediate(1, 'h'))
+    RE1 = insn('\xC3')
+    RE116_i = insn('\xC2', immediate(1, 'h'))
 
-    PUSH_r = insn(rex_nw, register(1), '\x50')
-    PUSH_b = insn(rex_nw, '\xFF', orbyte(6<<3), stack_bp(1))
-    PUSH_m = insn(rex_nw, '\xFF', orbyte(6<<3), mem_reg_plus_const(1))
-    PUSH_i8 = insn('\x6A', immediate(1, 'b'))
-    PUSH_i32 = insn('\x68', immediate(1, 'i'))
-    def PUSH_i(mc, immed):
+    def RET(self):
+        self.check_stack_size_at_ret()
+        self.RE1()
+
+    def RET16_i(self, immed):
+        self.check_stack_size_at_ret()
+        self.RE116_i(immed)
+
+    PUS1_r = insn(rex_nw, register(1), '\x50')
+    PUS1_b = insn(rex_nw, '\xFF', orbyte(6<<3), stack_bp(1))

From noreply at buildbot.pypy.org  Thu Feb  5 16:52:02 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  5 Feb 2015 16:52:02 +0100 (CET)
Subject: [pypy-commit] pypy default: Rename the --output option to
 --really-force-output, and document it
Message-ID: <20150205155202.D3E071C0459@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r75726:babfa1dd27a5
Date: 2015-02-05 16:51 +0100
http://bitbucket.org/pypy/pypy/changeset/babfa1dd27a5/

Log:	Rename the --output option to --really-force-output, and document it
	as "don't use with PyPy".

diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -151,7 +151,9 @@
                default=False, cmdline="--dont-write-c-files"),
     ArbitraryOption("instrumentctl", "internal",
                default=None),
-    StrOption("output", "Output file name", cmdline="--output"),
+    StrOption("output", "Output file name (don't change for PyPy!"
+              " doesn't work with virtualenv+shared: issue 1971)",
+              cmdline="--really-force-output"),
     StrOption("secondaryentrypoints",
             "Comma separated list of keys choosing secondary entrypoints",
             cmdline="--entrypoints", default="main"),

From noreply at buildbot.pypy.org  Thu Feb  5 17:57:47 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  5 Feb 2015 17:57:47 +0100 (CET)
Subject: [pypy-commit] pypy default: avoid race condition here
Message-ID: <20150205165747.90AE71C0976@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r75727:a3521647efba
Date: 2015-02-05 18:56 +0200
http://bitbucket.org/pypy/pypy/changeset/a3521647efba/

Log:	avoid race condition here

diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
--- a/pypy/module/_vmprof/src/vmprof.c
+++ b/pypy/module/_vmprof/src/vmprof.c
@@ -289,9 +289,9 @@
 
 void vmprof_set_mainloop(void* func, ptrdiff_t sp_offset, 
                          vmprof_get_virtual_ip_t get_virtual_ip) {
-    vmprof_mainloop_func = func;
     mainloop_sp_offset = sp_offset;
     mainloop_get_virtual_ip = get_virtual_ip;
+    vmprof_mainloop_func = func;
 }
 
 int vmprof_enable(int fd, long period_usec, int write_header, char *s,

From noreply at buildbot.pypy.org  Thu Feb  5 17:57:48 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  5 Feb 2015 17:57:48 +0100 (CET)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20150205165748.CA3BF1C0976@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r75728:75f1f2de8de1
Date: 2015-02-05 18:57 +0200
http://bitbucket.org/pypy/pypy/changeset/75f1f2de8de1/

Log:	merge

diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -151,7 +151,9 @@
                default=False, cmdline="--dont-write-c-files"),
     ArbitraryOption("instrumentctl", "internal",
                default=None),
-    StrOption("output", "Output file name", cmdline="--output"),
+    StrOption("output", "Output file name (don't change for PyPy!"
+              " doesn't work with virtualenv+shared: issue 1971)",
+              cmdline="--really-force-output"),
     StrOption("secondaryentrypoints",
             "Comma separated list of keys choosing secondary entrypoints",
             cmdline="--entrypoints", default="main"),

From noreply at buildbot.pypy.org  Thu Feb  5 23:28:26 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Thu,  5 Feb 2015 23:28:26 +0100 (CET)
Subject: [pypy-commit] pypy default: try to clarify how to get the proper
	backend for cppyy
Message-ID: <20150205222826.3A0861C0EFC@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r75729:22dc5cd0d3c1
Date: 2015-02-06 00:29 +0200
http://bitbucket.org/pypy/pypy/changeset/22dc5cd0d3c1/

Log:	try to clarify how to get the proper backend for cppyy

diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
--- a/pypy/doc/cppyy.rst
+++ b/pypy/doc/cppyy.rst
@@ -74,7 +74,9 @@
 See the :doc:`backend documentation `.
 
 A standalone version of Reflex that also provides the dynamically loadable
-backend is available for `download`_.
+backend is available for `download`_. Note this is currently the only way to
+get the dynamically loadable backend, so use this first.
+
 That version, as well as any other distribution of Reflex (e.g. the one that
 comes with `ROOT`_, which may be part of your Linux distribution as part of
 the selection of scientific software) will also work for a build with the

From noreply at buildbot.pypy.org  Thu Feb  5 23:37:39 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Thu,  5 Feb 2015 23:37:39 +0100 (CET)
Subject: [pypy-commit] pypy default: numpy nightly testing asserted,
 I could not reproduce. Raise an exception instead
Message-ID: <20150205223739.2D08E1C114E@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r75730:a9264f529424
Date: 2015-02-06 00:38 +0200
http://bitbucket.org/pypy/pypy/changeset/a9264f529424/

Log:	numpy nightly testing asserted, I could not reproduce. Raise an
	exception instead

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -77,12 +77,15 @@
             else:
                 new_strides = calc_new_strides(new_shape, self.get_shape(),
                                                self.get_strides(), self.order)
+                if new_strides is None or len(new_strides) != len(new_shape):
+                    return None
         if new_strides is not None:
             # We can create a view, strides somehow match up.
             new_backstrides = calc_backstrides(new_strides, new_shape)
             assert isinstance(orig_array, W_NDimArray) or orig_array is None
             return SliceArray(self.start, new_strides, new_backstrides,
                               new_shape, self, orig_array)
+        return None
 
     def get_view(self, space, orig_array, dtype, new_shape):
         strides, backstrides = calc_strides(new_shape, dtype,
@@ -452,7 +455,7 @@
         new_strides = calc_new_strides(new_shape, self.get_shape(),
                                        self.get_strides(),
                                        self.order)
-        if new_strides is None:
+        if new_strides is None or len(new_strides) != len(new_shape):
             raise oefmt(space.w_AttributeError,
                 "incompatible shape for a non-contiguous array")
         new_backstrides = [0] * len(new_shape)
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -334,8 +334,12 @@
         # Create copy with contiguous data
         arr = self.descr_copy(space)
         if arr.get_size() > 0:
-            arr.implementation = arr.implementation.reshape(self, new_shape)
-            assert arr.implementation
+            new_implementation = arr.implementation.reshape(self, new_shape)
+            if new_implementation is None:
+                raise oefmt(space.w_ValueError,
+                            'could not reshape array of size %d to shape %s',
+                            arr.get_size(), str(new_shape))
+            arr.implementation = new_implementation
         else:
             arr.implementation.shape = new_shape
         return arr
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -427,7 +427,6 @@
                 if oldI >= -len(old_shape):
                     cur_step = steps[oldI]
                     n_old_elems_to_use *= old_shape[oldI]
-    assert len(new_strides) == len(new_shape)
     return new_strides[:]
 
 @jit.unroll_safe

From noreply at buildbot.pypy.org  Fri Feb  6 01:07:52 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Fri,  6 Feb 2015 01:07:52 +0100 (CET)
Subject: [pypy-commit] pypy global-cell-cache: cache global cells when
	interpreting
Message-ID: <20150206000752.27A341C0FE9@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: global-cell-cache
Changeset: r75731:b9b462fe9ada
Date: 2015-02-06 01:07 +0100
http://bitbucket.org/pypy/pypy/changeset/b9b462fe9ada/

Log:	cache global cells when interpreting

	makes non-jitted richards 7% faster, but needs tests

diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -123,6 +123,9 @@
         if self.space.config.objspace.std.withmapdict:
             from pypy.objspace.std.mapdict import init_mapdict_cache
             init_mapdict_cache(self)
+        if self.space.config.objspace.std.withcelldict:
+            from pypy.objspace.std.celldict import init_celldict_cache
+            init_celldict_cache(self)
 
     def _cleanup_(self):
         if (self.magic == cpython_magic and
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -872,8 +872,12 @@
         self.space.delattr(w_obj, w_attributename)
 
     def STORE_GLOBAL(self, nameindex, next_instr):
+        w_newvalue = self.popvalue()
+        if self.space.config.objspace.std.withcelldict and jit.we_are_jitted():
+            from pypy.objspace.std.celldict import STORE_GLOBAL_celldict
+            STORE_GLOBAL_celldict(self.space, self, nameindex, w_newvalue)
+            return
         varname = self.getname_u(nameindex)
-        w_newvalue = self.popvalue()
         self.space.setitem_str(self.w_globals, varname, w_newvalue)
 
     def DELETE_GLOBAL(self, nameindex, next_instr):
@@ -905,7 +909,12 @@
     _load_global_failed._dont_inline_ = True
 
     def LOAD_GLOBAL(self, nameindex, next_instr):
-        self.pushvalue(self._load_global(self.getname_u(nameindex)))
+        if self.space.config.objspace.std.withcelldict and jit.we_are_jitted():
+            from pypy.objspace.std.celldict import LOAD_GLOBAL_celldict
+            w_result = LOAD_GLOBAL_celldict(self.space, self, nameindex)
+        else:
+            w_result = self._load_global(self.getname_u(nameindex))
+        self.pushvalue(w_result)
     LOAD_GLOBAL._always_inline_ = True
 
     def DELETE_FAST(self, varindex, next_instr):
diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py
--- a/pypy/objspace/std/celldict.py
+++ b/pypy/objspace/std/celldict.py
@@ -172,3 +172,79 @@
 
 
 create_iterator_classes(ModuleDictStrategy)
+
+class CelldictCache(object):
+    def __init__(self):
+        self.version = None
+        self.value = None
+        self.builtin_version = None
+        self.builtin_value = None
+INVALID_CACHE_ENTRY = CelldictCache()
+
+def init_celldict_cache(pycode):
+    num_entries = len(pycode.co_names_w)
+    pycode._celldict_cache = [INVALID_CACHE_ENTRY] * num_entries
+
+def _finditem_with_cache(space, frame, nameindex, pycode, w_dict, entry_version, entry_value, builtin=False):
+    from pypy.objspace.std.dictmultiobject import W_DictMultiObject
+    if (isinstance(w_dict, W_DictMultiObject) and
+            not w_dict.user_overridden_class):
+        strategy = w_dict.strategy
+        if isinstance(strategy, ModuleDictStrategy):
+            # it's enough to check that the version is the same
+            # if the version is the same, that means that both the same globals
+            # object is used, and that that object did not change
+            version = strategy.version
+            if version is entry_version:
+                result = entry_value
+            else:
+                # need to fill the cache
+                result = strategy.getitem_str(
+                        w_dict, frame.getname_u(nameindex))
+                entry = pycode._celldict_cache[nameindex]
+                if entry is INVALID_CACHE_ENTRY:
+                    entry = pycode._celldict_cache[nameindex] = CelldictCache()
+                if builtin:
+                    entry.builtin_version = version
+                    entry.builtin_value = result
+                else:
+                    entry.version = version
+                    entry.value = result
+            return result
+    return space.finditem_str(w_dict, frame.getname_u(nameindex))
+
+def LOAD_GLOBAL_celldict(space, frame, nameindex):
+    from pypy.interpreter.mixedmodule import MixedModule
+    pycode = frame.getcode()
+    w_globals = frame.w_globals
+    entry = pycode._celldict_cache[nameindex]
+    cell = _finditem_with_cache(space, frame, nameindex, pycode, w_globals,
+                                  entry.version, entry.value)
+    if cell is None:
+        assert not space.config.objspace.honor__builtins__
+        # not in the globals, now look in the built-ins
+        builtin = frame.get_builtin()
+        assert isinstance(builtin, MixedModule)
+        cell = _finditem_with_cache(space, frame, nameindex, pycode, builtin.w_dict,
+                                      entry.builtin_version, entry.builtin_value,
+                                      builtin=True)
+        if cell is None and builtin.lazy:
+            w_result = builtin._load_lazily(space, frame.getname_u(nameindex))
+        else:
+            w_result = unwrap_cell(space, cell)
+        if w_result is None:
+            frame._load_global_failed(frame.getname_u(nameindex))
+    else:
+        w_result = unwrap_cell(space, cell)
+    return w_result
+
+def STORE_GLOBAL_celldict(space, frame, nameindex, w_value):
+    pycode = frame.getcode()
+    w_globals = frame.w_globals
+    entry = pycode._celldict_cache[nameindex]
+    cell = _finditem_with_cache(space, frame, nameindex, pycode, w_globals,
+                                entry.version, entry.value)
+    w_newvalue = write_cell(space, cell, w_value)
+    if w_newvalue is None:
+        return
+    space.setitem_str(w_globals, frame.getname_u(nameindex), w_value)
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -45,6 +45,7 @@
 
 def write_cell(space, w_cell, w_value):
     from pypy.objspace.std.intobject import W_IntObject
+    assert w_value is not None
     if w_cell is None:
         # attribute does not exist at all, write it without a cell first
         return w_value

From noreply at buildbot.pypy.org  Fri Feb  6 10:29:03 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  6 Feb 2015 10:29:03 +0100 (CET)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20150206092903.656F71C1156@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r75733:13cf744cbd9a
Date: 2015-02-06 11:28 +0200
http://bitbucket.org/pypy/pypy/changeset/13cf744cbd9a/

Log:	merge

diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
--- a/pypy/doc/cppyy.rst
+++ b/pypy/doc/cppyy.rst
@@ -74,7 +74,9 @@
 See the :doc:`backend documentation `.
 
 A standalone version of Reflex that also provides the dynamically loadable
-backend is available for `download`_.
+backend is available for `download`_. Note this is currently the only way to
+get the dynamically loadable backend, so use this first.
+
 That version, as well as any other distribution of Reflex (e.g. the one that
 comes with `ROOT`_, which may be part of your Linux distribution as part of
 the selection of scientific software) will also work for a build with the
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -77,12 +77,15 @@
             else:
                 new_strides = calc_new_strides(new_shape, self.get_shape(),
                                                self.get_strides(), self.order)
+                if new_strides is None or len(new_strides) != len(new_shape):
+                    return None
         if new_strides is not None:
             # We can create a view, strides somehow match up.
             new_backstrides = calc_backstrides(new_strides, new_shape)
             assert isinstance(orig_array, W_NDimArray) or orig_array is None
             return SliceArray(self.start, new_strides, new_backstrides,
                               new_shape, self, orig_array)
+        return None
 
     def get_view(self, space, orig_array, dtype, new_shape):
         strides, backstrides = calc_strides(new_shape, dtype,
@@ -452,7 +455,7 @@
         new_strides = calc_new_strides(new_shape, self.get_shape(),
                                        self.get_strides(),
                                        self.order)
-        if new_strides is None:
+        if new_strides is None or len(new_strides) != len(new_shape):
             raise oefmt(space.w_AttributeError,
                 "incompatible shape for a non-contiguous array")
         new_backstrides = [0] * len(new_shape)
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -334,8 +334,12 @@
         # Create copy with contiguous data
         arr = self.descr_copy(space)
         if arr.get_size() > 0:
-            arr.implementation = arr.implementation.reshape(self, new_shape)
-            assert arr.implementation
+            new_implementation = arr.implementation.reshape(self, new_shape)
+            if new_implementation is None:
+                raise oefmt(space.w_ValueError,
+                            'could not reshape array of size %d to shape %s',
+                            arr.get_size(), str(new_shape))
+            arr.implementation = new_implementation
         else:
             arr.implementation.shape = new_shape
         return arr
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -427,7 +427,6 @@
                 if oldI >= -len(old_shape):
                     cur_step = steps[oldI]
                     n_old_elems_to_use *= old_shape[oldI]
-    assert len(new_strides) == len(new_shape)
     return new_strides[:]
 
 @jit.unroll_safe

From noreply at buildbot.pypy.org  Fri Feb  6 10:29:02 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri,  6 Feb 2015 10:29:02 +0100 (CET)
Subject: [pypy-commit] pypy default: backout vmprof merge,
 clearly not ready for the prime time
Message-ID: <20150206092902.02DCC1C1156@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r75732:93acf6aa53b7
Date: 2015-02-06 11:28 +0200
http://bitbucket.org/pypy/pypy/changeset/93acf6aa53b7/

Log:	backout vmprof merge, clearly not ready for the prime time

diff too long, truncating to 2000 out of 2674 lines

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -420,10 +420,3 @@
 the terms of the GPL license version 2 or any later version.  Thus the
 gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed
 under the terms of the GPL license as well.
-
-License for 'pypy/module/_vmprof/src'
---------------------------------------
-
-The code is based on gperftools. You may see a copy of the License for it at
-
-    https://code.google.com/p/gperftools/source/browse/COPYING
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -38,9 +38,6 @@
     "_csv", "cppyy", "_pypyjson"
 ])
 
-if sys.platform.startswith('linux') and sys.maxint > 2147483647:
-    working_modules.add('_vmprof')
-
 translation_modules = default_modules.copy()
 translation_modules.update([
     "fcntl", "time", "select", "signal", "_rawffi", "zlib", "struct", "_md5",
@@ -102,7 +99,6 @@
     "_hashlib"  : ["pypy.module._ssl.interp_ssl"],
     "_minimal_curses": ["pypy.module._minimal_curses.fficurses"],
     "_continuation": ["rpython.rlib.rstacklet"],
-    "_vmprof" : ["pypy.module._vmprof.interp_vmprof"],
     }
 
 def get_module_validator(modname):
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -9,7 +9,6 @@
 from rpython.rlib.signature import signature
 from rpython.rlib.rarithmetic import r_uint, SHRT_MIN, SHRT_MAX, \
     INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX
-from rpython.rlib.rweaklist import RWeakListMixin
 
 from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag,
     UserDelAction)
@@ -367,10 +366,6 @@
 
 # ____________________________________________________________
 
-class CodeObjWeakList(RWeakListMixin):
-    def __init__(self):
-        self.initialize()
-
 class ObjSpace(object):
     """Base class for the interpreter-level implementations of object spaces.
     http://pypy.readthedocs.org/en/latest/objspace.html"""
@@ -394,7 +389,6 @@
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
         self._code_of_sys_exc_info = None
-        self.all_code_objs = CodeObjWeakList()
 
         # can be overridden to a subclass
         self.initialize()
@@ -672,16 +666,6 @@
             assert ec is not None
             return ec
 
-    def register_code_object(self, pycode):
-        callback = self.getexecutioncontext().register_code_callback
-        if callback is not None:
-            callback(self, pycode)
-        self.all_code_objs.add_handle(pycode)
-
-    def set_code_callback(self, callback):
-        ec = self.getexecutioncontext()
-        ec.register_code_callback = callback
-        
     def _freeze_(self):
         return True
 
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -33,11 +33,6 @@
         self.profilefunc = None
         self.w_profilefuncarg = None
         self.thread_disappeared = False   # might be set to True after os.fork()
-        self.register_code_callback = None
-        if sys.maxint == 2147483647:
-            self._code_unique_id = 0 # XXX this is wrong, it won't work on 32bit
-        else:
-            self._code_unique_id = 0x7000000000000000
 
     @staticmethod
     def _mark_thread_disappeared(space):
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -14,10 +14,9 @@
     CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED,
     CO_GENERATOR, CO_KILL_DOCSTRING, CO_YIELD_INSIDE_TRY)
 from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT
-from rpython.rlib.rarithmetic import intmask, r_longlong
+from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.objectmodel import compute_hash
 from rpython.rlib import jit
-from rpython.rlib.debug import debug_start, debug_stop, debug_print
 
 
 class BytecodeCorruption(Exception):
@@ -55,9 +54,8 @@
     "CPython-style code objects."
     _immutable_ = True
     _immutable_fields_ = ["co_consts_w[*]", "co_names_w[*]", "co_varnames[*]",
-                          "co_freevars[*]", "co_cellvars[*]",
-                          "_args_as_cellvars[*]"]
-    
+                          "co_freevars[*]", "co_cellvars[*]", "_args_as_cellvars[*]"]
+
     def __init__(self, space,  argcount, nlocals, stacksize, flags,
                      code, consts, names, varnames, filename,
                      name, firstlineno, lnotab, freevars, cellvars,
@@ -85,7 +83,6 @@
         self.magic = magic
         self._signature = cpython_code_signature(self)
         self._initialize()
-        space.register_code_object(self)
 
     def _initialize(self):
         if self.co_cellvars:
@@ -127,15 +124,6 @@
             from pypy.objspace.std.mapdict import init_mapdict_cache
             init_mapdict_cache(self)
 
-        ec = self.space.getexecutioncontext()
-        self._unique_id = ec._code_unique_id
-        ec._code_unique_id += 2 # so we have one bit that we can mark stuff
-        # with
-
-    def _get_full_name(self):
-        return "py:%s:%d:%s" % (self.co_name, self.co_firstlineno,
-                                self.co_filename)
-
     def _cleanup_(self):
         if (self.magic == cpython_magic and
             '__pypy__' not in sys.builtin_module_names):
diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py
deleted file mode 100644
--- a/pypy/module/_vmprof/__init__.py
+++ /dev/null
@@ -1,18 +0,0 @@
-from pypy.interpreter.mixedmodule import MixedModule
-
-class Module(MixedModule):
-    """
-    Write me :)
-    """
-    appleveldefs = {
-    }
-
-    interpleveldefs = {
-        'enable': 'interp_vmprof.enable',
-        'disable': 'interp_vmprof.disable',
-    }
-
-    def setup_after_space_initialization(self):
-        # force the __extend__ hacks to occur early
-        from pypy.module._vmprof.interp_vmprof import VMProf
-        self.vmprof = VMProf()
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
deleted file mode 100644
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ /dev/null
@@ -1,223 +0,0 @@
-import py, os, sys
-from rpython.rtyper.lltypesystem import lltype, rffi, llmemory
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rtyper.annlowlevel import cast_instance_to_gcref, cast_base_ptr_to_instance
-from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib import jit, rposix, entrypoint
-from rpython.rtyper.tool import rffi_platform as platform
-from rpython.rlib.rstring import StringBuilder
-from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.error import oefmt, wrap_oserror, OperationError
-from pypy.interpreter.gateway import unwrap_spec
-from pypy.interpreter.pyframe import PyFrame
-
-ROOT = py.path.local(__file__).join('..')
-SRC = ROOT.join('src')
-
-# by default, we statically link vmprof.c into pypy; however, if you set
-# DYNAMIC_VMPROF to True, it will be dynamically linked to the libvmprof.so
-# which is expected to be inside pypy/module/_vmprof/src: this is very useful
-# during development. Note that you have to manually build libvmprof by
-# running make inside the src dir
-DYNAMIC_VMPROF = False
-
-eci_kwds = dict(
-    include_dirs = [SRC],
-    includes = ['vmprof.h', 'trampoline.h'],
-    separate_module_files = [SRC.join('trampoline.asmgcc.s')],
-    libraries = ['unwind'],
-    
-    post_include_bits=["""
-        void* pypy_vmprof_get_virtual_ip(void*);
-        void pypy_vmprof_init(void);
-    """],
-    
-    separate_module_sources=["""
-        void pypy_vmprof_init(void) {
-            vmprof_set_mainloop(pypy_execute_frame_trampoline, 0,
-                                pypy_vmprof_get_virtual_ip);
-        }
-    """],
-    )
-
-
-if DYNAMIC_VMPROF:
-    eci_kwds['libraries'] += ['vmprof']
-    eci_kwds['link_extra'] = ['-Wl,-rpath,%s' % SRC, '-L%s' % SRC]
-else:
-    eci_kwds['separate_module_files'] += [SRC.join('vmprof.c')]
-
-eci = ExternalCompilationInfo(**eci_kwds)
-
-check_eci = eci.merge(ExternalCompilationInfo(separate_module_files=[
-    SRC.join('fake_pypy_api.c')]))
-
-platform.verify_eci(check_eci)
-
-pypy_execute_frame_trampoline = rffi.llexternal(
-    "pypy_execute_frame_trampoline",
-    [llmemory.GCREF, llmemory.GCREF, llmemory.GCREF],
-    llmemory.GCREF,
-    compilation_info=eci,
-    _nowrapper=True, sandboxsafe=True,
-    random_effects_on_gcobjs=True)
-
-pypy_vmprof_init = rffi.llexternal("pypy_vmprof_init", [], lltype.Void,
-                                   compilation_info=eci)
-vmprof_enable = rffi.llexternal("vmprof_enable",
-                                [rffi.INT, rffi.LONG, rffi.INT,
-                                 rffi.CCHARP, rffi.INT],
-                                rffi.INT, compilation_info=eci,
-                                save_err=rffi.RFFI_SAVE_ERRNO)
-vmprof_disable = rffi.llexternal("vmprof_disable", [], rffi.INT,
-                                 compilation_info=eci,
-                                save_err=rffi.RFFI_SAVE_ERRNO)
-
-vmprof_register_virtual_function = rffi.llexternal(
-    "vmprof_register_virtual_function",
-    [rffi.CCHARP, rffi.VOIDP, rffi.VOIDP], lltype.Void,
-    compilation_info=eci, _nowrapper=True)
-
-original_execute_frame = PyFrame.execute_frame.im_func
-original_execute_frame.c_name = 'pypy_pyframe_execute_frame'
-original_execute_frame._dont_inline_ = True
-
-class __extend__(PyFrame):
-    def execute_frame(frame, w_inputvalue=None, operr=None):
-        # go through the asm trampoline ONLY if we are translated but not being JITted.
-        #
-        # If we are not translated, we obviously don't want to go through the
-        # trampoline because there is no C function it can call.
-        #
-        # If we are being JITted, we want to skip the trampoline, else the JIT
-        # cannot see throug it
-        if we_are_translated() and not jit.we_are_jitted():
-            # if we are translated, call the trampoline
-            gc_frame = cast_instance_to_gcref(frame)
-            gc_inputvalue = cast_instance_to_gcref(w_inputvalue)
-            gc_operr = cast_instance_to_gcref(operr)
-            gc_result = pypy_execute_frame_trampoline(gc_frame, gc_inputvalue, gc_operr)
-            return cast_base_ptr_to_instance(W_Root, gc_result)
-        else:
-            return original_execute_frame(frame, w_inputvalue, operr)
-
-
- at entrypoint.entrypoint_lowlevel('main', [llmemory.GCREF],
-                                'pypy_vmprof_get_virtual_ip', True)
-def get_virtual_ip(gc_frame):
-    frame = cast_base_ptr_to_instance(PyFrame, gc_frame)
-    if jit._get_virtualizable_token(frame):
-        return rffi.cast(rffi.VOIDP, 0)
-    virtual_ip = do_get_virtual_ip(frame)
-    return rffi.cast(rffi.VOIDP, virtual_ip)
-
-def do_get_virtual_ip(frame):
-    return frame.pycode._unique_id
-
-def write_long_to_string_builder(l, b):
-    if sys.maxint == 2147483647:
-        b.append(chr(l & 0xff))
-        b.append(chr((l >> 8) & 0xff))
-        b.append(chr((l >> 16) & 0xff))
-        b.append(chr((l >> 24) & 0xff))
-    else:
-        b.append(chr(l & 0xff))
-        b.append(chr((l >> 8) & 0xff))
-        b.append(chr((l >> 16) & 0xff))
-        b.append(chr((l >> 24) & 0xff))
-        b.append(chr((l >> 32) & 0xff))
-        b.append(chr((l >> 40) & 0xff))
-        b.append(chr((l >> 48) & 0xff))
-        b.append(chr((l >> 56) & 0xff))
-
-class VMProf(object):
-    def __init__(self):
-        self.is_enabled = False
-        self.ever_enabled = False
-        self.mapping_so_far = [] # stored mapping in between runs
-        self.fileno = -1
-
-    def enable(self, space, fileno, period):
-        if self.is_enabled:
-            raise oefmt(space.w_ValueError, "_vmprof already enabled")
-        self.fileno = fileno
-        self.is_enabled = True
-        self.write_header(fileno, period)
-        if not self.ever_enabled:
-            if we_are_translated():
-                pypy_vmprof_init()
-            self.ever_enabled = True
-        for weakcode in space.all_code_objs.get_all_handles():
-            code = weakcode()
-            if code:
-                self.register_code(space, code)
-        space.set_code_callback(vmprof_register_code)
-        if we_are_translated():
-            # does not work untranslated
-            res = vmprof_enable(fileno, period, 0,
-                                lltype.nullptr(rffi.CCHARP.TO), 0)
-        else:
-            res = 0
-        if res == -1:
-            raise wrap_oserror(space, OSError(rposix.get_saved_errno(),
-                                              "_vmprof.enable"))
-
-    def write_header(self, fileno, period):
-        if period == -1:
-            period_usec = 1000000 / 100 #  100hz
-        else:
-            period_usec = period
-        b = StringBuilder()
-        write_long_to_string_builder(0, b)
-        write_long_to_string_builder(3, b)
-        write_long_to_string_builder(0, b)
-        write_long_to_string_builder(period_usec, b)
-        write_long_to_string_builder(0, b)
-        os.write(fileno, b.build())
-
-    def register_code(self, space, code):
-        if self.fileno == -1:
-            raise OperationError(space.w_RuntimeError,
-                                 space.wrap("vmprof not running"))
-        name = code._get_full_name()
-        b = StringBuilder()
-        b.append('\x02')
-        write_long_to_string_builder(code._unique_id, b)
-        write_long_to_string_builder(len(name), b)
-        b.append(name)
-        os.write(self.fileno, b.build())
-
-    def disable(self, space):
-        if not self.is_enabled:
-            raise oefmt(space.w_ValueError, "_vmprof not enabled")
-        self.is_enabled = False
-        self.fileno = -1
-        if we_are_translated():
-           # does not work untranslated
-            res = vmprof_disable()
-        else:
-            res = 0
-        space.set_code_callback(None)
-        if res == -1:
-            raise wrap_oserror(space, OSError(rposix.get_saved_errno(),
-                                              "_vmprof.disable"))
-
-def vmprof_register_code(space, code):
-    from pypy.module._vmprof import Module
-    mod_vmprof = space.getbuiltinmodule('_vmprof')
-    assert isinstance(mod_vmprof, Module)
-    mod_vmprof.vmprof.register_code(space, code)
-        
- at unwrap_spec(fileno=int, period=int)
-def enable(space, fileno, period=-1):
-    from pypy.module._vmprof import Module
-    mod_vmprof = space.getbuiltinmodule('_vmprof')
-    assert isinstance(mod_vmprof, Module)
-    mod_vmprof.vmprof.enable(space, fileno, period)
-
-def disable(space):
-    from pypy.module._vmprof import Module
-    mod_vmprof = space.getbuiltinmodule('_vmprof')
-    assert isinstance(mod_vmprof, Module)
-    mod_vmprof.vmprof.disable(space)
-
diff --git a/pypy/module/_vmprof/src/config.h b/pypy/module/_vmprof/src/config.h
deleted file mode 100644
--- a/pypy/module/_vmprof/src/config.h
+++ /dev/null
@@ -1,2 +0,0 @@
-#define HAVE_SYS_UCONTEXT_H
-#define PC_FROM_UCONTEXT uc_mcontext.gregs[REG_RIP]
diff --git a/pypy/module/_vmprof/src/fake_pypy_api.c b/pypy/module/_vmprof/src/fake_pypy_api.c
deleted file mode 100644
--- a/pypy/module/_vmprof/src/fake_pypy_api.c
+++ /dev/null
@@ -1,29 +0,0 @@
-
-long pypy_jit_start_addr(void)
-{
-	return 3;
-}
-
-long pypy_jit_end_addr(void)
-{
-	return 3;
-}
-
-long pypy_jit_stack_depth_at_loc(long x)
-{
-	return 0;
-}
-
-long pypy_find_codemap_at_addr(long x)
-{
-	return 0;
-}
-
-long pypy_yield_codemap_at_addr(long x, long y, long *a)
-{
-	return 0;
-}
-
-void pypy_pyframe_execute_frame(void)
-{
-}
diff --git a/pypy/module/_vmprof/src/get_custom_offset.c b/pypy/module/_vmprof/src/get_custom_offset.c
deleted file mode 100644
--- a/pypy/module/_vmprof/src/get_custom_offset.c
+++ /dev/null
@@ -1,46 +0,0 @@
-
-long pypy_jit_start_addr();
-long pypy_jit_end_addr();
-long pypy_jit_stack_depth_at_loc(long);
-long pypy_find_codemap_at_addr(long);
-long pypy_yield_codemap_at_addr(long, long, long*);
-
-void vmprof_set_tramp_range(void* start, void* end)
-{
-}
-
-static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, unw_cursor_t *cp) {
-	intptr_t ip_l = (intptr_t)ip;
-
-	if (ip_l < pypy_jit_start_addr() || ip_l > pypy_jit_end_addr()) {
-		return -1;
-	}
-	return (void*)pypy_jit_stack_depth_at_loc(ip_l);
-}
-
-static long vmprof_write_header_for_jit_addr(void **result, long n,
-											 void *ip, int max_depth)
-{
-	long codemap_pos;
-	long current_pos = 0;
-	intptr_t id;
-	intptr_t addr = (intptr_t)ip;
-
-	if (addr < pypy_jit_start_addr() || addr > pypy_jit_end_addr()) {
-		return n;
-	}
-	codemap_pos = pypy_find_codemap_at_addr(addr);
-	if (codemap_pos == -1) {
-		return n;
-	}
-	while (1) {
-		id = pypy_yield_codemap_at_addr(codemap_pos, addr, ¤t_pos);
-		if (id == 0) {
-			return n;
-		}
-		result[n++] = id;
-		if (n >= max_depth) {
-			return n;
-		}
-	}
-}
diff --git a/pypy/module/_vmprof/src/getpc.h b/pypy/module/_vmprof/src/getpc.h
deleted file mode 100644
--- a/pypy/module/_vmprof/src/getpc.h
+++ /dev/null
@@ -1,187 +0,0 @@
-// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*-
-// Copyright (c) 2005, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-//     * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-//     * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-//     * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-// ---
-// Author: Craig Silverstein
-//
-// This is an internal header file used by profiler.cc.  It defines
-// the single (inline) function GetPC.  GetPC is used in a signal
-// handler to figure out the instruction that was being executed when
-// the signal-handler was triggered.
-//
-// To get this, we use the ucontext_t argument to the signal-handler
-// callback, which holds the full context of what was going on when
-// the signal triggered.  How to get from a ucontext_t to a Program
-// Counter is OS-dependent.
-
-#ifndef BASE_GETPC_H_
-#define BASE_GETPC_H_
-
-#include "config.h"
-
-// On many linux systems, we may need _GNU_SOURCE to get access to
-// the defined constants that define the register we want to see (eg
-// REG_EIP).  Note this #define must come first!
-#define _GNU_SOURCE 1
-// If #define _GNU_SOURCE causes problems, this might work instead.
-// It will cause problems for FreeBSD though!, because it turns off
-// the needed __BSD_VISIBLE.
-//#define _XOPEN_SOURCE 500
-
-#include          // for memcmp
-#if defined(HAVE_SYS_UCONTEXT_H)
-#include 
-#elif defined(HAVE_UCONTEXT_H)
-#include        // for ucontext_t (and also mcontext_t)
-#elif defined(HAVE_CYGWIN_SIGNAL_H)
-#include 
-typedef ucontext ucontext_t;
-#endif
-
-
-// Take the example where function Foo() calls function Bar().  For
-// many architectures, Bar() is responsible for setting up and tearing
-// down its own stack frame.  In that case, it's possible for the
-// interrupt to happen when execution is in Bar(), but the stack frame
-// is not properly set up (either before it's done being set up, or
-// after it's been torn down but before Bar() returns).  In those
-// cases, the stack trace cannot see the caller function anymore.
-//
-// GetPC can try to identify this situation, on architectures where it
-// might occur, and unwind the current function call in that case to
-// avoid false edges in the profile graph (that is, edges that appear
-// to show a call skipping over a function).  To do this, we hard-code
-// in the asm instructions we might see when setting up or tearing
-// down a stack frame.
-//
-// This is difficult to get right: the instructions depend on the
-// processor, the compiler ABI, and even the optimization level.  This
-// is a best effort patch -- if we fail to detect such a situation, or
-// mess up the PC, nothing happens; the returned PC is not used for
-// any further processing.
-struct CallUnrollInfo {
-  // Offset from (e)ip register where this instruction sequence
-  // should be matched. Interpreted as bytes. Offset 0 is the next
-  // instruction to execute. Be extra careful with negative offsets in
-  // architectures of variable instruction length (like x86) - it is
-  // not that easy as taking an offset to step one instruction back!
-  int pc_offset;
-  // The actual instruction bytes. Feel free to make it larger if you
-  // need a longer sequence.
-  unsigned char ins[16];
-  // How many bytes to match from ins array?
-  int ins_size;
-  // The offset from the stack pointer (e)sp where to look for the
-  // call return address. Interpreted as bytes.
-  int return_sp_offset;
-};
-
-
-// The dereferences needed to get the PC from a struct ucontext were
-// determined at configure time, and stored in the macro
-// PC_FROM_UCONTEXT in config.h.  The only thing we need to do here,
-// then, is to do the magic call-unrolling for systems that support it.
-
-// -- Special case 1: linux x86, for which we have CallUnrollInfo
-#if defined(__linux) && defined(__i386) && defined(__GNUC__)
-static const CallUnrollInfo callunrollinfo[] = {
-  // Entry to a function:  push %ebp;  mov  %esp,%ebp
-  // Top-of-stack contains the caller IP.
-  { 0,
-    {0x55, 0x89, 0xe5}, 3,
-    0
-  },
-  // Entry to a function, second instruction:  push %ebp;  mov  %esp,%ebp
-  // Top-of-stack contains the old frame, caller IP is +4.
-  { -1,
-    {0x55, 0x89, 0xe5}, 3,
-    4
-  },
-  // Return from a function: RET.
-  // Top-of-stack contains the caller IP.
-  { 0,
-    {0xc3}, 1,
-    0
-  }
-};
-
-inline void* GetPC(ucontext_t *signal_ucontext) {
-  // See comment above struct CallUnrollInfo.  Only try instruction
-  // flow matching if both eip and esp looks reasonable.
-  const int eip = signal_ucontext->uc_mcontext.gregs[REG_EIP];
-  const int esp = signal_ucontext->uc_mcontext.gregs[REG_ESP];
-  if ((eip & 0xffff0000) != 0 && (~eip & 0xffff0000) != 0 &&
-      (esp & 0xffff0000) != 0) {
-    char* eip_char = reinterpret_cast(eip);
-    for (int i = 0; i < sizeof(callunrollinfo)/sizeof(*callunrollinfo); ++i) {
-      if (!memcmp(eip_char + callunrollinfo[i].pc_offset,
-                  callunrollinfo[i].ins, callunrollinfo[i].ins_size)) {
-        // We have a match.
-        void **retaddr = (void**)(esp + callunrollinfo[i].return_sp_offset);
-        return *retaddr;
-      }
-    }
-  }
-  return (void*)eip;
-}
-
-// Special case #2: Windows, which has to do something totally different.
-#elif defined(_WIN32) || defined(__CYGWIN__) || defined(__CYGWIN32__) || defined(__MINGW32__)
-// If this is ever implemented, probably the way to do it is to have
-// profiler.cc use a high-precision timer via timeSetEvent:
-//    http://msdn2.microsoft.com/en-us/library/ms712713.aspx
-// We'd use it in mode TIME_CALLBACK_FUNCTION/TIME_PERIODIC.
-// The callback function would be something like prof_handler, but
-// alas the arguments are different: no ucontext_t!  I don't know
-// how we'd get the PC (using StackWalk64?)
-//    http://msdn2.microsoft.com/en-us/library/ms680650.aspx
-
-#include "base/logging.h"   // for RAW_LOG
-#ifndef HAVE_CYGWIN_SIGNAL_H
-typedef int ucontext_t;
-#endif
-
-inline void* GetPC(ucontext_t *signal_ucontext) {
-  RAW_LOG(ERROR, "GetPC is not yet implemented on Windows\n");
-  return NULL;
-}
-
-// Normal cases.  If this doesn't compile, it's probably because
-// PC_FROM_UCONTEXT is the empty string.  You need to figure out
-// the right value for your system, and add it to the list in
-// configure.ac (or set it manually in your config.h).
-#else
-inline void* GetPC(ucontext_t *signal_ucontext) {
-  return (void*)signal_ucontext->PC_FROM_UCONTEXT;   // defined in config.h
-}
-
-#endif
-
-#endif  // BASE_GETPC_H_
diff --git a/pypy/module/_vmprof/src/trampoline.asmgcc.s b/pypy/module/_vmprof/src/trampoline.asmgcc.s
deleted file mode 100644
--- a/pypy/module/_vmprof/src/trampoline.asmgcc.s
+++ /dev/null
@@ -1,17 +0,0 @@
-// NOTE: you need to use TABs, not spaces!
-        
-	.text
-	.p2align 4,,-1
-	.globl	pypy_execute_frame_trampoline
-	.type	pypy_execute_frame_trampoline, @function
-pypy_execute_frame_trampoline:
-	.cfi_startproc
-	pushq	%rdi
-	.cfi_def_cfa_offset 16
-	call pypy_pyframe_execute_frame at PLT
-	/* GCROOT 0(%rsp) */
-	popq	%rdi
-	.cfi_def_cfa_offset 8
-	ret
-	.cfi_endproc
-	.size	pypy_execute_frame_trampoline, .-pypy_execute_frame_trampoline
diff --git a/pypy/module/_vmprof/src/trampoline.h b/pypy/module/_vmprof/src/trampoline.h
deleted file mode 100644
--- a/pypy/module/_vmprof/src/trampoline.h
+++ /dev/null
@@ -1,1 +0,0 @@
-void* pypy_execute_frame_trampoline(void*, void*, void*);
diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
deleted file mode 100644
--- a/pypy/module/_vmprof/src/vmprof.c
+++ /dev/null
@@ -1,340 +0,0 @@
-/* VMPROF
- *
- * statistical sampling profiler specifically designed to profile programs
- * which run on a Virtual Machine and/or bytecode interpreter, such as Python,
- * etc.
- *
- * The logic to dump the C stack traces is partly stolen from the code in gperftools.
- * The file "getpc.h" has been entirely copied from gperftools.
- *
- * Tested only on gcc, linux, x86_64.
- *
- * Copyright (C) 2014-2015
- *   Antonio Cuni - anto.cuni at gmail.com
- *   Maciej Fijalkowski - fijall at gmail.com
- *
- */
-
-
-#include "getpc.h"      // should be first to get the _GNU_SOURCE dfn
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#define UNW_LOCAL_ONLY
-#include 
-
-#include "vmprof.h"
-
-#define _unused(x) ((void)x)
-
-#define MAX_FUNC_NAME 128
-#define MAX_STACK_DEPTH 64
-
-static FILE* profile_file = NULL;
-void* vmprof_mainloop_func;
-static ptrdiff_t mainloop_sp_offset;
-static vmprof_get_virtual_ip_t mainloop_get_virtual_ip;
-
-
-/* *************************************************************
- * functions to write a profile file compatible with gperftools
- * *************************************************************
- */
-
-#define MARKER_STACKTRACE '\x01'
-#define MARKER_VIRTUAL_IP '\x02'
-#define MARKER_TRAILER '\x03'
-
-static void prof_word(FILE* f, long x) {
-    fwrite(&x, sizeof(x), 1, f);
-}
-
-static void prof_header(FILE* f, long period_usec) {
-    prof_word(f, 0);
-    prof_word(f, 3);
-    prof_word(f, 0);
-    prof_word(f, period_usec);
-    prof_word(f, 0);
-}
-
-static void prof_write_stacktrace(FILE* f, void** stack, int depth, int count) {
-    int i;
-	char marker = MARKER_STACKTRACE;
-
-	fwrite(&marker, 1, 1, f);
-    prof_word(f, count);
-    prof_word(f, depth);
-    for(i=0; isp = bp;
-		bp -= sizeof(void*);
-        cp2->ip = ((void**)bp)[0];
-        // the ret is on the top of the stack minus WORD
-        return 1;
-    }
-}
-
-
-/* *************************************************************
- * functions to dump the stack trace
- * *************************************************************
- */
-
-// stolen from pprof:
-// Sometimes, we can try to get a stack trace from within a stack
-// trace, because libunwind can call mmap (maybe indirectly via an
-// internal mmap based memory allocator), and that mmap gets trapped
-// and causes a stack-trace request.  If were to try to honor that
-// recursive request, we'd end up with infinite recursion or deadlock.
-// Luckily, it's safe to ignore those subsequent traces.  In such
-// cases, we return 0 to indicate the situation.
-//static __thread int recursive;
-static int recursive; // XXX antocuni: removed __thread
-
-int get_stack_trace(void** result, int max_depth, ucontext_t *ucontext) {
-    void *ip;
-    int n = 0;
-    unw_cursor_t cursor;
-    unw_context_t uc = *ucontext;
-    if (recursive) {
-        return 0;
-    }
-    ++recursive;
-
-    int ret = unw_init_local(&cursor, &uc);
-    assert(ret >= 0);
-    _unused(ret);
-
-    while (n < max_depth) {
-        if (unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *) &ip) < 0) {
-            break;
-        }
-
-        unw_proc_info_t pip;
-        unw_get_proc_info(&cursor, &pip);
-
-        /* char funcname[4096]; */
-        /* unw_word_t offset; */
-        /* unw_get_proc_name(&cursor, funcname, 4096, &offset); */
-        /* printf("%s+%#lx <%p>\n", funcname, offset, ip); */
-
-        /* if n==0, it means that the signal handler interrupted us while we
-           were in the trampoline, so we are not executing (yet) the real main
-           loop function; just skip it */
-        if (vmprof_mainloop_func && 
-            (void*)pip.start_ip == (void*)vmprof_mainloop_func &&
-            n > 0) {
-          // found main loop stack frame
-          void* sp;
-          unw_get_reg(&cursor, UNW_REG_SP, (unw_word_t *) &sp);
-          void *arg_addr = (char*)sp + mainloop_sp_offset;
-          void **arg_ptr = (void**)arg_addr;
-          // fprintf(stderr, "stacktrace mainloop: rsp %p   &f2 %p   offset %ld\n", 
-          //         sp, arg_addr, mainloop_sp_offset);
-          ip = mainloop_get_virtual_ip(*arg_ptr);
-        }
-
-        result[n++] = ip;
-		n = vmprof_write_header_for_jit_addr(result, n, ip, max_depth);
-        if (vmprof_unw_step(&cursor) <= 0) {
-            break;
-        }
-    }
-    --recursive;
-    return n;
-}
-
-
-static int __attribute__((noinline)) frame_forcer(int rv) {
-    return rv;
-}
-
-static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext) {
-    void* stack[MAX_STACK_DEPTH];
-    stack[0] = GetPC((ucontext_t*)ucontext);
-    int depth = frame_forcer(get_stack_trace(stack+1, MAX_STACK_DEPTH-1, ucontext));
-    depth++;  // To account for pc value in stack[0];
-    prof_write_stacktrace(profile_file, stack, depth, 1);
-}
-
-/* *************************************************************
- * functions to enable/disable the profiler
- * *************************************************************
- */
-
-static int open_profile(int fd, long period_usec, int write_header, char *s,
-						int slen) {
-	if ((fd = dup(fd)) == -1) {
-		return -1;
-	}
-    profile_file = fdopen(fd, "wb");
-	if (!profile_file) {
-		return -1;
-	}
-	if (write_header)
-		prof_header(profile_file, period_usec);
-	if (s)
-		fwrite(s, slen, 1, profile_file);
-	return 0;
-}
-
-static int close_profile(void) {
-	// XXX all of this can happily fail
-    FILE* src;
-    char buf[BUFSIZ];
-    size_t size;
-	int marker = MARKER_TRAILER;
-	fwrite(&marker, 1, 1, profile_file);
-
-    // copy /proc/PID/maps to the end of the profile file
-    sprintf(buf, "/proc/%d/maps", getpid());
-    src = fopen(buf, "r");    
-    while ((size = fread(buf, 1, BUFSIZ, src))) {
-        fwrite(buf, 1, size, profile_file);
-    }
-    fclose(src);
-    fclose(profile_file);
-	return 0;
-}
-
-
-static int install_sigprof_handler(void) {
-    struct sigaction sa;
-    memset(&sa, 0, sizeof(sa));
-    sa.sa_sigaction = sigprof_handler;
-    sa.sa_flags = SA_RESTART | SA_SIGINFO;
-    if (sigemptyset(&sa.sa_mask) == -1 ||
-		sigaction(SIGPROF, &sa, NULL) == -1) {
-		return -1;
-	}
-	return 0;
-}
-
-static int remove_sigprof_handler(void) {
-    //sighandler_t res = signal(SIGPROF, SIG_DFL);
-	//if (res == SIG_ERR) {
-	//	return -1;
-	//}
-	return 0;
-};
-
-static int install_sigprof_timer(long period_usec) {
-    static struct itimerval timer;
-    timer.it_interval.tv_sec = 0;
-    timer.it_interval.tv_usec = period_usec;
-    timer.it_value = timer.it_interval;
-    if (setitimer(ITIMER_PROF, &timer, NULL) != 0) {
-		return -1;
-    }
-	return 0;
-}
-
-static int remove_sigprof_timer(void) {
-    static struct itimerval timer;
-    timer.it_interval.tv_sec = 0;
-    timer.it_interval.tv_usec = 0;
-    timer.it_value = timer.it_interval;
-    if (setitimer(ITIMER_PROF, &timer, NULL) != 0) {
-		return -1;
-    }
-	return 0;
-}
-
-/* *************************************************************
- * public API
- * *************************************************************
- */
-
-void vmprof_set_mainloop(void* func, ptrdiff_t sp_offset, 
-                         vmprof_get_virtual_ip_t get_virtual_ip) {
-    mainloop_sp_offset = sp_offset;
-    mainloop_get_virtual_ip = get_virtual_ip;
-    vmprof_mainloop_func = func;
-}
-
-int vmprof_enable(int fd, long period_usec, int write_header, char *s,
-				  int slen)
-{
-    if (period_usec == -1)
-        period_usec = 1000000 / 100; /* 100hz */
-    if (open_profile(fd, period_usec, write_header, s, slen) == -1) {
-		return -1;
-	}
-    if (install_sigprof_handler() == -1) {
-		return -1;
-	}
-    if (install_sigprof_timer(period_usec) == -1) {
-		return -1;
-	}
-	return 0;
-}
-
-int vmprof_disable(void) {
-    if (remove_sigprof_timer() == -1) {
-		return -1;
-	}
-    if (remove_sigprof_handler() == -1) {
-		return -1;
-	}
-    if (close_profile() == -1) {
-		return -1;
-	}
-	return 0;
-}
-
-void vmprof_register_virtual_function(const char* name, void* start, void* end) {
-    // for now *end is simply ignored
-	char buf[1024];
-	int lgt = strlen(name) + 2 * sizeof(long) + 1;
-
-	if (lgt > 1024) {
-		lgt = 1024;
-	}
-	buf[0] = MARKER_VIRTUAL_IP;
-	((void **)(((void*)buf) + 1))[0] = start;
-	((long *)(((void*)buf) + 1 + sizeof(long)))[0] = lgt - 2 * sizeof(long) - 1;
-	strncpy(buf + 2 * sizeof(long) + 1, name, 1024 - 2 * sizeof(long) - 1);
-	fwrite(buf, lgt, 1, profile_file);
-}
diff --git a/pypy/module/_vmprof/src/vmprof.h b/pypy/module/_vmprof/src/vmprof.h
deleted file mode 100644
--- a/pypy/module/_vmprof/src/vmprof.h
+++ /dev/null
@@ -1,22 +0,0 @@
-#ifndef VMPROF_VMPROF_H_
-#define VMPROF_VMPROF_H_
-
-#include 
-
-typedef void* (*vmprof_get_virtual_ip_t)(void*);
-
-extern void* vmprof_mainloop_func;
-void vmprof_set_mainloop(void* func, ptrdiff_t sp_offset, 
-                         vmprof_get_virtual_ip_t get_virtual_ip);
-
-void vmprof_register_virtual_function(const char* name, void* start, void* end);
-
-
-int vmprof_enable(int fd, long period_usec, int write_header, char* vips,
-				  int vips_len);
-int vmprof_disable(void);
-
-// XXX: this should be part of _vmprof (the CPython extension), not vmprof (the library)
-void vmprof_set_tramp_range(void* start, void* end);
-
-#endif
diff --git a/pypy/module/_vmprof/test/__init__.py b/pypy/module/_vmprof/test/__init__.py
deleted file mode 100644
diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
deleted file mode 100644
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ /dev/null
@@ -1,55 +0,0 @@
-
-import tempfile
-from pypy.tool.pytest.objspace import gettestobjspace
-
-class AppTestVMProf(object):
-    def setup_class(cls):
-        cls.space = gettestobjspace(usemodules=['_vmprof', 'struct'])
-        cls.tmpfile = tempfile.NamedTemporaryFile()
-        cls.w_tmpfileno = cls.space.wrap(cls.tmpfile.fileno())
-        cls.w_tmpfilename = cls.space.wrap(cls.tmpfile.name)
-        cls.tmpfile2 = tempfile.NamedTemporaryFile()
-        cls.w_tmpfileno2 = cls.space.wrap(cls.tmpfile2.fileno())
-        cls.w_tmpfilename2 = cls.space.wrap(cls.tmpfile2.name)
-
-    def test_import_vmprof(self):
-        import struct, sys
-
-        WORD = struct.calcsize('l')
-        
-        def count(s):
-            i = 0
-            count = 0
-            i += 5 * WORD # header
-            while i < len(s):
-                assert s[i] == '\x02'
-                i += 1
-                _, size = struct.unpack("ll", s[i:i + 2 * WORD])
-                count += 1
-                i += 2 * WORD + size
-            return count
-        
-        import _vmprof
-        _vmprof.enable(self.tmpfileno)
-        _vmprof.disable()
-        s = open(self.tmpfilename).read()
-        no_of_codes = count(s)
-        assert no_of_codes > 10
-        d = {}
-
-        exec """def foo():
-            pass
-        """ in d
-
-        _vmprof.enable(self.tmpfileno2)
-
-        exec """def foo2():
-            pass
-        """ in d
-
-        _vmprof.disable()
-        s = open(self.tmpfilename2).read()
-        no_of_codes2 = count(s)
-        assert "py:foo:" in s
-        assert "py:foo2:" in s
-        assert no_of_codes2 >= no_of_codes + 2 # some extra codes from tests
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -35,9 +35,6 @@
     name = opcode_method_names[ord(bytecode.co_code[next_instr])]
     return '%s #%d %s' % (bytecode.get_repr(), next_instr, name)
 
-def get_unique_id(next_instr, is_being_profiled, bytecode):
-    return bytecode._unique_id
-
 
 def should_unroll_one_iteration(next_instr, is_being_profiled, bytecode):
     return (bytecode.co_flags & CO_GENERATOR) != 0
@@ -48,7 +45,6 @@
     virtualizables = ['frame']
 
 pypyjitdriver = PyPyJitDriver(get_printable_location = get_printable_location,
-                              get_unique_id = get_unique_id,
                               should_unroll_one_iteration =
                               should_unroll_one_iteration,
                               name='pypyjit')
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -57,7 +57,6 @@
         BaseAssembler.setup_once(self)
 
     def setup(self, looptoken):
-        BaseAssembler.setup(self, looptoken)
         assert self.memcpy_addr != 0, 'setup_once() not called?'
         if we_are_translated():
             self.debug = False
@@ -72,6 +71,7 @@
         self.mc.datablockwrapper = self.datablockwrapper
         self.target_tokens_currently_compiling = {}
         self.frame_depth_to_patch = []
+        self._finish_gcmap = lltype.nullptr(jitframe.GCMAP)
 
     def teardown(self):
         self.current_clt = None
@@ -663,7 +663,6 @@
             assert len(set(inputargs)) == len(inputargs)
 
         self.setup(original_loop_token)
-        self.codemap.inherit_code_from_position(faildescr.adr_jump_offset)
         descr_number = compute_unique_id(faildescr)
         if log:
             operations = self._inject_debugging_code(faildescr, operations,
@@ -884,12 +883,8 @@
         self.datablockwrapper.done()      # finish using cpu.asmmemmgr
         self.datablockwrapper = None
         allblocks = self.get_asmmemmgr_blocks(looptoken)
-        size = self.mc.get_relative_pos() 
-        res = self.mc.materialize(self.cpu.asmmemmgr, allblocks,
+        return self.mc.materialize(self.cpu.asmmemmgr, allblocks,
                                    self.cpu.gc_ll_descr.gcrootmap)
-        self.cpu.asmmemmgr.register_codemap(
-            self.codemap.get_final_bytecode(res, size))
-        return res
 
     def update_frame_depth(self, frame_depth):
         baseofs = self.cpu.get_baseofs_of_frame_field()
diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py
--- a/rpython/jit/backend/arm/regalloc.py
+++ b/rpython/jit/backend/arm/regalloc.py
@@ -392,9 +392,6 @@
         else:
             self.rm._sync_var(v)
 
-    def prepare_op_debug_merge_point(self, op, fcond):
-        self.assembler.codemap.debug_merge_point(op)
-
     def _prepare_op_int_add(self, op, fcond):
         boxes = op.getarglist()
         a0, a1 = boxes
diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py
--- a/rpython/jit/backend/llsupport/asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/asmmemmgr.py
@@ -4,10 +4,7 @@
 from rpython.rlib import rmmap
 from rpython.rlib.debug import debug_start, debug_print, debug_stop
 from rpython.rlib.debug import have_debug_prints
-from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib.rbisect import bisect, bisect_tuple
-
-_memmngr = None # global reference so we can use @entrypoint :/
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 
 
 class AsmMemoryManager(object):
@@ -27,12 +24,6 @@
         self.free_blocks = {}      # map {start: stop}
         self.free_blocks_end = {}  # map {stop: start}
         self.blocks_by_size = [[] for i in range(self.num_indices)]
-        # two lists of jit addresses (sorted) and the corresponding stack
-        # depths
-        self.jit_addr_map = []
-        self.jit_frame_depth_map = []
-        self.jit_codemap = []
-        # see codemap.py
 
     def malloc(self, minsize, maxsize):
         """Allocate executable memory, between minsize and maxsize bytes,
@@ -54,13 +45,6 @@
         if r_uint is not None:
             self.total_mallocs -= r_uint(stop - start)
         self._add_free_block(start, stop)
-        # fix up jit_addr_map
-        jit_adr_start = bisect(self.jit_addr_map, start)
-        jit_adr_stop = bisect(self.jit_addr_map, stop)
-        self.jit_addr_map = (self.jit_addr_map[:jit_adr_start] +
-                             self.jit_addr_map[jit_adr_stop:])
-        self.jit_frame_depth_map = (self.jit_frame_depth_map[:jit_adr_start] +
-                                    self.jit_frame_depth_map[jit_adr_stop:])
 
     def open_malloc(self, minsize):
         """Allocate at least minsize bytes.  Returns (start, stop)."""
@@ -167,35 +151,6 @@
         del self.free_blocks_end[stop]
         return (start, stop)
 
-    def register_frame_depth_map(self, rawstart, frame_positions,
-                                 frame_assignments):
-        if not frame_positions:
-            return
-        if not self.jit_addr_map or rawstart > self.jit_addr_map[-1]:
-            start = len(self.jit_addr_map)
-            self.jit_addr_map += [0] * len(frame_positions)
-            self.jit_frame_depth_map += [0] * len(frame_positions)
-        else:
-            start = bisect(self.jit_addr_map, rawstart)
-            self.jit_addr_map = (self.jit_addr_map[:start] +
-                                 [0] * len(frame_positions) +
-                                 self.jit_addr_map[start:])
-            self.jit_frame_depth_map = (self.jit_frame_depth_map[:start] +
-                                 [0] * len(frame_positions) +
-                                 self.jit_frame_depth_map[start:])
-        for i, pos in enumerate(frame_positions):
-            self.jit_addr_map[i + start] = pos + rawstart
-            self.jit_frame_depth_map[i + start] = frame_assignments[i]
-
-    def register_codemap(self, codemap):
-        start = codemap[0]
-        pos = bisect_tuple(self.jit_codemap, start)
-        if pos == len(self.jit_codemap): # common case
-            self.jit_codemap.append(codemap)
-        else:
-            self.jit_codemap = (self.jit_codemap[:pos] + [codemap] +
-                                self.jit_codemap[pos:])
-
     def _delete(self):
         "NOT_RPYTHON"
         if self._allocated:
@@ -255,9 +210,6 @@
 
     gcroot_markers = None
 
-    frame_positions = None
-    frame_assignments = None
-
     def __init__(self, translated=None):
         if translated is None:
             translated = we_are_translated()
@@ -359,10 +311,6 @@
             assert gcrootmap is not None
             for pos, mark in self.gcroot_markers:
                 gcrootmap.register_asm_addr(rawstart + pos, mark)
-        asmmemmgr.register_frame_depth_map(rawstart, self.frame_positions,
-                                           self.frame_assignments)
-        self.frame_positions = None
-        self.frame_assignments = None
         return rawstart
 
     def _become_a_plain_block_builder(self):
diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -1,7 +1,6 @@
 from rpython.jit.backend.llsupport import jitframe
 from rpython.jit.backend.llsupport.memcpy import memcpy_fn, memset_fn
 from rpython.jit.backend.llsupport.symbolic import WORD
-from rpython.jit.backend.llsupport.codemap import CodemapBuilder
 from rpython.jit.metainterp.history import (INT, REF, FLOAT, JitCellToken,
     ConstInt, BoxInt, AbstractFailDescr)
 from rpython.jit.metainterp.resoperation import ResOperation, rop
@@ -129,10 +128,6 @@
                                               track_allocation=False)
         self.gcmap_for_finish[0] = r_uint(1)
 
-    def setup(self, looptoken):
-        self.codemap = CodemapBuilder()
-        self._finish_gcmap = lltype.nullptr(jitframe.GCMAP)
-
     def set_debug(self, v):
         r = self._debug
         self._debug = v
@@ -199,9 +194,6 @@
         guardtok.faildescr.rd_locs = positions
         return fail_descr, target
 
-    def debug_merge_point(self, op):
-        self.codemap.debug_merge_point(op, self.mc.get_relative_pos())
-
     def call_assembler(self, op, guard_op, argloc, vloc, result_loc, tmploc):
         self._store_force_index(guard_op)
         descr = op.getdescr()
@@ -284,9 +276,6 @@
         # YYY very minor leak -- we need the counters to stay alive
         # forever, just because we want to report them at the end
         # of the process
-
-        # XXX the numbers here are ALMOST unique, but not quite, use a counter
-        #     or something
         struct = lltype.malloc(DEBUG_COUNTER, flavor='raw',
                                track_allocation=False)
         struct.i = 0
diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py
deleted file mode 100644
--- a/rpython/jit/backend/llsupport/codemap.py
+++ /dev/null
@@ -1,142 +0,0 @@
-
-""" Bytecode for storage in asmmemmgr.jit_codemap. Format is as follows:
-
- list of tuples of shape (addr, machine code size, bytecode info)
- where bytecode info is a string made up of:
-    8 bytes unique_id, 4 bytes start_addr (relative), 4 bytes size (relative),
-    2 bytes how many items to skip to go to the next on similar level
-    [so far represented by a list of integers for simplicity]
-
-"""
-
-from rpython.rlib import rgc
-from rpython.rlib.entrypoint import jit_entrypoint
-from rpython.jit.backend.llsupport import asmmemmgr
-from rpython.rlib.rbisect import bisect, bisect_tuple
-from rpython.rtyper.lltypesystem import lltype, rffi
-
- at jit_entrypoint([lltype.Signed], lltype.Signed,
-                c_name='pypy_jit_stack_depth_at_loc')
- at rgc.no_collect
-def stack_depth_at_loc(loc):
-    _memmngr = asmmemmgr._memmngr
-
-    pos = bisect(_memmngr.jit_addr_map, loc)
-    if pos == 0 or pos == len(_memmngr.jit_addr_map):
-        return -1
-    return _memmngr.jit_frame_depth_map[pos-1]
-
- at jit_entrypoint([], lltype.Signed, c_name='pypy_jit_start_addr')
-def jit_start_addr():
-    _memmngr = asmmemmgr._memmngr
-
-    return _memmngr.jit_addr_map[0]
-
- at jit_entrypoint([], lltype.Signed, c_name='pypy_jit_end_addr')
-def jit_end_addr():
-    _memmngr = asmmemmgr._memmngr
-
-    return _memmngr.jit_addr_map[-1]
-
- at jit_entrypoint([lltype.Signed], lltype.Signed,
-                c_name='pypy_find_codemap_at_addr')
-def find_codemap_at_addr(addr):
-    _memmngr = asmmemmgr._memmngr
-
-    res = bisect_tuple(_memmngr.jit_codemap, addr) - 1
-    if res == len(_memmngr.jit_codemap):
-        return -1
-    return res
-
- at jit_entrypoint([lltype.Signed, lltype.Signed,
-                 rffi.CArrayPtr(lltype.Signed)], lltype.Signed,
-                 c_name='pypy_yield_codemap_at_addr')
-def yield_bytecode_at_addr(codemap_no, addr, current_pos_addr):
-    """ will return consecutive unique_ids from codemap, starting from position
-    `pos` until addr
-    """
-    _memmngr = asmmemmgr._memmngr
-
-    codemap = _memmngr.jit_codemap[codemap_no]
-    current_pos = current_pos_addr[0]
-    start_addr = codemap[0]
-    rel_addr = addr - start_addr
-    while True:
-        if current_pos >= len(codemap[2]):
-            return 0
-        next_start = codemap[2][current_pos + 1]
-        if next_start > rel_addr:
-            return 0
-        next_stop = codemap[2][current_pos + 2]
-        if next_stop > rel_addr:
-            current_pos_addr[0] = current_pos + 4
-            return codemap[2][current_pos]
-        # we need to skip potentially more than one
-        current_pos = codemap[2][current_pos + 3]
-
-def unpack_traceback(addr):
-    codemap_pos = find_codemap_at_addr(addr)
-    assert codemap_pos >= 0
-    storage = lltype.malloc(rffi.CArray(lltype.Signed), 1, flavor='raw')
-    storage[0] = 0
-    res = []
-    while True:
-        item = yield_bytecode_at_addr(codemap_pos, addr, storage)
-        if item == 0:
-            break
-        res.append(item)
-    lltype.free(storage, flavor='raw')
-    return res
-
-
-class CodemapBuilder(object):
-    def __init__(self):
-        self.l = []
-        self.patch_position = []
-        self.last_call_depth = -1
-
-    def debug_merge_point(self, op, pos):
-        call_depth = op.getarg(1).getint()
-        if call_depth != self.last_call_depth:
-            unique_id = op.getarg(3).getint()
-            if unique_id == 0: # uninteresting case
-                return
-            assert unique_id & 1 == 0
-            if call_depth > self.last_call_depth:
-                self.l.append(unique_id)
-                self.l.append(pos) # <- this is a relative pos
-                self.patch_position.append(len(self.l))
-                self.l.append(0) # marker
-                self.l.append(0) # second marker
-            else:
-                for i in range(self.last_call_depth - call_depth):
-                    to_patch = self.patch_position.pop()
-                    self.l[to_patch] = pos
-                    self.l[to_patch + 1] = len(self.l)
-            self.last_call_depth = call_depth
-
-    def inherit_code_from_position(self, pos):
-        lst = unpack_traceback(pos)
-        self.last_call_depth = len(lst) - 1
-        for item in lst:
-            self.l.append(item)
-            self.l.append(0)
-            self.patch_position.append(len(self.l))
-            self.l.append(0) # marker
-            self.l.append(0) # second marker
-
-    def get_final_bytecode(self, addr, size):
-        while self.patch_position:
-            pos = self.patch_position.pop()
-            self.l[pos] = size
-            self.l[pos + 1] = len(self.l)
-        # at the end there should be no zeros
-        for i in range(len(self.l) / 4):
-            item = self.l[i * 4] # unique_id
-            assert item > 0 # no zeros here
-            item = self.l[i * 4 + 2] # end in asm
-            assert item > 0
-            item = self.l[i * 4 + 3] # end in l
-            assert item > 0
-        return (addr, size, self.l) # XXX compact self.l
-
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -16,7 +16,7 @@
     FieldDescr, ArrayDescr, CallDescr, InteriorFieldDescr,
     FLAG_POINTER, FLAG_FLOAT)
 from rpython.jit.backend.llsupport.memcpy import memset_fn
-from rpython.jit.backend.llsupport import asmmemmgr
+from rpython.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
 from rpython.rlib.unroll import unrolling_iterable
 
 
@@ -48,8 +48,7 @@
             self._setup_exception_handling_translated()
         else:
             self._setup_exception_handling_untranslated()
-        self.asmmemmgr = asmmemmgr.AsmMemoryManager()
-        asmmemmgr._memmngr = self.asmmemmgr
+        self.asmmemmgr = AsmMemoryManager()
         self._setup_frame_realloc(translate_support_code)
         ad = self.gc_ll_descr.getframedescrs(self).arraydescr
         self.signedarraydescr = ad
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -17,6 +17,8 @@
 class GcRewriterAssembler(object):
     """ This class performs the following rewrites on the list of operations:
 
+     - Remove the DEBUG_MERGE_POINTs.
+
      - Turn all NEW_xxx to either a CALL_MALLOC_GC, or a CALL_MALLOC_NURSERY
        followed by SETFIELDs in order to initialize their GC fields.  The
        two advantages of CALL_MALLOC_NURSERY is that it inlines the common
@@ -58,6 +60,8 @@
         #
         for i in range(len(operations)):
             op = operations[i]
+            if op.getopnum() == rop.DEBUG_MERGE_POINT:
+                continue
             # ---------- turn NEWxxx into CALL_MALLOC_xxx ----------
             if op.is_malloc():
                 self.handle_malloc_operation(op)
diff --git a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -2,7 +2,6 @@
 from rpython.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
 from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
 from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin
-from rpython.jit.backend.llsupport import asmmemmgr
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import debug
 
@@ -158,7 +157,6 @@
         class FakeGcRootMap:
             def register_asm_addr(self, retaddr, mark):
                 puts.append((retaddr, mark))
-
         #
         mc = BlockBuilderMixin()
         mc.writechar('X')
@@ -264,16 +262,3 @@
     md.done()
     assert allblocks == [(1597, 1697), (1797, 1835)]
     assert ops == [('free', 1835, 1897)]
-
-def test_find_jit_frame_depth():
-    mgr = AsmMemoryManager()
-    mgr.register_frame_depth_map(11, [0, 5, 10], [1, 2, 3])
-    mgr.register_frame_depth_map(30, [0, 5, 10], [4, 5, 6])
-    mgr.register_frame_depth_map(0, [0, 5, 10], [7, 8, 9])
-    asmmemmgr._memmngr = mgr
-    assert asmmemmgr.stack_depth_at_loc(13) == 1
-    assert asmmemmgr.stack_depth_at_loc(-3) == -1
-    assert asmmemmgr.stack_depth_at_loc(41) == -1
-    assert asmmemmgr.stack_depth_at_loc(5) == 8
-    assert asmmemmgr.stack_depth_at_loc(17) == 2
-    assert asmmemmgr.stack_depth_at_loc(38) == 5
diff --git a/rpython/jit/backend/llsupport/test/ztranslation_test.py b/rpython/jit/backend/llsupport/test/ztranslation_test.py
--- a/rpython/jit/backend/llsupport/test/ztranslation_test.py
+++ b/rpython/jit/backend/llsupport/test/ztranslation_test.py
@@ -2,7 +2,7 @@
 from rpython.tool.udir import udir
 from rpython.rlib.jit import JitDriver, unroll_parameters, set_param
 from rpython.rlib.jit import PARAMETERS, dont_look_inside
-from rpython.rlib.jit import promote, _get_virtualizable_token
+from rpython.rlib.jit import promote
 from rpython.rlib import jit_hooks, rposix
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rthread import ThreadLocalReference, ThreadLocalField
@@ -28,15 +28,12 @@
         # - floats neg and abs
         # - llexternal with macro=True
 
-        class BasicFrame(object):
+        class Frame(object):
             _virtualizable_ = ['i']
 
             def __init__(self, i):
                 self.i = i
 
-        class Frame(BasicFrame):
-            pass
-
         eci = ExternalCompilationInfo(post_include_bits=['''
 #define pypy_my_fabs(x)  fabs(x)
 '''])
@@ -62,7 +59,6 @@
             while frame.i > 3:
                 jitdriver.can_enter_jit(frame=frame, total=total, j=j)
                 jitdriver.jit_merge_point(frame=frame, total=total, j=j)
-                _get_virtualizable_token(frame)
                 total += frame.i
                 if frame.i >= 20:
                     frame.i -= 2
diff --git a/rpython/jit/backend/x86/arch.py b/rpython/jit/backend/x86/arch.py
--- a/rpython/jit/backend/x86/arch.py
+++ b/rpython/jit/backend/x86/arch.py
@@ -47,6 +47,3 @@
     THREADLOCAL_OFS = (FRAME_FIXED_SIZE - 1) * WORD
 
 assert PASS_ON_MY_FRAME >= 12       # asmgcc needs at least JIT_USE_WORDS + 3
-
-# return address, followed by FRAME_FIXED_SIZE words
-DEFAULT_FRAME_BYTES = (1 + FRAME_FIXED_SIZE) * WORD
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -18,8 +18,7 @@
 from rpython.jit.backend.llsupport.regalloc import (get_scale, valid_addressing_size)
 from rpython.jit.backend.x86.arch import (FRAME_FIXED_SIZE, WORD, IS_X86_64,
                                        JITFRAME_FIXED_SIZE, IS_X86_32,
-                                       PASS_ON_MY_FRAME, THREADLOCAL_OFS,
-                                       DEFAULT_FRAME_BYTES)
+                                       PASS_ON_MY_FRAME, THREADLOCAL_OFS)
 from rpython.jit.backend.x86.regloc import (eax, ecx, edx, ebx, esp, ebp, esi,
     xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, r8, r9, r10, r11, edi,
     r12, r13, r14, r15, X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG,
@@ -66,7 +65,6 @@
             self._build_float_constants()
 
     def setup(self, looptoken):
-        BaseAssembler.setup(self, looptoken)
         assert self.memcpy_addr != 0, "setup_once() not called?"
         self.current_clt = looptoken.compiled_loop_token
         self.pending_guard_tokens = []
@@ -81,6 +79,7 @@
                                                         allblocks)
         self.target_tokens_currently_compiling = {}
         self.frame_depth_to_patch = []
+        self._finish_gcmap = lltype.nullptr(jitframe.GCMAP)
 
     def teardown(self):
         self.pending_guard_tokens = None
@@ -267,10 +266,6 @@
         # the correct "ret" arg
         offset = mc.get_relative_pos() - jz_location
         mc.overwrite32(jz_location-4, offset)
-        # From now on this function is basically "merged" with
-        # its caller and so contains DEFAULT_FRAME_BYTES bytes
-        # plus my own return address, which we'll ignore next
-        mc.force_frame_size(DEFAULT_FRAME_BYTES + WORD)
         mc.ADD_ri(esp.value, WORD)
         mc.JMP(imm(self.propagate_exception_path))
         #
@@ -282,7 +277,6 @@
             return      # not supported (for tests, or non-translated)
         #
         self.mc = codebuf.MachineCodeBlockWrapper()
-        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         #
         # read and reset the current exception
 
@@ -336,10 +330,7 @@
         offset = mc.get_relative_pos() - jnz_location
         assert 0 < offset <= 127
         mc.overwrite(jnz_location-1, chr(offset))
-        # From now on this function is basically "merged" with
-        # its caller and so contains DEFAULT_FRAME_BYTES bytes
-        # plus my own return address, which we'll ignore next
-        mc.force_frame_size(DEFAULT_FRAME_BYTES + WORD)
+        # adjust the esp to point back to the previous return
         mc.ADD_ri(esp.value, WORD)
         mc.JMP(imm(self.propagate_exception_path))
         #
@@ -417,8 +408,6 @@
                 mc.LEA_rs(esp.value, 2 * WORD)
             self._pop_all_regs_from_frame(mc, [], withfloats, callee_only=True)
             mc.RET16_i(WORD)
-            # Note that wb_slowpath[0..3] end with a RET16_i, which must be
-            # taken care of in the caller by stack_frame_size_delta(-WORD)
         else:
             if IS_X86_32:
                 mc.MOV_rs(edx.value, 4 * WORD)
@@ -524,8 +513,6 @@
             assert len(set(inputargs)) == len(inputargs)
 
         self.setup(original_loop_token)
-        self.codemap.inherit_code_from_position(faildescr.adr_jump_offset)
-        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         descr_number = compute_unique_id(faildescr)
         if log:
             operations = self._inject_debugging_code(faildescr, operations,
@@ -686,12 +673,8 @@
         self.datablockwrapper.done()      # finish using cpu.asmmemmgr
         self.datablockwrapper = None
         allblocks = self.get_asmmemmgr_blocks(looptoken)
-        size = self.mc.get_relative_pos()
-        res = self.mc.materialize(self.cpu.asmmemmgr, allblocks,
-                                  self.cpu.gc_ll_descr.gcrootmap)
-        self.cpu.asmmemmgr.register_codemap(
-            self.codemap.get_final_bytecode(res, size))
-        return res
+        return self.mc.materialize(self.cpu.asmmemmgr, allblocks,
+                                   self.cpu.gc_ll_descr.gcrootmap)
 
     def patch_jump_for_descr(self, faildescr, adr_new_target):
         adr_jump_offset = faildescr.adr_jump_offset
@@ -702,7 +685,6 @@
         # place, but clobber the recovery stub with a jump to the real
         # target.
         mc = codebuf.MachineCodeBlockWrapper()
-        mc.force_frame_size(DEFAULT_FRAME_BYTES)
         if rx86.fits_in_32bits(offset):
             mc.writeimm32(offset)
             mc.copy_to_raw_memory(adr_jump_offset)
@@ -1773,7 +1755,6 @@
 
     def generate_propagate_error_64(self):
         assert WORD == 8
-        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         startpos = self.mc.get_relative_pos()
         self.mc.JMP(imm(self.propagate_exception_path))
         return startpos
@@ -1781,7 +1762,6 @@
     def generate_quick_failure(self, guardtok):
         """ Gather information about failure
         """
-        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         startpos = self.mc.get_relative_pos()
         fail_descr, target = self.store_info_on_descr(startpos, guardtok)
         self.mc.PUSH(imm(fail_descr))
@@ -1857,9 +1837,6 @@
 
     def _build_failure_recovery(self, exc, withfloats=False):
         mc = codebuf.MachineCodeBlockWrapper()
-        # this is jumped to, from a stack that has DEFAULT_FRAME_BYTES
-        # followed by 2 extra words just pushed
-        mc.force_frame_size(DEFAULT_FRAME_BYTES + 2 * WORD)
         self.mc = mc
 
         self._push_all_regs_to_frame(mc, [], withfloats)
@@ -1931,7 +1908,6 @@
             self.mc.J_il(rx86.Conditions[condition], 0)
         else:
             self.mc.JMP_l(0)
-            self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         guard_token.pos_jump_offset = self.mc.get_relative_pos() - 4
         self.pending_guard_tokens.append(guard_token)
 
@@ -2026,7 +2002,6 @@
         offset = jmp_location - je_location
         assert 0 < offset <= 127
         self.mc.overwrite(je_location - 1, chr(offset))
-        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         #
         return jmp_location
 
@@ -2111,8 +2086,6 @@
         if is_frame and align_stack:
             mc.SUB_ri(esp.value, 16 - WORD) # erase the return address
         mc.CALL(imm(self.wb_slowpath[helper_num]))
-        if not is_frame:
-            mc.stack_frame_size_delta(-WORD)
         if is_frame and align_stack:
             mc.ADD_ri(esp.value, 16 - WORD) # erase the return address
 
@@ -2349,7 +2322,6 @@
         offset = self.mc.get_relative_pos() - jmp_adr1
         assert 0 < offset <= 127
         self.mc.overwrite(jmp_adr1-1, chr(offset))
-        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         # write down the tid, but not if it's the result of the CALL
         self.mc.MOV(mem(eax, 0), imm(arraydescr.tid))
         # while we're at it, this line is not needed if we've done the CALL
diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py
--- a/rpython/jit/backend/x86/callbuilder.py
+++ b/rpython/jit/backend/x86/callbuilder.py
@@ -46,7 +46,7 @@
         if not self.fnloc_is_immediate:
             self.fnloc = None
             self.arglocs = arglocs + [fnloc]
-        self.start_frame_size = self.mc._frame_size
+        self.current_esp = 0     # 0 or (usually) negative, counted in bytes
 
     def select_call_release_gil_mode(self):
         AbstractCallBuilder.select_call_release_gil_mode(self)
@@ -58,15 +58,13 @@
     def subtract_esp_aligned(self, count):
         if count > 0:
             align = align_stack_words(count)
+            self.current_esp -= align * WORD
             self.mc.SUB_ri(esp.value, align * WORD)
 
-    def get_current_esp(self):
-        return self.start_frame_size - self.mc._frame_size
-
     def restore_stack_pointer(self, target_esp=0):
-        current_esp = self.get_current_esp()
-        if current_esp != target_esp:
-            self.mc.ADD_ri(esp.value, target_esp - current_esp)
+        if self.current_esp != target_esp:
+            self.mc.ADD_ri(esp.value, target_esp - self.current_esp)
+            self.current_esp = target_esp
 
     def load_result(self):
         """Overridden in CallBuilder32 and CallBuilder64"""
@@ -89,10 +87,9 @@
         # after the rearrangements done just before, ignoring the return
         # value eax, if necessary
         assert not self.is_call_release_gil
-        current_esp = self.get_current_esp()
-        self.change_extra_stack_depth = (current_esp != 0)
+        self.change_extra_stack_depth = (self.current_esp != 0)
         if self.change_extra_stack_depth:
-            self.asm.set_extra_stack_depth(self.mc, -current_esp)
+            self.asm.set_extra_stack_depth(self.mc, -self.current_esp)
         noregs = self.asm.cpu.gc_ll_descr.is_shadow_stack()
         gcmap = self.asm._regalloc.get_gcmap([eax], noregs=noregs)
         self.asm.push_gcmap(self.mc, gcmap, store=True)
@@ -133,7 +130,7 @@
             # and 5/7 words as described for asmgcroot.ASM_FRAMEDATA, for a
             # total size of JIT_USE_WORDS.  This structure is found at
             # [ESP+css].
-            css = -self.get_current_esp() + (
+            css = -self.current_esp + (
                 WORD * (PASS_ON_MY_FRAME - asmgcroot.JIT_USE_WORDS))
             assert css >= 2 * WORD
             # Save ebp
@@ -169,7 +166,7 @@
             else:
                 self.tlofs_reg = r12
             self.mc.MOV_rs(self.tlofs_reg.value,
-                           THREADLOCAL_OFS - self.get_current_esp())
+                           THREADLOCAL_OFS - self.current_esp)
         return self.tlofs_reg
 
     def save_stack_position(self):
@@ -428,10 +425,7 @@
         else:
             self.mc.CALL(self.fnloc)
             if self.callconv != FFI_DEFAULT_ABI:
-                # in the STDCALL ABI, the CALL above has an effect on
-                # the stack depth.  Adjust 'mc._frame_size'.
-                delta = self._fix_stdcall(self.callconv)
-                self.mc.stack_frame_size_delta(-delta)
+                self.current_esp += self._fix_stdcall(self.callconv)
 
     def _fix_stdcall(self, callconv):
         from rpython.rlib.clibffi import FFI_STDCALL
diff --git a/rpython/jit/backend/x86/codebuf.py b/rpython/jit/backend/x86/codebuf.py
--- a/rpython/jit/backend/x86/codebuf.py
+++ b/rpython/jit/backend/x86/codebuf.py
@@ -23,7 +23,6 @@
                               codebuilder_cls):
     def __init__(self):
         self.init_block_builder()
-        codebuilder_cls.__init__(self)
         # a list of relative positions; for each position p, the bytes
         # at [p-4:p] encode an absolute address that will need to be
         # made relative.  Only works on 32-bit!
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -12,7 +12,7 @@
      valid_addressing_size)
 from rpython.jit.backend.x86 import rx86
 from rpython.jit.backend.x86.arch import (WORD, JITFRAME_FIXED_SIZE, IS_X86_32,
-    IS_X86_64, DEFAULT_FRAME_BYTES)
+    IS_X86_64)
 from rpython.jit.backend.x86.jump import remap_frame_layout_mixed
 from rpython.jit.backend.x86.regloc import (FrameLoc, RegLoc, ConstFloatLoc,
     FloatImmedLoc, ImmedLoc, imm, imm0, imm1, ecx, eax, edx, ebx, esi, edi,
@@ -314,7 +314,6 @@
         while i < len(operations):
             op = operations[i]
             self.assembler.mc.mark_op(op)
-            assert self.assembler.mc._frame_size == DEFAULT_FRAME_BYTES
             self.rm.position = i
             self.xrm.position = i
             if op.has_no_side_effect() and op.result not in self.longevity:
@@ -1298,7 +1297,7 @@
         assembler.closing_jump(self.jump_target_descr)
 
     def consider_debug_merge_point(self, op):
-        self.assembler.debug_merge_point(op)
+        pass
 
     def consider_jit_debug(self, op):
         pass
diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -447,11 +447,6 @@
 class AbstractX86CodeBuilder(object):
     """Abstract base class."""
 
-    def __init__(self):
-        self.frame_positions = []
-        self.frame_assignments = []
-        self.force_frame_size(self.WORD)
-
     def writechar(self, char):
         raise NotImplementedError
 
@@ -469,23 +464,6 @@
         self.writechar(chr((imm >> 16) & 0xFF))
         self.writechar(chr((imm >> 24) & 0xFF))
 
-    def force_frame_size(self, frame_size):
-        self.frame_positions.append(self.get_relative_pos())
-        self.frame_assignments.append(frame_size)
-        self._frame_size = frame_size
-
-    def stack_frame_size_delta(self, delta):
-        "Called when we generate an instruction that changes the value of ESP"
-        self._frame_size += delta
-        self.frame_positions.append(self.get_relative_pos()) 
-        self.frame_assignments.append(self._frame_size)
-        assert self._frame_size >= self.WORD
-
-    def check_stack_size_at_ret(self):
-        assert self._frame_size == self.WORD
-        if not we_are_translated():
-            self._frame_size = None
-
     # ------------------------------ MOV ------------------------------
 
     MOV_ri = insn(register(1), '\xB8', immediate(2))
@@ -496,24 +474,14 @@
     INC_m = insn(rex_w, '\xFF', orbyte(0), mem_reg_plus_const(1))
     INC_j = insn(rex_w, '\xFF', orbyte(0), abs_(1))
 
-    AD1_ri,ADD_rr,ADD_rb,_,_,ADD_rm,ADD_rj,_,_ = common_modes(0)
+    ADD_ri,ADD_rr,ADD_rb,_,_,ADD_rm,ADD_rj,_,_ = common_modes(0)
     OR_ri, OR_rr, OR_rb, _,_,OR_rm, OR_rj, _,_ = common_modes(1)
     AND_ri,AND_rr,AND_rb,_,_,AND_rm,AND_rj,_,_ = common_modes(4)
-    SU1_ri,SUB_rr,SUB_rb,_,_,SUB_rm,SUB_rj,SUB_ji8,SUB_mi8 = common_modes(5)
+    SUB_ri,SUB_rr,SUB_rb,_,_,SUB_rm,SUB_rj,SUB_ji8,SUB_mi8 = common_modes(5)
     SBB_ri,SBB_rr,SBB_rb,_,_,SBB_rm,SBB_rj,_,_ = common_modes(3)
     XOR_ri,XOR_rr,XOR_rb,_,_,XOR_rm,XOR_rj,_,_ = common_modes(6)
     CMP_ri,CMP_rr,CMP_rb,CMP_bi,CMP_br,CMP_rm,CMP_rj,_,_ = common_modes(7)
 
-    def ADD_ri(self, reg, immed):
-        self.AD1_ri(reg, immed)
-        if reg == R.esp:
-            self.stack_frame_size_delta(-immed)
-
-    def SUB_ri(self, reg, immed):
-        self.SU1_ri(reg, immed)
-        if reg == R.esp:
-            self.stack_frame_size_delta(+immed)
-
     CMP_mi8 = insn(rex_w, '\x83', orbyte(7<<3), mem_reg_plus_const(1), immediate(2, 'b'))
     CMP_mi32 = insn(rex_w, '\x81', orbyte(7<<3), mem_reg_plus_const(1), immediate(2))
     CMP_mi = select_8_or_32_bit_immed(CMP_mi8, CMP_mi32)
@@ -563,65 +531,30 @@
     # ------------------------------ Misc stuff ------------------------------
 
     NOP = insn('\x90')
-    RE1 = insn('\xC3')
-    RE116_i = insn('\xC2', immediate(1, 'h'))
+    RET = insn('\xC3')
+    RET16_i = insn('\xC2', immediate(1, 'h'))
 
-    def RET(self):
-        self.check_stack_size_at_ret()
-        self.RE1()
+    PUSH_r = insn(rex_nw, register(1), '\x50')
+    PUSH_b = insn(rex_nw, '\xFF', orbyte(6<<3), stack_bp(1))
+    PUSH_m = insn(rex_nw, '\xFF', orbyte(6<<3), mem_reg_plus_const(1))
+    PUSH_i8 = insn('\x6A', immediate(1, 'b'))
+    PUSH_i32 = insn('\x68', immediate(1, 'i'))
+    def PUSH_i(mc, immed):
+        if single_byte(immed):
+            mc.PUSH_i8(immed)
+        else:
+            mc.PUSH_i32(immed)
 
-    def RET16_i(self, immed):
-        self.check_stack_size_at_ret()

From noreply at buildbot.pypy.org  Fri Feb  6 12:06:56 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  6 Feb 2015 12:06:56 +0100 (CET)
Subject: [pypy-commit] pypy default: update the error message now that we
 don't have --output any more
Message-ID: <20150206110656.A1DDA1C02A4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r75734:a2df20419f11
Date: 2015-02-06 12:06 +0100
http://bitbucket.org/pypy/pypy/changeset/a2df20419f11/

Log:	update the error message now that we don't have --output any more

diff --git a/rpython/translator/goal/translate.py b/rpython/translator/goal/translate.py
--- a/rpython/translator/goal/translate.py
+++ b/rpython/translator/goal/translate.py
@@ -308,7 +308,9 @@
                 samefile = this_exe.samefile(exe_name)
                 assert not samefile, (
                     'Output file %s is the currently running '
-                    'interpreter (use --output=...)' % exe_name)
+                    'interpreter (please move the executable, and '
+                    'possibly its associated libpypy-c, somewhere else '
+                    'before you execute it)' % exe_name)
             except EnvironmentError:
                 pass
 

From noreply at buildbot.pypy.org  Fri Feb  6 12:44:30 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  6 Feb 2015 12:44:30 +0100 (CET)
Subject: [pypy-commit] pypy stackroot-speedup-2: ready to merge
Message-ID: <20150206114430.D38B71C02A4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stackroot-speedup-2
Changeset: r75735:3a5badc0148c
Date: 2015-02-06 12:20 +0100
http://bitbucket.org/pypy/pypy/changeset/3a5badc0148c/

Log:	ready to merge


From noreply at buildbot.pypy.org  Fri Feb  6 12:44:32 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  6 Feb 2015 12:44:32 +0100 (CET)
Subject: [pypy-commit] pypy default: Yet another attempt to fix issue 1971.
Message-ID: <20150206114432.11F101C02A4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r75736:e51a71b67c43
Date: 2015-02-06 12:44 +0100
http://bitbucket.org/pypy/pypy/changeset/e51a71b67c43/

Log:	Yet another attempt to fix issue 1971.

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -235,6 +235,11 @@
 
         config.translation.suggest(check_str_without_nul=True)
         config.translation.suggest(shared=True)
+        if config.translation.shared:
+            if config.translation.output is not None:
+                raise Exception("Cannot use the --output option with PyPy "
+                                "when --shared is on (it is by default). "
+                                "See issue #1971.")
 
         if config.translation.thread:
             config.objspace.usemodules.thread = True
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -151,9 +151,7 @@
                default=False, cmdline="--dont-write-c-files"),
     ArbitraryOption("instrumentctl", "internal",
                default=None),
-    StrOption("output", "Output file name (don't change for PyPy!"
-              " doesn't work with virtualenv+shared: issue 1971)",
-              cmdline="--really-force-output"),
+    StrOption("output", "Output file name", cmdline="--output"),
     StrOption("secondaryentrypoints",
             "Comma separated list of keys choosing secondary entrypoints",
             cmdline="--entrypoints", default="main"),

From noreply at buildbot.pypy.org  Fri Feb  6 12:51:51 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  6 Feb 2015 12:51:51 +0100 (CET)
Subject: [pypy-commit] pypy default: hg merge stackroot-speedup-2
Message-ID: <20150206115151.111EF1C115A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r75737:2f0b1cc091f5
Date: 2015-02-06 12:50 +0100
http://bitbucket.org/pypy/pypy/changeset/2f0b1cc091f5/

Log:	hg merge stackroot-speedup-2

	This is a quick hack to avoid walking the complete stack in every
	minor GC collection. It only works with asmgcc so far. We'll see
	if it shows some speed-ups. (Adapting shadowstack for it should not
	be hard.)

diff --git a/rpython/jit/backend/x86/arch.py b/rpython/jit/backend/x86/arch.py
--- a/rpython/jit/backend/x86/arch.py
+++ b/rpython/jit/backend/x86/arch.py
@@ -35,7 +35,9 @@
     PASS_ON_MY_FRAME = 15
     JITFRAME_FIXED_SIZE = 6 + 8 * 2 # 6 GPR + 8 XMM * 2 WORDS/float
     # 'threadlocal_addr' is passed as 2nd argument on the stack,
-    # and it can be left here for when it is needed
+    # and it can be left here for when it is needed.  As an additional hack,
+    # with asmgcc, it is made odd-valued to mean "already seen this frame
+    # during the previous minor collection".
     THREADLOCAL_OFS = (FRAME_FIXED_SIZE + 2) * WORD
 else:
     # rbp + rbx + r12 + r13 + r14 + r15 + threadlocal + 12 extra words = 19
@@ -43,7 +45,9 @@
     PASS_ON_MY_FRAME = 12
     JITFRAME_FIXED_SIZE = 28 # 13 GPR + 15 XMM
     # 'threadlocal_addr' is passed as 2nd argument in %esi,
-    # and is moved into this frame location
+    # and is moved into this frame location.  As an additional hack,
+    # with asmgcc, it is made odd-valued to mean "already seen this frame
+    # during the previous minor collection".
     THREADLOCAL_OFS = (FRAME_FIXED_SIZE - 1) * WORD
 
 assert PASS_ON_MY_FRAME >= 12       # asmgcc needs at least JIT_USE_WORDS + 3
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1980,6 +1980,23 @@
 
     def _call_assembler_emit_call(self, addr, argloc, _):
         threadlocal_loc = RawEspLoc(THREADLOCAL_OFS, INT)
+        if self._is_asmgcc():
+            # We need to remove the bit "already seen during the
+            # previous minor collection" instead of passing this
+            # value directly.
+            if IS_X86_64:
+                tmploc = esi    # already the correct place
+                if argloc is tmploc:
+                    self.mc.MOV_rr(esi.value, edi.value)
+                    argloc = edi
+            else:
+                tmploc = eax
+                if tmploc is argloc:
+                    tmploc = edx
+            self.mc.MOV(tmploc, threadlocal_loc)
+            self.mc.AND_ri(tmploc.value, ~1)
+            threadlocal_loc = tmploc
+        #
         self.simple_call(addr, [argloc, threadlocal_loc])
 
     def _call_assembler_emit_helper_call(self, addr, arglocs, result_loc):
@@ -2355,6 +2372,8 @@
         assert self.cpu.translate_support_code
         assert isinstance(resloc, RegLoc)
         self.mc.MOV_rs(resloc.value, THREADLOCAL_OFS)
+        if self._is_asmgcc():
+            self.mc.AND_ri(resloc.value, ~1)
         self.load_from_mem(resloc, addr_add_const(resloc, offset),
                            imm(size), imm(sign))
 
diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py
--- a/rpython/jit/backend/x86/callbuilder.py
+++ b/rpython/jit/backend/x86/callbuilder.py
@@ -167,6 +167,8 @@
                 self.tlofs_reg = r12
             self.mc.MOV_rs(self.tlofs_reg.value,
                            THREADLOCAL_OFS - self.current_esp)
+            if self.asm._is_asmgcc():
+                self.mc.AND_ri(self.tlofs_reg.value, ~1)
         return self.tlofs_reg
 
     def save_stack_position(self):
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -375,6 +375,10 @@
         # the nursery.
         self.pinned_objects_in_nursery = 0
         #
+        # This flag is set if the previous minor collection found at least
+        # one pinned object alive.
+        self.any_pinned_object_kept = False
+        #
         # Keeps track of old objects pointing to pinned objects. These objects
         # must be traced every minor collection. Without tracing them the
         # referenced pinned object wouldn't be visited and therefore collected.
@@ -1489,7 +1493,9 @@
         # The following counter keeps track of alive and pinned young objects
         # inside the nursery. We reset it here and increace it in
         # '_trace_drag_out()'.
+        any_pinned_object_from_earlier = self.any_pinned_object_kept
         self.pinned_objects_in_nursery = 0
+        self.any_pinned_object_kept = False
         #
         # Before everything else, remove from 'old_objects_pointing_to_young'
         # the young arrays.
@@ -1513,7 +1519,7 @@
         # are copied out or flagged.  They are also added to the list
         # 'old_objects_pointing_to_young'.
         self.nursery_surviving_size = 0
-        self.collect_roots_in_nursery()
+        self.collect_roots_in_nursery(any_pinned_object_from_earlier)
         #
         # visit all objects that are known for pointing to pinned
         # objects. This way we populate 'surviving_pinned_objects'
@@ -1649,7 +1655,7 @@
     def _visit_old_objects_pointing_to_pinned(self, obj, ignore):
         self.trace(obj, self._trace_drag_out, obj)
 
-    def collect_roots_in_nursery(self):
+    def collect_roots_in_nursery(self, any_pinned_object_from_earlier):
         # we don't need to trace prebuilt GcStructs during a minor collect:
         # if a prebuilt GcStruct contains a pointer to a young object,
         # then the write_barrier must have ensured that the prebuilt
@@ -1659,10 +1665,19 @@
             callback = IncrementalMiniMarkGC._trace_drag_out1_marking_phase
         else:
             callback = IncrementalMiniMarkGC._trace_drag_out1
+        #
+        # Note a subtlety: if the nursery contains pinned objects "from
+        # earlier", i.e. created earlier than the previous minor
+        # collection, then we can't use the "is_minor=True" optimization.
+        # We really need to walk the complete stack to be sure we still
+        # see them.
+        use_jit_frame_stoppers = not any_pinned_object_from_earlier
+        #
         self.root_walker.walk_roots(
             callback,     # stack roots
             callback,     # static in prebuilt non-gc
-            None)         # static in prebuilt gc
+            None,         # static in prebuilt gc
+            is_minor=use_jit_frame_stoppers)
         debug_stop("gc-minor-walkroots")
 
     def collect_cardrefs_to_nursery(self):
@@ -1844,6 +1859,7 @@
             self.surviving_pinned_objects.append(
                 llarena.getfakearenaaddress(obj - size_gc_header))
             self.pinned_objects_in_nursery += 1
+            self.any_pinned_object_kept = True
             return
         else:
             # First visit to an object that has already a shadow.
diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -1322,7 +1322,8 @@
         self.root_walker.walk_roots(
             MiniMarkGC._trace_drag_out1,  # stack roots
             MiniMarkGC._trace_drag_out1,  # static in prebuilt non-gc
-            None)                         # static in prebuilt gc
+            None,                         # static in prebuilt gc
+            is_minor=True)
         debug_stop("gc-minor-walkroots")
 
     def collect_cardrefs_to_nursery(self):
diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py
--- a/rpython/memory/gc/test/test_direct.py
+++ b/rpython/memory/gc/test/test_direct.py
@@ -34,7 +34,8 @@
 
     def walk_roots(self, collect_stack_root,
                    collect_static_in_prebuilt_nongc,
-                   collect_static_in_prebuilt_gc):
+                   collect_static_in_prebuilt_gc,
+                   is_minor=False):
         gc = self.tester.gc
         layoutbuilder = self.tester.layoutbuilder
         if collect_static_in_prebuilt_gc:
diff --git a/rpython/memory/gctransform/asmgcroot.py b/rpython/memory/gctransform/asmgcroot.py
--- a/rpython/memory/gctransform/asmgcroot.py
+++ b/rpython/memory/gctransform/asmgcroot.py
@@ -340,9 +340,10 @@
         # called first, to initialize self.belongs_to_current_thread.
         assert not hasattr(self, 'gc_detach_callback_pieces_ptr')
 
-    def walk_stack_roots(self, collect_stack_root):
+    def walk_stack_roots(self, collect_stack_root, is_minor=False):
         gcdata = self.gcdata
         gcdata._gc_collect_stack_root = collect_stack_root
+        gcdata._gc_collect_is_minor = is_minor
         pypy_asm_stackwalk(llhelper(ASM_CALLBACK_PTR, self._asm_callback),
                            gcrootanchor)
 
@@ -468,6 +469,13 @@
             if gc.points_to_valid_gc_object(addr):
                 collect_stack_root(gc, addr)
         #
+        # small hack: the JIT reserves THREADLOCAL_OFS's last bit for
+        # us.  We use it to store an "already traced past this frame"
+        # flag.
+        if self._with_jit and self.gcdata._gc_collect_is_minor:
+            if self.mark_jit_frame_can_stop(callee):
+                return False
+        #
         # track where the caller_frame saved the registers from its own
         # caller
         #
@@ -548,6 +556,19 @@
         else:  # kind == LOC_EBP_MINUS:   at -N(%ebp)
             return ebp_in_caller - offset
 
+    def mark_jit_frame_can_stop(self, callee):
+        location = self._shape_decompressor.get_threadlocal_loc()
+        if location == LOC_NOWHERE:
+            return False
+        addr = self.getlocation(callee, llmemory.NULL, location)
+        #
+        x = addr.signed[0]
+        if x & 1:
+            return True            # this JIT stack frame is already marked!
+        else:
+            addr.signed[0] = x | 1    # otherwise, mark it but don't stop
+            return False
+
 
 LOC_REG       = 0
 LOC_ESP_PLUS  = 1
@@ -729,6 +750,17 @@
             llop.debug_fatalerror(lltype.Void, "asmgcroot: invalid index")
             return 0   # annotator fix
 
+    def get_threadlocal_loc(self):
+        index = self.jit_index
+        if index < 0:
+            return LOC_NOWHERE     # case "outside the jit"
+        else:
+            # case "in the jit"
+            from rpython.jit.backend.x86.arch import THREADLOCAL_OFS, WORD
+            return (LOC_ESP_PLUS |
+                    ((THREADLOCAL_OFS // WORD + self.extra_stack_depth) << 2))
+
+
 # ____________________________________________________________
 
 #
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -1462,7 +1462,8 @@
 
     def walk_roots(self, collect_stack_root,
                    collect_static_in_prebuilt_nongc,
-                   collect_static_in_prebuilt_gc):
+                   collect_static_in_prebuilt_gc,
+                   is_minor=False):
         gcdata = self.gcdata
         gc = self.gc
         if collect_static_in_prebuilt_nongc:
@@ -1482,7 +1483,7 @@
                     collect_static_in_prebuilt_gc(gc, result)
                 addr += sizeofaddr
         if collect_stack_root:
-            self.walk_stack_roots(collect_stack_root)     # abstract
+            self.walk_stack_roots(collect_stack_root, is_minor)     # abstract
 
     def finished_minor_collection(self):
         func = self.finished_minor_collection_func
diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py
--- a/rpython/memory/gctransform/shadowstack.py
+++ b/rpython/memory/gctransform/shadowstack.py
@@ -99,7 +99,7 @@
         self.shadow_stack_pool.initial_setup()
         BaseRootWalker.setup_root_walker(self)
 
-    def walk_stack_roots(self, collect_stack_root):
+    def walk_stack_roots(self, collect_stack_root, is_minor=False):
         gcdata = self.gcdata
         self.rootstackhook(collect_stack_root,
                            gcdata.root_stack_base, gcdata.root_stack_top)
diff --git a/rpython/memory/gcwrapper.py b/rpython/memory/gcwrapper.py
--- a/rpython/memory/gcwrapper.py
+++ b/rpython/memory/gcwrapper.py
@@ -191,7 +191,8 @@
 
     def walk_roots(self, collect_stack_root,
                    collect_static_in_prebuilt_nongc,
-                   collect_static_in_prebuilt_gc):
+                   collect_static_in_prebuilt_gc,
+                   is_minor=False):
         gcheap = self.gcheap
         gc = gcheap.gc
         if collect_static_in_prebuilt_gc:
@@ -203,7 +204,7 @@
                 if self.gcheap.gc.points_to_valid_gc_object(addrofaddr):
                     collect_static_in_prebuilt_nongc(gc, addrofaddr)
         if collect_stack_root:
-            for addrofaddr in gcheap.llinterp.find_roots():
+            for addrofaddr in gcheap.llinterp.find_roots(is_minor):
                 if self.gcheap.gc.points_to_valid_gc_object(addrofaddr):
                     collect_stack_root(gc, addrofaddr)
 
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -146,13 +146,21 @@
                            }
             return self._tlobj
 
-    def find_roots(self):
+    def find_roots(self, is_minor=False):
         """Return a list of the addresses of the roots."""
         #log.findroots("starting")
         roots = []
-        for frame in self.frame_stack:
+        for frame in reversed(self.frame_stack):
             #log.findroots("graph", frame.graph.name)
             frame.find_roots(roots)
+            # If a call is done with 'is_minor=True', we can stop after the
+            # first frame in the stack that was already seen by the previous
+            # call with 'is_minor=True'.  (We still need to trace that frame,
+            # but not its callers.)
+            if is_minor:
+                if getattr(frame, '_find_roots_already_seen', False):
+                    break
+                frame._find_roots_already_seen = True
         return roots
 
     def find_exception(self, exc):
diff --git a/rpython/translator/c/gcc/test/test_asmgcroot.py b/rpython/translator/c/gcc/test/test_asmgcroot.py
--- a/rpython/translator/c/gcc/test/test_asmgcroot.py
+++ b/rpython/translator/c/gcc/test/test_asmgcroot.py
@@ -251,13 +251,17 @@
     def define_callback_with_collect(cls):
         return lambda: 0
 
-class TestAsmGCRootWithSemiSpaceGC_Shared(TestAsmGCRootWithSemiSpaceGC):
-    @classmethod
-    def make_config(cls):
-        config = TestAsmGCRootWithSemiSpaceGC.make_config()
-        config.translation.shared = True
-        return config
+#class TestAsmGCRootWithSemiSpaceGC_Shared(TestAsmGCRootWithSemiSpaceGC):
+#    @classmethod
+#    def make_config(cls):
+#        config = TestAsmGCRootWithSemiSpaceGC.make_config()
+#        config.translation.shared = True
+#        return config
 
 class TestAsmGCRootWithHybridTagged(AbstractTestAsmGCRoot,
                                     test_newgc.TestHybridTaggedPointers):
     pass
+
+class TestAsmGCRootWithIncrementalMinimark(AbstractTestAsmGCRoot,
+                                    test_newgc.TestIncrementalMiniMarkGC):
+    pass

From noreply at buildbot.pypy.org  Fri Feb  6 16:12:18 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri,  6 Feb 2015 16:12:18 +0100 (CET)
Subject: [pypy-commit] stmgc default: Change back to accessing read-markers
 through a non-aliasing type to allow for
Message-ID: <20150206151218.94D761C0100@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: 
Changeset: r1609:ddd64fd996c7
Date: 2015-02-06 16:13 +0100
http://bitbucket.org/pypy/stmgc/changeset/ddd64fd996c7/

Log:	Change back to accessing read-markers through a non-aliasing type to
	allow for better optimizations by the c compiler. It should be safe
	now as we do not ever rely on read markers in the signal handler any
	more (by doing an stm_validate()).

diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -521,7 +521,8 @@
         long i;
         for (i = 1; i < NB_SEGMENTS; i++) {
             /* reset read marker */
-            *((char *)(get_segment_base(i) + (((uintptr_t)obj) >> 4))) = 0;
+            ((struct stm_read_marker_s *)
+             (get_segment_base(i) + (((uintptr_t)obj) >> 4)))->rm = 0;
         }
         return false;
     }
@@ -547,7 +548,8 @@
         long i;
         for (i = 1; i < NB_SEGMENTS; i++) {
             /* reset read marker */
-            *((char *)(get_segment_base(i) + (((uintptr_t)obj) >> 4))) = 0;
+            ((struct stm_read_marker_s *)
+             (get_segment_base(i) + (((uintptr_t)obj) >> 4)))->rm = 0;
         }
         return false;
     }
diff --git a/c8/stm/misc.c b/c8/stm/misc.c
--- a/c8/stm/misc.c
+++ b/c8/stm/misc.c
@@ -31,7 +31,8 @@
 
 bool _stm_was_read(object_t *obj)
 {
-    uint8_t rm = *((char *)(STM_SEGMENT->segment_base + (((uintptr_t)obj) >> 4)));
+    uint8_t rm = ((struct stm_read_marker_s *)
+                  (STM_SEGMENT->segment_base + (((uintptr_t)obj) >> 4)))->rm;
     assert(rm <= STM_SEGMENT->transaction_read_version);
     return rm == STM_SEGMENT->transaction_read_version;
 }
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -266,7 +266,8 @@
 
                 /* mark slot as unread (it can only have the read marker
                    in this segment) */
-                *((char *)(pseg->pub.segment_base + (((uintptr_t)obj) >> 4))) = 0;
+                ((struct stm_read_marker_s *)
+                 (pseg->pub.segment_base + (((uintptr_t)obj) >> 4)))->rm = 0;
 
                 _stm_large_free(stm_object_pages + item->addr);
             } TREE_LOOP_END;
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -24,8 +24,19 @@
 
 typedef TLPREFIX struct object_s object_t;
 typedef TLPREFIX struct stm_segment_info_s stm_segment_info_t;
+typedef TLPREFIX struct stm_read_marker_s stm_read_marker_t;
 typedef TLPREFIX char stm_char;
 
+struct stm_read_marker_s {
+    /* In every segment, every object has a corresponding read marker.
+       We assume that objects are at least 16 bytes long, and use
+       their address divided by 16.  The read marker is equal to
+       'STM_SEGMENT->transaction_read_version' if and only if the
+       object was read in the current transaction.  The nurseries
+       also have corresponding read markers, but they are never used. */
+    uint8_t rm;
+};
+
 struct stm_segment_info_s {
     uint8_t transaction_read_version;
     int segment_num;
@@ -80,7 +91,6 @@
 
 bool _stm_is_accessible_page(uintptr_t pagenum);
 
-long stm_can_move(object_t *obj);
 void _stm_test_switch(stm_thread_local_t *tl);
 void _stm_test_switch_segment(int segnum);
 void _push_obj_to_other_segments(object_t *obj);
@@ -149,7 +159,8 @@
 __attribute__((always_inline))
 static inline void stm_read(object_t *obj)
 {
-    *((stm_char *)(((uintptr_t)obj) >> 4)) = STM_SEGMENT->transaction_read_version;
+    ((stm_read_marker_t *)(((uintptr_t)obj) >> 4))->rm =
+        STM_SEGMENT->transaction_read_version;
 }
 
 __attribute__((always_inline))
@@ -226,6 +237,8 @@
 long stm_id(object_t *obj);
 void stm_set_prebuilt_identityhash(object_t *obj, long hash);
 
+long stm_can_move(object_t *obj);
+
 object_t *stm_setup_prebuilt(object_t *);
 object_t *stm_setup_prebuilt_weakref(object_t *);
 

From noreply at buildbot.pypy.org  Fri Feb  6 17:28:11 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  6 Feb 2015 17:28:11 +0100 (CET)
Subject: [pypy-commit] stmgc default: remove unused type
Message-ID: <20150206162811.0AE5F1C0100@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1610:ee8dd5569dea
Date: 2015-02-06 17:28 +0100
http://bitbucket.org/pypy/stmgc/changeset/ee8dd5569dea/

Log:	remove unused type

diff --git a/c7/stmgc.h b/c7/stmgc.h
--- a/c7/stmgc.h
+++ b/c7/stmgc.h
@@ -25,7 +25,6 @@
 typedef TLPREFIX struct object_s object_t;
 typedef TLPREFIX struct stm_segment_info_s stm_segment_info_t;
 typedef TLPREFIX struct stm_read_marker_s stm_read_marker_t;
-typedef TLPREFIX struct stm_creation_marker_s stm_creation_marker_t;
 typedef TLPREFIX char stm_char;
 
 struct stm_read_marker_s {

From noreply at buildbot.pypy.org  Fri Feb  6 17:29:58 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  6 Feb 2015 17:29:58 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c8: hg merge stmgc-c7, and import stmgc/c8
Message-ID: <20150206162958.97B461C0100@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c8
Changeset: r75738:053bac719b8c
Date: 2015-02-06 17:29 +0100
http://bitbucket.org/pypy/pypy/changeset/053bac719b8c/

Log:	hg merge stmgc-c7, and import stmgc/c8

diff too long, truncating to 2000 out of 4037 lines

diff --git a/lib_pypy/atomic.py b/lib_pypy/atomic.py
deleted file mode 100644
--- a/lib_pypy/atomic.py
+++ /dev/null
@@ -1,24 +0,0 @@
-"""
-API for accessing the multithreading extensions of PyPy
-"""
-import thread
-
-try:
-    from __pypy__ import thread as _thread
-    from __pypy__.thread import (atomic, getsegmentlimit,
-                                 hint_commit_soon, is_atomic)
-except ImportError:
-    # Not a STM-enabled PyPy.  We can still provide a version of 'atomic'
-    # that is good enough for our purposes.  With this limited version,
-    # an atomic block in thread X will not prevent running thread Y, if
-    # thread Y is not within an atomic block at all.
-    atomic = thread.allocate_lock()
-
-    def getsegmentlimit():
-        return 1
-
-    def hint_commit_soon():
-        pass
-
-    def is_atomic():
-        return atomic.locked()
diff --git a/lib_pypy/pypy_test/test_transaction.py b/lib_pypy/pypy_test/test_transaction.py
--- a/lib_pypy/pypy_test/test_transaction.py
+++ b/lib_pypy/pypy_test/test_transaction.py
@@ -1,3 +1,4 @@
+import py
 from lib_pypy import transaction
 
 N = 1000
@@ -7,9 +8,9 @@
 def test_simple_random_order():
     for x in range(N):
         lst = []
-        for i in range(10):
-            transaction.add(lst.append, i)
-        transaction.run()
+        with transaction.TransactionQueue():
+            for i in range(10):
+                transaction.add(lst.append, i)
         if VERBOSE:
             print lst
         assert sorted(lst) == range(10), lst
@@ -22,8 +23,8 @@
             i += 1
             if i < 10:
                 transaction.add(do_stuff, i)
-        transaction.add(do_stuff, 0)
-        transaction.run()
+        with transaction.TransactionQueue():
+            transaction.add(do_stuff, 0)
         if VERBOSE:
             print lst
         assert lst == range(10), lst
@@ -36,9 +37,9 @@
             j += 1
             if j < 10:
                 transaction.add(do_stuff, i, j)
-        for i in range(5):
-            transaction.add(do_stuff, i, 0)
-        transaction.run()
+        with transaction.TransactionQueue():
+            for i in range(5):
+                transaction.add(do_stuff, i, 0)
         if VERBOSE:
             print lsts
         assert lsts == (range(10),) * 5, lsts
@@ -56,10 +57,10 @@
             else:
                 lsts[i].append('foo')
                 raise FooError
-        for i in range(10):
-            transaction.add(do_stuff, i, 0)
         try:
-            transaction.run()
+            with transaction.TransactionQueue():
+                for i in range(10):
+                    transaction.add(do_stuff, i, 0)
         except FooError:
             pass
         else:
@@ -77,8 +78,9 @@
 
 
 def test_number_of_transactions_reported():
-    transaction.add(lambda: None)
-    transaction.run()
+    py.test.skip("not reimplemented")
+    with transaction.TransactionQueue():
+        transaction.add(lambda: None)
     assert transaction.number_of_transactions_in_last_run() == 1
 
     def add_transactions(l):
@@ -86,11 +88,52 @@
             for x in range(l[0]):
                 transaction.add(add_transactions, l[1:])
 
-    transaction.add(add_transactions, [10, 10, 10])
-    transaction.run()
+    with transaction.TransactionQueue():
+        transaction.add(add_transactions, [10, 10, 10])
     assert transaction.number_of_transactions_in_last_run() == 1111
 
 
+def test_stmidset():
+    s = transaction.stmidset()
+    key1 = []
+    key2 = []
+    s.add(key1)
+    assert key1 in s
+    assert key2 not in s
+    s.add(key2)
+    assert key1 in s
+    assert key2 in s
+    s.remove(key1)
+    assert key1 not in s
+    assert key2 in s
+    py.test.raises(KeyError, s.remove, key1)
+    s.discard(key1)
+    assert key1 not in s
+    assert key2 in s
+    s.discard(key2)
+    assert key2 not in s
+
+def test_stmiddict():
+    d = transaction.stmiddict()
+    key1 = []
+    key2 = []
+    py.test.raises(KeyError, "d[key1]")
+    d[key1] = 5
+    assert d[key1] == 5
+    assert key1 in d
+    assert d.get(key1) == 5
+    assert d.get(key1, 42) == 5
+    del d[key1]
+    py.test.raises(KeyError, "d[key1]")
+    assert key1 not in d
+    assert d.get(key1) is None
+    assert d.get(key1, 42) == 42
+    assert d.setdefault(key1, 42) == 42
+    assert d.setdefault(key1, 43) == 42
+    assert d.setdefault(key2) is None
+    assert d[key2] is None
+
+
 def run_tests():
     for name in sorted(globals().keys()):
         if name.startswith('test_'):
diff --git a/lib_pypy/transaction.py b/lib_pypy/transaction.py
--- a/lib_pypy/transaction.py
+++ b/lib_pypy/transaction.py
@@ -6,16 +6,16 @@
 give a simple-to-use API.
 
 Note that some rough edges still need to be sorted out; for now you
-have to explicitly set the number of threads to use by calling
-set_num_threads(), or you get a default of 4.
-
+have to explicitly set the number of threads to use by passing the
+'nb_segments' argument to TransactionQueue(), or you get a default of 4
+(or whatever the compiled-in maximum is).
 """
 
 from __future__ import with_statement
 import sys, thread, collections, cStringIO, linecache
 
 try:
-    from __pypy__.thread import atomic, is_atomic
+    from pypystm import atomic, is_atomic
 except ImportError:
     # Not a STM-enabled PyPy.  We can use a regular lock for 'atomic',
     # which is good enough for our purposes.  With this limited version,
@@ -37,63 +37,117 @@
     signals_enabled = _SignalsEnabled()
 
 try:
-    from __pypy__.thread import last_abort_info
+    from pypystm import hint_commit_soon
 except ImportError:
     # Not a STM-enabled PyPy.
-    def last_abort_info():
+    def hint_commit_soon():
         return None
 
+try:
+    from pypystm import getsegmentlimit
+except ImportError:
+    # Not a STM-enabled PyPy.
+    def getsegmentlimit():
+        return 1
 
-def set_num_threads(num):
-    """Set the number of threads to use."""
-    if num < 1:
-        raise ValueError("'num' must be at least 1, got %r" % (num,))
-    if _thread_pool.in_transaction:
-        raise TransactionError("cannot change the number of threads "
-                               "while running transactions")
-    _thread_pool.num_threads = num
+try:
+    from pypystm import hashtable
+except ImportError:
+    # Not a STM-enabled PyPy.
+    hashtable = dict
+
+class stmidset(object):
+    def __init__(self):
+        self._hashtable = hashtable()
+
+    def add(self, key):
+        self._hashtable[id(key)] = key
+
+    def __contains__(self, key):
+        return id(key) in self._hashtable
+
+    def remove(self, key):
+        del self._hashtable[id(key)]
+
+    def discard(self, key):
+        try:
+            del self._hashtable[id(key)]
+        except KeyError:
+            pass
+
+class stmiddict(object):
+    def __init__(self):
+        self._hashtable = hashtable()
+
+    def __getitem__(self, key):
+        return self._hashtable[id(key)][1]
+
+    def __setitem__(self, key, value):
+        self._hashtable[id(key)] = (key, value)
+
+    def __delitem__(self, key):
+        del self._hashtable[id(key)]
+
+    def __contains__(self, key):
+        return id(key) in self._hashtable
+
+    def get(self, key, default=None):
+        try:
+            return self._hashtable[id(key)][1]
+        except KeyError:
+            return default
+
+    def setdefault(self, key, default=None):
+        return self._hashtable.setdefault(id(key), (key, default))[1]
+
+
+# ------------------------------------------------------------
 
 
 class TransactionError(Exception):
     pass
 
 
-# XXX right now uses the same API as the old pypy-stm.  This will
-# be redesigned later.
-
 def add(f, *args, **kwds):
-    """Register the call 'f(*args, **kwds)' as running a new
-    transaction.  If we are currently running in a transaction too, the
-    new transaction will only start after the end of the current
-    transaction.  Note that if the current transaction or another running
-    in the meantime raises an exception, all pending transactions are
-    cancelled.
+    """Register a new transaction that will be done by 'f(*args, **kwds)'.
+    Must be called within the transaction in the "with TransactionQueue()"
+    block, or within a transaction started by this one, directly or
+    indirectly.
     """
     _thread_local.pending.append((f, args, kwds))
 
 
-def run():
-    """Run the pending transactions, as well as all transactions started
-    by them, and so on.  The order is random and undeterministic.  Must
-    be called from the main program, i.e. not from within another
-    transaction.  If at some point all transactions are done, returns.
-    If a transaction raises an exception, it propagates here; in this
-    case all pending transactions are cancelled.
+class TransactionQueue(object):
+    """Use in 'with TransactionQueue():'.  Creates a queue of
+    transactions.  The first transaction in the queue is the content of
+    the 'with:' block, which is immediately started.
+
+    Any transaction can register new transactions that will be run
+    after the current one is finished, using the global function add().
     """
-    tpool = _thread_pool
-    if tpool.in_transaction:
-        raise TransactionError("recursive invocation of transaction.run()")
-    if not _thread_local.pending:
-        return     # nothing to do
-    try:
-        tpool.setup()
-        tpool.run()
-    finally:
-        tpool.teardown()
-    tpool.reraise()
 
-def number_of_transactions_in_last_run():
-    return _thread_pool.transactions_run
+    def __init__(self, nb_segments=0):
+        if nb_segments <= 0:
+            nb_segments = getsegmentlimit()
+        _thread_pool.ensure_threads(nb_segments)
+
+    def __enter__(self):
+        if hasattr(_thread_local, "pending"):
+            raise TransactionError(
+                "recursive invocation of TransactionQueue()")
+        if is_atomic():
+            raise TransactionError(
+                "invocation of TransactionQueue() from an atomic context")
+        _thread_local.pending = []
+        atomic.__enter__()
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        atomic.__exit__(exc_type, exc_value, traceback)
+        pending = _thread_local.pending
+        del _thread_local.pending
+        if exc_type is None and len(pending) > 0:
+            _thread_pool.run(pending)
+
 
 # ____________________________________________________________
 
@@ -101,159 +155,118 @@
 class _ThreadPool(object):
 
     def __init__(self):
-        try:
-            from __pypy__.thread import getsegmentlimit
-            self.num_threads = getsegmentlimit()
-        except ImportError:
-            self.num_threads = 4
-        self.in_transaction = False
-        self.transactions_run = None
+        self.lock_running = thread.allocate_lock()
+        self.lock_done_running = thread.allocate_lock()
+        self.lock_done_running.acquire()
+        self.nb_threads = 0
+        self.deque = collections.deque()
+        self.locks = []
+        self.lock_deque = thread.allocate_lock()
+        self.exception = []
 
-    def setup(self):
-        # a mutex to protect parts of _grab_next_thing_to_do()
-        self.lock_mutex = thread.allocate_lock()
-        # this lock is released if and only if there are things to do in
-        # 'self.pending'; both are modified together, with the lock_mutex.
-        self.lock_pending = thread.allocate_lock()
-        # this lock is released when we are finished at the end
-        self.lock_if_released_then_finished = thread.allocate_lock()
-        self.lock_if_released_then_finished.acquire()
+    def ensure_threads(self, n):
+        if n > self.nb_threads:
+            with self.lock_running:
+                for i in range(self.nb_threads, n):
+                    assert len(self.locks) == self.nb_threads
+                    self.nb_threads += 1
+                    thread.start_new_thread(self.thread_runner, ())
+                    # The newly started thread should run immediately into
+                    # the case 'if len(self.locks) == self.nb_threads:'
+                    # and release this lock.  Wait until it does.
+                    self.lock_done_running.acquire()
+
+    def run(self, pending):
+        # For now, can't run multiple threads with each an independent
+        # TransactionQueue(): they are serialized.
+        with self.lock_running:
+            assert self.exception == []
+            assert len(self.deque) == 0
+            deque = self.deque
+            with self.lock_deque:
+                deque.extend(pending)
+                try:
+                    for i in range(len(pending)):
+                        self.locks.pop().release()
+                except IndexError:     # pop from empty list
+                    pass
+            #
+            self.lock_done_running.acquire()
+            #
+            if self.exception:
+                exc_type, exc_value, exc_traceback = self.exception
+                del self.exception[:]
+                raise exc_type, exc_value, exc_traceback
+
+    def thread_runner(self):
+        deque = self.deque
+        lock = thread.allocate_lock()
+        lock.acquire()
+        pending = []
+        _thread_local.pending = pending
+        lock_deque = self.lock_deque
+        exception = self.exception
         #
-        self.pending = _thread_local.pending
-        # there must be pending items at the beginning, which means that
-        # 'lock_pending' can indeed be released
-        assert self.pending
-        _thread_local.pending = None
-        #
-        self.num_waiting_threads = 0
-        self.transactions_run = 0
-        self.finished = False
-        self.got_exception = []
-        self.in_transaction = True
+        while True:
+            #
+            # Look at the deque and try to fetch the next item on the left.
+            # If empty, we add our lock to the 'locks' list.
+            lock_deque.acquire()
+            if deque:
+                next_transaction = deque.popleft()
+                lock_deque.release()
+            else:
+                self.locks.append(lock)
+                if len(self.locks) == self.nb_threads:
+                    self.lock_done_running.release()
+                lock_deque.release()
+                #
+                # Now wait until our lock is released.
+                lock.acquire()
+                continue
+            #
+            # Now we have a next_transaction.  Run it.
+            assert len(pending) == 0
+            while True:
+                f, args, kwds = next_transaction
+                with atomic:
+                    if len(exception) == 0:
+                        try:
+                            f(*args, **kwds)
+                        except:
+                            exception.extend(sys.exc_info())
+                del next_transaction
+                #
+                # If no new 'pending' transactions have been added, exit
+                # this loop and go back to fetch more from the deque.
+                if len(pending) == 0:
+                    break
+                #
+                # If we have some new 'pending' transactions, add them
+                # to the right of the deque and pop the next one from
+                # the left.  As we do this atomically with the
+                # 'lock_deque', we are sure that the deque cannot be
+                # empty before the popleft().  (We do that even when
+                # 'len(pending) == 1' instead of simply assigning the
+                # single item to 'next_transaction', because it looks
+                # like a good idea to preserve some first-in-first-out
+                # approximation.)
+                with self.lock_deque:
+                    deque.extend(pending)
+                    next_transaction = deque.popleft()
+                    try:
+                        for i in range(1, len(pending)):
+                            self.locks.pop().release()
+                    except IndexError:     # pop from empty list
+                        pass
+                del pending[:]
 
-    def run(self):
-        # start the N threads
-        task_counters = [[0] for i in range(self.num_threads)]
-        for counter in task_counters:
-            thread.start_new_thread(self._run_thread, (counter,))
-        # now wait.  When we manage to acquire the following lock, then
-        # we are finished.
-        self.lock_if_released_then_finished.acquire()
-        self.transactions_run = sum(x[0] for x in task_counters)
-
-    def teardown(self):
-        self.in_transaction = False
-        self.pending = None
-        self.lock_if_released_then_finished = None
-        self.lock_pending = None
-        self.lock_mutex = None
-        _thread_local.pending = collections.deque()
-
-    def reraise(self):
-        exc = self.got_exception
-        self.got_exception = None
-        if exc:
-            raise exc[0], exc[1], exc[2]    # exception, value, traceback
-
-    def _run_thread(self, counter):
-        tloc_pending = _thread_local.pending
-        got_exception = self.got_exception
-        try:
-            while True:
-                self._do_it(self._grab_next_thing_to_do(tloc_pending),
-                            got_exception)
-                counter[0] += 1
-        except _Done:
-            pass
-
-    def _grab_next_thing_to_do(self, tloc_pending):
-        if tloc_pending:
-            # grab the next thing to do from the thread-local deque
-            next = tloc_pending.popleft()
-            # add the rest, if any, to the global 'pending'
-            if tloc_pending:
-                #
-                self.lock_mutex.acquire()
-                if not self.pending:
-                    # self.pending is empty so far, but we are adding stuff.
-                    # we have to release the following lock.
-                    self.lock_pending.release()
-                self.pending.extend(tloc_pending)
-                self.lock_mutex.release()
-                #
-                tloc_pending.clear()
-            return next
-        #
-        self.lock_mutex.acquire()
-        while True:
-            try:
-                next = self.pending.popleft()
-            except IndexError:
-                # self.pending is empty: wait until it no longer is.
-                pass
-            else:
-                # self.pending was not empty.  If now it is empty, then
-                # fix the status of 'lock_pending'.
-                if not self.pending:
-                    self.lock_pending.acquire()
-                self.lock_mutex.release()
-                return next
-            #
-            # first check if all N threads are waiting here.
-            assert not self.finished
-            self.num_waiting_threads += 1
-            if self.num_waiting_threads == self.num_threads:
-                # yes, so finished!  unlock this to wake up the other
-                # threads, which are all waiting on the following acquire().
-                self.finished = True
-                self.lock_pending.release()
-            #
-            self.lock_mutex.release()
-            self.lock_pending.acquire()
-            self.lock_pending.release()
-            self.lock_mutex.acquire()
-            #
-            self.num_waiting_threads -= 1
-            if self.finished:
-                last_one_to_leave = self.num_waiting_threads == 0
-                self.lock_mutex.release()
-                if last_one_to_leave:
-                    self.lock_if_released_then_finished.release()
-                raise _Done
-
-    @staticmethod
-    def _do_it((f, args, kwds), got_exception):
-        # this is a staticmethod in order to make sure that we don't
-        # accidentally use 'self' in the atomic block.
-        try:
-            while True:
-                with signals_enabled:
-                    with atomic:
-                        info = last_abort_info()
-                        if info is None:
-                            if not got_exception:
-                                f(*args, **kwds)
-                            # else return early if already an exc to reraise
-                            return
-                report_abort_info(info)
-        except:
-            got_exception[:] = sys.exc_info()
 
 _thread_pool = _ThreadPool()
+_thread_local = thread._local()
 
 
-class _Done(Exception):
-    pass
-
-
-class _ThreadLocal(thread._local):
-    def __init__(self):
-        self.pending = collections.deque()
-
-_thread_local = _ThreadLocal()
-
-
-def report_abort_info(info):
+def XXXreport_abort_info(info):
     header = info[0]
     f = cStringIO.StringIO()
     if len(info) > 1:
@@ -279,3 +292,27 @@
         header[1], 'atom '*header[3], 'inev '*(header[4]>1),
         header[5], header[6])
     sys.stderr.write(f.getvalue())
+
+
+class threadlocalproperty(object):
+    def __init__(self, *default):
+        self.tl_default = default
+        self.tl_name = intern(str(id(self)))
+
+    def tl_get(self, obj):
+        try:
+            return obj._threadlocalproperties
+        except AttributeError:
+            return obj.__dict__.setdefault('_threadlocalproperties',
+                                           thread._local())
+
+    def __get__(self, obj, cls=None):
+        if obj is None:
+            return self
+        return getattr(self.tl_get(obj), self.tl_name, *self.tl_default)
+
+    def __set__(self, obj, value):
+        setattr(self.tl_get(obj), self.tl_name, value)
+
+    def __delete__(self, obj):
+        delattr(self.tl_get(obj), self.tl_name)
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -226,9 +226,10 @@
         # expose the following variables to ease debugging
         global space, entry_point
 
-        if config.translation.stm:
+        if config.translation.stm or config.objspace.usemodules.pypystm:
+            config.translation.stm = True
             config.translation.thread = True
-            config.objspace.usemodules._stm = True
+            config.objspace.usemodules.pypystm = True
 
         if config.objspace.allworkingmodules:
             from pypy.config.pypyoption import enable_allworkingmodules
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1022,6 +1022,9 @@
     def newlist_unicode(self, list_u):
         return self.newlist([self.wrap(u) for u in list_u])
 
+    def newlist_int(self, list_i):
+        return self.newlist([self.wrap(i) for i in list_i])
+
     def newlist_hint(self, sizehint):
         from pypy.objspace.std.listobject import make_empty_list_with_size
         return make_empty_list_with_size(self, sizehint)
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -34,7 +34,7 @@
         self.w_profilefuncarg = None
         #
         if self.space.config.translation.stm:
-            from pypy.module._stm.ec import initialize_execution_context
+            from pypy.module.pypystm.ec import initialize_execution_context
             initialize_execution_context(self)
         self.thread_disappeared = False   # might be set to True after os.fork()
 
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -29,22 +29,11 @@
 class ThreadModule(MixedModule):
     appleveldefs = {
         'signals_enabled': 'app_signal.signals_enabled',
-        'atomic':                  'app_atomic.atomic',
-        'exclusive_atomic':        'app_atomic.exclusive_atomic',
     }
     interpleveldefs = {
         '_signals_enter':  'interp_signal.signals_enter',
         '_signals_exit':   'interp_signal.signals_exit',
-        '_atomic_enter':           'interp_atomic.atomic_enter',
-        '_exclusive_atomic_enter': 'interp_atomic.exclusive_atomic_enter',
-        '_atomic_exit':            'interp_atomic.atomic_exit',
-        'getsegmentlimit':         'interp_atomic.getsegmentlimit',
-        'hint_commit_soon':        'interp_atomic.hint_commit_soon',
-        'is_atomic':               'interp_atomic.is_atomic',
-        'error': 'space.fromcache(pypy.module.thread.error.Cache).w_error',
     }
-    def activate(self, space):
-        return self.space.config.objspace.usemodules.thread
 
 
 class IntOpModule(MixedModule):
diff --git a/pypy/module/__pypy__/test/test_signal.py b/pypy/module/__pypy__/test/test_signal.py
--- a/pypy/module/__pypy__/test/test_signal.py
+++ b/pypy/module/__pypy__/test/test_signal.py
@@ -4,7 +4,7 @@
 
 
 class AppTestMinimal:
-    spaceconfig = dict(usemodules=['__pypy__'])
+    spaceconfig = dict(usemodules=['__pypy__', 'thread'])
 
     def test_signal(self):
         from __pypy__ import thread
diff --git a/pypy/module/_stm/__init__.py b/pypy/module/pypystm/__init__.py
rename from pypy/module/_stm/__init__.py
rename to pypy/module/pypystm/__init__.py
--- a/pypy/module/_stm/__init__.py
+++ b/pypy/module/pypystm/__init__.py
@@ -4,12 +4,24 @@
 
 class Module(MixedModule):
     appleveldefs = {
+        'atomic':                  'app_atomic.atomic',
+        'exclusive_atomic':        'app_atomic.exclusive_atomic',
     }
 
     interpleveldefs = {
+        '_atomic_enter':           'interp_atomic.atomic_enter',
+        '_exclusive_atomic_enter': 'interp_atomic.exclusive_atomic_enter',
+        '_atomic_exit':            'interp_atomic.atomic_exit',
+        'getsegmentlimit':         'interp_atomic.getsegmentlimit',
+        'hint_commit_soon':        'interp_atomic.hint_commit_soon',
+        'is_atomic':               'interp_atomic.is_atomic',
+        'error': 'space.fromcache(pypy.module.thread.error.Cache).w_error',
+
         'local': 'local.STMLocal',
         'count': 'count.count',
         'hashtable': 'hashtable.W_Hashtable',
         'time': 'time.time',
         'clock': 'time.clock',
+        'stmset': 'stmset.W_STMSet',
+        'stmdict': 'stmdict.W_STMDict',
     }
diff --git a/pypy/module/__pypy__/app_atomic.py b/pypy/module/pypystm/app_atomic.py
rename from pypy/module/__pypy__/app_atomic.py
rename to pypy/module/pypystm/app_atomic.py
--- a/pypy/module/__pypy__/app_atomic.py
+++ b/pypy/module/pypystm/app_atomic.py
@@ -1,12 +1,12 @@
-from __pypy__ import thread
+import pypystm
 
 class Atomic(object):
-    __enter__ = thread._atomic_enter
-    __exit__  = thread._atomic_exit
+    __enter__ = pypystm._atomic_enter
+    __exit__  = pypystm._atomic_exit
 
 class ExclusiveAtomic(object):
-    __enter__ = thread._exclusive_atomic_enter
-    __exit__ = thread._atomic_exit
+    __enter__ = pypystm._exclusive_atomic_enter
+    __exit__ = pypystm._atomic_exit
 
 atomic = Atomic()
 exclusive_atomic = ExclusiveAtomic()
diff --git a/pypy/module/_stm/count.py b/pypy/module/pypystm/count.py
rename from pypy/module/_stm/count.py
rename to pypy/module/pypystm/count.py
--- a/pypy/module/_stm/count.py
+++ b/pypy/module/pypystm/count.py
@@ -1,5 +1,5 @@
 """
-_stm.count()
+pypystm.count()
 """
 
 from rpython.rlib import rstm
diff --git a/pypy/module/_stm/ec.py b/pypy/module/pypystm/ec.py
rename from pypy/module/_stm/ec.py
rename to pypy/module/pypystm/ec.py
--- a/pypy/module/_stm/ec.py
+++ b/pypy/module/pypystm/ec.py
@@ -20,7 +20,7 @@
     """Called from ExecutionContext.__init__()."""
     if ec.space.config.translation.rweakref:
         from rpython.rlib import rweakref
-        from pypy.module._stm.local import STMLocal
+        from pypy.module.pypystm.local import STMLocal
         ec._thread_local_dicts = rweakref.RWeakKeyDictionary(STMLocal, W_Root)
     else:
         ec._thread_local_dicts = FakeWeakKeyDictionary()
diff --git a/pypy/module/_stm/hashtable.py b/pypy/module/pypystm/hashtable.py
rename from pypy/module/_stm/hashtable.py
rename to pypy/module/pypystm/hashtable.py
--- a/pypy/module/_stm/hashtable.py
+++ b/pypy/module/pypystm/hashtable.py
@@ -1,12 +1,13 @@
 """
-The class _stm.hashtable, mapping integers to objects.
+The class pypystm.hashtable, mapping integers to objects.
 """
 
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef
-from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
 
 from rpython.rlib import rstm
+from rpython.rlib.rarithmetic import intmask
 from rpython.rtyper.annlowlevel import cast_gcref_to_instance
 from rpython.rtyper.annlowlevel import cast_instance_to_gcref
 
@@ -25,21 +26,68 @@
 
     @unwrap_spec(key=int)
     def setitem_w(self, key, w_value):
-        gcref = cast_instance_to_gcref(w_value)
-        self.h.set(key, gcref)
+        entry = self.h.lookup(key)
+        entry.object = cast_instance_to_gcref(w_value)
 
     @unwrap_spec(key=int)
     def delitem_w(self, space, key):
-        gcref = self.h.get(key)
-        if not gcref:
+        entry = self.h.lookup(key)
+        if not entry.object:
             space.raise_key_error(space.wrap(key))
-        self.h.set(key, rstm.NULL_GCREF)
+        entry.object = rstm.NULL_GCREF
 
     @unwrap_spec(key=int)
     def contains_w(self, space, key):
         gcref = self.h.get(key)
         return space.newbool(not not gcref)
 
+    @unwrap_spec(key=int, w_default=WrappedDefault(None))
+    def get_w(self, space, key, w_default):
+        gcref = self.h.get(key)
+        if not gcref:
+            return w_default
+        return cast_gcref_to_instance(W_Root, gcref)
+
+    @unwrap_spec(key=int, w_default=WrappedDefault(None))
+    def setdefault_w(self, space, key, w_default):
+        entry = self.h.lookup(key)
+        gcref = entry.object
+        if not gcref:
+            entry.object = cast_instance_to_gcref(w_default)
+            return w_default
+        return cast_gcref_to_instance(W_Root, gcref)
+
+    def len_w(self, space):
+        return space.wrap(self.h.len())
+
+    def keys_w(self, space):
+        array, count = self.h.list()
+        try:
+            lst = [intmask(array[i].index) for i in range(count)]
+        finally:
+            self.h.freelist(array)
+        return space.newlist_int(lst)
+
+    def values_w(self, space):
+        array, count = self.h.list()
+        try:
+            lst_w = [cast_gcref_to_instance(W_Root, array[i].object)
+                     for i in range(count)]
+        finally:
+            self.h.freelist(array)
+        return space.newlist(lst_w)
+
+    def items_w(self, space):
+        array, count = self.h.list()
+        try:
+            lst_w = [space.newtuple([
+                         space.wrap(intmask(array[i].index)),
+                         cast_gcref_to_instance(W_Root, array[i].object)])
+                     for i in range(count)]
+        finally:
+            self.h.freelist(array)
+        return space.newlist(lst_w)
+
 
 def W_Hashtable___new__(space, w_subtype):
     r = space.allocate_instance(W_Hashtable, w_subtype)
@@ -47,10 +95,17 @@
     return space.wrap(r)
 
 W_Hashtable.typedef = TypeDef(
-    '_stm.hashtable',
+    'pypystm.hashtable',
     __new__ = interp2app(W_Hashtable___new__),
     __getitem__ = interp2app(W_Hashtable.getitem_w),
     __setitem__ = interp2app(W_Hashtable.setitem_w),
     __delitem__ = interp2app(W_Hashtable.delitem_w),
     __contains__ = interp2app(W_Hashtable.contains_w),
-    )
+    get = interp2app(W_Hashtable.get_w),
+    setdefault = interp2app(W_Hashtable.setdefault_w),
+
+    __len__ = interp2app(W_Hashtable.len_w),
+    keys    = interp2app(W_Hashtable.keys_w),
+    values  = interp2app(W_Hashtable.values_w),
+    items   = interp2app(W_Hashtable.items_w),
+)
diff --git a/pypy/module/__pypy__/interp_atomic.py b/pypy/module/pypystm/interp_atomic.py
rename from pypy/module/__pypy__/interp_atomic.py
rename to pypy/module/pypystm/interp_atomic.py
diff --git a/pypy/module/_stm/local.py b/pypy/module/pypystm/local.py
rename from pypy/module/_stm/local.py
rename to pypy/module/pypystm/local.py
--- a/pypy/module/_stm/local.py
+++ b/pypy/module/pypystm/local.py
@@ -1,5 +1,5 @@
 """
-The '_stm.local' class, used for 'thread._local' with STM.
+The 'pypystm.local' class, used for 'thread._local' with STM.
 """
 
 from pypy.interpreter.gateway import W_Root, interp2app
@@ -10,7 +10,7 @@
 
 def _fill_untranslated(ec):
     if not we_are_translated() and not hasattr(ec, '_thread_local_dicts'):
-        from pypy.module._stm.ec import initialize_execution_context
+        from pypy.module.pypystm.ec import initialize_execution_context
         initialize_execution_context(ec)
 
 
@@ -67,7 +67,7 @@
         # No arguments allowed
         pass
 
-STMLocal.typedef = TypeDef("_stm.local",
+STMLocal.typedef = TypeDef("pypystm.local",
                      __doc__ = "Thread-local data",
                      __new__ = interp2app(STMLocal.descr_local__new__.im_func),
                      __init__ = interp2app(STMLocal.descr_local__init__),
diff --git a/pypy/module/pypystm/stmdict.py b/pypy/module/pypystm/stmdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypystm/stmdict.py
@@ -0,0 +1,243 @@
+"""
+The class pypystm.stmdict, giving a part of the regular 'dict' interface
+"""
+
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.typedef import TypeDef
+from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
+
+from rpython.rlib import rstm, jit, rgc
+from rpython.rlib.objectmodel import specialize, we_are_translated
+from rpython.rtyper.annlowlevel import cast_gcref_to_instance
+from rpython.rtyper.annlowlevel import cast_instance_to_gcref
+from rpython.rtyper.lltypesystem import lltype, llmemory
+
+ARRAY = lltype.GcArray(llmemory.GCREF)
+PARRAY = lltype.Ptr(ARRAY)
+
+
+def find_equal_item(space, array, w_key):
+    w_item = cast_gcref_to_instance(W_Root, array[0])
+    if space.eq_w(w_key, w_item):
+        return 0
+    if len(array) > 2:
+        return _run_next_iterations(space, array, w_key)
+    return -1
+
+ at jit.dont_look_inside
+def _run_next_iterations(space, array, w_key):
+    i = 2
+    limit = len(array)
+    while True:
+        w_item = cast_gcref_to_instance(W_Root, array[i])
+        if space.eq_w(w_key, w_item):
+            return i
+        i += 2
+        if i >= limit:
+            return -1
+
+def ll_arraycopy(source, dest, source_start, dest_start, length):
+    if we_are_translated():
+        rgc.ll_arraycopy(source, dest, source_start, dest_start, length)
+    else:
+        for i in range(length):
+            dest[dest_start + i] = source[source_start + i]
+
+def pop_from_entry(entry, space, w_key):
+    array = lltype.cast_opaque_ptr(PARRAY, entry.object)
+    if not array:
+        return None
+    i = find_equal_item(space, array, w_key)
+    if i < 0:
+        return None
+    # found
+    w_value = cast_gcref_to_instance(W_Root, array[i + 1])
+    L = len(array) - 2
+    if L == 0:
+        narray = lltype.nullptr(ARRAY)
+    else:
+        narray = lltype.malloc(ARRAY, L)
+        ll_arraycopy(array, narray, 0, 0, i)
+        ll_arraycopy(array, narray, i + 2, i, L - i)
+    entry.object = lltype.cast_opaque_ptr(llmemory.GCREF, narray)
+    return w_value
+
+
+class W_STMDict(W_Root):
+
+    def __init__(self):
+        self.h = rstm.create_hashtable()
+
+    def getitem_w(self, space, w_key):
+        hkey = space.hash_w(w_key)
+        gcref = self.h.get(hkey)
+        array = lltype.cast_opaque_ptr(PARRAY, gcref)
+        if array:
+            i = find_equal_item(space, array, w_key)
+            if i >= 0:
+                return cast_gcref_to_instance(W_Root, array[i + 1])
+        space.raise_key_error(w_key)
+
+    def setitem_w(self, space, w_key, w_value):
+        hkey = space.hash_w(w_key)
+        entry = self.h.lookup(hkey)
+        array = lltype.cast_opaque_ptr(PARRAY, entry.object)
+        if array:
+            i = find_equal_item(space, array, w_key)
+            if i >= 0:
+                # already there, update the value
+                array[i + 1] = cast_instance_to_gcref(w_value)
+                return
+            L = len(array)
+            narray = lltype.malloc(ARRAY, L + 2)
+            ll_arraycopy(array, narray, 0, 0, L)
+        else:
+            narray = lltype.malloc(ARRAY, 2)
+            L = 0
+        narray[L] = cast_instance_to_gcref(w_key)
+        narray[L + 1] = cast_instance_to_gcref(w_value)
+        entry.object = lltype.cast_opaque_ptr(llmemory.GCREF, narray)
+
+    def delitem_w(self, space, w_key):
+        hkey = space.hash_w(w_key)
+        entry = self.h.lookup(hkey)
+        if pop_from_entry(entry, space, w_key) is None:
+            space.raise_key_error(w_key)
+
+    def contains_w(self, space, w_key):
+        hkey = space.hash_w(w_key)
+        gcref = self.h.get(hkey)
+        array = lltype.cast_opaque_ptr(PARRAY, gcref)
+        if array and find_equal_item(space, array, w_key) >= 0:
+            return space.w_True
+        return space.w_False
+
+    @unwrap_spec(w_default=WrappedDefault(None))
+    def get_w(self, space, w_key, w_default):
+        hkey = space.hash_w(w_key)
+        gcref = self.h.get(hkey)
+        array = lltype.cast_opaque_ptr(PARRAY, gcref)
+        if array:
+            i = find_equal_item(space, array, w_key)
+            if i >= 0:
+                return cast_gcref_to_instance(W_Root, array[i + 1])
+        return w_default
+
+    def pop_w(self, space, w_key, w_default=None):
+        hkey = space.hash_w(w_key)
+        entry = self.h.lookup(hkey)
+        w_value = pop_from_entry(entry, space, w_key)
+        if w_value is not None:
+            return w_value
+        elif w_default is not None:
+            return w_default
+        else:
+            space.raise_key_error(w_key)
+
+    @unwrap_spec(w_default=WrappedDefault(None))
+    def setdefault_w(self, space, w_key, w_default):
+        hkey = space.hash_w(w_key)
+        entry = self.h.lookup(hkey)
+        array = lltype.cast_opaque_ptr(PARRAY, entry.object)
+        if array:
+            i = find_equal_item(space, array, w_key)
+            if i >= 0:
+                # already there, return the existing value
+                return cast_gcref_to_instance(W_Root, array[i + 1])
+            L = len(array)
+            narray = lltype.malloc(ARRAY, L + 2)
+            ll_arraycopy(array, narray, 0, 0, L)
+        else:
+            narray = lltype.malloc(ARRAY, 2)
+            L = 0
+        narray[L] = cast_instance_to_gcref(w_key)
+        narray[L + 1] = cast_instance_to_gcref(w_default)
+        entry.object = lltype.cast_opaque_ptr(llmemory.GCREF, narray)
+        return w_default
+
+    def get_length(self):
+        array, count = self.h.list()
+        try:
+            total_length_times_two = 0
+            for i in range(count):
+                subarray = lltype.cast_opaque_ptr(PARRAY, array[i].object)
+                assert subarray
+                total_length_times_two += len(subarray)
+        finally:
+            self.h.freelist(array)
+        return total_length_times_two >> 1
+
+    def get_keys_values_w(self, offset):
+        array, count = self.h.list()
+        try:
+            result_list_w = []
+            for i in range(count):
+                subarray = lltype.cast_opaque_ptr(PARRAY, array[i].object)
+                assert subarray
+                j = offset
+                limit = len(subarray)
+                while j < limit:
+                    w_item = cast_gcref_to_instance(W_Root, subarray[j])
+                    result_list_w.append(w_item)
+                    j += 2
+        finally:
+            self.h.freelist(array)
+        return result_list_w
+
+    def get_items_w(self, space):
+        array, count = self.h.list()
+        try:
+            result_list_w = []
+            for i in range(count):
+                subarray = lltype.cast_opaque_ptr(PARRAY, array[i].object)
+                assert subarray
+                j = 0
+                limit = len(subarray)
+                while j < limit:
+                    w_key = cast_gcref_to_instance(W_Root, subarray[j])
+                    w_value = cast_gcref_to_instance(W_Root, subarray[j + 1])
+                    result_list_w.append(space.newtuple([w_key, w_value]))
+                    j += 2
+        finally:
+            self.h.freelist(array)
+        return result_list_w
+
+    def len_w(self, space):
+        return space.wrap(self.get_length())
+
+    def iter_w(self, space):
+        # not a real lazy iterator!
+        return space.iter(self.keys_w(space))
+
+    def keys_w(self, space):
+        return space.newlist(self.get_keys_values_w(offset=0))
+
+    def values_w(self, space):
+        return space.newlist(self.get_keys_values_w(offset=1))
+
+    def items_w(self, space):
+        return space.newlist(self.get_items_w(space))
+
+
+def W_STMDict___new__(space, w_subtype):
+    r = space.allocate_instance(W_STMDict, w_subtype)
+    r.__init__()
+    return space.wrap(r)
+
+W_STMDict.typedef = TypeDef(
+    'pypystm.stmdict',
+    __new__ = interp2app(W_STMDict___new__),
+    __getitem__ = interp2app(W_STMDict.getitem_w),
+    __setitem__ = interp2app(W_STMDict.setitem_w),
+    __delitem__ = interp2app(W_STMDict.delitem_w),
+    __contains__ = interp2app(W_STMDict.contains_w),
+    get = interp2app(W_STMDict.get_w),
+    pop = interp2app(W_STMDict.pop_w),
+    setdefault = interp2app(W_STMDict.setdefault_w),
+
+    __len__  = interp2app(W_STMDict.len_w),
+    __iter__ = interp2app(W_STMDict.iter_w),
+    keys     = interp2app(W_STMDict.keys_w),
+    values   = interp2app(W_STMDict.values_w),
+    items    = interp2app(W_STMDict.items_w),
+    )
diff --git a/pypy/module/pypystm/stmset.py b/pypy/module/pypystm/stmset.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypystm/stmset.py
@@ -0,0 +1,146 @@
+"""
+The class pypystm.stmset, giving a part of the regular 'set' interface
+"""
+
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.typedef import TypeDef
+from pypy.interpreter.gateway import interp2app
+
+from rpython.rlib import rstm, jit
+from rpython.rlib.rgc import ll_arraycopy
+from rpython.rlib.objectmodel import specialize
+from rpython.rtyper.annlowlevel import cast_gcref_to_instance
+from rpython.rtyper.annlowlevel import cast_instance_to_gcref
+from rpython.rtyper.lltypesystem import lltype, llmemory
+
+ARRAY = lltype.GcArray(llmemory.GCREF)
+PARRAY = lltype.Ptr(ARRAY)
+
+
+def find_equal_item(space, array, w_key):
+    w_item = cast_gcref_to_instance(W_Root, array[0])
+    if space.eq_w(w_key, w_item):
+        return 0
+    if len(array) > 1:
+        return _run_next_iterations(space, array, w_key)
+    return -1
+
+ at jit.dont_look_inside
+def _run_next_iterations(space, array, w_key):
+    i = 1
+    limit = len(array)
+    while True:
+        w_item = cast_gcref_to_instance(W_Root, array[i])
+        if space.eq_w(w_key, w_item):
+            return i
+        i += 1
+        if i >= limit:
+            return -1
+
+
+class W_STMSet(W_Root):
+
+    def __init__(self):
+        self.h = rstm.create_hashtable()
+
+    def contains_w(self, space, w_key):
+        hkey = space.hash_w(w_key)
+        gcref = self.h.get(hkey)
+        array = lltype.cast_opaque_ptr(PARRAY, gcref)
+        if array and find_equal_item(space, array, w_key) >= 0:
+            return space.w_True
+        return space.w_False
+
+    def add_w(self, space, w_key):
+        hkey = space.hash_w(w_key)
+        entry = self.h.lookup(hkey)
+        array = lltype.cast_opaque_ptr(PARRAY, entry.object)
+        if array:
+            if find_equal_item(space, array, w_key) >= 0:
+                return      # already there
+            L = len(array)
+            narray = lltype.malloc(ARRAY, L + 1)
+            ll_arraycopy(array, narray, 0, 0, L)
+        else:
+            narray = lltype.malloc(ARRAY, 1)
+            L = 0
+        narray[L] = cast_instance_to_gcref(w_key)
+        entry.object = lltype.cast_opaque_ptr(llmemory.GCREF, narray)
+
+    def try_remove(self, space, w_key):
+        hkey = space.hash_w(w_key)
+        entry = self.h.lookup(hkey)
+        array = lltype.cast_opaque_ptr(PARRAY, entry.object)
+        if not array:
+            return False
+        i = find_equal_item(space, array, w_key)
+        if i < 0:
+            return False
+        # found
+        L = len(array) - 1
+        if L == 0:
+            narray = lltype.nullptr(ARRAY)
+        else:
+            narray = lltype.malloc(ARRAY, L)
+            ll_arraycopy(array, narray, 0, 0, i)
+            ll_arraycopy(array, narray, i + 1, i, L - i)
+        entry.object = lltype.cast_opaque_ptr(llmemory.GCREF, narray)
+        return True
+
+    def remove_w(self, space, w_key):
+        if not self.try_remove(space, w_key):
+            space.raise_key_error(w_key)
+
+    def discard_w(self, space, w_key):
+        self.try_remove(space, w_key)
+
+    def get_length(self):
+        array, count = self.h.list()
+        try:
+            total_length = 0
+            for i in range(count):
+                subarray = lltype.cast_opaque_ptr(PARRAY, array[i].object)
+                assert subarray
+                total_length += len(subarray)
+        finally:
+            self.h.freelist(array)
+        return total_length
+
+    def get_items_w(self):
+        array, count = self.h.list()
+        try:
+            result_list_w = []
+            for i in range(count):
+                subarray = lltype.cast_opaque_ptr(PARRAY, array[i].object)
+                assert subarray
+                for j in range(len(subarray)):
+                    w_item = cast_gcref_to_instance(W_Root, subarray[j])
+                    result_list_w.append(w_item)
+        finally:
+            self.h.freelist(array)
+        return result_list_w
+
+    def len_w(self, space):
+        return space.wrap(self.get_length())
+
+    def iter_w(self, space):
+        # not a real lazy iterator!
+        return space.iter(space.newlist(self.get_items_w()))
+
+
+def W_STMSet___new__(space, w_subtype):
+    r = space.allocate_instance(W_STMSet, w_subtype)
+    r.__init__()
+    return space.wrap(r)
+
+W_STMSet.typedef = TypeDef(
+    'pypystm.stmset',
+    __new__ = interp2app(W_STMSet___new__),
+    __contains__ = interp2app(W_STMSet.contains_w),
+    add = interp2app(W_STMSet.add_w),
+    remove = interp2app(W_STMSet.remove_w),
+    discard = interp2app(W_STMSet.discard_w),
+
+    __len__ = interp2app(W_STMSet.len_w),
+    __iter__ = interp2app(W_STMSet.iter_w),
+    )
diff --git a/pypy/module/_stm/test/__init__.py b/pypy/module/pypystm/test/__init__.py
rename from pypy/module/_stm/test/__init__.py
rename to pypy/module/pypystm/test/__init__.py
diff --git a/pypy/module/__pypy__/test/test_atomic.py b/pypy/module/pypystm/test/test_atomic.py
rename from pypy/module/__pypy__/test/test_atomic.py
rename to pypy/module/pypystm/test/test_atomic.py
--- a/pypy/module/__pypy__/test/test_atomic.py
+++ b/pypy/module/pypystm/test/test_atomic.py
@@ -1,32 +1,13 @@
-from __future__ import with_statement
-from pypy.module.thread.test.support import GenericTestThread
-from rpython.rtyper.lltypesystem import rffi
 
 
-def test_bdecode(space):
-    from pypy.module.__pypy__.interp_atomic import bdecode
-    def bdec(s, expected):
-        p = rffi.str2charp(s)
-        w_obj, q = bdecode(space, p)
-        assert q == rffi.ptradd(p, len(s))
-        rffi.free_charp(p)
-        w_expected = space.wrap(expected)
-        assert space.eq_w(w_obj, w_expected)
-
-    bdec("i123e", 123)
-    bdec("i-123e", -123)
-    bdec('12:+"*-%&/()=?\x00', '+"*-%&/()=?\x00')
-    bdec("li123eli456eee", [123, [456]])
-    bdec("l5:abcdei2ee", ["abcde", 2])
-
-
-class AppTestAtomic(GenericTestThread):
+class AppTestAtomic:
+    spaceconfig = dict(usemodules=['pypystm', 'thread'])
 
     def test_simple(self):
-        from __pypy__ import thread
-        for atomic in thread.atomic, thread.exclusive_atomic:
+        import pypystm
+        for atomic in pypystm.atomic, pypystm.exclusive_atomic:
             with atomic:
-                assert thread.is_atomic()
+                assert pypystm.is_atomic()
             try:
                 with atomic:
                     raise ValueError
@@ -34,40 +15,40 @@
                 pass
 
     def test_nest_composable_atomic(self):
-        from __pypy__ import thread
-        with thread.atomic:
-            with thread.atomic:
-                assert thread.is_atomic()
-            assert thread.is_atomic()
-        assert not thread.is_atomic()
+        import pypystm
+        with pypystm.atomic:
+            with pypystm.atomic:
+                assert pypystm.is_atomic()
+            assert pypystm.is_atomic()
+        assert not pypystm.is_atomic()
 
     def test_nest_composable_below_exclusive(self):
-        from __pypy__ import thread
-        with thread.exclusive_atomic:
-            with thread.atomic:
-                with thread.atomic:
-                    assert thread.is_atomic()
-                assert thread.is_atomic()
-            assert thread.is_atomic()
-        assert not thread.is_atomic()
+        import pypystm
+        with pypystm.exclusive_atomic:
+            with pypystm.atomic:
+                with pypystm.atomic:
+                    assert pypystm.is_atomic()
+                assert pypystm.is_atomic()
+            assert pypystm.is_atomic()
+        assert not pypystm.is_atomic()
 
     def test_nest_exclusive_fails(self):
-        from __pypy__ import thread
+        import pypystm
         try:
-            with thread.exclusive_atomic:
-                with thread.exclusive_atomic:
-                    assert thread.is_atomic()
-        except thread.error, e:
-            assert not thread.is_atomic()
+            with pypystm.exclusive_atomic:
+                with pypystm.exclusive_atomic:
+                    assert pypystm.is_atomic()
+        except pypystm.error, e:
+            assert not pypystm.is_atomic()
             assert e.message == "exclusive_atomic block can't be entered inside another atomic block"
 
     def test_nest_exclusive_fails2(self):
-        from __pypy__ import thread
+        import pypystm
         try:
-            with thread.atomic:
-                with thread.exclusive_atomic:
-                    assert thread.is_atomic()
-                assert thread.is_atomic()
-        except thread.error, e:
-            assert not thread.is_atomic()
+            with pypystm.atomic:
+                with pypystm.exclusive_atomic:
+                    assert pypystm.is_atomic()
+                assert pypystm.is_atomic()
+        except pypystm.error, e:
+            assert not pypystm.is_atomic()
             assert e.message == "exclusive_atomic block can't be entered inside another atomic block"
diff --git a/pypy/module/_stm/test/test_count.py b/pypy/module/pypystm/test/test_count.py
rename from pypy/module/_stm/test/test_count.py
rename to pypy/module/pypystm/test/test_count.py
--- a/pypy/module/_stm/test/test_count.py
+++ b/pypy/module/pypystm/test/test_count.py
@@ -1,10 +1,10 @@
 
 
 class AppTestCount:
-    spaceconfig = dict(usemodules=['_stm'])
+    spaceconfig = dict(usemodules=['pypystm'])
 
     def test_count(self):
-        import _stm
-        x = _stm.count()
-        y = _stm.count()
+        import pypystm
+        x = pypystm.count()
+        y = pypystm.count()
         assert y == x + 1
diff --git a/pypy/module/_stm/test/test_hashtable.py b/pypy/module/pypystm/test/test_hashtable.py
rename from pypy/module/_stm/test/test_hashtable.py
rename to pypy/module/pypystm/test/test_hashtable.py
--- a/pypy/module/_stm/test/test_hashtable.py
+++ b/pypy/module/pypystm/test/test_hashtable.py
@@ -1,11 +1,11 @@
 
 
 class AppTestHashtable:
-    spaceconfig = dict(usemodules=['_stm'])
+    spaceconfig = dict(usemodules=['pypystm'])
 
     def test_simple(self):
-        import _stm
-        h = _stm.hashtable()
+        import pypystm
+        h = pypystm.hashtable()
         h[42+65536] = "bar"
         raises(KeyError, "h[42]")
         h[42] = "foo"
@@ -16,3 +16,42 @@
         raises(KeyError, "h[42]")
         assert h[42+65536] == "bar"
         raises(KeyError, "del h[42]")
+
+    def test_get_setdefault(self):
+        import pypystm
+        h = pypystm.hashtable()
+        assert h.get(42) is None
+        assert h.get(-43, None) is None
+        assert h.get(44, 81) == 81
+        raises(KeyError, "h[42]")
+        raises(KeyError, "h[-43]")
+        raises(KeyError, "h[44]")
+        assert h.setdefault(42) is None
+        assert h[42] is None
+        assert h.setdefault(42, "81") is None
+        assert h[42] is None
+        assert h.setdefault(44, "-81") == "-81"
+        assert h[44] == "-81"
+        assert h[42] is None
+
+    def test_len(self):
+        import pypystm
+        h = pypystm.hashtable()
+        assert len(h) == 0
+        h[42] = "foo"
+        assert len(h) == 1
+        h[43] = "bar"
+        assert len(h) == 2
+        h[42] = "baz"
+        assert len(h) == 2
+        del h[42]
+        assert len(h) == 1
+
+    def test_keys_values_items(self):
+        import pypystm
+        h = pypystm.hashtable()
+        h[42] = "foo"
+        h[43] = "bar"
+        assert sorted(h.keys()) == [42, 43]
+        assert sorted(h.values()) == ["bar", "foo"]
+        assert sorted(h.items()) == [(42, "foo"), (43, "bar")]
diff --git a/pypy/module/_stm/test/test_local.py b/pypy/module/pypystm/test/test_local.py
rename from pypy/module/_stm/test/test_local.py
rename to pypy/module/pypystm/test/test_local.py
--- a/pypy/module/_stm/test/test_local.py
+++ b/pypy/module/pypystm/test/test_local.py
@@ -3,11 +3,11 @@
 
 class AppTestSTMLocal(test_local.AppTestLocal):
     spaceconfig = test_local.AppTestLocal.spaceconfig.copy()
-    spaceconfig['usemodules'] += ('_stm',)
+    spaceconfig['usemodules'] += ('pypystm',)
 
     def setup_class(cls):
         test_local.AppTestLocal.setup_class.im_func(cls)
         cls.w__local = cls.space.appexec([], """():
-            import _stm
-            return _stm.local
+            import pypystm
+            return pypystm.local
         """)
diff --git a/pypy/module/pypystm/test/test_stmdict.py b/pypy/module/pypystm/test/test_stmdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypystm/test/test_stmdict.py
@@ -0,0 +1,118 @@
+
+
+class AppTestDict:
+    spaceconfig = dict(usemodules=['pypystm'])
+
+    def test_simple(self):
+        import pypystm
+        d = pypystm.stmdict()
+        raises(KeyError, "d[42.0]")
+        d[42.0] = "5"
+        assert d[42.0] == "5"
+        assert 42.0 in d
+        assert d.get(42.0) == "5"
+        assert d.get(42.0, 42) == "5"
+        del d[42.0]
+        raises(KeyError, "d[42.0]")
+        assert 42.0 not in d
+        assert d.get(42.0) is None
+        assert d.get(42.0, "41") == "41"
+        assert d.setdefault(42.0, "42") == "42"
+        assert d.setdefault(42.0, "43") == "42"
+        assert d.setdefault(42.5) is None
+        assert d[42.5] is None
+        assert d[42.0] == "42"
+        d[42.0] = "foo"
+        assert d[42.0] == "foo"
+
+    def test_hash_collision(self):
+        import pypystm
+        key1 = 5L
+        key2 = 5L + 2**64 - 1
+        key3 = 5L + 2**65 - 2
+        assert hash(key1) == hash(key2) == hash(key3)
+        d = pypystm.stmdict()
+        d[key1] = 1.0
+        d[key2] = 2.0
+        assert d[key1] == 1.0
+        assert key2 in d
+        assert key3 not in d
+        raises(KeyError, "d[key3]")
+        assert d.get(key3) is None
+        del d[key1]
+        assert key1 not in d
+        assert d[key2] == 2.0
+        assert key3 not in d
+        raises(KeyError, "del d[key1]")
+        del d[key2]
+        assert key1 not in d
+        assert key2 not in d
+        assert key3 not in d
+        raises(KeyError, "del d[key3]")
+        assert d.setdefault(key1, 5.0) == 5.0
+        assert d.setdefault(key2, 7.5) == 7.5
+        assert d.setdefault(key1, 2.3) == 5.0
+
+    def test_must_be_hashable(self):
+        import pypystm
+        d = pypystm.stmdict()
+        raises(TypeError, "d[[]]")
+        raises(TypeError, "d[[]] = 5")
+        raises(TypeError, "del d[[]]")
+        raises(TypeError, "[] in d")
+        raises(TypeError, "d.get([])")
+        raises(TypeError, "d.setdefault([], 0)")
+
+    def test_equal_elements(self):
+        import pypystm
+        d = pypystm.stmdict()
+        d[42.0] = "hello"
+        assert d[42] == "hello"
+        assert d.get(42L) == "hello"
+        assert d.get(42.001) is None
+
+    def test_list_from_dict(self):
+        import pypystm
+        d = pypystm.stmdict()
+        assert len(d) == 0
+        assert tuple(d) == ()
+        d[42.5] = "foo"
+        d[42.0] = ["bar"]
+        assert sorted(d) == [42.0, 42.5]
+        assert len(d) == 2
+        del d[42]
+        assert len(d) == 1
+        assert list(d) == [42.5]
+        #
+        class Key(object):
+            def __hash__(self):
+                return hash(42.5)
+        key3 = Key()
+        d[key3] = "other"
+        assert len(d) == 2
+        items = list(d)
+        assert items == [42.5, key3] or items == [key3, 42.5]
+
+    def test_keys_values_items(self):
+        import pypystm
+        d = pypystm.stmdict()
+        d[42.5] = "bar"
+        d[42.0] = "foo"
+        assert sorted(d.keys()) == [42.0, 42.5]
+        assert sorted(d.values()) == ["bar", "foo"]
+        assert sorted(d.items()) == [(42.0, "foo"), (42.5, "bar")]
+
+    def test_pop(self):
+        import pypystm
+        d = pypystm.stmdict()
+        raises(KeyError, d.pop, 42.0)
+        assert d.pop(42.0, "foo") == "foo"
+        raises(KeyError, "d[42.0]")
+        d[42.0] = "bar"
+        res = d.pop(42.0)
+        assert res == "bar"
+        raises(KeyError, "d[42.0]")
+        d[42.0] = "bar"
+        res = d.pop(42.0, "foo")
+        assert res == "bar"
+        raises(KeyError, "d[42.0]")
diff --git a/pypy/module/pypystm/test/test_stmset.py b/pypy/module/pypystm/test/test_stmset.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypystm/test/test_stmset.py
@@ -0,0 +1,85 @@
+
+
+class AppTestSet:
+    spaceconfig = dict(usemodules=['pypystm'])
+
+    def test_simple(self):
+        import pypystm
+        s = pypystm.stmset()
+        s.add(42.0)
+        assert 42.0 in s
+        assert 42.5 not in s
+        s.add(42.5)
+        assert 42.0 in s
+        assert 42.5 in s
+        s.add(42.5)
+        assert 42.0 in s
+        assert 42.5 in s
+        s.remove(42.0)
+        assert 42.0 not in s
+        assert 42.5 in s
+        raises(KeyError, s.remove, 42.0)
+        s.discard(42.0)
+        assert 42.0 not in s
+        assert 42.5 in s
+        s.discard(42.5)
+        assert 42.5 not in s
+
+    def test_hash_collision(self):
+        import pypystm
+        class Key(object):
+            def __hash__(self):
+                return 42
+        key1 = Key()
+        key2 = Key()
+        key3 = Key()
+        s = pypystm.stmset()
+        s.add(key1)
+        s.add(key2)
+        assert key1 in s
+        assert key2 in s
+        assert key3 not in s
+        s.remove(key1)
+        assert key1 not in s
+        assert key2 in s
+        assert key3 not in s
+        s.remove(key2)
+        assert key1 not in s
+        assert key2 not in s
+        assert key3 not in s
+
+    def test_must_be_hashable(self):
+        import pypystm
+        s = pypystm.stmset()
+        raises(TypeError, s.add, [])
+        raises(TypeError, s.remove, [])
+        raises(TypeError, s.discard, [])
+
+    def test_equal_elements(self):
+        import pypystm
+        s = pypystm.stmset()
+        s.add(42.0)
+        assert 42 in s
+        assert 42L in s
+        assert 42.001 not in s
+
+    def test_list_from_set(self):
+        import pypystm
+        s = pypystm.stmset()
+        assert len(s) == 0
+        assert tuple(s) == ()
+        s.add(42.5)
+        s.add(42.0)
+        assert sorted(s) == [42.0, 42.5]
+        assert len(s) == 2
+        s.remove(42.0)
+        assert list(s) == [42.5]
+        #
+        class Key(object):
+            def __hash__(self):
+                return hash(42.5)
+        key3 = Key()
+        s.add(key3)
+        assert len(s) == 2
+        items = list(s)
+        assert items == [42.5, key3] or items == [key3, 42.5]
diff --git a/pypy/module/_stm/test/test_time.py b/pypy/module/pypystm/test/test_time.py
rename from pypy/module/_stm/test/test_time.py
rename to pypy/module/pypystm/test/test_time.py
--- a/pypy/module/_stm/test/test_time.py
+++ b/pypy/module/pypystm/test/test_time.py
@@ -1,13 +1,13 @@
 
 
 class AppTestHashtable:
-    spaceconfig = dict(usemodules=['_stm'])
+    spaceconfig = dict(usemodules=['pypystm'])
 
     def test_simple(self):
-        import _stm
-        t1 = _stm.time()
-        t2 = _stm.time()
+        import pypystm
+        t1 = pypystm.time()
+        t2 = pypystm.time()
         assert t1 < t2 < t1 + 1
-        t1 = _stm.clock()
-        t2 = _stm.clock()
+        t1 = pypystm.clock()
+        t2 = pypystm.clock()
         assert t1 < t2 < t1 + 1
diff --git a/pypy/module/pypystm/test_pypy_c/__init__.py b/pypy/module/pypystm/test_pypy_c/__init__.py
new file mode 100644
diff --git a/pypy/module/pypystm/test_pypy_c/support.py b/pypy/module/pypystm/test_pypy_c/support.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypystm/test_pypy_c/support.py
@@ -0,0 +1,146 @@
+import py
+import sys, os, subprocess, types
+from rpython.tool.udir import udir
+
+
+class BaseTestSTM(object):
+
+    HEADER = """
+import thread, pypystm
+
+NUM_THREADS = 3
+
+_b_to_go = NUM_THREADS
+_b_done = False
+_b_lock = thread.allocate_lock()
+_b_locks = [thread.allocate_lock() for _i in range(NUM_THREADS)]
+for _bl in _b_locks:
+    _bl.acquire()
+
+class BarrierThreadsDone(Exception):
+    pass
+
+def barrier(tnum, done=False):
+    '''Waits until NUM_THREADS call this function, and then returns
+    in all these threads at once.'''
+    global _b_to_go, _b_done
+    _b_lock.acquire()
+    if done:
+        _b_done = True
+    _b_to_go -= 1
+    if _b_to_go > 0:
+        _b_lock.release()
+        _b_locks[tnum].acquire()
+    else:
+        _b_to_go = NUM_THREADS
+        for i in range(NUM_THREADS):
+            if i != tnum:
+                _b_locks[i].release()
+        _b_lock.release()
+    if _b_done:
+        raise BarrierThreadsDone
+
+def _run(tnum, lock, result, function, args):
+    start = pypystm.time()
+    try:
+        try:
+            while True:
+                function(*args)
+                if pypystm.time() - start >= 3.0:
+                    break
+        except BarrierThreadsDone:
+            pass
+        result.append(1)
+    finally:
+        lock.release()
+    while len(result) != NUM_THREADS:
+        barrier(tnum, done=True)
+
+def run_in_threads(function, arg_thread_num=False, arg_class=None):
+    locks = []
+    result = []
+    for i in range(NUM_THREADS):
+        lock = thread.allocate_lock()
+        lock.acquire()
+        args = ()
+        if arg_thread_num:
+            args += (i,)
+        if arg_class:
+            args += (arg_class(),)
+        thread.start_new_thread(_run, (i, lock, result, function, args))
+        locks.append(lock)
+    for lock in locks:
+        lock._py3k_acquire(timeout=30)
+    if len(result) < len(locks):
+        raise Exception("not all threads completed successfully")
+"""
+
+    def setup_class(cls):
+        if '__pypy__' not in sys.builtin_module_names:
+            py.test.skip("must run this test with pypy")
+        try:
+            import pypystm
+        except ImportError:
+            py.test.skip("must give a pypy-c with stm enabled")
+        cls.tmpdir = udir.join('test-pypy-stm')
+        cls.tmpdir.ensure(dir=True)
+
+    def setup_method(self, meth):
+        self.filepath = self.tmpdir.join(meth.im_func.func_name + '.py')
+        self.logfile = self.filepath.new(ext='.log')
+
+    def _write_source(self, func_or_src, args=[]):
+        src = py.code.Source(func_or_src)
+        if isinstance(func_or_src, types.FunctionType):
+            funcname = func_or_src.func_name
+        else:
+            funcname = 'main'
+        arglist = ', '.join(map(repr, args))
+        with self.filepath.open("w") as f:
+            f.write(self.HEADER)
+            f.write(str(src) + '\n')
+            f.write("print %s(%s)\n" % (funcname, arglist))
+
+    def _execute(self, import_site=False):
+        cmdline = [sys.executable]
+        if not import_site:
+            cmdline.append('-S')
+        cmdline.append(str(self.filepath))
+        env = os.environ.copy()
+        env['PYPYSTM'] = str(self.logfile)
+        #
+        pipe = subprocess.Popen(cmdline,
+                                env=env,
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE)
+        stdout, stderr = pipe.communicate()
+        if pipe.returncode > 0:
+            raise IOError("subprocess error %d:\n%s" % (pipe.returncode,
+                                                        stderr))
+        if pipe.returncode < 0:
+            raise IOError("subprocess was killed by signal %d" % (
+                pipe.returncode,))
+
+    def _parse_log(self):
+        from pypy.stm.print_stm_log import StmLog
+        return StmLog(str(self.logfile))
+
+    def _check_count_conflicts(self, func_or_src, args=[]):
+        self._write_source(func_or_src, args)
+        self._execute()
+        stmlog = self._parse_log()
+        count = stmlog.get_total_aborts_and_pauses()
+        print 'stmlog.get_total_aborts_and_pauses():', count
+        return count
+
+    def check_almost_no_conflict(self, *args):
+        count = self._check_count_conflicts(*args)
+        assert count < 500
+
+    def check_MANY_conflicts(self, *args):
+        count = self._check_count_conflicts(*args)
+        assert count > 20000
+
+    def check_SOME_conflicts(self, *args):
+        count = self._check_count_conflicts(*args)
+        assert count > 1000
diff --git a/pypy/module/pypystm/test_pypy_c/test_conflict.py b/pypy/module/pypystm/test_pypy_c/test_conflict.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypystm/test_pypy_c/test_conflict.py
@@ -0,0 +1,46 @@
+from pypy.module.pypystm.test_pypy_c.support import BaseTestSTM
+
+
+class TestConflict(BaseTestSTM):
+
+    def test_obvious(self):
+        def f():
+            class X(object):
+                pass
+            x = X()     # shared
+            x.a = 0
+            def g():
+                x.a += 1
+            run_in_threads(g)
+        #
+        self.check_MANY_conflicts(f)
+
+    def test_plain_dict_access(self):
+        def f():
+            d = {}     # shared
+            def g(n):
+                d[n] = d.get(n, 0) + 1
+            run_in_threads(g, arg_thread_num=True)
+        #
+        self.check_MANY_conflicts(f)
+
+    def test_write_to_many_objects_in_order(self):
+        def f():
+            import weakref
+
+            class X(object):
+                pass
+
+            lst = []     # shared
+
+            def g(tnum):
+                if tnum == 0:
+                    lst[:] = [X() for i in range(1000)]
+                barrier(tnum)
+                for x in lst:
+                    x.a = 5
+                barrier(tnum)
+
+            run_in_threads(g, arg_thread_num=True)
+        #
+        self.check_SOME_conflicts(f)
diff --git a/pypy/module/pypystm/test_pypy_c/test_no_conflict.py b/pypy/module/pypystm/test_pypy_c/test_no_conflict.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypystm/test_pypy_c/test_no_conflict.py
@@ -0,0 +1,52 @@
+import py
+from pypy.module.pypystm.test_pypy_c.support import BaseTestSTM

From noreply at buildbot.pypy.org  Fri Feb  6 17:32:49 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  6 Feb 2015 17:32:49 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Fix warning
Message-ID: <20150206163249.5BC971C0473@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r75739:ec151f9ee5d7
Date: 2015-02-06 17:32 +0100
http://bitbucket.org/pypy/pypy/changeset/ec151f9ee5d7/

Log:	Fix warning

diff --git a/rpython/translator/stm/src_stm/extracode.h b/rpython/translator/stm/src_stm/extracode.h
--- a/rpython/translator/stm/src_stm/extracode.h
+++ b/rpython/translator/stm/src_stm/extracode.h
@@ -1,3 +1,5 @@
+#include 
+
 
 static void _stm_call_finalizer(object_t *obj)
 {

From noreply at buildbot.pypy.org  Fri Feb  6 17:32:50 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  6 Feb 2015 17:32:50 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c8: hg merge stmgc-c7
Message-ID: <20150206163250.76ECB1C0473@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c8
Changeset: r75740:8e3beb94417a
Date: 2015-02-06 17:32 +0100
http://bitbucket.org/pypy/pypy/changeset/8e3beb94417a/

Log:	hg merge stmgc-c7

diff --git a/rpython/translator/stm/src_stm/extracode.h b/rpython/translator/stm/src_stm/extracode.h
--- a/rpython/translator/stm/src_stm/extracode.h
+++ b/rpython/translator/stm/src_stm/extracode.h
@@ -1,3 +1,5 @@
+#include 
+
 
 static void _stm_call_finalizer(object_t *obj)
 {

From noreply at buildbot.pypy.org  Fri Feb  6 20:12:22 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri,  6 Feb 2015 20:12:22 +0100 (CET)
Subject: [pypy-commit] pypy default: Remove unused 'annotator' parameter
 from functions in rpython.translator.unsimplify.
Message-ID: <20150206191222.D708B1C00EF@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: 
Changeset: r75741:80ab83e0a2f3
Date: 2015-02-06 20:12 +0100
http://bitbucket.org/pypy/pypy/changeset/80ab83e0a2f3/

Log:	Remove unused 'annotator' parameter from functions in
	rpython.translator.unsimplify.

diff --git a/rpython/flowspace/generator.py b/rpython/flowspace/generator.py
--- a/rpython/flowspace/generator.py
+++ b/rpython/flowspace/generator.py
@@ -107,7 +107,7 @@
     # First, always run simplify_graph in order to reduce the number of
     # variables passed around
     simplify_graph(graph)
-    insert_empty_startblock(None, graph)
+    insert_empty_startblock(graph)
     _insert_reads(graph.startblock, Entry.varnames)
     Entry.block = graph.startblock
     #
@@ -130,7 +130,7 @@
             if hlop.opname == 'yield_':
                 [v_yielded_value] = hlop.args
                 del block.operations[index]
-                newlink = split_block(None, block, index)
+                newlink = split_block(block, index)
                 newblock = newlink.target
                 #
                 class Resume(AbstractPosition):
diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -107,7 +107,7 @@
     """
     # split the block just before the jit_merge_point()
     if portalopindex > 0:
-        link = split_block(None, portalblock, portalopindex)
+        link = split_block(portalblock, portalopindex)
         portalblock = link.target
     portalop = portalblock.operations[0]
     # split again, this time enforcing the order of the live vars
@@ -115,7 +115,7 @@
     assert portalop.opname == 'jit_marker'
     assert portalop.args[0].value == 'jit_merge_point'
     greens_v, reds_v = decode_hp_hint_args(portalop)
-    link = split_block(None, portalblock, 0, greens_v + reds_v)
+    link = split_block(portalblock, 0, greens_v + reds_v)
     return link.target
 
 def sort_vars(args_v):
diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py
--- a/rpython/memory/gctransform/transform.py
+++ b/rpython/memory/gctransform/transform.py
@@ -221,7 +221,7 @@
         # for sanity, we need an empty block at the start of the graph
         inserted_empty_startblock = False
         if not starts_with_empty_block(graph):
-            insert_empty_startblock(self.translator.annotator, graph)
+            insert_empty_startblock(graph)
             inserted_empty_startblock = True
         is_borrowed = self.compute_borrowed_vars(graph)
 
@@ -239,7 +239,7 @@
                 if link.prevblock.exitswitch is None:
                     link.prevblock.operations.extend(llops)
                 else:
-                    insert_empty_block(self.translator.annotator, link, llops)
+                    insert_empty_block(link, llops)
 
         # remove the empty block at the start of the graph, which should
         # still be empty (but let's check)
diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py
--- a/rpython/rtyper/rtyper.py
+++ b/rpython/rtyper/rtyper.py
@@ -371,9 +371,7 @@
                 assert 0 <= pos < len(newops) - 1
                 extraops = block.operations[pos+1:]
                 del block.operations[pos+1:]
-                extrablock = insert_empty_block(self.annotator,
-                                                noexclink,
-                                                newops = extraops)
+                extrablock = insert_empty_block(noexclink, newops=extraops)
 
         if extrablock is None:
             self.insert_link_conversions(block)
@@ -447,10 +445,9 @@
                     # cannot insert conversion operations around a single
                     # link, unless it is the only exit of this block.
                     # create a new block along the link...
-                    newblock = insert_empty_block(self.annotator,
-                                                  link,
+                    newblock = insert_empty_block(link,
                     # ...and store the conversions there.
-                                               newops=newops)
+                                                  newops=newops)
                     link = newblock.exits[0]
             for i, new_a1 in newlinkargs.items():
                 link.args[i] = new_a1
diff --git a/rpython/translator/backendopt/constfold.py b/rpython/translator/backendopt/constfold.py
--- a/rpython/translator/backendopt/constfold.py
+++ b/rpython/translator/backendopt/constfold.py
@@ -171,7 +171,7 @@
             v_result.concretetype = nextop.result.concretetype
             constants[nextop.result] = v_result
             callop = SpaceOperation('direct_call', callargs, v_result)
-            newblock = insert_empty_block(None, link, [callop])
+            newblock = insert_empty_block(link, [callop])
             [link] = newblock.exits
             assert link.target is block
             folded_count += 1
@@ -197,7 +197,7 @@
                 splitlink = block.exits[0]
             else:
                 # split the block at the given position
-                splitlink = split_block(None, block, position)
+                splitlink = split_block(block, position)
                 assert list(block.exits) == [splitlink]
             assert link.target is block
             assert splitlink.prevblock is block
diff --git a/rpython/translator/backendopt/inline.py b/rpython/translator/backendopt/inline.py
--- a/rpython/translator/backendopt/inline.py
+++ b/rpython/translator/backendopt/inline.py
@@ -396,7 +396,7 @@
         copiedexceptblock.recloseblock(Link(linkargs, blocks[0]))
 
     def do_inline(self, block, index_operation):
-        splitlink = split_block(None, block, index_operation)
+        splitlink = split_block(block, index_operation)
         afterblock = splitlink.target
         # these variables have to be passed along all the links in the inlined
         # graph because the original function needs them in the blocks after
diff --git a/rpython/translator/exceptiontransform.py b/rpython/translator/exceptiontransform.py
--- a/rpython/translator/exceptiontransform.py
+++ b/rpython/translator/exceptiontransform.py
@@ -259,7 +259,7 @@
             if not self.raise_analyzer.can_raise(op):
                 continue
 
-            splitlink = split_block(None, block, i+1)
+            splitlink = split_block(block, i+1)
             afterblock = splitlink.target
             if lastblock is block:
                 lastblock = afterblock
@@ -432,7 +432,7 @@
 
         if insert_zeroing_op:
             if normalafterblock is None:
-                normalafterblock = insert_empty_block(None, l0)
+                normalafterblock = insert_empty_block(l0)
             v_result = spaceop.result
             if v_result in l0.args:
                 result_i = l0.args.index(v_result)
diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -1072,7 +1072,7 @@
                         link.target in stopblocks):
                         hints['exactlength'] = True
                     chints = Constant(hints)
-                    newblock = unsimplify.insert_empty_block(None, link)
+                    newblock = unsimplify.insert_empty_block(link)
                     index = link.args.index(vlist)
                     vlist2 = newblock.inputargs[index]
                     vlist3 = Variable(vlist2)
diff --git a/rpython/translator/test/test_unsimplify.py b/rpython/translator/test/test_unsimplify.py
--- a/rpython/translator/test/test_unsimplify.py
+++ b/rpython/translator/test/test_unsimplify.py
@@ -21,7 +21,7 @@
             w = x * y
             return z + w
         graph, t = translate(f, [int, int])
-        split_block(t.annotator, graph.startblock, i)
+        split_block(graph.startblock, i)
         checkgraph(graph)
         interp = LLInterpreter(t.rtyper)
         result = interp.eval_graph(graph, [1, 2])
@@ -35,7 +35,7 @@
             else:
                 return y + 2
         graph, t = translate(f, [int, int])
-        split_block(t.annotator, graph.startblock, i)
+        split_block(graph.startblock, i)
         checkgraph(graph)
         interp = LLInterpreter(t.rtyper)
         result = interp.eval_graph(graph, [-12, 2])
@@ -61,7 +61,7 @@
                 return 1
             return x
         graph, t = translate(catches, [int])
-        split_block(t.annotator, graph.startblock, i)
+        split_block(graph.startblock, i)
         checkgraph(graph)
         interp = LLInterpreter(t.rtyper)
         result = interp.eval_graph(graph, [0])
diff --git a/rpython/translator/unsimplify.py b/rpython/translator/unsimplify.py
--- a/rpython/translator/unsimplify.py
+++ b/rpython/translator/unsimplify.py
@@ -7,7 +7,7 @@
     var.concretetype = concretetype
     return var
 
-def insert_empty_block(annotator, link, newops=[]):
+def insert_empty_block(link, newops=[]):
     """Insert and return a new block along the given link."""
     vars = {}
     for v in link.args:
@@ -30,7 +30,7 @@
     link.target = newblock
     return newblock
 
-def insert_empty_startblock(annotator, graph):
+def insert_empty_startblock(graph):
     vars = [v.copy() for v in graph.startblock.inputargs]
     newblock = Block(vars)
     newblock.closeblock(Link(vars, graph.startblock))
@@ -41,7 +41,7 @@
             and graph.startblock.exitswitch is None
             and graph.startblock.exits[0].args == graph.getargs())
 
-def split_block(annotator, block, index, _forcelink=None):
+def split_block(block, index, _forcelink=None):
     """return a link where prevblock is the block leading up but excluding the
     index'th operation and target is a new block with the neccessary variables
     passed on.

From noreply at buildbot.pypy.org  Fri Feb  6 22:27:45 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  6 Feb 2015 22:27:45 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: ssl: fix error message in
 load_cert_chain() password.
Message-ID: <20150206212745.9DC621C00EF@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: stdlib-2.7.9
Changeset: r75742:0a58fd236ab2
Date: 2015-02-06 22:26 +0100
http://bitbucket.org/pypy/pypy/changeset/0a58fd236ab2/

Log:	ssl: fix error message in load_cert_chain() password.

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -1092,8 +1092,14 @@
     password = ""
     if pw_info.w_callable:
         try:
-            password = pw_info.space.str_w(
-                space.call_function(pw_info.w_callable))
+            w_result = space.call_function(pw_info.w_callable)
+            try:
+                password = pw_info.space.bufferstr_w(w_result)
+            except OperationError as e:
+                if not e.match(space, space.w_TypeError):
+                    raise
+                raise oefmt(space.w_TypeError, 
+                            "password callback must return a string")
         except OperationError as e:
             pw_info.operationerror = e
             return rffi.cast(rffi.INT, -1)
@@ -1328,14 +1334,19 @@
             if space.is_true(space.callable(w_password)):
                 pw_info.w_callable = w_password
             else:
-                pw_info.password = space.str_w(w_password)
+                try:
+                    pw_info.password = space.bufferstr_w(w_password)
+                except OperationError as e:
+                    if not e.match(space, space.w_TypeError):
+                        raise
+                    raise oefmt(space.w_TypeError, 
+                                "password should be a string or callable")
 
             libssl_SSL_CTX_set_default_passwd_cb(
                 self.ctx, _password_callback)
             libssl_SSL_CTX_set_default_passwd_cb_userdata(
                 self.ctx, rffi.cast(rffi.VOIDP, index))
 
-
         try:
             set_errno(0)
             ret = libssl_SSL_CTX_use_certificate_chain_file(self.ctx, certfile)

From noreply at buildbot.pypy.org  Fri Feb  6 22:54:01 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri,  6 Feb 2015 22:54:01 +0100 (CET)
Subject: [pypy-commit] pypy llvm-translation-backend: hg merge default
Message-ID: <20150206215401.9457C1C0100@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: llvm-translation-backend
Changeset: r75743:6000e86e39ba
Date: 2015-02-06 20:17 +0100
http://bitbucket.org/pypy/pypy/changeset/6000e86e39ba/

Log:	hg merge default

diff too long, truncating to 2000 out of 19187 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -7,10 +7,7 @@
 9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
 ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm
 20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0
-20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0
-0000000000000000000000000000000000000000 release-2.3.0
 394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3
 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1
 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1
-32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1
-0000000000000000000000000000000000000000 release-2.2=3.1
+10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -42,19 +42,19 @@
   Amaury Forgeot d'Arc
   Samuele Pedroni
   Alex Gaynor
+  Brian Kearns
+  Matti Picus
+  Philip Jenvey
   Michael Hudson
   David Schneider
-  Matti Picus
-  Brian Kearns
-  Philip Jenvey
   Holger Krekel
   Christian Tismer
   Hakan Ardo
   Benjamin Peterson
   Manuel Jacob
+  Ronan Lamy
   Anders Chrigstrom
   Eric van Riet Paap
-  Ronan Lamy
   Wim Lavrijsen
   Richard Emslie
   Alexander Schremmer
@@ -68,9 +68,9 @@
   Camillo Bruni
   Laura Creighton
   Toon Verwaest
+  Romain Guillebert
   Leonardo Santagada
   Seo Sanghyeon
-  Romain Guillebert
   Justin Peel
   Ronny Pfannschmidt
   David Edelsohn
@@ -91,15 +91,16 @@
   Michal Bendowski
   Jan de Mooij
   stian
+  Tyler Wade
   Michael Foord
   Stephan Diehl
-  Tyler Wade
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
   Patrick Maupin
   Bob Ippolito
   Bruno Gola
+  David Malcolm
   Jean-Paul Calderone
   Timo Paulssen
   Squeaky
@@ -108,18 +109,19 @@
   Marius Gedminas
   Martin Matusiak
   Konstantin Lopuhin
+  Wenzhu Man
   John Witulski
-  Wenzhu Man
+  Laurence Tratt
+  Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
   Mark Pearse
   Simon Cross
-  Ivan Sichmann Freitas
   Andreas Stührk
+  Stefano Rivera
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Stefano Rivera
   Paweł Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -129,7 +131,6 @@
   tav
   Taavi Burns
   Georg Brandl
-  Laurence Tratt
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -141,13 +142,12 @@
   Jeremy Thurgood
   Rami Chowdhury
   Tobias Pape
-  David Malcolm
   Eugene Oden
   Henry Mason
   Vasily Kuznetsov
   Preston Timmons
+  David Ripton
   Jeff Terrace
-  David Ripton
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
@@ -166,13 +166,16 @@
   Gintautas Miliauskas
   Michael Twomey
   Lucian Branescu Mihaila
+  Yichao Yu
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
   Karl Bartel
+  Wouter van Heyst
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  anatoly techtonik
   Stuart Williams
   Jasper Schulz
   Christian Hudon
@@ -182,12 +185,11 @@
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
-  anatoly techtonik
   Neil Shepperd
+  Stanislaw Halik
   Mikael Schönenberg
   Elmo M?ntynen
   Jonathan David Riehl
-  Stanislaw Halik
   Anders Qvist
   Corbin Simpson
   Chirag Jadwani
@@ -196,10 +198,13 @@
   Vincent Legoll
   Alan McIntyre
   Alexander Sedov
+  Attila Gobi
   Christopher Pope
   Christian Tismer 
   Marc Abramowitz
   Dan Stromberg
+  Arjun Naik
+  Valentina Mukhamedzhanova
   Stefano Parmesan
   Alexis Daboville
   Jens-Uwe Mager
@@ -213,8 +218,6 @@
   Sylvain Thenault
   Nathan Taylor
   Vladimir Kryachko
-  Arjun Naik
-  Attila Gobi
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -222,22 +225,23 @@
   Ryan Gonzalez
   Ian Foote
   Kristjan Valur Jonsson
+  David Lievens
   Neil Blakey-Milner
   Lutz Paelike
   Lucio Torre
   Lars Wassermann
-  Valentina Mukhamedzhanova
   Henrik Vendelbo
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
   Sergey Kishchenko
-  Yichao Yu
   Ignas Mikalajunas
   Christoph Gerum
   Martin Blais
   Lene Wagner
   Tomo Cocoa
+  Toni Mattis
+  Lucas Stadler
   roberto at goyle
   Yury V. Zaytsev
   Anna Katrina Dominguez
@@ -265,23 +269,30 @@
   Stephan Busemann
   Rafał Gałczyński
   Christian Muirhead
+  Berker Peksag
   James Lan
   shoma hosaka
-  Daniel Neuh?user
-  Matthew Miller
+  Daniel Neuhäuser
+  Ben Mather
+  halgari
+  Boglarka Vezer
+  Chris Pressey
   Buck Golemon
   Konrad Delong
   Dinu Gherman
   Chris Lambacher
   coolbutuseless at gmail.com
+  Jim Baker
   Rodrigo Araújo
-  Jim Baker
+  Nikolaos-Digenis Karagiannis
   James Robert
   Armin Ronacher
   Brett Cannon
+  Donald Stufft
   yrttyr
   aliceinwire
   OlivierBlanvillain
+  Dan Sanders
   Zooko Wilcox-O Hearn
   Tomer Chachamu
   Christopher Groskopf
@@ -295,6 +306,7 @@
   Markus Unterwaditzer
   Even Wiik Thomassen
   jbs
+  squeaky
   soareschen
   Kurt Griffiths
   Mike Bayer
@@ -306,6 +318,7 @@
   Anna Ravencroft
   Dan Crosta
   Julien Phalip
+  Roman Podoliaka
   Dan Loewenherz
 
   Heinrich-Heine University, Germany 
diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -17,6 +17,10 @@
 except ImportError:
     assert '__pypy__' not in _sys.builtin_module_names
     newdict = lambda _ : {}
+try:
+    from __pypy__ import reversed_dict
+except ImportError:
+    reversed_dict = lambda d: reversed(d.keys())
 
 try:
     from thread import get_ident as _get_ident
@@ -29,142 +33,35 @@
 ################################################################################
 
 class OrderedDict(dict):
-    'Dictionary that remembers insertion order'
-    # An inherited dict maps keys to values.
-    # The inherited dict provides __getitem__, __len__, __contains__, and get.
-    # The remaining methods are order-aware.
-    # Big-O running times for all methods are the same as regular dictionaries.
+    '''Dictionary that remembers insertion order.
 
-    # The internal self.__map dict maps keys to links in a doubly linked list.
-    # The circular doubly linked list starts and ends with a sentinel element.
-    # The sentinel element never gets deleted (this simplifies the algorithm).
-    # Each link is stored as a list of length three:  [PREV, NEXT, KEY].
+    In PyPy all dicts are ordered anyway.  This is mostly useful as a
+    placeholder to mean "this dict must be ordered even on CPython".
 
-    def __init__(self, *args, **kwds):
-        '''Initialize an ordered dictionary.  The signature is the same as
-        regular dictionaries, but keyword arguments are not recommended because
-        their insertion order is arbitrary.
-
-        '''
-        if len(args) > 1:
-            raise TypeError('expected at most 1 arguments, got %d' % len(args))
-        try:
-            self.__root
-        except AttributeError:
-            self.__root = root = []                     # sentinel node
-            root[:] = [root, root, None]
-            self.__map = {}
-        self.__update(*args, **kwds)
-
-    def __setitem__(self, key, value, dict_setitem=dict.__setitem__):
-        'od.__setitem__(i, y) <==> od[i]=y'
-        # Setting a new item creates a new link at the end of the linked list,
-        # and the inherited dictionary is updated with the new key/value pair.
-        if key not in self:
-            root = self.__root
-            last = root[0]
-            last[1] = root[0] = self.__map[key] = [last, root, key]
-        return dict_setitem(self, key, value)
-
-    def __delitem__(self, key, dict_delitem=dict.__delitem__):
-        'od.__delitem__(y) <==> del od[y]'
-        # Deleting an existing item uses self.__map to find the link which gets
-        # removed by updating the links in the predecessor and successor nodes.
-        dict_delitem(self, key)
-        link_prev, link_next, _ = self.__map.pop(key)
-        link_prev[1] = link_next                        # update link_prev[NEXT]
-        link_next[0] = link_prev                        # update link_next[PREV]
-
-    def __iter__(self):
-        'od.__iter__() <==> iter(od)'
-        # Traverse the linked list in order.
-        root = self.__root
-        curr = root[1]                                  # start at the first node
-        while curr is not root:
-            yield curr[2]                               # yield the curr[KEY]
-            curr = curr[1]                              # move to next node
+    Known difference: iterating over an OrderedDict which is being
+    concurrently modified raises RuntimeError in PyPy.  In CPython
+    instead we get some behavior that appears reasonable in some
+    cases but is nonsensical in other cases.  This is officially
+    forbidden by the CPython docs, so we forbid it explicitly for now.
+    '''
 
     def __reversed__(self):
-        'od.__reversed__() <==> reversed(od)'
-        # Traverse the linked list in reverse order.
-        root = self.__root
-        curr = root[0]                                  # start at the last node
-        while curr is not root:
-            yield curr[2]                               # yield the curr[KEY]
-            curr = curr[0]                              # move to previous node
-
-    def clear(self):
-        'od.clear() -> None.  Remove all items from od.'
-        root = self.__root
-        root[:] = [root, root, None]
-        self.__map.clear()
-        dict.clear(self)
-
-    # -- the following methods do not depend on the internal structure --
-
-    def keys(self):
-        'od.keys() -> list of keys in od'
-        return list(self)
-
-    def values(self):
-        'od.values() -> list of values in od'
-        return [self[key] for key in self]
-
-    def items(self):
-        'od.items() -> list of (key, value) pairs in od'
-        return [(key, self[key]) for key in self]
-
-    def iterkeys(self):
-        'od.iterkeys() -> an iterator over the keys in od'
-        return iter(self)
-
-    def itervalues(self):
-        'od.itervalues -> an iterator over the values in od'
-        for k in self:
-            yield self[k]
-
-    def iteritems(self):
-        'od.iteritems -> an iterator over the (key, value) pairs in od'
-        for k in self:
-            yield (k, self[k])
-
-    update = MutableMapping.update
-
-    __update = update # let subclasses override update without breaking __init__
-
-    __marker = object()
-
-    def pop(self, key, default=__marker):
-        '''od.pop(k[,d]) -> v, remove specified key and return the corresponding
-        value.  If key is not found, d is returned if given, otherwise KeyError
-        is raised.
-
-        '''
-        if key in self:
-            result = self[key]
-            del self[key]
-            return result
-        if default is self.__marker:
-            raise KeyError(key)
-        return default
-
-    def setdefault(self, key, default=None):
-        'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
-        if key in self:
-            return self[key]
-        self[key] = default
-        return default
+        return reversed_dict(self)
 
     def popitem(self, last=True):
         '''od.popitem() -> (k, v), return and remove a (key, value) pair.
         Pairs are returned in LIFO order if last is true or FIFO order if false.
 
         '''
-        if not self:
-            raise KeyError('dictionary is empty')
-        key = next(reversed(self) if last else iter(self))
-        value = self.pop(key)
-        return key, value
+        if last:
+            return dict.popitem(self)
+        else:
+            it = dict.__iter__(self)
+            try:
+                k = it.next()
+            except StopIteration:
+                raise KeyError('dictionary is empty')
+            return (k, self.pop(k))
 
     def __repr__(self, _repr_running={}):
         'od.__repr__() <==> repr(od)'
@@ -183,8 +80,6 @@
         'Return state information for pickling'
         items = [[k, self[k]] for k in self]
         inst_dict = vars(self).copy()
-        for k in vars(OrderedDict()):
-            inst_dict.pop(k, None)
         if inst_dict:
             return (self.__class__, (items,), inst_dict)
         return self.__class__, (items,)
@@ -193,17 +88,6 @@
         'od.copy() -> a shallow copy of od'
         return self.__class__(self)
 
-    @classmethod
-    def fromkeys(cls, iterable, value=None):
-        '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S.
-        If not specified, the value defaults to None.
-
-        '''
-        self = cls()
-        for key in iterable:
-            self[key] = value
-        return self
-
     def __eq__(self, other):
         '''od.__eq__(y) <==> od==y.  Comparison to another OD is order-sensitive
         while comparison to a regular mapping is order-insensitive.
diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py
@@ -2,7 +2,6 @@
 import array
 import gc
 import unittest
-from ctypes.test import xfail
 
 class X(Structure):
     _fields_ = [("c_int", c_int)]
@@ -11,7 +10,6 @@
         self._init_called = True
 
 class Test(unittest.TestCase):
-    @xfail
     def test_fom_buffer(self):
         a = array.array("i", range(16))
         x = (c_int * 16).from_buffer(a)
@@ -34,10 +32,9 @@
         del a; gc.collect(); gc.collect(); gc.collect()
         self.assertEqual(x[:], expected)
 
-        self.assertRaises(TypeError,
+        self.assertRaises((TypeError, ValueError),
                           (c_char * 16).from_buffer, "a" * 16)
 
-    @xfail
     def test_fom_buffer_with_offset(self):
         a = array.array("i", range(16))
         x = (c_int * 15).from_buffer(a, sizeof(c_int))
@@ -46,7 +43,6 @@
         self.assertRaises(ValueError, lambda: (c_int * 16).from_buffer(a, sizeof(c_int)))
         self.assertRaises(ValueError, lambda: (c_int * 1).from_buffer(a, 16 * sizeof(c_int)))
 
-    @xfail
     def test_from_buffer_copy(self):
         a = array.array("i", range(16))
         x = (c_int * 16).from_buffer_copy(a)
@@ -71,7 +67,6 @@
         x = (c_char * 16).from_buffer_copy("a" * 16)
         self.assertEqual(x[:], "a" * 16)
 
-    @xfail
     def test_fom_buffer_copy_with_offset(self):
         a = array.array("i", range(16))
         x = (c_int * 15).from_buffer_copy(a, sizeof(c_int))
diff --git a/lib-python/2.7/test/test_collections.py b/lib-python/2.7/test/test_collections.py
--- a/lib-python/2.7/test/test_collections.py
+++ b/lib-python/2.7/test/test_collections.py
@@ -578,7 +578,12 @@
             def __repr__(self):
                 return "MySet(%s)" % repr(list(self))
         s = MySet([5,43,2,1])
-        self.assertEqual(s.pop(), 1)
+        # changed from CPython 2.7: it was "s.pop() == 1" but I see
+        # nothing that guarantees a particular order here.  In the
+        # 'all_ordered_dicts' branch of PyPy (or with OrderedDict
+        # instead of sets), it consistently returns 5, but this test
+        # should not rely on this or any other order.
+        self.assert_(s.pop() in [5,43,2,1])
 
     def test_issue8750(self):
         empty = WithSet()
@@ -1010,8 +1015,9 @@
                                           c=3, e=5).items()), pairs)                # mixed input
 
         # make sure no positional args conflict with possible kwdargs
-        self.assertEqual(inspect.getargspec(OrderedDict.__dict__['__init__']).args,
-                         ['self'])
+        if '__init__' in OrderedDict.__dict__:   # absent in PyPy
+            self.assertEqual(inspect.getargspec(OrderedDict.__dict__['__init__']).args,
+                             ['self'])
 
         # Make sure that direct calls to __init__ do not clear previous contents
         d = OrderedDict([('a', 1), ('b', 2), ('c', 3), ('d', 44), ('e', 55)])
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -83,6 +83,37 @@
     def in_dll(self, dll, name):
         return self.from_address(dll._handle.getaddressindll(name))
 
+    def from_buffer(self, obj, offset=0):
+        size = self._sizeofinstances()
+        buf = buffer(obj, offset, size)
+        if len(buf) < size:
+            raise ValueError(
+                "Buffer size too small (%d instead of at least %d bytes)"
+                % (len(buf) + offset, size + offset))
+        raw_addr = buf._pypy_raw_address()
+        result = self.from_address(raw_addr)
+        result._ensure_objects()['ffffffff'] = obj
+        return result
+
+    def from_buffer_copy(self, obj, offset=0):
+        size = self._sizeofinstances()
+        buf = buffer(obj, offset, size)
+        if len(buf) < size:
+            raise ValueError(
+                "Buffer size too small (%d instead of at least %d bytes)"
+                % (len(buf) + offset, size + offset))
+        result = self()
+        dest = result._buffer.buffer
+        try:
+            raw_addr = buf._pypy_raw_address()
+        except ValueError:
+            _rawffi.rawstring2charp(dest, buf)
+        else:
+            from ctypes import memmove
+            memmove(dest, raw_addr, size)
+        return result
+
+
 class CArgObject(object):
     """ simple wrapper around buffer, just for the case of freeing
     it afterwards
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -1,3 +1,4 @@
+import sys
 import _rawffi
 from _ctypes.basics import _CData, _CDataMeta, keepalive_key,\
      store_reference, ensure_objects, CArgObject
@@ -178,6 +179,8 @@
         instance = StructOrUnion.__new__(self)
         if isinstance(address, _rawffi.StructureInstance):
             address = address.buffer
+        # fix the address: turn it into as unsigned, in case it is negative
+        address = address & (sys.maxint * 2 + 1)
         instance.__dict__['_buffer'] = self._ffistruct.fromaddress(address)
         return instance
 
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -6,3 +6,8 @@
 
 __version__ = "0.8.6"
 __version_info__ = (0, 8, 6)
+
+# The verifier module file names are based on the CRC32 of a string that
+# contains the following version number.  It may be older than __version__
+# if nothing is clearly incompatible.
+__version_verifier_modules__ = "0.8.6"
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -69,6 +69,7 @@
         self._function_caches = []
         self._libraries = []
         self._cdefsources = []
+        self._windows_unicode = None
         if hasattr(backend, 'set_ffi'):
             backend.set_ffi(self)
         for name in backend.__dict__:
@@ -77,6 +78,7 @@
         #
         with self._lock:
             self.BVoidP = self._get_cached_btype(model.voidp_type)
+            self.BCharA = self._get_cached_btype(model.char_array_type)
         if isinstance(backend, types.ModuleType):
             # _cffi_backend: attach these constants to the class
             if not hasattr(FFI, 'NULL'):
@@ -189,13 +191,16 @@
             cdecl = self._typeof(cdecl)
         return self._backend.alignof(cdecl)
 
-    def offsetof(self, cdecl, fieldname):
+    def offsetof(self, cdecl, *fields_or_indexes):
         """Return the offset of the named field inside the given
-        structure, which must be given as a C type name.
+        structure or array, which must be given as a C type name.  
+        You can give several field names in case of nested structures.
+        You can also give numeric values which correspond to array
+        items, in case of an array type.
         """
         if isinstance(cdecl, basestring):
             cdecl = self._typeof(cdecl)
-        return self._backend.typeoffsetof(cdecl, fieldname)[1]
+        return self._typeoffsetof(cdecl, *fields_or_indexes)[1]
 
     def new(self, cdecl, init=None):
         """Allocate an instance according to the specified C type and
@@ -264,6 +269,16 @@
         """
         return self._backend.buffer(cdata, size)
 
+    def from_buffer(self, python_buffer):
+        """Return a  that points to the data of the
+        given Python object, which must support the buffer interface.
+        Note that this is not meant to be used on the built-in types str,
+        unicode, or bytearray (you can build 'char[]' arrays explicitly)
+        but only on objects containing large quantities of raw data
+        in some other format, like 'array.array' or numpy arrays.
+        """
+        return self._backend.from_buffer(self.BCharA, python_buffer)
+
     def callback(self, cdecl, python_callable=None, error=None):
         """Return a callback object or a decorator making such a
         callback object.  'cdecl' must name a C function pointer type.
@@ -335,9 +350,23 @@
         which requires binary compatibility in the signatures.
         """
         from .verifier import Verifier, _caller_dir_pycache
+        #
+        # If set_unicode(True) was called, insert the UNICODE and
+        # _UNICODE macro declarations
+        if self._windows_unicode:
+            self._apply_windows_unicode(kwargs)
+        #
+        # Set the tmpdir here, and not in Verifier.__init__: it picks
+        # up the caller's directory, which we want to be the caller of
+        # ffi.verify(), as opposed to the caller of Veritier().
         tmpdir = tmpdir or _caller_dir_pycache()
+        #
+        # Make a Verifier() and use it to load the library.
         self.verifier = Verifier(self, source, tmpdir, **kwargs)
         lib = self.verifier.load_library()
+        #
+        # Save the loaded library for keep-alive purposes, even
+        # if the caller doesn't keep it alive itself (it should).
         self._libraries.append(lib)
         return lib
 
@@ -356,15 +385,29 @@
         with self._lock:
             return model.pointer_cache(self, ctype)
 
-    def addressof(self, cdata, field=None):
+    def addressof(self, cdata, *fields_or_indexes):
         """Return the address of a .
-        If 'field' is specified, return the address of this field.
+        If 'fields_or_indexes' are given, returns the address of that
+        field or array item in the structure or array, recursively in
+        case of nested structures.
         """
         ctype = self._backend.typeof(cdata)
-        ctype, offset = self._backend.typeoffsetof(ctype, field)
+        if fields_or_indexes:
+            ctype, offset = self._typeoffsetof(ctype, *fields_or_indexes)
+        else:
+            if ctype.kind == "pointer":
+                raise TypeError("addressof(pointer)")
+            offset = 0
         ctypeptr = self._pointer_to(ctype)
         return self._backend.rawaddressof(ctypeptr, cdata, offset)
 
+    def _typeoffsetof(self, ctype, field_or_index, *fields_or_indexes):
+        ctype, offset = self._backend.typeoffsetof(ctype, field_or_index)
+        for field1 in fields_or_indexes:
+            ctype, offset1 = self._backend.typeoffsetof(ctype, field1, 1)
+            offset += offset1
+        return ctype, offset
+
     def include(self, ffi_to_include):
         """Includes the typedefs, structs, unions and enums defined
         in another FFI instance.  Usage is similar to a #include in C,
@@ -387,6 +430,44 @@
     def from_handle(self, x):
         return self._backend.from_handle(x)
 
+    def set_unicode(self, enabled_flag):
+        """Windows: if 'enabled_flag' is True, enable the UNICODE and
+        _UNICODE defines in C, and declare the types like TCHAR and LPTCSTR
+        to be (pointers to) wchar_t.  If 'enabled_flag' is False,
+        declare these types to be (pointers to) plain 8-bit characters.
+        This is mostly for backward compatibility; you usually want True.
+        """
+        if self._windows_unicode is not None:
+            raise ValueError("set_unicode() can only be called once")
+        enabled_flag = bool(enabled_flag)
+        if enabled_flag:
+            self.cdef("typedef wchar_t TBYTE;"
+                      "typedef wchar_t TCHAR;"
+                      "typedef const wchar_t *LPCTSTR;"
+                      "typedef const wchar_t *PCTSTR;"
+                      "typedef wchar_t *LPTSTR;"
+                      "typedef wchar_t *PTSTR;"
+                      "typedef TBYTE *PTBYTE;"
+                      "typedef TCHAR *PTCHAR;")
+        else:
+            self.cdef("typedef char TBYTE;"
+                      "typedef char TCHAR;"
+                      "typedef const char *LPCTSTR;"
+                      "typedef const char *PCTSTR;"
+                      "typedef char *LPTSTR;"
+                      "typedef char *PTSTR;"
+                      "typedef TBYTE *PTBYTE;"
+                      "typedef TCHAR *PTCHAR;")
+        self._windows_unicode = enabled_flag
+
+    def _apply_windows_unicode(self, kwds):
+        defmacros = kwds.get('define_macros', ())
+        if not isinstance(defmacros, (list, tuple)):
+            raise TypeError("'define_macros' must be a list or tuple")
+        defmacros = list(defmacros) + [('UNICODE', '1'),
+                                       ('_UNICODE', '1')]
+        kwds['define_macros'] = defmacros
+
 
 def _load_backend_lib(backend, name, flags):
     if name is None:
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -169,6 +169,7 @@
 class CTypesGenericPtr(CTypesData):
     __slots__ = ['_address', '_as_ctype_ptr']
     _automatic_casts = False
+    kind = "pointer"
 
     @classmethod
     def _newp(cls, init):
@@ -370,10 +371,12 @@
                                 (CTypesPrimitive, type(source).__name__))
             return source
         #
+        kind1 = kind
         class CTypesPrimitive(CTypesGenericPrimitive):
             __slots__ = ['_value']
             _ctype = ctype
             _reftypename = '%s &' % name
+            kind = kind1
 
             def __init__(self, value):
                 self._value = value
@@ -703,12 +706,13 @@
         class struct_or_union(base_ctypes_class):
             pass
         struct_or_union.__name__ = '%s_%s' % (kind, name)
+        kind1 = kind
         #
         class CTypesStructOrUnion(CTypesBaseStructOrUnion):
             __slots__ = ['_blob']
             _ctype = struct_or_union
             _reftypename = '%s &' % (name,)
-            _kind = kind
+            _kind = kind = kind1
         #
         CTypesStructOrUnion._fix_class()
         return CTypesStructOrUnion
@@ -994,27 +998,42 @@
     def getcname(self, BType, replace_with):
         return BType._get_c_name(replace_with)
 
-    def typeoffsetof(self, BType, fieldname):
-        if fieldname is not None and issubclass(BType, CTypesGenericPtr):
-            BType = BType._BItem
-        if not issubclass(BType, CTypesBaseStructOrUnion):
-            raise TypeError("expected a struct or union ctype")
-        if fieldname is None:
-            return (BType, 0)
-        else:
+    def typeoffsetof(self, BType, fieldname, num=0):
+        if isinstance(fieldname, str):
+            if num == 0 and issubclass(BType, CTypesGenericPtr):
+                BType = BType._BItem
+            if not issubclass(BType, CTypesBaseStructOrUnion):
+                raise TypeError("expected a struct or union ctype")
             BField = BType._bfield_types[fieldname]
             if BField is Ellipsis:
                 raise TypeError("not supported for bitfields")
             return (BField, BType._offsetof(fieldname))
+        elif isinstance(fieldname, (int, long)):
+            if issubclass(BType, CTypesGenericArray):
+                BType = BType._CTPtr
+            if not issubclass(BType, CTypesGenericPtr):
+                raise TypeError("expected an array or ptr ctype")
+            BItem = BType._BItem
+            offset = BItem._get_size() * fieldname
+            if offset > sys.maxsize:
+                raise OverflowError
+            return (BItem, offset)
+        else:
+            raise TypeError(type(fieldname))
 
-    def rawaddressof(self, BTypePtr, cdata, offset):
+    def rawaddressof(self, BTypePtr, cdata, offset=None):
         if isinstance(cdata, CTypesBaseStructOrUnion):
             ptr = ctypes.pointer(type(cdata)._to_ctypes(cdata))
         elif isinstance(cdata, CTypesGenericPtr):
+            if offset is None or not issubclass(type(cdata)._BItem,
+                                                CTypesBaseStructOrUnion):
+                raise TypeError("unexpected cdata type")
+            ptr = type(cdata)._to_ctypes(cdata)
+        elif isinstance(cdata, CTypesGenericArray):
             ptr = type(cdata)._to_ctypes(cdata)
         else:
             raise TypeError("expected a ")
-        if offset != 0:
+        if offset:
             ptr = ctypes.cast(
                 ctypes.c_void_p(
                     ctypes.cast(ptr, ctypes.c_void_p).value + offset),
diff --git a/lib_pypy/cffi/commontypes.py b/lib_pypy/cffi/commontypes.py
--- a/lib_pypy/cffi/commontypes.py
+++ b/lib_pypy/cffi/commontypes.py
@@ -29,6 +29,9 @@
                 result = model.PointerType(resolve_common_type(result[:-2]))
         elif result in model.PrimitiveType.ALL_PRIMITIVE_TYPES:
             result = model.PrimitiveType(result)
+        elif result == 'set-unicode-needed':
+            raise api.FFIError("The Windows type %r is only available after "
+                               "you call ffi.set_unicode()" % (commontype,))
         else:
             if commontype == result:
                 raise api.FFIError("Unsupported type: %r.  Please file a bug "
@@ -86,8 +89,6 @@
         "ULONGLONG": "unsigned long long",
         "WCHAR": "wchar_t",
         "SHORT": "short",
-        "TBYTE": "WCHAR",
-        "TCHAR": "WCHAR",
         "UCHAR": "unsigned char",
         "UINT": "unsigned int",
         "UINT8": "unsigned char",
@@ -157,14 +158,12 @@
 
         "LPCVOID": model.const_voidp_type,
         "LPCWSTR": "const WCHAR *",
-        "LPCTSTR": "LPCWSTR",
         "LPDWORD": "DWORD *",
         "LPHANDLE": "HANDLE *",
         "LPINT": "int *",
         "LPLONG": "long *",
         "LPSTR": "CHAR *",
         "LPWSTR": "WCHAR *",
-        "LPTSTR": "LPWSTR",
         "LPVOID": model.voidp_type,
         "LPWORD": "WORD *",
         "LRESULT": "LONG_PTR",
@@ -173,7 +172,6 @@
         "PBYTE": "BYTE *",
         "PCHAR": "CHAR *",
         "PCSTR": "const CHAR *",
-        "PCTSTR": "LPCWSTR",
         "PCWSTR": "const WCHAR *",
         "PDWORD": "DWORD *",
         "PDWORDLONG": "DWORDLONG *",
@@ -200,9 +198,6 @@
         "PSIZE_T": "SIZE_T *",
         "PSSIZE_T": "SSIZE_T *",
         "PSTR": "CHAR *",
-        "PTBYTE": "TBYTE *",
-        "PTCHAR": "TCHAR *",
-        "PTSTR": "LPWSTR",
         "PUCHAR": "UCHAR *",
         "PUHALF_PTR": "UHALF_PTR *",
         "PUINT": "UINT *",
@@ -240,6 +235,15 @@
         "USN": "LONGLONG",
         "VOID": model.void_type,
         "WPARAM": "UINT_PTR",
+
+        "TBYTE": "set-unicode-needed",
+        "TCHAR": "set-unicode-needed",
+        "LPCTSTR": "set-unicode-needed",
+        "PCTSTR": "set-unicode-needed",
+        "LPTSTR": "set-unicode-needed",
+        "PTSTR": "set-unicode-needed",
+        "PTBYTE": "set-unicode-needed",
+        "PTCHAR": "set-unicode-needed",
         })
     return result
 
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -1,4 +1,3 @@
-
 from . import api, model
 from .commontypes import COMMON_TYPES, resolve_common_type
 try:
@@ -209,6 +208,8 @@
 
     def _add_constants(self, key, val):
         if key in self._int_constants:
+            if self._int_constants[key] == val:
+                return     # ignore identical double declarations
             raise api.FFIError(
                 "multiple declarations of constant: %s" % (key,))
         self._int_constants[key] = val
@@ -228,12 +229,18 @@
 
                 pyvalue = int(int_str, 0)
                 self._add_constants(key, pyvalue)
+                self._declare('macro ' + key, pyvalue)
             elif value == '...':
                 self._declare('macro ' + key, value)
             else:
-                raise api.CDefError('only supports the syntax "#define '
-                                    '%s ..." (literally) or "#define '
-                                    '%s 0x1FF" for now' % (key, key))
+                raise api.CDefError(
+                    'only supports one of the following syntax:\n'
+                    '  #define %s ...     (literally dot-dot-dot)\n'
+                    '  #define %s NUMBER  (with NUMBER an integer'
+                                    ' constant, decimal/hex/octal)\n'
+                    'got:\n'
+                    '  #define %s %s'
+                    % (key, key, key, value))
 
     def _parse_decl(self, decl):
         node = decl.type
@@ -460,6 +467,8 @@
             elif kind == 'union':
                 tp = model.UnionType(explicit_name, None, None, None)
             elif kind == 'enum':
+                if explicit_name == '__dotdotdot__':
+                    raise CDefError("Enums cannot be declared with ...")
                 tp = self._build_enum_type(explicit_name, type.values)
             else:
                 raise AssertionError("kind = %r" % (kind,))
@@ -532,9 +541,24 @@
 
     def _parse_constant(self, exprnode, partial_length_ok=False):
         # for now, limited to expressions that are an immediate number
-        # or negative number
+        # or positive/negative number
         if isinstance(exprnode, pycparser.c_ast.Constant):
-            return int(exprnode.value, 0)
+            s = exprnode.value
+            if s.startswith('0'):
+                if s.startswith('0x') or s.startswith('0X'):
+                    return int(s, 16)
+                return int(s, 8)
+            elif '1' <= s[0] <= '9':
+                return int(s, 10)
+            elif s[0] == "'" and s[-1] == "'" and (
+                    len(s) == 3 or (len(s) == 4 and s[1] == "\\")):
+                return ord(s[-2])
+            else:
+                raise api.CDefError("invalid constant %r" % (s,))
+        #
+        if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and
+                exprnode.op == '+'):
+            return self._parse_constant(exprnode.expr)
         #
         if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and
                 exprnode.op == '-'):
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -11,6 +11,9 @@
     """
 
 
+LIST_OF_FILE_NAMES = ['sources', 'include_dirs', 'library_dirs',
+                      'extra_objects', 'depends']
+
 def get_extension(srcfilename, modname, sources=(), **kwds):
     from distutils.core import Extension
     allsources = [srcfilename]
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -235,6 +235,8 @@
         BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist)
         return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length)
 
+char_array_type = ArrayType(PrimitiveType('char'), None)
+
 
 class StructOrUnionOrEnum(BaseTypeByIdentity):
     _attrs_ = ('name',)
@@ -478,7 +480,7 @@
     try:
         res = getattr(ffi._backend, funcname)(*args)
     except NotImplementedError as e:
-        raise NotImplementedError("%r: %s" % (srctype, e))
+        raise NotImplementedError("%s: %r: %s" % (funcname, srctype, e))
     # note that setdefault() on WeakValueDictionary is not atomic
     # and contains a rare bug (http://bugs.python.org/issue19542);
     # we have to use a lock and do it ourselves
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -65,7 +65,7 @@
         # The following two 'chained_list_constants' items contains
         # the head of these two chained lists, as a string that gives the
         # call to do, if any.
-        self._chained_list_constants = ['0', '0']
+        self._chained_list_constants = ['((void)lib,0)', '((void)lib,0)']
         #
         prnt = self._prnt
         # first paste some standard set of lines that are mostly '#define'
@@ -138,15 +138,22 @@
         prnt()
         prnt('#endif')
 
-    def load_library(self):
+    def load_library(self, flags=None):
         # XXX review all usages of 'self' here!
         # import it as a new extension module
+        if hasattr(sys, "getdlopenflags"):
+            previous_flags = sys.getdlopenflags()
         try:
+            if hasattr(sys, "setdlopenflags") and flags is not None:
+                sys.setdlopenflags(flags)
             module = imp.load_dynamic(self.verifier.get_module_name(),
                                       self.verifier.modulefilename)
         except ImportError as e:
             error = "importing %r: %s" % (self.verifier.modulefilename, e)
             raise ffiplatform.VerificationError(error)
+        finally:
+            if hasattr(sys, "setdlopenflags"):
+                sys.setdlopenflags(previous_flags)
         #
         # call loading_cpy_struct() to get the struct layout inferred by
         # the C compiler
@@ -228,7 +235,8 @@
                 converter = '_cffi_to_c_int'
                 extraarg = ', %s' % tp.name
             else:
-                converter = '_cffi_to_c_%s' % (tp.name.replace(' ', '_'),)
+                converter = '(%s)_cffi_to_c_%s' % (tp.get_c_name(''),
+                                                   tp.name.replace(' ', '_'))
             errvalue = '-1'
         #
         elif isinstance(tp, model.PointerType):
@@ -267,8 +275,8 @@
         self._prnt('  if (datasize != 0) {')
         self._prnt('    if (datasize < 0)')
         self._prnt('      %s;' % errcode)
-        self._prnt('    %s = alloca(datasize);' % (tovar,))
-        self._prnt('    memset((void *)%s, 0, datasize);' % (tovar,))
+        self._prnt('    %s = alloca((size_t)datasize);' % (tovar,))
+        self._prnt('    memset((void *)%s, 0, (size_t)datasize);' % (tovar,))
         self._prnt('    if (_cffi_convert_array_from_object('
                    '(char *)%s, _cffi_type(%d), %s) < 0)' % (
             tovar, self._gettypenum(tp), fromvar))
@@ -336,7 +344,7 @@
         prnt = self._prnt
         numargs = len(tp.args)
         if numargs == 0:
-            argname = 'no_arg'
+            argname = 'noarg'
         elif numargs == 1:
             argname = 'arg0'
         else:
@@ -386,6 +394,9 @@
         prnt('  Py_END_ALLOW_THREADS')
         prnt()
         #
+        prnt('  (void)self; /* unused */')
+        if numargs == 0:
+            prnt('  (void)noarg; /* unused */')
         if result_code:
             prnt('  return %s;' %
                  self._convert_expr_from_c(tp.result, 'result', 'result type'))
@@ -452,6 +463,7 @@
         prnt('static void %s(%s *p)' % (checkfuncname, cname))
         prnt('{')
         prnt('  /* only to generate compile-time warnings or errors */')
+        prnt('  (void)p;')
         for fname, ftype, fbitsize in tp.enumfields():
             if (isinstance(ftype, model.PrimitiveType)
                 and ftype.is_integer_type()) or fbitsize >= 0:
@@ -482,6 +494,8 @@
                 prnt('    sizeof(((%s *)0)->%s),' % (cname, fname))
         prnt('    -1')
         prnt('  };')
+        prnt('  (void)self; /* unused */')
+        prnt('  (void)noarg; /* unused */')
         prnt('  return _cffi_get_struct_layout(nums);')
         prnt('  /* the next line is not executed, but compiled */')
         prnt('  %s(0);' % (checkfuncname,))
@@ -578,7 +592,8 @@
     # constants, likely declared with '#define'
 
     def _generate_cpy_const(self, is_int, name, tp=None, category='const',
-                            vartp=None, delayed=True, size_too=False):
+                            vartp=None, delayed=True, size_too=False,
+                            check_value=None):
         prnt = self._prnt
         funcname = '_cffi_%s_%s' % (category, name)
         prnt('static int %s(PyObject *lib)' % funcname)
@@ -590,6 +605,9 @@
         else:
             assert category == 'const'
         #
+        if check_value is not None:
+            self._check_int_constant_value(name, check_value)
+        #
         if not is_int:
             if category == 'var':
                 realexpr = '&' + name
@@ -637,6 +655,27 @@
     # ----------
     # enums
 
+    def _check_int_constant_value(self, name, value, err_prefix=''):
+        prnt = self._prnt
+        if value <= 0:
+            prnt('  if ((%s) > 0 || (long)(%s) != %dL) {' % (
+                name, name, value))
+        else:
+            prnt('  if ((%s) <= 0 || (unsigned long)(%s) != %dUL) {' % (
+                name, name, value))
+        prnt('    char buf[64];')
+        prnt('    if ((%s) <= 0)' % name)
+        prnt('        snprintf(buf, 63, "%%ld", (long)(%s));' % name)
+        prnt('    else')
+        prnt('        snprintf(buf, 63, "%%lu", (unsigned long)(%s));' %
+             name)
+        prnt('    PyErr_Format(_cffi_VerificationError,')
+        prnt('                 "%s%s has the real value %s, not %s",')
+        prnt('                 "%s", "%s", buf, "%d");' % (
+            err_prefix, name, value))
+        prnt('    return -1;')
+        prnt('  }')
+
     def _enum_funcname(self, prefix, name):
         # "$enum_$1" => "___D_enum____D_1"
         name = name.replace('$', '___D_')
@@ -653,25 +692,8 @@
         prnt('static int %s(PyObject *lib)' % funcname)
         prnt('{')
         for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
-            if enumvalue < 0:
-                prnt('  if ((%s) >= 0 || (long)(%s) != %dL) {' % (
-                    enumerator, enumerator, enumvalue))
-            else:
-                prnt('  if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % (
-                    enumerator, enumerator, enumvalue))
-            prnt('    char buf[64];')
-            prnt('    if ((%s) < 0)' % enumerator)
-            prnt('        snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator)
-            prnt('    else')
-            prnt('        snprintf(buf, 63, "%%lu", (unsigned long)(%s));' %
-                 enumerator)
-            prnt('    PyErr_Format(_cffi_VerificationError,')
-            prnt('                 "enum %s: %s has the real value %s, '
-                 'not %s",')
-            prnt('                 "%s", "%s", buf, "%d");' % (
-                name, enumerator, enumvalue))
-            prnt('    return -1;')
-            prnt('  }')
+            self._check_int_constant_value(enumerator, enumvalue,
+                                           "enum %s: " % name)
         prnt('  return %s;' % self._chained_list_constants[True])
         self._chained_list_constants[True] = funcname + '(lib)'
         prnt('}')
@@ -695,8 +717,11 @@
     # macros: for now only for integers
 
     def _generate_cpy_macro_decl(self, tp, name):
-        assert tp == '...'
-        self._generate_cpy_const(True, name)
+        if tp == '...':
+            check_value = None
+        else:
+            check_value = tp     # an integer
+        self._generate_cpy_const(True, name, check_value=check_value)
 
     _generate_cpy_macro_collecttype = _generate_nothing
     _generate_cpy_macro_method = _generate_nothing
@@ -783,6 +808,24 @@
    typedef unsigned __int16 uint16_t;
    typedef unsigned __int32 uint32_t;
    typedef unsigned __int64 uint64_t;
+   typedef __int8 int_least8_t;
+   typedef __int16 int_least16_t;
+   typedef __int32 int_least32_t;
+   typedef __int64 int_least64_t;
+   typedef unsigned __int8 uint_least8_t;
+   typedef unsigned __int16 uint_least16_t;
+   typedef unsigned __int32 uint_least32_t;
+   typedef unsigned __int64 uint_least64_t;
+   typedef __int8 int_fast8_t;
+   typedef __int16 int_fast16_t;
+   typedef __int32 int_fast32_t;
+   typedef __int64 int_fast64_t;
+   typedef unsigned __int8 uint_fast8_t;
+   typedef unsigned __int16 uint_fast16_t;
+   typedef unsigned __int32 uint_fast32_t;
+   typedef unsigned __int64 uint_fast64_t;
+   typedef __int64 intmax_t;
+   typedef unsigned __int64 uintmax_t;
 # else
 #  include 
 # endif
@@ -828,12 +871,15 @@
             PyLong_FromLongLong((long long)(x)))
 
 #define _cffi_from_c_int(x, type)                                        \
-    (((type)-1) > 0 ?   /* unsigned */                                   \
-        (sizeof(type) < sizeof(long) ? PyInt_FromLong(x) :               \
-         sizeof(type) == sizeof(long) ? PyLong_FromUnsignedLong(x) :     \
-                                        PyLong_FromUnsignedLongLong(x))  \
-      : (sizeof(type) <= sizeof(long) ? PyInt_FromLong(x) :              \
-                                        PyLong_FromLongLong(x)))
+    (((type)-1) > 0 ? /* unsigned */                                     \
+        (sizeof(type) < sizeof(long) ?                                   \
+            PyInt_FromLong((long)x) :                                    \
+         sizeof(type) == sizeof(long) ?                                  \
+            PyLong_FromUnsignedLong((unsigned long)x) :                  \
+            PyLong_FromUnsignedLongLong((unsigned long long)x)) :        \
+        (sizeof(type) <= sizeof(long) ?                                  \
+            PyInt_FromLong((long)x) :                                    \
+            PyLong_FromLongLong((long long)x)))
 
 #define _cffi_to_c_int(o, type)                                          \
     (sizeof(type) == 1 ? (((type)-1) > 0 ? (type)_cffi_to_c_u8(o)        \
@@ -844,7 +890,7 @@
                                          : (type)_cffi_to_c_i32(o)) :    \
      sizeof(type) == 8 ? (((type)-1) > 0 ? (type)_cffi_to_c_u64(o)       \
                                          : (type)_cffi_to_c_i64(o)) :    \
-     (Py_FatalError("unsupported size for type " #type), 0))
+     (Py_FatalError("unsupported size for type " #type), (type)0))
 
 #define _cffi_to_c_i8                                                    \
                  ((int(*)(PyObject *))_cffi_exports[1])
@@ -907,6 +953,7 @@
 {
     PyObject *library;
     int was_alive = (_cffi_types != NULL);
+    (void)self; /* unused */
     if (!PyArg_ParseTuple(args, "OOO", &_cffi_types, &_cffi_VerificationError,
                                        &library))
         return NULL;
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -58,12 +58,12 @@
             modname = self.verifier.get_module_name()
             prnt("void %s%s(void) { }\n" % (prefix, modname))
 
-    def load_library(self):
+    def load_library(self, flags=0):
         # import it with the CFFI backend
         backend = self.ffi._backend
         # needs to make a path that contains '/', on Posix
         filename = os.path.join(os.curdir, self.verifier.modulefilename)
-        module = backend.load_library(filename)
+        module = backend.load_library(filename, flags)
         #
         # call loading_gen_struct() to get the struct layout inferred by
         # the C compiler
@@ -235,6 +235,7 @@
         prnt('static void %s(%s *p)' % (checkfuncname, cname))
         prnt('{')
         prnt('  /* only to generate compile-time warnings or errors */')
+        prnt('  (void)p;')
         for fname, ftype, fbitsize in tp.enumfields():
             if (isinstance(ftype, model.PrimitiveType)
                 and ftype.is_integer_type()) or fbitsize >= 0:
@@ -354,11 +355,20 @@
     # ----------
     # constants, likely declared with '#define'
 
-    def _generate_gen_const(self, is_int, name, tp=None, category='const'):
+    def _generate_gen_const(self, is_int, name, tp=None, category='const',
+                            check_value=None):
         prnt = self._prnt
         funcname = '_cffi_%s_%s' % (category, name)
         self.export_symbols.append(funcname)
-        if is_int:
+        if check_value is not None:
+            assert is_int
+            assert category == 'const'
+            prnt('int %s(char *out_error)' % funcname)
+            prnt('{')
+            self._check_int_constant_value(name, check_value)
+            prnt('  return 0;')
+            prnt('}')
+        elif is_int:
             assert category == 'const'
             prnt('int %s(long long *out_value)' % funcname)
             prnt('{')
@@ -367,6 +377,7 @@
             prnt('}')
         else:
             assert tp is not None
+            assert check_value is None
             prnt(tp.get_c_name(' %s(void)' % funcname, name),)
             prnt('{')
             if category == 'var':
@@ -383,9 +394,13 @@
 
     _loading_gen_constant = _loaded_noop
 
-    def _load_constant(self, is_int, tp, name, module):
+    def _load_constant(self, is_int, tp, name, module, check_value=None):
         funcname = '_cffi_const_%s' % name
-        if is_int:
+        if check_value is not None:
+            assert is_int
+            self._load_known_int_constant(module, funcname)
+            value = check_value
+        elif is_int:
             BType = self.ffi._typeof_locked("long long*")[0]
             BFunc = self.ffi._typeof_locked("int(*)(long long*)")[0]
             function = module.load_function(BFunc, funcname)
@@ -396,6 +411,7 @@
                 BLongLong = self.ffi._typeof_locked("long long")[0]
                 value += (1 << (8*self.ffi.sizeof(BLongLong)))
         else:
+            assert check_value is None
             BFunc = self.ffi._typeof_locked(tp.get_c_name('(*)(void)', name))[0]
             function = module.load_function(BFunc, funcname)
             value = function()
@@ -410,6 +426,36 @@
     # ----------
     # enums
 
+    def _check_int_constant_value(self, name, value):
+        prnt = self._prnt
+        if value <= 0:
+            prnt('  if ((%s) > 0 || (long)(%s) != %dL) {' % (
+                name, name, value))
+        else:
+            prnt('  if ((%s) <= 0 || (unsigned long)(%s) != %dUL) {' % (
+                name, name, value))
+        prnt('    char buf[64];')
+        prnt('    if ((%s) <= 0)' % name)
+        prnt('        sprintf(buf, "%%ld", (long)(%s));' % name)
+        prnt('    else')
+        prnt('        sprintf(buf, "%%lu", (unsigned long)(%s));' %
+             name)
+        prnt('    sprintf(out_error, "%s has the real value %s, not %s",')
+        prnt('            "%s", buf, "%d");' % (name[:100], value))
+        prnt('    return -1;')
+        prnt('  }')
+
+    def _load_known_int_constant(self, module, funcname):
+        BType = self.ffi._typeof_locked("char[]")[0]
+        BFunc = self.ffi._typeof_locked("int(*)(char*)")[0]
+        function = module.load_function(BFunc, funcname)
+        p = self.ffi.new(BType, 256)
+        if function(p) < 0:
+            error = self.ffi.string(p)
+            if sys.version_info >= (3,):
+                error = str(error, 'utf-8')
+            raise ffiplatform.VerificationError(error)
+
     def _enum_funcname(self, prefix, name):
         # "$enum_$1" => "___D_enum____D_1"
         name = name.replace('$', '___D_')
@@ -427,24 +473,7 @@
         prnt('int %s(char *out_error)' % funcname)
         prnt('{')
         for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
-            if enumvalue < 0:
-                prnt('  if ((%s) >= 0 || (long)(%s) != %dL) {' % (
-                    enumerator, enumerator, enumvalue))
-            else:
-                prnt('  if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % (
-                    enumerator, enumerator, enumvalue))
-            prnt('    char buf[64];')
-            prnt('    if ((%s) < 0)' % enumerator)
-            prnt('        sprintf(buf, "%%ld", (long)(%s));' % enumerator)
-            prnt('    else')
-            prnt('        sprintf(buf, "%%lu", (unsigned long)(%s));' %
-                 enumerator)
-            prnt('    sprintf(out_error,'
-                             ' "%s has the real value %s, not %s",')
-            prnt('            "%s", buf, "%d");' % (
-                enumerator[:100], enumvalue))
-            prnt('    return -1;')
-            prnt('  }')
+            self._check_int_constant_value(enumerator, enumvalue)
         prnt('  return 0;')
         prnt('}')
         prnt()
@@ -456,16 +485,8 @@
             tp.enumvalues = tuple(enumvalues)
             tp.partial_resolved = True
         else:
-            BType = self.ffi._typeof_locked("char[]")[0]
-            BFunc = self.ffi._typeof_locked("int(*)(char*)")[0]
             funcname = self._enum_funcname(prefix, name)
-            function = module.load_function(BFunc, funcname)
-            p = self.ffi.new(BType, 256)
-            if function(p) < 0:
-                error = self.ffi.string(p)
-                if sys.version_info >= (3,):
-                    error = str(error, 'utf-8')
-                raise ffiplatform.VerificationError(error)
+            self._load_known_int_constant(module, funcname)
 
     def _loaded_gen_enum(self, tp, name, module, library):
         for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
@@ -476,13 +497,21 @@
     # macros: for now only for integers
 
     def _generate_gen_macro_decl(self, tp, name):
-        assert tp == '...'
-        self._generate_gen_const(True, name)
+        if tp == '...':
+            check_value = None
+        else:
+            check_value = tp     # an integer
+        self._generate_gen_const(True, name, check_value=check_value)
 
     _loading_gen_macro = _loaded_noop
 
     def _loaded_gen_macro(self, tp, name, module, library):
-        value = self._load_constant(True, tp, name, module)
+        if tp == '...':
+            check_value = None
+        else:
+            check_value = tp     # an integer
+        value = self._load_constant(True, tp, name, module,
+                                    check_value=check_value)
         setattr(library, name, value)
         type(library)._cffi_dir.append(name)
 
@@ -565,6 +594,24 @@
    typedef unsigned __int16 uint16_t;
    typedef unsigned __int32 uint32_t;
    typedef unsigned __int64 uint64_t;
+   typedef __int8 int_least8_t;
+   typedef __int16 int_least16_t;
+   typedef __int32 int_least32_t;
+   typedef __int64 int_least64_t;
+   typedef unsigned __int8 uint_least8_t;
+   typedef unsigned __int16 uint_least16_t;
+   typedef unsigned __int32 uint_least32_t;
+   typedef unsigned __int64 uint_least64_t;
+   typedef __int8 int_fast8_t;
+   typedef __int16 int_fast16_t;
+   typedef __int32 int_fast32_t;
+   typedef __int64 int_fast64_t;
+   typedef unsigned __int8 uint_fast8_t;
+   typedef unsigned __int16 uint_fast16_t;
+   typedef unsigned __int32 uint_fast32_t;
+   typedef unsigned __int64 uint_fast64_t;
+   typedef __int64 intmax_t;
+   typedef unsigned __int64 uintmax_t;
 # else
 #  include 
 # endif
diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py
--- a/lib_pypy/cffi/verifier.py
+++ b/lib_pypy/cffi/verifier.py
@@ -1,12 +1,23 @@
-import sys, os, binascii, imp, shutil
-from . import __version__
+import sys, os, binascii, shutil
+from . import __version_verifier_modules__
 from . import ffiplatform
 
+if sys.version_info >= (3, 3):
+    import importlib.machinery
+    def _extension_suffixes():
+        return importlib.machinery.EXTENSION_SUFFIXES[:]
+else:
+    import imp
+    def _extension_suffixes():
+        return [suffix for suffix, _, type in imp.get_suffixes()
+                if type == imp.C_EXTENSION]
+
 
 class Verifier(object):
 
     def __init__(self, ffi, preamble, tmpdir=None, modulename=None,
-                 ext_package=None, tag='', force_generic_engine=False, **kwds):
+                 ext_package=None, tag='', force_generic_engine=False,
+                 source_extension='.c', flags=None, relative_to=None, **kwds):
         self.ffi = ffi
         self.preamble = preamble
         if not modulename:
@@ -14,14 +25,15 @@
         vengine_class = _locate_engine_class(ffi, force_generic_engine)
         self._vengine = vengine_class(self)
         self._vengine.patch_extension_kwds(kwds)
-        self.kwds = kwds
+        self.flags = flags
+        self.kwds = self.make_relative_to(kwds, relative_to)
         #
         if modulename:
             if tag:
                 raise TypeError("can't specify both 'modulename' and 'tag'")
         else:
-            key = '\x00'.join([sys.version[:3], __version__, preamble,
-                               flattened_kwds] +
+            key = '\x00'.join([sys.version[:3], __version_verifier_modules__,
+                               preamble, flattened_kwds] +
                               ffi._cdefsources)
             if sys.version_info >= (3,):
                 key = key.encode('utf-8')
@@ -33,7 +45,7 @@
                                               k1, k2)
         suffix = _get_so_suffixes()[0]
         self.tmpdir = tmpdir or _caller_dir_pycache()
-        self.sourcefilename = os.path.join(self.tmpdir, modulename + '.c')
+        self.sourcefilename = os.path.join(self.tmpdir, modulename + source_extension)
         self.modulefilename = os.path.join(self.tmpdir, modulename + suffix)
         self.ext_package = ext_package
         self._has_source = False
@@ -97,6 +109,20 @@
     def generates_python_module(self):
         return self._vengine._gen_python_module
 
+    def make_relative_to(self, kwds, relative_to):
+        if relative_to and os.path.dirname(relative_to):
+            dirname = os.path.dirname(relative_to)
+            kwds = kwds.copy()
+            for key in ffiplatform.LIST_OF_FILE_NAMES:
+                if key in kwds:
+                    lst = kwds[key]
+                    if not isinstance(lst, (list, tuple)):
+                        raise TypeError("keyword '%s' should be a list or tuple"
+                                        % (key,))
+                    lst = [os.path.join(dirname, fn) for fn in lst]
+                    kwds[key] = lst
+        return kwds
+
     # ----------
 
     def _locate_module(self):
@@ -148,7 +174,10 @@
 
     def _load_library(self):
         assert self._has_module
-        return self._vengine.load_library()
+        if self.flags is not None:
+            return self._vengine.load_library(self.flags)
+        else:
+            return self._vengine.load_library()
 
 # ____________________________________________________________
 
@@ -181,6 +210,9 @@
 def _caller_dir_pycache():
     if _TMPDIR:
         return _TMPDIR
+    result = os.environ.get('CFFI_TMPDIR')
+    if result:
+        return result
     filename = sys._getframe(2).f_code.co_filename
     return os.path.abspath(os.path.join(os.path.dirname(filename),
                            '__pycache__'))
@@ -222,11 +254,7 @@
             pass
 
 def _get_so_suffixes():
-    suffixes = []
-    for suffix, mode, type in imp.get_suffixes():
-        if type == imp.C_EXTENSION:
-            suffixes.append(suffix)
-
+    suffixes = _extension_suffixes()
     if not suffixes:
         # bah, no C_EXTENSION available.  Occurs on pypy without cpyext
         if sys.platform == 'win32':
diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
--- a/lib_pypy/greenlet.egg-info
+++ b/lib_pypy/greenlet.egg-info
@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: greenlet
-Version: 0.4.0
+Version: 0.4.5
 Summary: Lightweight in-process concurrent programming
 Home-page: https://github.com/python-greenlet/greenlet
 Author: Ralf Schmitt (for CPython), PyPy team
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -1,7 +1,7 @@
 import sys
 import _continuation
 
-__version__ = "0.4.0"
+__version__ = "0.4.5"
 
 # ____________________________________________________________
 # Exceptions
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -47,6 +47,11 @@
 
 Install build-time dependencies
 -------------------------------
+(**Note**: for some hints on how to translate the Python interpreter under
+Windows, see the `windows document`_)
+
+.. _`windows document`: windows.html
+
 
 To build PyPy on Unix using the C translation backend, you need at least a C
 compiler and ``make`` installed. Further, some optional modules have additional
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -65,9 +65,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '2.4'
+version = '2.5'
 # The full version, including alpha/beta/rc tags.
-release = '2.4.0'
+release = '2.5.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -12,19 +12,19 @@
   Amaury Forgeot d'Arc
   Samuele Pedroni
   Alex Gaynor
+  Brian Kearns
+  Matti Picus
+  Philip Jenvey
   Michael Hudson
   David Schneider
-  Matti Picus
-  Brian Kearns
-  Philip Jenvey
   Holger Krekel
   Christian Tismer
   Hakan Ardo
   Benjamin Peterson
   Manuel Jacob
+  Ronan Lamy
   Anders Chrigstrom
   Eric van Riet Paap
-  Ronan Lamy
   Wim Lavrijsen
   Richard Emslie
   Alexander Schremmer
@@ -38,9 +38,9 @@
   Camillo Bruni
   Laura Creighton
   Toon Verwaest
+  Romain Guillebert
   Leonardo Santagada
   Seo Sanghyeon
-  Romain Guillebert
   Justin Peel
   Ronny Pfannschmidt
   David Edelsohn
@@ -61,15 +61,16 @@
   Michal Bendowski
   Jan de Mooij
   stian
+  Tyler Wade
   Michael Foord
   Stephan Diehl
-  Tyler Wade
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
   Patrick Maupin
   Bob Ippolito
   Bruno Gola
+  David Malcolm
   Jean-Paul Calderone
   Timo Paulssen
   Squeaky
@@ -78,18 +79,19 @@
   Marius Gedminas
   Martin Matusiak
   Konstantin Lopuhin
+  Wenzhu Man
   John Witulski
-  Wenzhu Man
+  Laurence Tratt
+  Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
   Mark Pearse
   Simon Cross
-  Ivan Sichmann Freitas
   Andreas Stührk
+  Stefano Rivera
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Stefano Rivera
   Paweł Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -99,7 +101,6 @@
   tav
   Taavi Burns
   Georg Brandl
-  Laurence Tratt
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -111,13 +112,12 @@
   Jeremy Thurgood
   Rami Chowdhury
   Tobias Pape
-  David Malcolm
   Eugene Oden
   Henry Mason
   Vasily Kuznetsov
   Preston Timmons
+  David Ripton
   Jeff Terrace
-  David Ripton
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
@@ -136,13 +136,16 @@
   Gintautas Miliauskas
   Michael Twomey
   Lucian Branescu Mihaila
+  Yichao Yu
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
   Karl Bartel
+  Wouter van Heyst
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  anatoly techtonik
   Stuart Williams
   Jasper Schulz
   Christian Hudon
@@ -152,12 +155,11 @@
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
-  anatoly techtonik
   Neil Shepperd
+  Stanislaw Halik
   Mikael Schönenberg
   Elmo M?ntynen
   Jonathan David Riehl
-  Stanislaw Halik
   Anders Qvist
   Corbin Simpson
   Chirag Jadwani
@@ -166,10 +168,13 @@
   Vincent Legoll
   Alan McIntyre
   Alexander Sedov
+  Attila Gobi
   Christopher Pope
   Christian Tismer 
   Marc Abramowitz
   Dan Stromberg
+  Arjun Naik
+  Valentina Mukhamedzhanova
   Stefano Parmesan
   Alexis Daboville
   Jens-Uwe Mager
@@ -183,8 +188,6 @@
   Sylvain Thenault
   Nathan Taylor
   Vladimir Kryachko
-  Arjun Naik
-  Attila Gobi
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -192,22 +195,23 @@
   Ryan Gonzalez
   Ian Foote
   Kristjan Valur Jonsson
+  David Lievens
   Neil Blakey-Milner
   Lutz Paelike
   Lucio Torre
   Lars Wassermann
-  Valentina Mukhamedzhanova
   Henrik Vendelbo
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
   Sergey Kishchenko
-  Yichao Yu
   Ignas Mikalajunas
   Christoph Gerum
   Martin Blais
   Lene Wagner
   Tomo Cocoa
+  Toni Mattis
+  Lucas Stadler
   roberto at goyle
   Yury V. Zaytsev
   Anna Katrina Dominguez
@@ -235,23 +239,30 @@
   Stephan Busemann
   Rafał Gałczyński
   Christian Muirhead
+  Berker Peksag
   James Lan
   shoma hosaka
-  Daniel Neuh?user
-  Matthew Miller
+  Daniel Neuhäuser
+  Ben Mather
+  halgari
+  Boglarka Vezer
+  Chris Pressey
   Buck Golemon
   Konrad Delong
   Dinu Gherman
   Chris Lambacher
   coolbutuseless at gmail.com
+  Jim Baker
   Rodrigo Araújo
-  Jim Baker
+  Nikolaos-Digenis Karagiannis
   James Robert
   Armin Ronacher
   Brett Cannon
+  Donald Stufft
   yrttyr
   aliceinwire
   OlivierBlanvillain
+  Dan Sanders
   Zooko Wilcox-O Hearn
   Tomer Chachamu
   Christopher Groskopf
@@ -265,6 +276,7 @@
   Markus Unterwaditzer
   Even Wiik Thomassen
   jbs
+  squeaky
   soareschen
   Kurt Griffiths
   Mike Bayer
@@ -276,5 +288,6 @@
   Anna Ravencroft
   Dan Crosta
   Julien Phalip
+  Roman Podoliaka
   Dan Loewenherz
 
diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
--- a/pypy/doc/cppyy.rst
+++ b/pypy/doc/cppyy.rst
@@ -74,7 +74,9 @@
 See the :doc:`backend documentation `.
 
 A standalone version of Reflex that also provides the dynamically loadable
-backend is available for `download`_.
+backend is available for `download`_. Note this is currently the only way to
+get the dynamically loadable backend, so use this first.
+
 That version, as well as any other distribution of Reflex (e.g. the one that
 comes with `ROOT`_, which may be part of your Linux distribution as part of
 the selection of scientific software) will also work for a build with the
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -30,12 +30,10 @@
 
    Initialize threads. Only need to be called if there are any threads involved
 
-.. function:: long pypy_setup_home(char* home, int verbose);
+.. function:: int pypy_setup_home(char* home, int verbose);
 
    This function searches the PyPy standard library starting from the given
-   "PyPy home directory".  It is not strictly necessary to execute it before
-   running Python code, but without it you will not be able to import any
-   non-builtin module from the standard library.  The arguments are:
+   "PyPy home directory".  The arguments are:
 
    * ``home``: NULL terminated path to an executable inside the pypy directory
      (can be a .so name, can be made up)
@@ -84,17 +82,22 @@
 
     const char source[] = "print 'hello from pypy'";
 
-    int main()
+    int main(void)
     {
-      int res;
+        int res;
 
-      rpython_startup_code();
-      // pypy_setup_home() is not needed in this trivial example
-      res = pypy_execute_source((char*)source);
-      if (res) {
-        printf("Error calling pypy_execute_source!\n");
-      }
-      return res;
+        rpython_startup_code();
+        res = pypy_setup_home("/opt/pypy/bin/libpypy-c.so", 1);
+        if (res) {
+            printf("Error setting pypy home!\n");
+            return 1;
+        }
+
+        res = pypy_execute_source((char*)source);
+        if (res) {
+            printf("Error calling pypy_execute_source!\n");
+        }
+        return res;
     }
 
 If we save it as ``x.c`` now, compile it and run it (on linux) with::
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -3,6 +3,13 @@
 
 .. contents::
 
+See also: `Frequently ask questions about RPython.`__
+
+.. __: http://rpython.readthedocs.org/en/latest/faq.html
+
+---------------------------
+
+
 What is PyPy?
 -------------
 
@@ -190,6 +197,55 @@
 (now-dead) object are still true about the new object.
 
 
+
+Would type annotations help PyPy's performance?
+-----------------------------------------------
+
+Two examples of type annotations that are being proposed for improved
+performance are `Cython types`__ and `PEP 484 - Type Hints`__.
+
+.. __: http://docs.cython.org/src/reference/language_basics.html#declaring-data-types
+.. __: https://www.python.org/dev/peps/pep-0484/
+
+**Cython types** are, by construction, similar to C declarations.  For
+example, a local variable or an instance attribute can be declared
+``"cdef int"`` to force a machine word to be used.  This changes the
+usual Python semantics (e.g. no overflow checks, and errors when
+trying to write other types of objects there).  It gives some extra
+performance, but the exact benefits are unclear: right now
+(January 2015) for example we are investigating a technique that would
+store machine-word integers directly on instances, giving part of the
+benefits without the user-supplied ``"cdef int"``.
+
+**PEP 484 - Type Hints,** on the other hand, is almost entirely
+useless if you're looking at performance.  First, as the name implies,
+they are *hints:* they must still be checked at runtime, like PEP 484
+says.  Or maybe you're fine with a mode in which you get very obscure
+crashes when the type annotations are wrong; but even in that case the
+speed benefits would be extremely minor.
+
+There are several reasons for why.  One of them is that annotations
+are at the wrong level (e.g. a PEP 484 "int" corresponds to Python 3's
+int type, which does not necessarily fits inside one machine word;
+even worse, an "int" annotation allows arbitrary int subclasses).
+Another is that a lot more information is needed to produce good code

From noreply at buildbot.pypy.org  Fri Feb  6 22:54:02 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri,  6 Feb 2015 22:54:02 +0100 (CET)
Subject: [pypy-commit] pypy llvm-translation-backend: "Implement" new likely
 / unlikely operation pair by ignoring the hint.
Message-ID: <20150206215402.BAA261C0100@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: llvm-translation-backend
Changeset: r75744:94c8bfd0aaba
Date: 2015-02-06 20:20 +0100
http://bitbucket.org/pypy/pypy/changeset/94c8bfd0aaba/

Log:	"Implement" new likely / unlikely operation pair by ignoring the
	hint.

diff --git a/rpython/translator/llvm/genllvm.py b/rpython/translator/llvm/genllvm.py
--- a/rpython/translator/llvm/genllvm.py
+++ b/rpython/translator/llvm/genllvm.py
@@ -1539,6 +1539,12 @@
             database.f.write('declare i8* @_rpy_tls_addr()\n')
         self.w('{result.V} = call i8* @_rpy_tls_addr()'.format(**locals()))
 
+    def op_likely(self, result, cond):
+        self.w('{result.V} = bitcast {cond.TV} to {result.T}'.format(**locals()))
+
+    def op_unlikely(self, result, cond):
+        self.w('{result.V} = bitcast {cond.TV} to {result.T}'.format(**locals()))
+
 
 class GCPolicy(object):
     def __init__(self, genllvm):

From noreply at buildbot.pypy.org  Fri Feb  6 22:54:03 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri,  6 Feb 2015 22:54:03 +0100 (CET)
Subject: [pypy-commit] pypy llvm-translation-backend: Add ignored 'is_minor'
 parameter to LLVMStackRootWalker.walk_stack_roots() to match the abstract
 signature.
Message-ID: <20150206215403.E44861C0100@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: llvm-translation-backend
Changeset: r75745:18a0e4413b69
Date: 2015-02-06 20:51 +0100
http://bitbucket.org/pypy/pypy/changeset/18a0e4413b69/

Log:	Add ignored 'is_minor' parameter to
	LLVMStackRootWalker.walk_stack_roots() to match the abstract
	signature.

diff --git a/rpython/memory/gctransform/llvmgcroot.py b/rpython/memory/gctransform/llvmgcroot.py
--- a/rpython/memory/gctransform/llvmgcroot.py
+++ b/rpython/memory/gctransform/llvmgcroot.py
@@ -122,7 +122,7 @@
     def need_thread_support(self, gctransformer, getfn):
         pass
 
-    def walk_stack_roots(self, collect_stack_root):
+    def walk_stack_roots(self, collect_stack_root, is_minor=False):
         """Call `collect_stack_root()` for all gc roots on the stack.
 
         This is done by walking up the stack. For each safe point the hash

From noreply at buildbot.pypy.org  Fri Feb  6 22:54:05 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri,  6 Feb 2015 22:54:05 +0100 (CET)
Subject: [pypy-commit] pypy llvm-translation-backend: Implement
 length_of_simple_gcarray_from_opaque() operation.
Message-ID: <20150206215405.131F21C0100@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: llvm-translation-backend
Changeset: r75746:bda515ef13fa
Date: 2015-02-06 21:10 +0100
http://bitbucket.org/pypy/pypy/changeset/bda515ef13fa/

Log:	Implement length_of_simple_gcarray_from_opaque() operation.

diff --git a/rpython/translator/llvm/genllvm.py b/rpython/translator/llvm/genllvm.py
--- a/rpython/translator/llvm/genllvm.py
+++ b/rpython/translator/llvm/genllvm.py
@@ -1545,6 +1545,13 @@
     def op_unlikely(self, result, cond):
         self.w('{result.V} = bitcast {cond.TV} to {result.T}'.format(**locals()))
 
+    def op_length_of_simple_gcarray_from_opaque(self, result, ptr):
+        array_type = ArrayType()
+        array_type.setup(LLVMVoid, True)
+        tmp = self._tmp(PtrType.tmp(array_type))
+        self._cast(tmp, ptr)
+        self.op_getarraysize(result, tmp)
+
 
 class GCPolicy(object):
     def __init__(self, genllvm):

From noreply at buildbot.pypy.org  Fri Feb  6 23:36:17 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  6 Feb 2015 23:36:17 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: SSL: Use a non-moving buffer for
 the BIO functions. Fixes the last failure
Message-ID: <20150206223617.400671C13FF@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: stdlib-2.7.9
Changeset: r75747:5a8af5aeb0f3
Date: 2015-02-06 23:22 +0100
http://bitbucket.org/pypy/pypy/changeset/5a8af5aeb0f3/

Log:	SSL: Use a non-moving buffer for the BIO functions. Fixes the last
	failure

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -1443,53 +1443,8 @@
                     "cafile and capath cannot be both omitted"))
         # load from cadata
         if cadata is not None:
-            biobuf = libssl_BIO_new_mem_buf(cadata, len(cadata))
-            if not biobuf:
-                raise ssl_error(space, "Can't allocate buffer")
-            try:
-                store = libssl_SSL_CTX_get_cert_store(self.ctx)
-                loaded = 0
-                while True:
-                    if ca_file_type == SSL_FILETYPE_ASN1:
-                        cert = libssl_d2i_X509_bio(
-                            biobuf, None)
-                    else:
-                        cert = libssl_PEM_read_bio_X509(
-                            biobuf, None, None, None)
-                    if not cert:
-                        break
-                    try:
-                        r = libssl_X509_STORE_add_cert(store, cert)
-                    finally:
-                        libssl_X509_free(cert)
-                    if not r:
-                        err = libssl_ERR_peek_last_error()
-                        if (libssl_ERR_GET_LIB(err) == ERR_LIB_X509 and
-                            libssl_ERR_GET_REASON(err) ==
-                            X509_R_CERT_ALREADY_IN_HASH_TABLE):
-                            # cert already in hash table, not an error
-                            libssl_ERR_clear_error()
-                        else:
-                            break
-                    loaded += 1
-
-                err = libssl_ERR_peek_last_error()
-                if (ca_file_type == SSL_FILETYPE_ASN1 and
-                    loaded > 0 and
-                    libssl_ERR_GET_LIB(err) == ERR_LIB_ASN1 and
-                    libssl_ERR_GET_REASON(err) == ASN1_R_HEADER_TOO_LONG):
-                    # EOF ASN1 file, not an error
-                    libssl_ERR_clear_error()
-                elif (ca_file_type == SSL_FILETYPE_PEM and
-                      loaded > 0 and
-                      libssl_ERR_GET_LIB(err) == ERR_LIB_PEM and
-                      libssl_ERR_GET_REASON(err) == PEM_R_NO_START_LINE):
-                    # EOF PEM file, not an error
-                    libssl_ERR_clear_error()
-                else:
-                    raise _ssl_seterror(space, None, 0)
-            finally:
-                libssl_BIO_free(biobuf)
+            with rffi.scoped_nonmovingbuffer(cadata) as buf:
+                self._add_ca_certs(space, buf, len(cadata), ca_file_type)
             
         # load cafile or capath
         if cafile is not None or capath is not None:
@@ -1505,6 +1460,55 @@
                 else:
                     raise _ssl_seterror(space, None, -1)
 
+    def _add_ca_certs(self, space, data, size, ca_file_type):
+        biobuf = libssl_BIO_new_mem_buf(data, size)
+        if not biobuf:
+            raise ssl_error(space, "Can't allocate buffer")
+        try:
+            store = libssl_SSL_CTX_get_cert_store(self.ctx)
+            loaded = 0
+            while True:
+                if ca_file_type == SSL_FILETYPE_ASN1:
+                    cert = libssl_d2i_X509_bio(
+                        biobuf, None)
+                else:
+                    cert = libssl_PEM_read_bio_X509(
+                        biobuf, None, None, None)
+                if not cert:
+                    break
+                try:
+                    r = libssl_X509_STORE_add_cert(store, cert)
+                finally:
+                    libssl_X509_free(cert)
+                if not r:
+                    err = libssl_ERR_peek_last_error()
+                    if (libssl_ERR_GET_LIB(err) == ERR_LIB_X509 and
+                        libssl_ERR_GET_REASON(err) ==
+                        X509_R_CERT_ALREADY_IN_HASH_TABLE):
+                        # cert already in hash table, not an error
+                        libssl_ERR_clear_error()
+                    else:
+                        break
+                loaded += 1
+
+            err = libssl_ERR_peek_last_error()
+            if (ca_file_type == SSL_FILETYPE_ASN1 and
+                loaded > 0 and
+                libssl_ERR_GET_LIB(err) == ERR_LIB_ASN1 and
+                libssl_ERR_GET_REASON(err) == ASN1_R_HEADER_TOO_LONG):
+                # EOF ASN1 file, not an error
+                libssl_ERR_clear_error()
+            elif (ca_file_type == SSL_FILETYPE_PEM and
+                  loaded > 0 and
+                  libssl_ERR_GET_LIB(err) == ERR_LIB_PEM and
+                  libssl_ERR_GET_REASON(err) == PEM_R_NO_START_LINE):
+                # EOF PEM file, not an error
+                libssl_ERR_clear_error()
+            else:
+                raise _ssl_seterror(space, None, 0)
+        finally:
+            libssl_BIO_free(biobuf)
+
     def cert_store_stats_w(self, space):
         store = libssl_SSL_CTX_get_cert_store(self.ctx)
         x509 = 0

From noreply at buildbot.pypy.org  Fri Feb  6 23:36:18 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  6 Feb 2015 23:36:18 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: Attempt to fix tests on linux
	32bit.
Message-ID: <20150206223618.7E3811C13FF@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: stdlib-2.7.9
Changeset: r75748:6ba8d5a9355c
Date: 2015-02-06 23:35 +0100
http://bitbucket.org/pypy/pypy/changeset/6ba8d5a9355c/

Log:	Attempt to fix tests on linux 32bit.

diff --git a/pypy/module/_ssl/__init__.py b/pypy/module/_ssl/__init__.py
--- a/pypy/module/_ssl/__init__.py
+++ b/pypy/module/_ssl/__init__.py
@@ -1,3 +1,4 @@
+from rpython.rlib.rarithmetic import intmask
 from pypy.interpreter.mixedmodule import MixedModule
 from pypy.module._ssl import ssl_data
 
@@ -31,6 +32,8 @@
         from pypy.module._ssl.interp_ssl import constants, HAVE_OPENSSL_RAND
 
         for constant, value in constants.iteritems():
+            if constant.startswith('OP_'):
+                value = intmask(value)  # Convert to C long and wrap around.
             Module.interpleveldefs[constant] = "space.wrap(%r)" % (value,)
 
         if HAVE_OPENSSL_RAND:

From noreply at buildbot.pypy.org  Fri Feb  6 23:59:03 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  6 Feb 2015 23:59:03 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: hg merge default
Message-ID: <20150206225903.9D1271C0100@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: stdlib-2.7.9
Changeset: r75749:0005ffa6a65b
Date: 2015-02-06 23:58 +0100
http://bitbucket.org/pypy/pypy/changeset/0005ffa6a65b/

Log:	hg merge default + update interp_ssl to use the new
	rposix.get_saved_errno

diff too long, truncating to 2000 out of 10699 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -7,10 +7,7 @@
 9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
 ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm
 20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0
-20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0
-0000000000000000000000000000000000000000 release-2.3.0
 394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3
 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1
 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1
-32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1
-0000000000000000000000000000000000000000 release-2.2=3.1
+10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -42,19 +42,19 @@
   Amaury Forgeot d'Arc
   Samuele Pedroni
   Alex Gaynor
+  Brian Kearns
+  Matti Picus
+  Philip Jenvey
   Michael Hudson
   David Schneider
-  Matti Picus
-  Brian Kearns
-  Philip Jenvey
   Holger Krekel
   Christian Tismer
   Hakan Ardo
   Benjamin Peterson
   Manuel Jacob
+  Ronan Lamy
   Anders Chrigstrom
   Eric van Riet Paap
-  Ronan Lamy
   Wim Lavrijsen
   Richard Emslie
   Alexander Schremmer
@@ -68,9 +68,9 @@
   Camillo Bruni
   Laura Creighton
   Toon Verwaest
+  Romain Guillebert
   Leonardo Santagada
   Seo Sanghyeon
-  Romain Guillebert
   Justin Peel
   Ronny Pfannschmidt
   David Edelsohn
@@ -91,15 +91,16 @@
   Michal Bendowski
   Jan de Mooij
   stian
+  Tyler Wade
   Michael Foord
   Stephan Diehl
-  Tyler Wade
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
   Patrick Maupin
   Bob Ippolito
   Bruno Gola
+  David Malcolm
   Jean-Paul Calderone
   Timo Paulssen
   Squeaky
@@ -108,18 +109,19 @@
   Marius Gedminas
   Martin Matusiak
   Konstantin Lopuhin
+  Wenzhu Man
   John Witulski
-  Wenzhu Man
+  Laurence Tratt
+  Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
   Mark Pearse
   Simon Cross
-  Ivan Sichmann Freitas
   Andreas Stührk
+  Stefano Rivera
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Stefano Rivera
   Paweł Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -129,7 +131,6 @@
   tav
   Taavi Burns
   Georg Brandl
-  Laurence Tratt
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -141,13 +142,12 @@
   Jeremy Thurgood
   Rami Chowdhury
   Tobias Pape
-  David Malcolm
   Eugene Oden
   Henry Mason
   Vasily Kuznetsov
   Preston Timmons
+  David Ripton
   Jeff Terrace
-  David Ripton
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
@@ -166,13 +166,16 @@
   Gintautas Miliauskas
   Michael Twomey
   Lucian Branescu Mihaila
+  Yichao Yu
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
   Karl Bartel
+  Wouter van Heyst
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  anatoly techtonik
   Stuart Williams
   Jasper Schulz
   Christian Hudon
@@ -182,12 +185,11 @@
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
-  anatoly techtonik
   Neil Shepperd
+  Stanislaw Halik
   Mikael Schönenberg
   Elmo M?ntynen
   Jonathan David Riehl
-  Stanislaw Halik
   Anders Qvist
   Corbin Simpson
   Chirag Jadwani
@@ -196,10 +198,13 @@
   Vincent Legoll
   Alan McIntyre
   Alexander Sedov
+  Attila Gobi
   Christopher Pope
   Christian Tismer 
   Marc Abramowitz
   Dan Stromberg
+  Arjun Naik
+  Valentina Mukhamedzhanova
   Stefano Parmesan
   Alexis Daboville
   Jens-Uwe Mager
@@ -213,8 +218,6 @@
   Sylvain Thenault
   Nathan Taylor
   Vladimir Kryachko
-  Arjun Naik
-  Attila Gobi
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -222,22 +225,23 @@
   Ryan Gonzalez
   Ian Foote
   Kristjan Valur Jonsson
+  David Lievens
   Neil Blakey-Milner
   Lutz Paelike
   Lucio Torre
   Lars Wassermann
-  Valentina Mukhamedzhanova
   Henrik Vendelbo
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
   Sergey Kishchenko
-  Yichao Yu
   Ignas Mikalajunas
   Christoph Gerum
   Martin Blais
   Lene Wagner
   Tomo Cocoa
+  Toni Mattis
+  Lucas Stadler
   roberto at goyle
   Yury V. Zaytsev
   Anna Katrina Dominguez
@@ -265,23 +269,30 @@
   Stephan Busemann
   Rafał Gałczyński
   Christian Muirhead
+  Berker Peksag
   James Lan
   shoma hosaka
-  Daniel Neuh?user
-  Matthew Miller
+  Daniel Neuhäuser
+  Ben Mather
+  halgari
+  Boglarka Vezer
+  Chris Pressey
   Buck Golemon
   Konrad Delong
   Dinu Gherman
   Chris Lambacher
   coolbutuseless at gmail.com
+  Jim Baker
   Rodrigo Araújo
-  Jim Baker
+  Nikolaos-Digenis Karagiannis
   James Robert
   Armin Ronacher
   Brett Cannon
+  Donald Stufft
   yrttyr
   aliceinwire
   OlivierBlanvillain
+  Dan Sanders
   Zooko Wilcox-O Hearn
   Tomer Chachamu
   Christopher Groskopf
@@ -295,6 +306,7 @@
   Markus Unterwaditzer
   Even Wiik Thomassen
   jbs
+  squeaky
   soareschen
   Kurt Griffiths
   Mike Bayer
@@ -306,6 +318,7 @@
   Anna Ravencroft
   Dan Crosta
   Julien Phalip
+  Roman Podoliaka
   Dan Loewenherz
 
   Heinrich-Heine University, Germany 
diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py
@@ -2,7 +2,6 @@
 import array
 import gc
 import unittest
-from ctypes.test import xfail
 
 class X(Structure):
     _fields_ = [("c_int", c_int)]
@@ -11,7 +10,6 @@
         self._init_called = True
 
 class Test(unittest.TestCase):
-    @xfail
     def test_fom_buffer(self):
         a = array.array("i", range(16))
         x = (c_int * 16).from_buffer(a)
@@ -34,10 +32,9 @@
         del a; gc.collect(); gc.collect(); gc.collect()
         self.assertEqual(x[:], expected)
 
-        self.assertRaises(TypeError,
+        self.assertRaises((TypeError, ValueError),
                           (c_char * 16).from_buffer, "a" * 16)
 
-    @xfail
     def test_fom_buffer_with_offset(self):
         a = array.array("i", range(16))
         x = (c_int * 15).from_buffer(a, sizeof(c_int))
@@ -46,7 +43,6 @@
         self.assertRaises(ValueError, lambda: (c_int * 16).from_buffer(a, sizeof(c_int)))
         self.assertRaises(ValueError, lambda: (c_int * 1).from_buffer(a, 16 * sizeof(c_int)))
 
-    @xfail
     def test_from_buffer_copy(self):
         a = array.array("i", range(16))
         x = (c_int * 16).from_buffer_copy(a)
@@ -71,7 +67,6 @@
         x = (c_char * 16).from_buffer_copy("a" * 16)
         self.assertEqual(x[:], "a" * 16)
 
-    @xfail
     def test_fom_buffer_copy_with_offset(self):
         a = array.array("i", range(16))
         x = (c_int * 15).from_buffer_copy(a, sizeof(c_int))
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -83,6 +83,37 @@
     def in_dll(self, dll, name):
         return self.from_address(dll._handle.getaddressindll(name))
 
+    def from_buffer(self, obj, offset=0):
+        size = self._sizeofinstances()
+        buf = buffer(obj, offset, size)
+        if len(buf) < size:
+            raise ValueError(
+                "Buffer size too small (%d instead of at least %d bytes)"
+                % (len(buf) + offset, size + offset))
+        raw_addr = buf._pypy_raw_address()
+        result = self.from_address(raw_addr)
+        result._ensure_objects()['ffffffff'] = obj
+        return result
+
+    def from_buffer_copy(self, obj, offset=0):
+        size = self._sizeofinstances()
+        buf = buffer(obj, offset, size)
+        if len(buf) < size:
+            raise ValueError(
+                "Buffer size too small (%d instead of at least %d bytes)"
+                % (len(buf) + offset, size + offset))
+        result = self()
+        dest = result._buffer.buffer
+        try:
+            raw_addr = buf._pypy_raw_address()
+        except ValueError:
+            _rawffi.rawstring2charp(dest, buf)
+        else:
+            from ctypes import memmove
+            memmove(dest, raw_addr, size)
+        return result
+
+
 class CArgObject(object):
     """ simple wrapper around buffer, just for the case of freeing
     it afterwards
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -1,3 +1,4 @@
+import sys
 import _rawffi
 from _ctypes.basics import _CData, _CDataMeta, keepalive_key,\
      store_reference, ensure_objects, CArgObject
@@ -178,6 +179,8 @@
         instance = StructOrUnion.__new__(self)
         if isinstance(address, _rawffi.StructureInstance):
             address = address.buffer
+        # fix the address: turn it into as unsigned, in case it is negative
+        address = address & (sys.maxint * 2 + 1)
         instance.__dict__['_buffer'] = self._ffistruct.fromaddress(address)
         return instance
 
diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
--- a/lib_pypy/greenlet.egg-info
+++ b/lib_pypy/greenlet.egg-info
@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: greenlet
-Version: 0.4.0
+Version: 0.4.5
 Summary: Lightweight in-process concurrent programming
 Home-page: https://github.com/python-greenlet/greenlet
 Author: Ralf Schmitt (for CPython), PyPy team
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -1,7 +1,7 @@
 import sys
 import _continuation
 
-__version__ = "0.4.0"
+__version__ = "0.4.5"
 
 # ____________________________________________________________
 # Exceptions
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -65,9 +65,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '2.4'
+version = '2.5'
 # The full version, including alpha/beta/rc tags.
-release = '2.4.0'
+release = '2.5.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -12,19 +12,19 @@
   Amaury Forgeot d'Arc
   Samuele Pedroni
   Alex Gaynor
+  Brian Kearns
+  Matti Picus
+  Philip Jenvey
   Michael Hudson
   David Schneider
-  Matti Picus
-  Brian Kearns
-  Philip Jenvey
   Holger Krekel
   Christian Tismer
   Hakan Ardo
   Benjamin Peterson
   Manuel Jacob
+  Ronan Lamy
   Anders Chrigstrom
   Eric van Riet Paap
-  Ronan Lamy
   Wim Lavrijsen
   Richard Emslie
   Alexander Schremmer
@@ -38,9 +38,9 @@
   Camillo Bruni
   Laura Creighton
   Toon Verwaest
+  Romain Guillebert
   Leonardo Santagada
   Seo Sanghyeon
-  Romain Guillebert
   Justin Peel
   Ronny Pfannschmidt
   David Edelsohn
@@ -61,15 +61,16 @@
   Michal Bendowski
   Jan de Mooij
   stian
+  Tyler Wade
   Michael Foord
   Stephan Diehl
-  Tyler Wade
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
   Patrick Maupin
   Bob Ippolito
   Bruno Gola
+  David Malcolm
   Jean-Paul Calderone
   Timo Paulssen
   Squeaky
@@ -78,18 +79,19 @@
   Marius Gedminas
   Martin Matusiak
   Konstantin Lopuhin
+  Wenzhu Man
   John Witulski
-  Wenzhu Man
+  Laurence Tratt
+  Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
   Mark Pearse
   Simon Cross
-  Ivan Sichmann Freitas
   Andreas Stührk
+  Stefano Rivera
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Stefano Rivera
   Paweł Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -99,7 +101,6 @@
   tav
   Taavi Burns
   Georg Brandl
-  Laurence Tratt
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -111,13 +112,12 @@
   Jeremy Thurgood
   Rami Chowdhury
   Tobias Pape
-  David Malcolm
   Eugene Oden
   Henry Mason
   Vasily Kuznetsov
   Preston Timmons
+  David Ripton
   Jeff Terrace
-  David Ripton
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
@@ -136,13 +136,16 @@
   Gintautas Miliauskas
   Michael Twomey
   Lucian Branescu Mihaila
+  Yichao Yu
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
   Karl Bartel
+  Wouter van Heyst
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  anatoly techtonik
   Stuart Williams
   Jasper Schulz
   Christian Hudon
@@ -152,12 +155,11 @@
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
-  anatoly techtonik
   Neil Shepperd
+  Stanislaw Halik
   Mikael Schönenberg
   Elmo M?ntynen
   Jonathan David Riehl
-  Stanislaw Halik
   Anders Qvist
   Corbin Simpson
   Chirag Jadwani
@@ -166,10 +168,13 @@
   Vincent Legoll
   Alan McIntyre
   Alexander Sedov
+  Attila Gobi
   Christopher Pope
   Christian Tismer 
   Marc Abramowitz
   Dan Stromberg
+  Arjun Naik
+  Valentina Mukhamedzhanova
   Stefano Parmesan
   Alexis Daboville
   Jens-Uwe Mager
@@ -183,8 +188,6 @@
   Sylvain Thenault
   Nathan Taylor
   Vladimir Kryachko
-  Arjun Naik
-  Attila Gobi
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -192,22 +195,23 @@
   Ryan Gonzalez
   Ian Foote
   Kristjan Valur Jonsson
+  David Lievens
   Neil Blakey-Milner
   Lutz Paelike
   Lucio Torre
   Lars Wassermann
-  Valentina Mukhamedzhanova
   Henrik Vendelbo
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
   Sergey Kishchenko
-  Yichao Yu
   Ignas Mikalajunas
   Christoph Gerum
   Martin Blais
   Lene Wagner
   Tomo Cocoa
+  Toni Mattis
+  Lucas Stadler
   roberto at goyle
   Yury V. Zaytsev
   Anna Katrina Dominguez
@@ -235,23 +239,30 @@
   Stephan Busemann
   Rafał Gałczyński
   Christian Muirhead
+  Berker Peksag
   James Lan
   shoma hosaka
-  Daniel Neuh?user
-  Matthew Miller
+  Daniel Neuhäuser
+  Ben Mather
+  halgari
+  Boglarka Vezer
+  Chris Pressey
   Buck Golemon
   Konrad Delong
   Dinu Gherman
   Chris Lambacher
   coolbutuseless at gmail.com
+  Jim Baker
   Rodrigo Araújo
-  Jim Baker
+  Nikolaos-Digenis Karagiannis
   James Robert
   Armin Ronacher
   Brett Cannon
+  Donald Stufft
   yrttyr
   aliceinwire
   OlivierBlanvillain
+  Dan Sanders
   Zooko Wilcox-O Hearn
   Tomer Chachamu
   Christopher Groskopf
@@ -265,6 +276,7 @@
   Markus Unterwaditzer
   Even Wiik Thomassen
   jbs
+  squeaky
   soareschen
   Kurt Griffiths
   Mike Bayer
@@ -276,5 +288,6 @@
   Anna Ravencroft
   Dan Crosta
   Julien Phalip
+  Roman Podoliaka
   Dan Loewenherz
 
diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
--- a/pypy/doc/cppyy.rst
+++ b/pypy/doc/cppyy.rst
@@ -74,7 +74,9 @@
 See the :doc:`backend documentation `.
 
 A standalone version of Reflex that also provides the dynamically loadable
-backend is available for `download`_.
+backend is available for `download`_. Note this is currently the only way to
+get the dynamically loadable backend, so use this first.
+
 That version, as well as any other distribution of Reflex (e.g. the one that
 comes with `ROOT`_, which may be part of your Linux distribution as part of
 the selection of scientific software) will also work for a build with the
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -3,6 +3,13 @@
 
 .. contents::
 
+See also: `Frequently ask questions about RPython.`__
+
+.. __: http://rpython.readthedocs.org/en/latest/faq.html
+
+---------------------------
+
+
 What is PyPy?
 -------------
 
@@ -190,6 +197,55 @@
 (now-dead) object are still true about the new object.
 
 
+
+Would type annotations help PyPy's performance?
+-----------------------------------------------
+
+Two examples of type annotations that are being proposed for improved
+performance are `Cython types`__ and `PEP 484 - Type Hints`__.
+
+.. __: http://docs.cython.org/src/reference/language_basics.html#declaring-data-types
+.. __: https://www.python.org/dev/peps/pep-0484/
+
+**Cython types** are, by construction, similar to C declarations.  For
+example, a local variable or an instance attribute can be declared
+``"cdef int"`` to force a machine word to be used.  This changes the
+usual Python semantics (e.g. no overflow checks, and errors when
+trying to write other types of objects there).  It gives some extra
+performance, but the exact benefits are unclear: right now
+(January 2015) for example we are investigating a technique that would
+store machine-word integers directly on instances, giving part of the
+benefits without the user-supplied ``"cdef int"``.
+
+**PEP 484 - Type Hints,** on the other hand, is almost entirely
+useless if you're looking at performance.  First, as the name implies,
+they are *hints:* they must still be checked at runtime, like PEP 484
+says.  Or maybe you're fine with a mode in which you get very obscure
+crashes when the type annotations are wrong; but even in that case the
+speed benefits would be extremely minor.
+
+There are several reasons for why.  One of them is that annotations
+are at the wrong level (e.g. a PEP 484 "int" corresponds to Python 3's
+int type, which does not necessarily fits inside one machine word;
+even worse, an "int" annotation allows arbitrary int subclasses).
+Another is that a lot more information is needed to produce good code
+(e.g. "this ``f()`` called here really means this function there, and
+will never be monkey-patched" -- same with ``len()`` or ``list()``,
+btw).  The third reason is that some "guards" in PyPy's JIT traces
+don't really have an obvious corresponding type (e.g. "this dict is so
+far using keys which don't override ``__hash__`` so a more efficient
+implementation was used").  Many guards don't even have any correspondence
+with types at all ("this class attribute was not modified"; "the loop
+counter did not reach zero so we don't need to release the GIL"; and
+so on).
+
+As PyPy works right now, it is able to derive far more useful
+information than can ever be given by PEP 484, and it works
+automatically.  As far as we know, this is true even if we would add
+other techniques to PyPy, like a fast first-pass JIT.
+
+
+
 .. _`prolog and javascript`:
 
 Can I use PyPy's translation toolchain for other languages besides Python?
diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst
--- a/pypy/doc/getting-started-dev.rst
+++ b/pypy/doc/getting-started-dev.rst
@@ -4,6 +4,93 @@
 .. contents::
 
 
+Using Mercurial
+---------------
+
+PyPy development is based on Mercurial (hg).  If you are not used to
+version control, the cycle for a new PyPy contributor goes typically
+like this:
+
+* Make an account on bitbucket_.
+
+* Go to https://bitbucket.org/pypy/pypy/ and click "fork" (left
+  icons).  You get a fork of the repository, e.g. in
+  https://bitbucket.org/yourname/pypy/.
+
+* Clone this new repo (i.e. the fork) to your local machine with the command 
+  ``hg clone ssh://hg at bitbucket.org/yourname/pypy``.  It is a very slow
+  operation but only ever needs to be done once.  If you already cloned
+  ``https://bitbucket.org/pypy/pypy`` before, even if some time ago,
+  then you can reuse the same clone by editing the file ``.hg/hgrc`` in
+  your clone to contain the line ``default =
+  ssh://hg at bitbucket.org/yourname/pypy``, and then do ``hg pull && hg
+  up``.  If you already have such a clone but don't want to change it,
+  you can clone that copy with ``hg clone /path/to/other/copy``, and
+  then edit ``.hg/hgrc`` as above and do ``hg pull && hg up``.
+
+* Now you have a complete copy of the PyPy repo.  Make a branch
+  with a command like ``hg branch name_of_your_branch``.
+
+* Edit things.  Use ``hg diff`` to see what you changed.  Use ``hg add``
+  to make Mercurial aware of new files you added, e.g. new test files.
+  Use ``hg status`` to see if there are such files.  Run tests!  (See
+  the rest of this page.)
+
+* Commit regularly with ``hg commit``.  A one-line commit message is
+  fine.  We love to have tons of commits; make one as soon as you have
+  some progress, even if it is only some new test that doesn't pass yet,
+  or fixing things even if not all tests pass.  Step by step, you are
+  building the history of your changes, which is the point of a version
+  control system.  (There are commands like ``hg log`` and ``hg up``
+  that you should read about later, to learn how to navigate this
+  history.)
+
+* The commits stay on your machine until you do ``hg push`` to "push"
+  them back to the repo named in the file ``.hg/hgrc``.  Repos are
+  basically just collections of commits (a commit is also called a
+  changeset): there is one repo per url, plus one for each local copy on
+  each local machine.  The commands ``hg push`` and ``hg pull`` copy
+  commits around, with the goal that all repos in question end up with
+  the exact same set of commits.  By opposition, ``hg up`` only updates
+  the "working copy" by reading the local repository, i.e. it makes the
+  files that you see correspond to the latest (or any other) commit
+  locally present.
+
+* You should push often; there is no real reason not to.  Remember that
+  even if they are pushed, with the setup above, the commits are (1)
+  only in ``bitbucket.org/yourname/pypy``, and (2) in the branch you
+  named.  Yes, they are publicly visible, but don't worry about someone
+  walking around the thousands of repos on bitbucket saying "hah, look
+  at the bad coding style of that guy".  Try to get into the mindset
+  that your work is not secret and it's fine that way.  We might not
+  accept it as is for PyPy, asking you instead to improve some things,
+  but we are not going to judge you.
+
+* The final step is to open a pull request, so that we know that you'd
+  like to merge that branch back to the original ``pypy/pypy`` repo.
+  This can also be done several times if you have interesting
+  intermediate states, but if you get there, then we're likely to
+  proceed to the next stage, which is...
+
+* Get a regular account for pushing directly to
+  ``bitbucket.org/pypy/pypy`` (just ask and you'll get it, basically).
+  Once you have it you can rewrite your file ``.hg/hgrc`` to contain
+  ``default = ssh://hg at bitbucket.org/pypy/pypy``.  Your changes will
+  then be pushed directly to the official repo, but (if you follow these
+  rules) they are still on a branch, and we can still review the
+  branches you want to merge.
+
+* If you get closer to the regular day-to-day development, you'll notice
+  that we generally push small changes as one or a few commits directly
+  to the branch ``default``.  Also, we often collaborate even if we are
+  on other branches, which do not really "belong" to anyone.  At this
+  point you'll need ``hg merge`` and learn how to resolve conflicts that
+  sometimes occur when two people try to push different commits in
+  parallel on the same branch.  But it is likely an issue for later ``:-)``
+
+.. _bitbucket: https://bitbucket.org/
+
+
 Running PyPy's unit tests
 -------------------------
 
diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -22,12 +22,12 @@
   will capture the revision number of this change for the release;
   some of the next updates may be done before or after branching; make
   sure things are ported back to the trunk and to the branch as
-  necessary; also update the version number in pypy/doc/conf.py,
-  and in pypy/doc/index.rst
+  necessary; also update the version number in pypy/doc/conf.py.
 * update pypy/doc/contributor.rst (and possibly LICENSE)
   pypy/doc/tool/makecontributor.py generates the list of contributors
 * rename pypy/doc/whatsnew_head.rst to whatsnew_VERSION.rst
-  and create a fresh whatsnew_head.rst after the release
+  create a fresh whatsnew_head.rst after the release
+  and add the new file to  pypy/doc/index-of-whatsnew.rst
 * go to pypy/tool/release and run:
   force-builds.py 
 * wait for builds to complete, make sure there are no failures
@@ -38,10 +38,14 @@
     no JIT: windows, linux, os/x
     sandbox: linux, os/x
 
-* repackage and upload source tar.bz2 to bitbucket and to cobra, as some packagers 
-  prefer a clearly labeled source package
+* repackage and upload source "-src.tar.bz2" to bitbucket and to cobra, as some
+  packagers prefer a clearly labeled source package (download e.g.
+  https://bitbucket.org/pypy/pypy/get/release-2.5.x.tar.bz2, unpack,
+  rename the top-level directory to "pypy-2.5.0-src", repack, and upload)
+
 * write release announcement pypy/doc/release-x.y(.z).txt
   the release announcement should contain a direct link to the download page
+  and add new files to  pypy/doc/index-of-release-notes.rst
 * update pypy.org (under extradoc/pypy.org), rebuild and commit
 
 * post announcement on morepypy.blogspot.com
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-2.5.0.rst
    release-2.4.0.rst
    release-2.3.1.rst
    release-2.3.0.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-2.5.0.rst
    whatsnew-2.4.0.rst
    whatsnew-2.3.1.rst
    whatsnew-2.3.0.rst
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -1,14 +1,23 @@
 Potential project list
 ======================
 
-===========================
-Simple tasks for newcomers:
-===========================
+==========================
+Simple tasks for newcomers
+==========================
 
-Optimize random
----------------
+* Tkinter module missing support for threads:
+  https://bitbucket.org/pypy/pypy/issue/1929/tkinter-broken-for-threaded-python-on-both
 
-https://bitbucket.org/pypy/pypy/issue/1901/try-using-a-different-implementation-of
+* Optimize random:
+  https://bitbucket.org/pypy/pypy/issue/1901/try-using-a-different-implementation-of
+
+* Implement AF_XXX packet types of sockets:
+  https://bitbucket.org/pypy/pypy/issue/1942/support-for-af_xxx-sockets
+
+
+==================
+Mid-to-large tasks
+==================
 
 Below is a list of projects that are interesting for potential contributors
 who are seriously interested in the PyPy project. They mostly share common
@@ -33,15 +42,8 @@
 ------------------------
 
 PyPy's bytearray type is very inefficient. It would be an interesting
-task to look into possible optimizations on this.
-
-Implement AF_XXX packet types for PyPy
---------------------------------------
-
-PyPy is missing AF_XXX types of sockets. Implementing it is easy-to-medium
-task. `bug report`_
-
-.. _`bug report`: https://bitbucket.org/pypy/pypy/issue/1942/support-for-af_xxx-sockets#more
+task to look into possible optimizations on this.  (XXX current status
+unknown; ask on #pypy for updates on this.)
 
 Implement copy-on-write list slicing
 ------------------------------------
@@ -106,6 +108,8 @@
 Translation Toolchain
 ---------------------
 
+(XXX this is unlikely to be feasible.)
+
 * Incremental or distributed translation.
 * Allow separate compilation of extension modules.
 
@@ -187,3 +191,37 @@
 to make them work at all if they currently don't. A part of this work would
 be to get cpyext into a shape where it supports running Cython generated
 extensions.
+
+======================================
+Make more python modules pypy-friendly
+======================================
+
+Work has been started on a few popular python packages. Here is a partial
+list of good work that needs to be finished:
+
+**matplotlib** https://github.com/mattip/matplotlib
+
+    Status: the repo is an older version of matplotlib adapted to pypy and cpyext
+
+    TODO: A suggested first step would be to merge the differences into 
+    matplotlib/HEAD. The major problem is the use of a generic view into a
+    numpy ndarray. The int* fields would need to be converted into int[MAX_DIMS]
+    c-arrays and filled in.
+
+**wxPython** https://bitbucket.org/waedt/wxpython_cffi
+
+    Status: A GSOC 2013 project to adapt the Phoenix sip build system to cffi
+
+    TODO: Merge the latest version of the wrappers and finish the sip conversion
+
+**pygame** https://github.com/CTPUG/pygame_cffi
+
+    Status: see blog post 
+
+    TODO: see the end of the blog post
+
+**pyopengl** https://bitbucket.org/duangle/pyopengl-cffi
+
+    Status: unknown
+
+
diff --git a/pypy/doc/release-2.5.0.rst b/pypy/doc/release-2.5.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-2.5.0.rst
@@ -0,0 +1,104 @@
+==============================
+PyPy 2.5.0 - Pincushion Protea
+==============================
+
+We're pleased to announce PyPy 2.5, which contains significant performance
+enhancements and bug fixes.
+
+You can download the PyPy 2.5.0 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project, and for those who donate to our three sub-projects, as well as our
+volunteers and contributors (10 new commiters joined PyPy since the last
+release).
+We've shown quite a bit of progress, but we're slowly running out of funds.
+Please consider donating more, or even better convince your employer to donate,
+so we can finish those projects! The three sub-projects are:
+
+* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatible version
+   we call PyPy3 2.4.0, and are working toward a Python 3.3 compatible version
+
+* `STM`_ (software transactional memory): We have released a first working version,
+  and continue to try out new promising paths of achieving a fast multithreaded Python
+
+* `NumPy`_ which requires installation of our fork of upstream numpy,
+  available `on bitbucket`_
+
+.. _`Py3k`: http://pypy.org/py3donate.html
+.. _`STM`: http://pypy.org/tmdonate2.html
+.. _`NumPy`: http://pypy.org/numpydonate.html
+.. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`pypy and cpython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+This release supports **x86** machines on most common operating systems
+(Linux 32/64, Mac OS X 64, Windows, and OpenBSD),
+as well as newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux.
+
+While we support 32 bit python on Windows, work on the native Windows 64
+bit python is still stalling, we would welcome a volunteer
+to `handle that`_.
+
+.. _`pypy and cpython 2.7.x`: http://speed.pypy.org
+.. _`handle that`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation
+
+Highlights
+==========
+
+* The past months have seen pypy mature and grow, as rpython becomes the goto
+  solution for writing fast dynamic language interpreters. Our separation of
+  rpython and the python interpreter PyPy is now much clearer in the
+  `PyPy documentation`_  and we now have seperate `RPython documentation`_.
+
+* We have improved warmup time as well as jitted code performance: more than 10%
+  compared to pypy-2.4.0, due to internal cleanup and gc nursery improvements.
+  We no longer zero-out memory allocated in the gc nursery by default, work that
+  was started during a GSoC.
+
+* Passing objects between C and PyPy has been improved. We are now able to pass
+  raw pointers to C (without copying) using **pinning**. This improves I/O;
+  benchmarks that use networking intensively improved by about 50%. File()
+  operations still need some refactoring but are already showing a 20%
+  improvement on our benchmarks. Let us know if you see similar improvements.
+
+* Our integrated numpy support gained much of the GenericUfunc api in order to
+  support the lapack/blas linalg module of numpy. This dovetails with work in the
+  pypy/numpy repository to support linalg both through the (slower) cpyext capi
+  interface and also via (the faster) pure python cffi interface, using an
+  extended frompyfunc() api. We will soon post a seperate blog post specifically
+  about linalg and PyPy.
+
+* Dictionaries are now ordered by default, see the `blog post`_
+
+* Our nightly translations use --shared by default, including on OS/X and linux
+
+* We now more carefully handle errno (and GetLastError, WSAGetLastError) tying
+  the handlers as close as possible to the external function call, in non-jitted
+  as well as jitted code.
+
+* Issues reported with our previous release were resolved_ after reports from users on
+  our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at
+  #pypy.
+
+.. _`PyPy documentation`: http://doc.pypy.org
+.. _`RPython documentation`: http://rpython.readthedocs.org
+.. _`blog post`: http://morepypy.blogspot.com/2015/01/faster-more-memory-efficient-and-more.html
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-2.5.0.html
+
+We have further improvements on the way: rpython file handling,
+finishing numpy linalg compatibility, numpy object dtypes, a better profiler,
+as well as support for Python stdlib 2.7.9.
+
+Please try it out and let us know what you think. We especially welcome
+success stories, we know you are using PyPy, please tell us about it!
+
+Cheers
+
+The PyPy Team
diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py
--- a/pypy/doc/test/test_whatsnew.py
+++ b/pypy/doc/test/test_whatsnew.py
@@ -38,7 +38,11 @@
                                    current_branch, current_branch)
     cmd = r'hg log -R "%s" -r "%s" --template "{branches}\n"' % (path, revset)
     out = getoutput(cmd)
-    branches = set(map(str.strip, out.splitlines()))
+    branches = set()
+    for item in out.splitlines():
+        item = item.strip()
+        if not item.startswith('release-'):
+            branches.add(item)
     branches.discard("default")
     return branches, current_branch
 
diff --git a/pypy/doc/whatsnew-2.5.0.rst b/pypy/doc/whatsnew-2.5.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-2.5.0.rst
@@ -0,0 +1,139 @@
+=======================
+What's new in PyPy 2.5
+=======================
+
+.. this is a revision shortly after release-2.4.x
+.. startrev: 7026746cbb1b
+
+.. branch: win32-fixes5
+
+Fix c code generation for msvc so empty "{ }" are avoided in unions,
+Avoid re-opening files created with NamedTemporaryFile,
+Allocate by 4-byte chunks in rffi_platform,
+Skip testing objdump if it does not exist,
+and other small adjustments in own tests
+
+.. branch: rtyper-stuff
+
+Small internal refactorings in the rtyper.
+
+.. branch: var-in-Some
+
+Store annotations on the Variable objects, rather than in a big dict.
+Introduce a new framework for double-dispatched annotation implementations.
+
+.. branch: ClassRepr
+
+Refactor ClassRepr and make normalizecalls independent of the rtyper.
+
+.. branch: remove-remaining-smm
+
+Remove all remaining multimethods.
+
+.. branch: improve-docs
+
+Split RPython documentation from PyPy documentation and clean up.  There now is
+a clearer separation between documentation for users, developers and people
+interested in background information.
+
+.. branch: kill-multimethod
+
+Kill multimethod machinery, all multimethods were removed earlier.
+
+.. branch nditer-external_loop
+
+Implement `external_loop` arguement to numpy's nditer
+
+.. branch kill-rctime
+
+Rename pypy/module/rctime to pypy/module/time, since it contains the implementation of the 'time' module.
+
+.. branch: ssa-flow
+
+Use SSA form for flow graphs inside build_flow() and part of simplify_graph()
+
+.. branch: ufuncapi
+
+Implement most of the GenericUfunc api to support numpy linalg. The strategy is
+to encourage use of pure python or cffi ufuncs by extending frompyfunc().
+See the docstring of frompyfunc for more details. This dovetails with a branch
+of pypy/numpy - cffi-linalg which is a rewrite of the _umath_linalg module in
+python, calling lapack from cffi. The branch also support traditional use of
+cpyext GenericUfunc definitions in c.
+
+.. branch: all_ordered_dicts
+
+This makes ordered dicts the default dictionary implementation in
+RPython and in PyPy. It polishes the basic idea of rordereddict.py
+and then fixes various things, up to simplifying
+collections.OrderedDict.
+
+Note: Python programs can rely on the guaranteed dict order in PyPy
+now, but for compatibility with other Python implementations they
+should still use collections.OrderedDict where that really matters.
+Also, support for reversed() was *not* added to the 'dict' class;
+use OrderedDict.
+
+Benchmark results: in the noise. A few benchmarks see good speed
+improvements but the average is very close to parity.
+
+.. branch: berkerpeksag/fix-broken-link-in-readmerst-1415127402066
+.. branch: bigint-with-int-ops
+.. branch: dstufft/update-pip-bootstrap-location-to-the-new-1420760611527
+.. branch: float-opt
+.. branch: gc-incminimark-pinning
+
+This branch adds an interface rgc.pin which would (very temporarily)
+make object non-movable. That's used by rffi.alloc_buffer and
+rffi.get_nonmovable_buffer and improves performance considerably for
+IO operations.
+
+.. branch: gc_no_cleanup_nursery
+
+A branch started by Wenzhu Man (SoC'14) and then done by fijal. It
+removes the clearing of the nursery. The drawback is that new objects
+are not automatically filled with zeros any longer, which needs some
+care, mostly for GC references (which the GC tries to follow, so they
+must not contain garbage). The benefit is a quite large speed-up.
+
+.. branch: improve-gc-tracing-hooks
+.. branch: improve-ptr-conv-error
+.. branch: intern-not-immortal
+
+Fix intern() to return mortal strings, like in CPython.
+
+.. branch: issue1922-take2
+.. branch: kill-exported-symbols-list
+.. branch: kill-rctime
+.. branch: kill_ll_termios
+.. branch: look-into-all-modules
+.. branch: nditer-external_loop
+.. branch: numpy-generic-item
+.. branch: osx-shared
+
+``--shared`` support on OS/X (thanks wouter)
+
+.. branch: portable-threadlocal
+.. branch: pypy-dont-copy-ops
+.. branch: recursion_and_inlining
+.. branch: slim-down-resumedescr
+.. branch: squeaky/use-cflags-for-compiling-asm
+.. branch: unicode-fix
+.. branch: zlib_zdict
+
+.. branch: errno-again
+
+Changes how errno, GetLastError, and WSAGetLastError are handled.
+The idea is to tie reading the error status as close as possible to
+the external function call. This fixes some bugs, both of the very
+rare kind (e.g. errno on Linux might in theory be overwritten by
+mmap(), called rarely during major GCs, if such a major GC occurs at
+exactly the wrong time), and some of the less rare kind
+(particularly on Windows tests).
+
+.. branch: osx-package.py
+.. branch: package.py-helpful-error-message
+
+.. branch: typed-cells
+
+Improve performance of integer globals and class attributes.
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -2,121 +2,6 @@
 What's new in PyPy 2.5+
 =======================
 
-.. this is a revision shortly after release-2.4.x
-.. startrev: 7026746cbb1b
+.. this is a revision shortly after release-2.5.x
+.. startrev: 397b96217b85
 
-.. branch: win32-fixes5
-
-Fix c code generation for msvc so empty "{ }" are avoided in unions,
-Avoid re-opening files created with NamedTemporaryFile,
-Allocate by 4-byte chunks in rffi_platform,
-Skip testing objdump if it does not exist,
-and other small adjustments in own tests
-
-.. branch: rtyper-stuff
-
-Small internal refactorings in the rtyper.
-
-.. branch: var-in-Some
-
-Store annotations on the Variable objects, rather than in a big dict.
-Introduce a new framework for double-dispatched annotation implementations.
-
-.. branch: ClassRepr
-
-Refactor ClassRepr and make normalizecalls independent of the rtyper.
-
-.. branch: remove-remaining-smm
-
-Remove all remaining multimethods.
-
-.. branch: improve-docs
-
-Split RPython documentation from PyPy documentation and clean up.  There now is
-a clearer separation between documentation for users, developers and people
-interested in background information.
-
-.. branch: kill-multimethod
-
-Kill multimethod machinery, all multimethods were removed earlier.
-
-.. branch nditer-external_loop
-
-Implement `external_loop` arguement to numpy's nditer
-
-.. branch kill-rctime
-
-Rename pypy/module/rctime to pypy/module/time, since it contains the implementation of the 'time' module.
-
-.. branch: ssa-flow
-
-Use SSA form for flow graphs inside build_flow() and part of simplify_graph()
-
-.. branch: ufuncapi
-
-Implement most of the GenericUfunc api to support numpy linalg. The strategy is
-to encourage use of pure python or cffi ufuncs by extending frompyfunc().
-See the docstring of frompyfunc for more details. This dovetails with a branch
-of pypy/numpy - cffi-linalg which is a rewrite of the _umath_linalg module in
-python, calling lapack from cffi. The branch also support traditional use of
-cpyext GenericUfunc definitions in c.
-
-.. branch: all_ordered_dicts
-
-This makes ordered dicts the default dictionary implementation in
-RPython and in PyPy. It polishes the basic idea of rordereddict.py
-and then fixes various things, up to simplifying
-collections.OrderedDict.
-
-Note: Python programs can rely on the guaranteed dict order in PyPy
-now, but for compatibility with other Python implementations they
-should still use collections.OrderedDict where that really matters.
-Also, support for reversed() was *not* added to the 'dict' class;
-use OrderedDict.
-
-Benchmark results: in the noise. A few benchmarks see good speed
-improvements but the average is very close to parity.
-
-.. branch: berkerpeksag/fix-broken-link-in-readmerst-1415127402066
-.. branch: bigint-with-int-ops
-.. branch: dstufft/update-pip-bootstrap-location-to-the-new-1420760611527
-.. branch: float-opt
-.. branch: gc-incminimark-pinning
-
-This branch adds an interface rgc.pin which would (very temporarily)
-make object non-movable. That's used by rffi.alloc_buffer and
-rffi.get_nonmovable_buffer and improves performance considerably for
-IO operations.
-
-.. branch: gc_no_cleanup_nursery
-
-A branch started by Wenzhu Man (SoC'14) and then done by fijal. It
-removes the clearing of the nursery. The drawback is that new objects
-are not automatically filled with zeros any longer, which needs some
-care, mostly for GC references (which the GC tries to follow, so they
-must not contain garbage). The benefit is a quite large speed-up.
-
-.. branch: improve-gc-tracing-hooks
-.. branch: improve-ptr-conv-error
-.. branch: intern-not-immortal
-
-Fix intern() to return mortal strings, like in CPython.
-
-.. branch: issue1922-take2
-.. branch: kill-exported-symbols-list
-.. branch: kill-rctime
-.. branch: kill_ll_termios
-.. branch: look-into-all-modules
-.. branch: nditer-external_loop
-.. branch: numpy-generic-item
-.. branch: osx-shared
-
-``--shared`` support on OS/X (thanks wouter)
-
-.. branch: portable-threadlocal
-.. branch: pypy-dont-copy-ops
-.. branch: recursion_and_inlining
-.. branch: slim-down-resumedescr
-.. branch: squeaky/use-cflags-for-compiling-asm
-.. branch: unicode-fix
-.. branch: zlib_zdict
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -235,6 +235,11 @@
 
         config.translation.suggest(check_str_without_nul=True)
         config.translation.suggest(shared=True)
+        if config.translation.shared:
+            if config.translation.output is not None:
+                raise Exception("Cannot use the --output option with PyPy "
+                                "when --shared is on (it is by default). "
+                                "See issue #1971.")
 
         if config.translation.thread:
             config.objspace.usemodules.thread = True
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1017,6 +1017,9 @@
     def newlist_unicode(self, list_u):
         return self.newlist([self.wrap(u) for u in list_u])
 
+    def newlist_int(self, list_i):
+        return self.newlist([self.wrap(i) for i in list_i])
+
     def newlist_hint(self, sizehint):
         from pypy.objspace.std.listobject import make_empty_list_with_size
         return make_empty_list_with_size(self, sizehint)
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -486,10 +486,10 @@
                              w_exception_class=w_exception_class)
 wrap_oserror._annspecialcase_ = 'specialize:arg(3)'
 
-def exception_from_errno(space, w_type):
-    from rpython.rlib.rposix import get_errno
+def exception_from_saved_errno(space, w_type):
+    from rpython.rlib.rposix import get_saved_errno
 
-    errno = get_errno()
+    errno = get_saved_errno()
     msg = os.strerror(errno)
     w_error = space.call_function(w_type, space.wrap(errno), space.wrap(msg))
     return OperationError(w_type, w_error)
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -445,7 +445,10 @@
         f_back = space.interp_w(PyFrame, w_f_back, can_be_None=True)
         new_frame.f_backref = jit.non_virtual_ref(f_back)
 
-        new_frame.builtin = space.interp_w(Module, w_builtin)
+        if space.config.objspace.honor__builtins__:
+            new_frame.builtin = space.interp_w(Module, w_builtin)
+        else:
+            assert space.interp_w(Module, w_builtin) is space.builtin
         new_frame.set_blocklist([unpickle_block(space, w_blk)
                                  for w_blk in space.unpackiterable(w_blockstack)])
         values_w = maker.slp_from_tuple_with_nulls(space, w_valuestack)
@@ -530,9 +533,10 @@
 
         # cellvars are values exported to inner scopes
         # freevars are values coming from outer scopes
-        freevarnames = list(self.pycode.co_cellvars)
+        # (see locals2fast for why CO_OPTIMIZED)
+        freevarnames = self.pycode.co_cellvars
         if self.pycode.co_flags & consts.CO_OPTIMIZED:
-            freevarnames.extend(self.pycode.co_freevars)
+            freevarnames = freevarnames + self.pycode.co_freevars
         for i in range(len(freevarnames)):
             name = freevarnames[i]
             cell = self.cells[i]
@@ -561,7 +565,16 @@
 
         self.setfastscope(new_fastlocals_w)
 
-        freevarnames = self.pycode.co_cellvars + self.pycode.co_freevars
+        freevarnames = self.pycode.co_cellvars
+        if self.pycode.co_flags & consts.CO_OPTIMIZED:
+            freevarnames = freevarnames + self.pycode.co_freevars
+            # If the namespace is unoptimized, then one of the
+            # following cases applies:
+            # 1. It does not contain free variables, because it
+            #    uses import * or is a top-level namespace.
+            # 2. It is a class namespace.
+            # We don't want to accidentally copy free variables
+            # into the locals dict used by the class.
         for i in range(len(freevarnames)):
             name = freevarnames[i]
             cell = self.cells[i]
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -516,3 +516,21 @@
         assert seen == [(1, f, firstline + 6, 'line', None),
                         (1, f, firstline + 7, 'line', None),
                         (1, f, firstline + 8, 'line', None)]
+
+    def test_locals2fast_freevar_bug(self):
+        import sys
+        def f(n):
+            class A(object):
+                def g(self):
+                    return n
+                n = 42
+            return A()
+        res = f(10).g()
+        assert res == 10
+        #
+        def trace(*args):
+            return trace
+        sys.settrace(trace)
+        res = f(10).g()
+        sys.settrace(None)
+        assert res == 10
diff --git a/pypy/module/__pypy__/interp_time.py b/pypy/module/__pypy__/interp_time.py
--- a/pypy/module/__pypy__/interp_time.py
+++ b/pypy/module/__pypy__/interp_time.py
@@ -1,7 +1,7 @@
 from __future__ import with_statement
 import sys
 
-from pypy.interpreter.error import exception_from_errno
+from pypy.interpreter.error import exception_from_saved_errno
 from pypy.interpreter.gateway import unwrap_spec
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.tool import rffi_platform
@@ -48,11 +48,13 @@
 
     c_clock_gettime = rffi.llexternal("clock_gettime",
         [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT,
-        compilation_info=CConfig._compilation_info_, releasegil=False
+        compilation_info=CConfig._compilation_info_, releasegil=False,
+        save_err=rffi.RFFI_SAVE_ERRNO
     )
     c_clock_getres = rffi.llexternal("clock_getres",
         [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT,
-        compilation_info=CConfig._compilation_info_, releasegil=False
+        compilation_info=CConfig._compilation_info_, releasegil=False,
+        save_err=rffi.RFFI_SAVE_ERRNO
     )
 
     @unwrap_spec(clk_id="c_int")
@@ -60,7 +62,7 @@
         with lltype.scoped_alloc(TIMESPEC) as tp:
             ret = c_clock_gettime(clk_id, tp)
             if ret != 0:
-                raise exception_from_errno(space, space.w_IOError)
+                raise exception_from_saved_errno(space, space.w_IOError)
             return space.wrap(int(tp.c_tv_sec) + 1e-9 * int(tp.c_tv_nsec))
 
     @unwrap_spec(clk_id="c_int")
@@ -68,5 +70,5 @@
         with lltype.scoped_alloc(TIMESPEC) as tp:
             ret = c_clock_getres(clk_id, tp)
             if ret != 0:
-                raise exception_from_errno(space, space.w_IOError)
+                raise exception_from_saved_errno(space, space.w_IOError)
             return space.wrap(int(tp.c_tv_sec) + 1e-9 * int(tp.c_tv_nsec))
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -160,7 +160,7 @@
 
 
 @jit.jit_callback("CFFI")
-def invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata):
+def _invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata):
     """ Callback specification.
     ffi_cif - something ffi specific, don't care
     ll_args - rffi.VOIDPP - pointer to array of pointers to args
@@ -168,7 +168,6 @@
     ll_userdata - a special structure which holds necessary information
                   (what the real callback is for example), casted to VOIDP
     """
-    e = cerrno.get_real_errno()
     ll_res = rffi.cast(rffi.CCHARP, ll_res)
     unique_id = rffi.cast(lltype.Signed, ll_userdata)
     callback = global_callback_mapping.get(unique_id)
@@ -185,12 +184,9 @@
         return
     #
     must_leave = False
-    ec = None
     space = callback.space
     try:
         must_leave = space.threadlocals.try_enter_thread(space)
-        ec = cerrno.get_errno_container(space)
-        cerrno.save_errno_into(ec, e)
         extra_line = ''
         try:
             w_res = callback.invoke(ll_args)
@@ -212,5 +208,8 @@
         callback.write_error_return_value(ll_res)
     if must_leave:
         space.threadlocals.leave_thread(space)
-    if ec is not None:
-        cerrno.restore_errno_from(ec)
+
+def invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata):
+    cerrno._errno_after(rffi.RFFI_ERR_ALL)
+    _invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata)
+    cerrno._errno_before(rffi.RFFI_ERR_ALL)
diff --git a/pypy/module/_cffi_backend/cerrno.py b/pypy/module/_cffi_backend/cerrno.py
--- a/pypy/module/_cffi_backend/cerrno.py
+++ b/pypy/module/_cffi_backend/cerrno.py
@@ -2,7 +2,6 @@
 
 from rpython.rlib import rposix
 
-from pypy.interpreter.executioncontext import ExecutionContext
 from pypy.interpreter.gateway import unwrap_spec
 
 WIN32 = sys.platform == 'win32'
@@ -10,43 +9,22 @@
     from rpython.rlib import rwin32
 
 
-ExecutionContext._cffi_saved_errno = 0
-ExecutionContext._cffi_saved_LastError = 0
-
-
-def get_errno_container(space):
-    return space.getexecutioncontext()
-
-get_real_errno = rposix.get_errno
-
-
-def restore_errno_from(ec):
-    if WIN32:
-        rwin32.SetLastError(ec._cffi_saved_LastError)
-    rposix.set_errno(ec._cffi_saved_errno)
-
-def save_errno_into(ec, errno):
-    ec._cffi_saved_errno = errno
-    if WIN32:
-        ec._cffi_saved_LastError = rwin32.GetLastError()
-
+_errno_before = rposix._errno_before
+_errno_after  = rposix._errno_after
 
 def get_errno(space):
-    ec = get_errno_container(space)
-    return space.wrap(ec._cffi_saved_errno)
+    return space.wrap(rposix.get_saved_errno())
 
 @unwrap_spec(errno=int)
 def set_errno(space, errno):
-    ec = get_errno_container(space)
-    ec._cffi_saved_errno = errno
+    rposix.set_saved_errno(errno)
 
 # ____________________________________________________________
 
 @unwrap_spec(code=int)
 def getwinerror(space, code=-1):
-    from rpython.rlib.rwin32 import FormatError
+    from rpython.rlib.rwin32 import GetLastError_saved, FormatError
     if code == -1:
-        ec = get_errno_container(space)
-        code = ec._cffi_saved_LastError
+        code = GetLastError_saved()
     message = FormatError(code)
     return space.newtuple([space.wrap(code), space.wrap(message)])
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -155,13 +155,9 @@
                     # argtype is a pointer type, and w_obj a list/tuple/str
                     mustfree_max_plus_1 = i + 1
 
-            ec = cerrno.get_errno_container(space)
-            cerrno.restore_errno_from(ec)
             jit_libffi.jit_ffi_call(cif_descr,
                                     rffi.cast(rffi.VOIDP, funcaddr),
                                     buffer)
-            e = cerrno.get_real_errno()
-            cerrno.save_errno_into(ec, e)
 
             resultdata = rffi.ptradd(buffer, cif_descr.exchange_result)
             w_res = self.ctitem.copy_and_convert_to_object(resultdata)
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -347,7 +347,8 @@
 # ____________________________________________________________
 
 
-rffi_fdopen = rffi.llexternal("fdopen", [rffi.INT, rffi.CCHARP], rffi.CCHARP)
+rffi_fdopen = rffi.llexternal("fdopen", [rffi.INT, rffi.CCHARP], rffi.CCHARP,
+                              save_err=rffi.RFFI_SAVE_ERRNO)
 rffi_setbuf = rffi.llexternal("setbuf", [rffi.CCHARP, rffi.CCHARP], lltype.Void)
 rffi_fclose = rffi.llexternal("fclose", [rffi.CCHARP], rffi.INT)
 
@@ -357,7 +358,7 @@
     def __init__(self, fd, mode):
         self.llf = rffi_fdopen(fd, mode)
         if not self.llf:
-            raise OSError(rposix.get_errno(), "fdopen failed")
+            raise OSError(rposix.get_saved_errno(), "fdopen failed")
         rffi_setbuf(self.llf, lltype.nullptr(rffi.CCHARP.TO))
 
     def close(self):
diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -19,10 +19,16 @@
 
 
 def make_write_blocking_error(space, written):
+    # XXX CPython reads 'errno' here.  I *think* it doesn't make sense,
+    # because we might reach this point after calling a write() method
+    # that may be overridden by the user, if that method returns None.
+    # In that case what we get is a potentially nonsense errno.  But
+    # we'll use get_saved_errno() anyway, and hope (like CPython does)
+    # that we're getting a reasonable value at this point.
     w_type = space.gettypeobject(W_BlockingIOError.typedef)
     w_value = space.call_function(
         w_type,
-        space.wrap(rposix.get_errno()),
+        space.wrap(rposix.get_saved_errno()),
         space.wrap("write could not complete without blocking"),
         space.wrap(written))
     return OperationError(w_type, w_value)
diff --git a/pypy/module/_locale/interp_locale.py b/pypy/module/_locale/interp_locale.py
--- a/pypy/module/_locale/interp_locale.py
+++ b/pypy/module/_locale/interp_locale.py
@@ -300,7 +300,8 @@
         return space.wrap(result)
 
     _bindtextdomain = rlocale.external('bindtextdomain', [rffi.CCHARP, rffi.CCHARP],
-                                                                rffi.CCHARP)
+                                                                rffi.CCHARP,
+                                       save_err=rffi.RFFI_SAVE_ERRNO)
 
     @unwrap_spec(domain=str)
     def bindtextdomain(space, domain, w_dir):
@@ -325,7 +326,7 @@
                 rffi.free_charp(dir_c)
 
         if not dirname:
-            errno = rposix.get_errno()
+            errno = rposix.get_saved_errno()
             raise OperationError(space.w_OSError, space.wrap(errno))
         return space.wrap(rffi.charp2str(dirname))
 
diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ b/pypy/module/_multiprocessing/interp_connection.py
@@ -406,7 +406,7 @@
                 size, written_ptr, rffi.NULL)
 
             if (result == 0 and
-                rwin32.GetLastError() == ERROR_NO_SYSTEM_RESOURCES):
+                rwin32.GetLastError_saved() == ERROR_NO_SYSTEM_RESOURCES):
                 raise oefmt(space.w_ValueError,
                             "Cannot send %d bytes over connection", size)
         finally:
@@ -430,7 +430,7 @@
             if result:
                 return intmask(read_ptr[0]), lltype.nullptr(rffi.CCHARP.TO)
 
-            err = rwin32.GetLastError()
+            err = rwin32.GetLastError_saved()
             if err == ERROR_BROKEN_PIPE:
                 raise OperationError(space.w_EOFError, space.w_None)
             elif err != ERROR_MORE_DATA:
@@ -441,7 +441,7 @@
                                   lltype.nullptr(rwin32.LPDWORD.TO),
                                   lltype.nullptr(rwin32.LPDWORD.TO),
                                   left_ptr):
-                raise wrap_windowserror(space, rwin32.lastWindowsError())
+                raise wrap_windowserror(space, rwin32.lastSavedWindowsError())
 
             length = intmask(read_ptr[0] + left_ptr[0])
             if length > maxlength: # bad message, close connection
@@ -460,7 +460,7 @@
                                read_ptr, rffi.NULL)
             if not result:
                 rffi.free_charp(newbuf)
-                raise wrap_windowserror(space, rwin32.lastWindowsError())
+                raise wrap_windowserror(space, rwin32.lastSavedWindowsError())
 
             assert read_ptr[0] == left_ptr[0]
             return length, newbuf
@@ -480,7 +480,7 @@
                                   lltype.nullptr(rwin32.LPDWORD.TO),
                                   bytes_ptr,
                                   lltype.nullptr(rwin32.LPDWORD.TO)):
-                raise wrap_windowserror(space, rwin32.lastWindowsError())
+                raise wrap_windowserror(space, rwin32.lastSavedWindowsError())
             bytes = bytes_ptr[0]
         finally:
             lltype.free(bytes_ptr, flavor='raw')
@@ -506,7 +506,8 @@
                                       lltype.nullptr(rwin32.LPDWORD.TO),
                                       bytes_ptr,
                                       lltype.nullptr(rwin32.LPDWORD.TO)):
-                    raise wrap_windowserror(space, rwin32.lastWindowsError())
+                    raise wrap_windowserror(space,
+                                            rwin32.lastSavedWindowsError())
                 bytes = bytes_ptr[0]
             finally:
                 lltype.free(bytes_ptr, flavor='raw')
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -26,12 +26,14 @@
 
     _CreateSemaphore = rwin32.winexternal(
         'CreateSemaphoreA', [rffi.VOIDP, rffi.LONG, rffi.LONG, rwin32.LPCSTR],
-        rwin32.HANDLE)
-    _CloseHandle = rwin32.winexternal('CloseHandle', [rwin32.HANDLE],
+        rwin32.HANDLE,
+        save_err=rffi.RFFI_FULL_LASTERROR)
+    _CloseHandle_no_errno = rwin32.winexternal('CloseHandle', [rwin32.HANDLE],
         rwin32.BOOL, releasegil=False)
     _ReleaseSemaphore = rwin32.winexternal(
         'ReleaseSemaphore', [rwin32.HANDLE, rffi.LONG, rffi.LONGP],
-        rwin32.BOOL)
+        rwin32.BOOL,
+        save_err=rffi.RFFI_SAVE_LASTERROR)
 
 else:
     from rpython.rlib import rposix
@@ -81,51 +83,61 @@
 
     _sem_open = external('sem_open',
                          [rffi.CCHARP, rffi.INT, rffi.INT, rffi.UINT],
-                         SEM_T)
+                         SEM_T, save_err=rffi.RFFI_SAVE_ERRNO)
     # sem_close is releasegil=False to be able to use it in the __del__
-    _sem_close = external('sem_close', [SEM_T], rffi.INT, releasegil=False)
-    _sem_unlink = external('sem_unlink', [rffi.CCHARP], rffi.INT)
-    _sem_wait = external('sem_wait', [SEM_T], rffi.INT)
-    _sem_trywait = external('sem_trywait', [SEM_T], rffi.INT)
-    _sem_post = external('sem_post', [SEM_T], rffi.INT)
-    _sem_getvalue = external('sem_getvalue', [SEM_T, rffi.INTP], rffi.INT)
+    _sem_close_no_errno = external('sem_close', [SEM_T], rffi.INT,
+                                   releasegil=False)
+    _sem_close = external('sem_close', [SEM_T], rffi.INT, releasegil=False,
+                          save_err=rffi.RFFI_SAVE_ERRNO)
+    _sem_unlink = external('sem_unlink', [rffi.CCHARP], rffi.INT,
+                           save_err=rffi.RFFI_SAVE_ERRNO)
+    _sem_wait = external('sem_wait', [SEM_T], rffi.INT,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
+    _sem_trywait = external('sem_trywait', [SEM_T], rffi.INT,
+                            save_err=rffi.RFFI_SAVE_ERRNO)
+    _sem_post = external('sem_post', [SEM_T], rffi.INT,
+                         save_err=rffi.RFFI_SAVE_ERRNO)
+    _sem_getvalue = external('sem_getvalue', [SEM_T, rffi.INTP], rffi.INT,
+                             save_err=rffi.RFFI_SAVE_ERRNO)
 
-    _gettimeofday = external('gettimeofday', [TIMEVALP, rffi.VOIDP], rffi.INT)
+    _gettimeofday = external('gettimeofday', [TIMEVALP, rffi.VOIDP], rffi.INT,
+                             save_err=rffi.RFFI_SAVE_ERRNO)
 
     _select = external('select', [rffi.INT, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP,
-                                                          TIMEVALP], rffi.INT)
+                                                          TIMEVALP], rffi.INT,
+                       save_err=rffi.RFFI_SAVE_ERRNO)
 
     @jit.dont_look_inside
     def sem_open(name, oflag, mode, value):
         res = _sem_open(name, oflag, mode, value)
         if res == rffi.cast(SEM_T, SEM_FAILED):
-            raise OSError(rposix.get_errno(), "sem_open failed")
+            raise OSError(rposix.get_saved_errno(), "sem_open failed")
         return res
 
     def sem_close(handle):
         res = _sem_close(handle)
         if res < 0:
-            raise OSError(rposix.get_errno(), "sem_close failed")
+            raise OSError(rposix.get_saved_errno(), "sem_close failed")
 
     def sem_unlink(name):
         res = _sem_unlink(name)
         if res < 0:
-            raise OSError(rposix.get_errno(), "sem_unlink failed")
+            raise OSError(rposix.get_saved_errno(), "sem_unlink failed")
 
     def sem_wait(sem):
         res = _sem_wait(sem)
         if res < 0:
-            raise OSError(rposix.get_errno(), "sem_wait failed")
+            raise OSError(rposix.get_saved_errno(), "sem_wait failed")
 
     def sem_trywait(sem):
         res = _sem_trywait(sem)
         if res < 0:
-            raise OSError(rposix.get_errno(), "sem_trywait failed")
+            raise OSError(rposix.get_saved_errno(), "sem_trywait failed")
 
     def sem_timedwait(sem, deadline):
         res = _sem_timedwait(sem, deadline)
         if res < 0:
-            raise OSError(rposix.get_errno(), "sem_timedwait failed")
+            raise OSError(rposix.get_saved_errno(), "sem_timedwait failed")
 
     def _sem_timedwait_save(sem, deadline):
         delay = 0
@@ -135,7 +147,7 @@
                 # poll
                 if _sem_trywait(sem) == 0:
                     return 0
-                elif rposix.get_errno() != errno.EAGAIN:
+                elif rposix.get_saved_errno() != errno.EAGAIN:
                     return -1
 
                 now = gettimeofday()
@@ -143,7 +155,7 @@
                 c_tv_nsec = rffi.getintfield(deadline[0], 'c_tv_nsec')
                 if (c_tv_sec < now[0] or
                     (c_tv_sec == now[0] and c_tv_nsec <= now[1])):
-                    rposix.set_errno(errno.ETIMEDOUT)
+                    rposix.set_saved_errno(errno.ETIMEDOUT)
                     return -1
 
 
@@ -166,21 +178,21 @@
 
     if SEM_TIMED_WAIT:
         _sem_timedwait = external('sem_timedwait', [SEM_T, TIMESPECP],
-                                  rffi.INT)
+                                  rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
     else:
         _sem_timedwait = _sem_timedwait_save
 
     def sem_post(sem):
         res = _sem_post(sem)
         if res < 0:
-            raise OSError(rposix.get_errno(), "sem_post failed")
+            raise OSError(rposix.get_saved_errno(), "sem_post failed")
 
     def sem_getvalue(sem):
         sval_ptr = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
         try:
             res = _sem_getvalue(sem, sval_ptr)
             if res < 0:
-                raise OSError(rposix.get_errno(), "sem_getvalue failed")
+                raise OSError(rposix.get_saved_errno(), "sem_getvalue failed")
             return rffi.cast(lltype.Signed, sval_ptr[0])
         finally:
             lltype.free(sval_ptr, flavor='raw')
@@ -190,7 +202,7 @@
         try:
             res = _gettimeofday(now, None)
             if res < 0:
-                raise OSError(rposix.get_errno(), "gettimeofday failed")
+                raise OSError(rposix.get_saved_errno(), "gettimeofday failed")
             return (rffi.getintfield(now[0], 'c_tv_sec'),
                     rffi.getintfield(now[0], 'c_tv_usec'))
         finally:
@@ -216,18 +228,16 @@
 
 if sys.platform == 'win32':
     def create_semaphore(space, name, val, max):
-        rwin32.SetLastError(0)
+        rwin32.SetLastError_saved(0)
         handle = _CreateSemaphore(rffi.NULL, val, max, rffi.NULL)
         # On Windows we should fail on ERROR_ALREADY_EXISTS
-        err = rwin32.GetLastError()
+        err = rwin32.GetLastError_saved()
         if err != 0:
             raise WindowsError(err, "CreateSemaphore")
         return handle
 
     def delete_semaphore(handle):
-        if not _CloseHandle(handle):
-            err = rwin32.GetLastError()
-            raise WindowsError(err, "CloseHandle")
+        _CloseHandle_no_errno(handle)
 
     def semlock_acquire(self, space, block, w_timeout):
         if not block:
@@ -286,7 +296,7 @@
     def semlock_release(self, space):
         if not _ReleaseSemaphore(self.handle, 1,
                                  lltype.nullptr(rffi.LONGP.TO)):
-            err = rwin32.GetLastError()
+            err = rwin32.GetLastError_saved()
             if err == 0x0000012a: # ERROR_TOO_MANY_POSTS
                 raise OperationError(
                     space.w_ValueError,
@@ -300,7 +310,7 @@
         previous_ptr = lltype.malloc(rffi.LONGP.TO, 1, flavor='raw')
         try:
             if not _ReleaseSemaphore(self.handle, 1, previous_ptr):
-                raise rwin32.lastWindowsError("ReleaseSemaphore")
+                raise rwin32.lastSavedWindowsError("ReleaseSemaphore")
             return previous_ptr[0] + 1
         finally:
             lltype.free(previous_ptr, flavor='raw')
@@ -320,7 +330,7 @@
         return sem
 
     def delete_semaphore(handle):
-        sem_close(handle)
+        _sem_close_no_errno(handle)
 
     def semlock_acquire(self, space, block, w_timeout):
         if not block:
diff --git a/pypy/module/_multiprocessing/interp_win32.py b/pypy/module/_multiprocessing/interp_win32.py
--- a/pypy/module/_multiprocessing/interp_win32.py
+++ b/pypy/module/_multiprocessing/interp_win32.py
@@ -41,20 +41,24 @@
         rwin32.DWORD, rwin32.DWORD, rwin32.DWORD,
         rwin32.DWORD, rwin32.DWORD, rwin32.DWORD,
         rffi.VOIDP],
-    rwin32.HANDLE)
+    rwin32.HANDLE,
+    save_err=rffi.RFFI_SAVE_LASTERROR)
 
 _ConnectNamedPipe = rwin32.winexternal(
-    'ConnectNamedPipe', [rwin32.HANDLE, rffi.VOIDP], rwin32.BOOL)
+    'ConnectNamedPipe', [rwin32.HANDLE, rffi.VOIDP], rwin32.BOOL,
+    save_err=rffi.RFFI_SAVE_LASTERROR)
 
 _SetNamedPipeHandleState = rwin32.winexternal(
     'SetNamedPipeHandleState', [
         rwin32.HANDLE,
         rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD],
-    rwin32.BOOL)
+    rwin32.BOOL,
+    save_err=rffi.RFFI_SAVE_LASTERROR)
 
 _WaitNamedPipe = rwin32.winexternal(
     'WaitNamedPipeA', [rwin32.LPCSTR, rwin32.DWORD],
-    rwin32.BOOL)
+    rwin32.BOOL,
+    save_err=rffi.RFFI_SAVE_LASTERROR)
 
 _PeekNamedPipe = rwin32.winexternal(
     'PeekNamedPipe', [
@@ -62,31 +66,36 @@
         rffi.VOIDP,
         rwin32.DWORD,
         rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD],
-    rwin32.BOOL) 
+    rwin32.BOOL,
+    save_err=rffi.RFFI_SAVE_LASTERROR)

From noreply at buildbot.pypy.org  Sat Feb  7 10:34:22 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat,  7 Feb 2015 10:34:22 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: "coding:utf8" cookie: Don't read
 the second line if the first is not a comment.
Message-ID: <20150207093422.BCAE71C10F7@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: stdlib-2.7.9
Changeset: r75750:84cced3cc507
Date: 2015-02-07 10:34 +0100
http://bitbucket.org/pypy/pypy/changeset/84cced3cc507/

Log:	"coding:utf8" cookie: Don't read the second line if the first is not
	a comment.

diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py
--- a/pypy/interpreter/pyparser/pyparse.py
+++ b/pypy/interpreter/pyparser/pyparse.py
@@ -34,14 +34,14 @@
 def _check_for_encoding(s):
     eol = s.find('\n')
     if eol < 0:
-        return _check_line_for_encoding(s)
-    enc = _check_line_for_encoding(s[:eol])
-    if enc:
+        return _check_line_for_encoding(s)[0]
+    enc, again = _check_line_for_encoding(s[:eol])
+    if enc or not again:
         return enc
     eol2 = s.find('\n', eol + 1)
     if eol2 < 0:
-        return _check_line_for_encoding(s[eol + 1:])
-    return _check_line_for_encoding(s[eol + 1:eol2])
+        return _check_line_for_encoding(s[eol + 1:])[0]
+    return _check_line_for_encoding(s[eol + 1:eol2])[0]
 
 
 def _check_line_for_encoding(line):
@@ -51,8 +51,8 @@
         if line[i] == '#':
             break
         if line[i] not in ' \t\014':
-            return None
-    return pytokenizer.match_encoding_declaration(line[i:])
+            return None, False  # Not a comment, don't read the second line.
+    return pytokenizer.match_encoding_declaration(line[i:]), True
 
 
 class CompileInfo(object):

From noreply at buildbot.pypy.org  Sat Feb  7 11:45:14 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat,  7 Feb 2015 11:45:14 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: "import re" import more modules
	than before
Message-ID: <20150207104514.6044E1C02A4@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: stdlib-2.7.9
Changeset: r75751:43111b46f92d
Date: 2015-02-07 11:44 +0100
http://bitbucket.org/pypy/pypy/changeset/43111b46f92d/

Log:	"import re" import more modules than before (_locale for example)

diff --git a/pypy/module/thread/test/test_import_lock.py b/pypy/module/thread/test/test_import_lock.py
--- a/pypy/module/thread/test/test_import_lock.py
+++ b/pypy/module/thread/test/test_import_lock.py
@@ -105,7 +105,7 @@
         assert importlock.count == 0
         # A new module
         importhook(space, 're')
-        assert importlock.count == 7
+        assert importlock.count == 9
         # Import it again
         previous_count = importlock.count
         importhook(space, 're')

From noreply at buildbot.pypy.org  Sat Feb  7 13:21:47 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  7 Feb 2015 13:21:47 +0100 (CET)
Subject: [pypy-commit] pypy default: Don't put trigraphs in the C code,
 even if nowdays they seem
Message-ID: <20150207122147.DD2171C0884@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r75753:9d5b04c35d89
Date: 2015-02-07 13:21 +0100
http://bitbucket.org/pypy/pypy/changeset/9d5b04c35d89/

Log:	Don't put trigraphs in the C code, even if nowdays they seem to be
	ignored (clang gives a warning)

diff --git a/rpython/translator/c/support.py b/rpython/translator/c/support.py
--- a/rpython/translator/c/support.py
+++ b/rpython/translator/c/support.py
@@ -89,7 +89,8 @@
            ''')
 
 def _char_repr(c):
-    if c in '\\"': return '\\' + c
+    # escape with a '\' the characters '\', '"' or (for trigraphs) '?'
+    if c in '\\"?': return '\\' + c
     if ' ' <= c < '\x7F': return c
     return '\\%03o' % ord(c)
 

From noreply at buildbot.pypy.org  Sat Feb  7 13:21:46 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  7 Feb 2015 13:21:46 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Small attempt to speed-up tracing in
 the GC: force trace() to be inlined
Message-ID: <20150207122146.C15F61C0884@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r75752:f7205d08f1ba
Date: 2015-02-07 10:13 +0100
http://bitbucket.org/pypy/pypy/changeset/f7205d08f1ba/

Log:	Small attempt to speed-up tracing in the GC: force trace() to be
	inlined here.

diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -1,5 +1,6 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, rffi
 from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rlib.objectmodel import func_with_new_name
 from rpython.rlib.debug import ll_assert
 from rpython.memory.gcheader import GCHeaderBuilder
 from rpython.memory.support import DEFAULT_CHUNK_SIZE
@@ -217,6 +218,10 @@
             i += 1
     trace._annspecialcase_ = 'specialize:arg(2)'
 
+    tracei = func_with_new_name(trace, 'tracei')
+    tracei._annspecialcase_ = 'specialize:arg(2)'
+    tracei._always_inline_ = True
+
     def _trace_slow_path(self, obj, callback, arg):
         typeid = self.get_type_id(obj)
         if self.has_gcptr_in_varsize(typeid):
diff --git a/rpython/memory/gctransform/stmframework.py b/rpython/memory/gctransform/stmframework.py
--- a/rpython/memory/gctransform/stmframework.py
+++ b/rpython/memory/gctransform/stmframework.py
@@ -55,7 +55,7 @@
                   annmodel.SomeInteger()))
         #
         def pypy_stmcb_trace(obj, visit_fn):
-            gc.trace(obj, invokecallback, visit_fn)
+            gc.tracei(obj, invokecallback, visit_fn)
         pypy_stmcb_trace.c_name = "pypy_stmcb_trace"
         self.autoregister_ptrs.append(
             getfn(pypy_stmcb_trace, [llannotation.SomeAddress(),

From noreply at buildbot.pypy.org  Sat Feb  7 13:23:11 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  7 Feb 2015 13:23:11 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Don't default to '--shared' with stm:
 it has a lot of thread-locals, 
Message-ID: <20150207122311.6F1211C0884@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r75754:735a3d34474c
Date: 2015-02-07 13:23 +0100
http://bitbucket.org/pypy/pypy/changeset/735a3d34474c/

Log:	Don't default to '--shared' with stm: it has a lot of thread-locals,
	and reading thread-locals is much slower with --shared

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -239,7 +239,9 @@
             enable_translationmodules(config)
 
         config.translation.suggest(check_str_without_nul=True)
-        if sys.platform.startswith('linux'):
+        # Don't default to '--shared' with stm: it has a lot of thread-locals,
+        # and reading thread-locals is much slower with --shared
+        if sys.platform.startswith('linux') and not config.translation.stm:
             config.translation.suggest(shared=True)
 
         if config.translation.thread:

From noreply at buildbot.pypy.org  Sat Feb  7 14:44:24 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  7 Feb 2015 14:44:24 +0100 (CET)
Subject: [pypy-commit] pypy default: Add documentation to the check_*()
 methods in the jit tests
Message-ID: <20150207134424.F0BAE1C02A4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r75755:ae6f25df2dfe
Date: 2015-02-07 14:44 +0100
http://bitbucket.org/pypy/pypy/changeset/ae6f25df2dfe/

Log:	Add documentation to the check_*() methods in the jit tests

diff --git a/rpython/jit/metainterp/test/support.py b/rpython/jit/metainterp/test/support.py
--- a/rpython/jit/metainterp/test/support.py
+++ b/rpython/jit/metainterp/test/support.py
@@ -162,41 +162,67 @@
 class JitMixin:
     basic = True
 
+    # Basic terminology: the JIT produces "loops" and "bridges".
+    # Bridges are always attached to failing guards.  Every loop is
+    # the "trunk" of a tree of compiled code, which is formed by first
+    # compiling a loop and then incrementally adding some number of
+    # bridges to it.  Each loop and each bridge ends with either a
+    # FINISH or a JUMP instruction (the name "loop" is not really
+    # adapted any more).  The JUMP instruction jumps to any LABEL
+    # pseudo-instruction, which can be anywhere, within the same tree
+    # or another one.
+
     def check_resops(self, expected=None, **check):
+        """Check the instructions in all loops and bridges, ignoring
+        the ones that end in FINISH.  Either pass a dictionary (then
+        the check must match exactly), or some keyword arguments (then
+        the check is only about the instructions named)."""
         get_stats().check_resops(expected=expected, **check)
 
     def check_simple_loop(self, expected=None, **check):
+        """Useful in the simplest case when we have only one loop
+        ending with a jump back to itself and possibly a few bridges.
+        Only the operations within the loop formed by that single jump
+        will be counted; the bridges are all ignored.  If several loops
+        were compiled, complains."""
         get_stats().check_simple_loop(expected=expected, **check)
 
     def check_trace_count(self, count): # was check_loop_count
-        # The number of traces compiled
+        """Check the number of loops and bridges compiled."""
         assert get_stats().compiled_count == count
 
     def check_trace_count_at_most(self, count):
+        """Check the number of loops and bridges compiled."""
         assert get_stats().compiled_count <= count
 
     def check_jitcell_token_count(self, count): # was check_tree_loop_count
+        """This should check the number of independent trees of code.
+        (xxx it is not 100% clear that the count is correct)"""
         assert len(get_stats().jitcell_token_wrefs) == count
 
     def check_target_token_count(self, count):
+        """(xxx unknown)"""
         tokens = get_stats().get_all_jitcell_tokens()
         n = sum([len(t.target_tokens) for t in tokens])
         assert n == count
 
     def check_enter_count(self, count):
+        """Check the number of times pyjitpl ran.  (Every time, it
+        should have produced either one loop or one bridge, or aborted;
+        but it is not 100% clear that this is still correct in the
+        presence of unrolling.)"""
         assert get_stats().enter_count == count
 
     def check_enter_count_at_most(self, count):
+        """Check the number of times pyjitpl ran."""
         assert get_stats().enter_count <= count
 
-    def check_jumps(self, maxcount):
-        return # FIXME
-        assert get_stats().exec_jumps <= maxcount
-
     def check_aborted_count(self, count):
+        """Check the number of times pyjitpl was aborted."""
         assert get_stats().aborted_count == count
 
     def check_aborted_count_at_least(self, count):
+        """Check the number of times pyjitpl was aborted."""
         assert get_stats().aborted_count >= count
 
     def meta_interp(self, *args, **kwds):
diff --git a/rpython/jit/metainterp/test/test_send.py b/rpython/jit/metainterp/test/test_send.py
--- a/rpython/jit/metainterp/test/test_send.py
+++ b/rpython/jit/metainterp/test/test_send.py
@@ -202,7 +202,7 @@
         # the final one.
         self.check_trace_count(1)
         self.check_resops(guard_class=1, int_add=4, int_sub=4)
-        self.check_jumps(14)
+        #self.check_jumps(14)
 
     def test_oosend_guard_failure_2(self):
         # same as above, but using prebuilt objects 'w1' and 'w2'
@@ -244,7 +244,7 @@
         assert res == f(4, 28)
         self.check_trace_count(1)
         self.check_resops(guard_class=1, int_add=4, int_sub=4)
-        self.check_jumps(14)
+        #self.check_jumps(14)
 
     def test_oosend_different_initial_class(self):
         myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'w'])

From noreply at buildbot.pypy.org  Sat Feb  7 20:11:54 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  7 Feb 2015 20:11:54 +0100 (CET)
Subject: [pypy-commit] pypy default: Clarify a bit this logic
Message-ID: <20150207191154.885CF1C11BC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r75756:f252c05d7308
Date: 2015-02-07 20:11 +0100
http://bitbucket.org/pypy/pypy/changeset/f252c05d7308/

Log:	Clarify a bit this logic

diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -304,13 +304,20 @@
 REX_B = 1
 
 @specialize.arg(2)
-def encode_rex(mc, rexbyte, basevalue, orbyte):
+def encode_rex(mc, rexbyte, w, orbyte):
     if mc.WORD == 8:
         assert 0 <= rexbyte < 8
-        if basevalue != 0 or rexbyte != 0:
-            if basevalue == 0:
-                basevalue = 0x40
-            mc.writechar(chr(basevalue | rexbyte))
+        mc.writechar(chr(0x40 | w | rexbyte))
+    else:
+        assert rexbyte == 0
+    return 0
+
+ at specialize.arg(2)
+def encode_rex_opt(mc, rexbyte, _, orbyte):
+    if mc.WORD == 8:
+        assert 0 <= rexbyte < 8
+        if rexbyte != 0:
+            mc.writechar(chr(0x40 | rexbyte))
     else:
         assert rexbyte == 0
     return 0
@@ -322,9 +329,9 @@
 # the REX prefix in all cases.  It is only useful on instructions which
 # have an 8-bit register argument, to force access to the "sil" or "dil"
 # registers (as opposed to "ah-dh").
-rex_w  = encode_rex, 0, (0x40 | REX_W), None      # a REX.W prefix
-rex_nw = encode_rex, 0, 0, None                   # an optional REX prefix
-rex_fw = encode_rex, 0, 0x40, None                # a forced REX prefix
+rex_w  = encode_rex, 0, REX_W, None       # a REX.W prefix
+rex_nw = encode_rex_opt, 0, 0, None       # an optional REX prefix
+rex_fw = encode_rex, 0, 0, None           # a forced REX prefix
 
 # ____________________________________________________________
 

From noreply at buildbot.pypy.org  Sat Feb  7 20:51:54 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sat,  7 Feb 2015 20:51:54 +0100 (CET)
Subject: [pypy-commit] pypy default: extend matrix test to expose indexing
 error since reshape() call subtype __array_finalize__, fix.
Message-ID: <20150207195154.A40EC1C04CD@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r75757:07ae676d67c6
Date: 2015-02-06 08:30 +0200
http://bitbucket.org/pypy/pypy/changeset/07ae676d67c6/

Log:	extend matrix test to expose indexing error since reshape() call
	subtype __array_finalize__, fix.

diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -81,7 +81,10 @@
                 return w_object.descr_copy(space, w_order)
             elif not copy and (subok or type(w_object) is W_NDimArray):
                 return w_object
-        # we have a ndarray, but need to copy or change dtype or create W_NDimArray
+        if subok:
+            raise oefmt(space.w_NotImplementedError, 
+                "array(..., subok=True) only partially implemented")
+        # we have a ndarray, but need to copy or change dtype 
         if dtype is None:
             dtype = w_object.get_dtype()
         if dtype != w_object.get_dtype():
@@ -89,13 +92,12 @@
             copy = True
         if copy:
             shape = w_object.get_shape()
-            _elems_w = w_object.reshape(space, space.wrap(-1))
             elems_w = [None] * w_object.get_size()
-            for i in range(len(elems_w)):
-                elems_w[i] = _elems_w.descr_getitem(space, space.wrap(i))
-        elif subok:
-            raise oefmt(space.w_NotImplementedError, 
-                "array(...copy=False, subok=True) not implemented yet")
+            elsize = w_object.get_dtype().elsize
+            # TODO - use w_object.implementation without copying to a list
+            # unfortunately that causes a union error in translation
+            for i in range(w_object.get_size()):
+                elems_w[i] = w_object.implementation.getitem(i * elsize)
         else:
             sz = support.product(w_object.get_shape()) * dtype.elsize
             return W_NDimArray.from_shape_and_storage(space,
@@ -113,7 +115,7 @@
             dtype = descriptor.variable_dtype(space, dtype.char + '1')
 
     w_arr = W_NDimArray.from_shape(space, shape, dtype, order=order)
-    if len(elems_w) == 1:
+    if support.product(shape) < 2:
         w_arr.set_scalar_value(dtype.coerce(space, elems_w[0]))
     else:
         loop.assign(space, w_arr, elems_w)
diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py
--- a/pypy/module/micronumpy/test/test_subtype.py
+++ b/pypy/module/micronumpy/test/test_subtype.py
@@ -272,40 +272,103 @@
         import numpy as N
         # numpy's matrix class caused an infinite loop
         class matrix(N.ndarray):
-            getcnt = 0
             def __new__(subtype, data, dtype=None, copy=True):
+                print('matrix __new__')
+                if isinstance(data, matrix):
+                    dtype2 = data.dtype
+                    if (dtype is None):
+                        dtype = dtype2
+                    if (dtype2 == dtype) and (not copy):
+                        return data
+                    return data.astype(dtype)
+
+                if isinstance(data, N.ndarray):
+                    if dtype is None:
+                        intype = data.dtype
+                    else:
+                        intype = N.dtype(dtype)
+                    new = data.view(subtype)
+                    if intype != data.dtype:
+                        return new.astype(intype)
+                    if copy: return new.copy()
+                    else: return new
+
+                if isinstance(data, str):
+                    data = _convert_from_string(data)
+
+                # now convert data to an array
                 arr = N.array(data, dtype=dtype, copy=copy)
+                ndim = arr.ndim
                 shape = arr.shape
+                if (ndim > 2):
+                    raise ValueError("matrix must be 2-dimensional")
+                elif ndim == 0:
+                    shape = (1, 1)
+                elif ndim == 1:
+                    shape = (1, shape[0])
+
+                order = False
+                if (ndim == 2) and arr.flags.fortran:
+                    order = True
+
+                if not (order or arr.flags.contiguous):
+                    arr = arr.copy()
 
                 ret = N.ndarray.__new__(subtype, shape, arr.dtype,
                                         buffer=arr,
-                                        order=True)
+                                        order=order)
                 return ret
 
+            def __array_finalize__(self, obj):
+                print('matrix __array_finalize__')
+                self._getitem = False
+                if (isinstance(obj, matrix) and obj._getitem): return
+                ndim = self.ndim
+                if (ndim == 2):
+                    return
+                if (ndim > 2):
+                    newshape = tuple([x for x in self.shape if x > 1])
+                    ndim = len(newshape)
+                    if ndim == 2:
+                        self.shape = newshape
+                        return
+                    elif (ndim > 2):
+                        raise ValueError("shape too large to be a matrix.")
+                else:
+                    newshape = self.shape
+                if ndim == 0:
+                    self.shape = (1, 1)
+                elif ndim == 1:
+                    self.shape = (1, newshape[0])
+                return
+
             def __getitem__(self, index):
-                matrix.getcnt += 1
-                if matrix.getcnt > 10:
-                    # XXX strides.find_shape_and_elems is sensitive
-                    # to shape modification
-                    xxx
-                out = N.ndarray.__getitem__(self, index)
+                print('matrix __getitem__')
+                self._getitem = True
+
+                try:
+                    out = N.ndarray.__getitem__(self, index)
+                finally:
+                    self._getitem = False
 
                 if not isinstance(out, N.ndarray):
                     return out
+
+                if out.ndim == 0:
+                    return out[()]
+                if out.ndim == 1:
+                    sh = out.shape[0]
                     # Determine when we should have a column array
-                old_shape = out.shape
-                if out.ndim < 2:
-                    sh = out.shape[0]
                     try:
                         n = len(index)
                     except:
                         n = 0
-                    if n > 1:
+                    if n > 1 and isscalar(index[1]):
                         out.shape = (sh, 1)
                     else:
                         out.shape = (1, sh)
-                #print 'out, shape was',old_shape,'now',out.shape,'out',out
                 return out
+
         a = matrix([[1., 2.], [3., 4.]])
         b = N.array([a])
         assert (b == a).all()
@@ -318,6 +381,17 @@
         assert len(b.shape) == 2
         assert (b == a).all()
 
+        b = N.array(a, copy=True, dtype=int)
+        assert len(b.shape) == 2
+        assert (b == a).all()
+
+        c = matrix(a, copy=False)
+        assert c.base is not None
+        c[0, 0] = 100
+        assert a[0, 0] == 100
+        b = N.array(c, copy=True)
+        assert (b == a).all()
+
     def test_setstate_no_version(self):
         # Some subclasses of ndarray, like MaskedArray, do not use
         # version in __setstare__

From noreply at buildbot.pypy.org  Sun Feb  8 05:56:59 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun,  8 Feb 2015 05:56:59 +0100 (CET)
Subject: [pypy-commit] pypy default: fix for empty ndarrays
Message-ID: <20150208045659.8F0CD1C11BC@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r75758:dfffd5d7cc7e
Date: 2015-02-08 06:56 +0200
http://bitbucket.org/pypy/pypy/changeset/dfffd5d7cc7e/

Log:	fix for empty ndarrays

diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -115,7 +115,7 @@
             dtype = descriptor.variable_dtype(space, dtype.char + '1')
 
     w_arr = W_NDimArray.from_shape(space, shape, dtype, order=order)
-    if support.product(shape) < 2:
+    if support.product(shape) == 1:
         w_arr.set_scalar_value(dtype.coerce(space, elems_w[0]))
     else:
         loop.assign(space, w_arr, elems_w)

From noreply at buildbot.pypy.org  Sun Feb  8 10:09:23 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  8 Feb 2015 10:09:23 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: re-add 'signals_enabled'
Message-ID: <20150208090923.CAA4D1C0092@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r75759:fdd9315038c5
Date: 2015-02-08 10:09 +0100
http://bitbucket.org/pypy/pypy/changeset/fdd9315038c5/

Log:	re-add 'signals_enabled'

diff --git a/lib_pypy/transaction.py b/lib_pypy/transaction.py
--- a/lib_pypy/transaction.py
+++ b/lib_pypy/transaction.py
@@ -232,7 +232,8 @@
                 with atomic:
                     if len(exception) == 0:
                         try:
-                            f(*args, **kwds)
+                            with signals_enabled:
+                                f(*args, **kwds)
                         except:
                             exception.extend(sys.exc_info())
                 del next_transaction

From noreply at buildbot.pypy.org  Sun Feb  8 17:38:22 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun,  8 Feb 2015 17:38:22 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Fix a test
Message-ID: <20150208163822.2DA5A1C0455@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: py3k
Changeset: r75762:06870a7c11d8
Date: 2015-02-08 17:37 +0100
http://bitbucket.org/pypy/pypy/changeset/06870a7c11d8/

Log:	Fix a test

diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -706,7 +706,7 @@
             pass
         x = X(); x.a = 10; x.b = 20; x.c = 30
         d = x.__dict__
-        assert list(__pypy__.reversed_dict(d)) == d.keys()[::-1]
+        assert list(__pypy__.reversed_dict(d)) == list(d.keys())[::-1]
 
 
 class AppTestWithMapDictAndCounters(object):

From noreply at buildbot.pypy.org  Sun Feb  8 17:38:20 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun,  8 Feb 2015 17:38:20 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Translation fixes
Message-ID: <20150208163820.E8DC01C0455@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: py3k
Changeset: r75761:e11bc12d0270
Date: 2015-02-08 17:14 +0100
http://bitbucket.org/pypy/pypy/changeset/e11bc12d0270/

Log:	Translation fixes

diff --git a/pypy/module/_posixsubprocess/interp_subprocess.py b/pypy/module/_posixsubprocess/interp_subprocess.py
--- a/pypy/module/_posixsubprocess/interp_subprocess.py
+++ b/pypy/module/_posixsubprocess/interp_subprocess.py
@@ -7,7 +7,7 @@
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
 from pypy.interpreter.error import (
-    OperationError, exception_from_errno, oefmt, wrap_oserror)
+    OperationError, exception_from_saved_errno, oefmt, wrap_oserror)
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.module.posix.interp_posix import run_fork_hooks
 
@@ -50,7 +50,8 @@
     'pypy_subprocess_cloexec_pipe',
     [rffi.CArrayPtr(rffi.INT)], rffi.INT,
     compilation_info=eci,
-    releasegil=True)
+    releasegil=True,
+    save_err=rffi.RFFI_SAVE_ERRNO)
 c_init = rffi.llexternal(
     'pypy_subprocess_init',
     [], lltype.Void,
@@ -225,7 +226,7 @@
     with lltype.scoped_alloc(rffi.CArrayPtr(rffi.INT).TO, 2) as fds:
         res = c_cloexec_pipe(fds)
         if res != 0:
-            raise exception_from_errno(space, space.w_OSError)
+            raise exception_from_saved_errno(space, space.w_OSError)
 
         return space.newtuple([space.wrap(fds[0]),
                                space.wrap(fds[1]),
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rpoll, rsocket
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.ropenssl import *
-from rpython.rlib.rposix import get_errno, set_errno
+from rpython.rlib.rposix import get_saved_errno
 from rpython.rtyper.lltypesystem import lltype, rffi
 
 from pypy.interpreter.baseobjspace import W_Root
@@ -188,11 +188,9 @@
         else:
             keyfile = space.str_w(w_keyfile)
 
-        set_errno(0)
-
         ret = libssl_SSL_CTX_use_certificate_chain_file(self.ctx, certfile)
         if ret != 1:
-            errno = get_errno()
+            errno = get_saved_errno()
             if errno:
                 libssl_ERR_clear_error()
                 raise wrap_oserror(space, OSError(errno, ''),
@@ -203,7 +201,7 @@
         ret = libssl_SSL_CTX_use_PrivateKey_file(self.ctx, keyfile,
                                                  SSL_FILETYPE_PEM)
         if ret != 1:
-            errno = get_errno()
+            errno = get_saved_errno()
             if errno:
                 libssl_ERR_clear_error()
                 raise wrap_oserror(space, OSError(errno, ''),
@@ -227,11 +225,10 @@
         if cafile is None and capath is None:
             raise OperationError(space.w_TypeError, space.wrap(
                     "cafile and capath cannot be both omitted"))
-        set_errno(0)
         ret = libssl_SSL_CTX_load_verify_locations(
             self.ctx, cafile, capath)
         if ret != 1:
-            errno = get_errno()
+            errno = get_saved_errno()
             if errno:
                 libssl_ERR_clear_error()
                 raise wrap_oserror(space, OSError(errno, ''),
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -199,8 +199,10 @@
 ssl_external('SSLv2_method', [], SSL_METHOD)
 ssl_external('SSLv3_method', [], SSL_METHOD)
 ssl_external('SSLv23_method', [], SSL_METHOD)
-ssl_external('SSL_CTX_use_PrivateKey_file', [SSL_CTX, rffi.CCHARP, rffi.INT], rffi.INT)
-ssl_external('SSL_CTX_use_certificate_chain_file', [SSL_CTX, rffi.CCHARP], rffi.INT)
+ssl_external('SSL_CTX_use_PrivateKey_file', [SSL_CTX, rffi.CCHARP, rffi.INT], rffi.INT,
+             save_err=rffi.RFFI_FULL_ERRNO_ZERO)
+ssl_external('SSL_CTX_use_certificate_chain_file', [SSL_CTX, rffi.CCHARP], rffi.INT,
+             save_err=rffi.RFFI_FULL_ERRNO_ZERO)
 ssl_external('SSL_CTX_get_options', [SSL_CTX], rffi.LONG, macro=True)
 ssl_external('SSL_CTX_set_options', [SSL_CTX, rffi.LONG], rffi.LONG, macro=True)
 if HAVE_SSL_CTX_CLEAR_OPTIONS:
@@ -213,7 +215,7 @@
 ssl_external('SSL_CTX_set_cipher_list', [SSL_CTX, rffi.CCHARP], rffi.INT)
 ssl_external('SSL_CTX_load_verify_locations',
              [SSL_CTX, rffi.CCHARP, rffi.CCHARP], rffi.INT,
-             save_err=SAVE_ERR)
+             save_err=rffi.RFFI_FULL_ERRNO_ZERO)
 ssl_external('SSL_CTX_check_private_key', [SSL_CTX], rffi.INT)
 ssl_external('SSL_CTX_set_session_id_context', [SSL_CTX, rffi.CCHARP, rffi.UINT], rffi.INT)
 SSL_CTX_STATS_NAMES = """

From noreply at buildbot.pypy.org  Sun Feb  8 17:38:19 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun,  8 Feb 2015 17:38:19 +0100 (CET)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <20150208163819.BC1B71C0455@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: py3k
Changeset: r75760:a4b363d6f714
Date: 2015-02-07 19:18 +0100
http://bitbucket.org/pypy/pypy/changeset/a4b363d6f714/

Log:	hg merge default Probably a lot of issues.

diff too long, truncating to 2000 out of 19140 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -7,10 +7,7 @@
 9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
 ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm
 20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0
-20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0
-0000000000000000000000000000000000000000 release-2.3.0
 394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3
 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1
 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1
-32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1
-0000000000000000000000000000000000000000 release-2.2=3.1
+10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -42,19 +42,19 @@
   Amaury Forgeot d'Arc
   Samuele Pedroni
   Alex Gaynor
+  Brian Kearns
+  Matti Picus
+  Philip Jenvey
   Michael Hudson
   David Schneider
-  Matti Picus
-  Brian Kearns
-  Philip Jenvey
   Holger Krekel
   Christian Tismer
   Hakan Ardo
   Benjamin Peterson
   Manuel Jacob
+  Ronan Lamy
   Anders Chrigstrom
   Eric van Riet Paap
-  Ronan Lamy
   Wim Lavrijsen
   Richard Emslie
   Alexander Schremmer
@@ -68,9 +68,9 @@
   Camillo Bruni
   Laura Creighton
   Toon Verwaest
+  Romain Guillebert
   Leonardo Santagada
   Seo Sanghyeon
-  Romain Guillebert
   Justin Peel
   Ronny Pfannschmidt
   David Edelsohn
@@ -91,15 +91,16 @@
   Michal Bendowski
   Jan de Mooij
   stian
+  Tyler Wade
   Michael Foord
   Stephan Diehl
-  Tyler Wade
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
   Patrick Maupin
   Bob Ippolito
   Bruno Gola
+  David Malcolm
   Jean-Paul Calderone
   Timo Paulssen
   Squeaky
@@ -108,18 +109,19 @@
   Marius Gedminas
   Martin Matusiak
   Konstantin Lopuhin
+  Wenzhu Man
   John Witulski
-  Wenzhu Man
+  Laurence Tratt
+  Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
   Mark Pearse
   Simon Cross
-  Ivan Sichmann Freitas
   Andreas Stührk
+  Stefano Rivera
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Stefano Rivera
   Paweł Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -129,7 +131,6 @@
   tav
   Taavi Burns
   Georg Brandl
-  Laurence Tratt
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -141,13 +142,12 @@
   Jeremy Thurgood
   Rami Chowdhury
   Tobias Pape
-  David Malcolm
   Eugene Oden
   Henry Mason
   Vasily Kuznetsov
   Preston Timmons
+  David Ripton
   Jeff Terrace
-  David Ripton
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
@@ -166,13 +166,16 @@
   Gintautas Miliauskas
   Michael Twomey
   Lucian Branescu Mihaila
+  Yichao Yu
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
   Karl Bartel
+  Wouter van Heyst
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  anatoly techtonik
   Stuart Williams
   Jasper Schulz
   Christian Hudon
@@ -182,12 +185,11 @@
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
-  anatoly techtonik
   Neil Shepperd
+  Stanislaw Halik
   Mikael Schönenberg
   Elmo M?ntynen
   Jonathan David Riehl
-  Stanislaw Halik
   Anders Qvist
   Corbin Simpson
   Chirag Jadwani
@@ -196,10 +198,13 @@
   Vincent Legoll
   Alan McIntyre
   Alexander Sedov
+  Attila Gobi
   Christopher Pope
   Christian Tismer 
   Marc Abramowitz
   Dan Stromberg
+  Arjun Naik
+  Valentina Mukhamedzhanova
   Stefano Parmesan
   Alexis Daboville
   Jens-Uwe Mager
@@ -213,8 +218,6 @@
   Sylvain Thenault
   Nathan Taylor
   Vladimir Kryachko
-  Arjun Naik
-  Attila Gobi
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -222,22 +225,23 @@
   Ryan Gonzalez
   Ian Foote
   Kristjan Valur Jonsson
+  David Lievens
   Neil Blakey-Milner
   Lutz Paelike
   Lucio Torre
   Lars Wassermann
-  Valentina Mukhamedzhanova
   Henrik Vendelbo
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
   Sergey Kishchenko
-  Yichao Yu
   Ignas Mikalajunas
   Christoph Gerum
   Martin Blais
   Lene Wagner
   Tomo Cocoa
+  Toni Mattis
+  Lucas Stadler
   roberto at goyle
   Yury V. Zaytsev
   Anna Katrina Dominguez
@@ -265,23 +269,30 @@
   Stephan Busemann
   Rafał Gałczyński
   Christian Muirhead
+  Berker Peksag
   James Lan
   shoma hosaka
-  Daniel Neuh?user
-  Matthew Miller
+  Daniel Neuhäuser
+  Ben Mather
+  halgari
+  Boglarka Vezer
+  Chris Pressey
   Buck Golemon
   Konrad Delong
   Dinu Gherman
   Chris Lambacher
   coolbutuseless at gmail.com
+  Jim Baker
   Rodrigo Araújo
-  Jim Baker
+  Nikolaos-Digenis Karagiannis
   James Robert
   Armin Ronacher
   Brett Cannon
+  Donald Stufft
   yrttyr
   aliceinwire
   OlivierBlanvillain
+  Dan Sanders
   Zooko Wilcox-O Hearn
   Tomer Chachamu
   Christopher Groskopf
@@ -295,6 +306,7 @@
   Markus Unterwaditzer
   Even Wiik Thomassen
   jbs
+  squeaky
   soareschen
   Kurt Griffiths
   Mike Bayer
@@ -306,6 +318,7 @@
   Anna Ravencroft
   Dan Crosta
   Julien Phalip
+  Roman Podoliaka
   Dan Loewenherz
 
   Heinrich-Heine University, Germany 
diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -17,6 +17,10 @@
 except ImportError:
     assert '__pypy__' not in _sys.builtin_module_names
     newdict = lambda _ : {}
+try:
+    from __pypy__ import reversed_dict
+except ImportError:
+    reversed_dict = lambda d: reversed(d.keys())
 
 try:
     from thread import get_ident as _get_ident
@@ -29,142 +33,35 @@
 ################################################################################
 
 class OrderedDict(dict):
-    'Dictionary that remembers insertion order'
-    # An inherited dict maps keys to values.
-    # The inherited dict provides __getitem__, __len__, __contains__, and get.
-    # The remaining methods are order-aware.
-    # Big-O running times for all methods are the same as regular dictionaries.
+    '''Dictionary that remembers insertion order.
 
-    # The internal self.__map dict maps keys to links in a doubly linked list.
-    # The circular doubly linked list starts and ends with a sentinel element.
-    # The sentinel element never gets deleted (this simplifies the algorithm).
-    # Each link is stored as a list of length three:  [PREV, NEXT, KEY].
+    In PyPy all dicts are ordered anyway.  This is mostly useful as a
+    placeholder to mean "this dict must be ordered even on CPython".
 
-    def __init__(self, *args, **kwds):
-        '''Initialize an ordered dictionary.  The signature is the same as
-        regular dictionaries, but keyword arguments are not recommended because
-        their insertion order is arbitrary.
-
-        '''
-        if len(args) > 1:
-            raise TypeError('expected at most 1 arguments, got %d' % len(args))
-        try:
-            self.__root
-        except AttributeError:
-            self.__root = root = []                     # sentinel node
-            root[:] = [root, root, None]
-            self.__map = {}
-        self.__update(*args, **kwds)
-
-    def __setitem__(self, key, value, dict_setitem=dict.__setitem__):
-        'od.__setitem__(i, y) <==> od[i]=y'
-        # Setting a new item creates a new link at the end of the linked list,
-        # and the inherited dictionary is updated with the new key/value pair.
-        if key not in self:
-            root = self.__root
-            last = root[0]
-            last[1] = root[0] = self.__map[key] = [last, root, key]
-        return dict_setitem(self, key, value)
-
-    def __delitem__(self, key, dict_delitem=dict.__delitem__):
-        'od.__delitem__(y) <==> del od[y]'
-        # Deleting an existing item uses self.__map to find the link which gets
-        # removed by updating the links in the predecessor and successor nodes.
-        dict_delitem(self, key)
-        link_prev, link_next, _ = self.__map.pop(key)
-        link_prev[1] = link_next                        # update link_prev[NEXT]
-        link_next[0] = link_prev                        # update link_next[PREV]
-
-    def __iter__(self):
-        'od.__iter__() <==> iter(od)'
-        # Traverse the linked list in order.
-        root = self.__root
-        curr = root[1]                                  # start at the first node
-        while curr is not root:
-            yield curr[2]                               # yield the curr[KEY]
-            curr = curr[1]                              # move to next node
+    Known difference: iterating over an OrderedDict which is being
+    concurrently modified raises RuntimeError in PyPy.  In CPython
+    instead we get some behavior that appears reasonable in some
+    cases but is nonsensical in other cases.  This is officially
+    forbidden by the CPython docs, so we forbid it explicitly for now.
+    '''
 
     def __reversed__(self):
-        'od.__reversed__() <==> reversed(od)'
-        # Traverse the linked list in reverse order.
-        root = self.__root
-        curr = root[0]                                  # start at the last node
-        while curr is not root:
-            yield curr[2]                               # yield the curr[KEY]
-            curr = curr[0]                              # move to previous node
-
-    def clear(self):
-        'od.clear() -> None.  Remove all items from od.'
-        root = self.__root
-        root[:] = [root, root, None]
-        self.__map.clear()
-        dict.clear(self)
-
-    # -- the following methods do not depend on the internal structure --
-
-    def keys(self):
-        'od.keys() -> list of keys in od'
-        return list(self)
-
-    def values(self):
-        'od.values() -> list of values in od'
-        return [self[key] for key in self]
-
-    def items(self):
-        'od.items() -> list of (key, value) pairs in od'
-        return [(key, self[key]) for key in self]
-
-    def iterkeys(self):
-        'od.iterkeys() -> an iterator over the keys in od'
-        return iter(self)
-
-    def itervalues(self):
-        'od.itervalues -> an iterator over the values in od'
-        for k in self:
-            yield self[k]
-
-    def iteritems(self):
-        'od.iteritems -> an iterator over the (key, value) pairs in od'
-        for k in self:
-            yield (k, self[k])
-
-    update = MutableMapping.update
-
-    __update = update # let subclasses override update without breaking __init__
-
-    __marker = object()
-
-    def pop(self, key, default=__marker):
-        '''od.pop(k[,d]) -> v, remove specified key and return the corresponding
-        value.  If key is not found, d is returned if given, otherwise KeyError
-        is raised.
-
-        '''
-        if key in self:
-            result = self[key]
-            del self[key]
-            return result
-        if default is self.__marker:
-            raise KeyError(key)
-        return default
-
-    def setdefault(self, key, default=None):
-        'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
-        if key in self:
-            return self[key]
-        self[key] = default
-        return default
+        return reversed_dict(self)
 
     def popitem(self, last=True):
         '''od.popitem() -> (k, v), return and remove a (key, value) pair.
         Pairs are returned in LIFO order if last is true or FIFO order if false.
 
         '''
-        if not self:
-            raise KeyError('dictionary is empty')
-        key = next(reversed(self) if last else iter(self))
-        value = self.pop(key)
-        return key, value
+        if last:
+            return dict.popitem(self)
+        else:
+            it = dict.__iter__(self)
+            try:
+                k = it.next()
+            except StopIteration:
+                raise KeyError('dictionary is empty')
+            return (k, self.pop(k))
 
     def __repr__(self, _repr_running={}):
         'od.__repr__() <==> repr(od)'
@@ -183,8 +80,6 @@
         'Return state information for pickling'
         items = [[k, self[k]] for k in self]
         inst_dict = vars(self).copy()
-        for k in vars(OrderedDict()):
-            inst_dict.pop(k, None)
         if inst_dict:
             return (self.__class__, (items,), inst_dict)
         return self.__class__, (items,)
@@ -193,17 +88,6 @@
         'od.copy() -> a shallow copy of od'
         return self.__class__(self)
 
-    @classmethod
-    def fromkeys(cls, iterable, value=None):
-        '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S.
-        If not specified, the value defaults to None.
-
-        '''
-        self = cls()
-        for key in iterable:
-            self[key] = value
-        return self
-
     def __eq__(self, other):
         '''od.__eq__(y) <==> od==y.  Comparison to another OD is order-sensitive
         while comparison to a regular mapping is order-insensitive.
diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py
@@ -2,7 +2,6 @@
 import array
 import gc
 import unittest
-from ctypes.test import xfail
 
 class X(Structure):
     _fields_ = [("c_int", c_int)]
@@ -11,7 +10,6 @@
         self._init_called = True
 
 class Test(unittest.TestCase):
-    @xfail
     def test_fom_buffer(self):
         a = array.array("i", range(16))
         x = (c_int * 16).from_buffer(a)
@@ -34,10 +32,9 @@
         del a; gc.collect(); gc.collect(); gc.collect()
         self.assertEqual(x[:], expected)
 
-        self.assertRaises(TypeError,
+        self.assertRaises((TypeError, ValueError),
                           (c_char * 16).from_buffer, "a" * 16)
 
-    @xfail
     def test_fom_buffer_with_offset(self):
         a = array.array("i", range(16))
         x = (c_int * 15).from_buffer(a, sizeof(c_int))
@@ -46,7 +43,6 @@
         self.assertRaises(ValueError, lambda: (c_int * 16).from_buffer(a, sizeof(c_int)))
         self.assertRaises(ValueError, lambda: (c_int * 1).from_buffer(a, 16 * sizeof(c_int)))
 
-    @xfail
     def test_from_buffer_copy(self):
         a = array.array("i", range(16))
         x = (c_int * 16).from_buffer_copy(a)
@@ -71,7 +67,6 @@
         x = (c_char * 16).from_buffer_copy("a" * 16)
         self.assertEqual(x[:], "a" * 16)
 
-    @xfail
     def test_fom_buffer_copy_with_offset(self):
         a = array.array("i", range(16))
         x = (c_int * 15).from_buffer_copy(a, sizeof(c_int))
diff --git a/lib-python/2.7/test/test_collections.py b/lib-python/2.7/test/test_collections.py
--- a/lib-python/2.7/test/test_collections.py
+++ b/lib-python/2.7/test/test_collections.py
@@ -578,7 +578,12 @@
             def __repr__(self):
                 return "MySet(%s)" % repr(list(self))
         s = MySet([5,43,2,1])
-        self.assertEqual(s.pop(), 1)
+        # changed from CPython 2.7: it was "s.pop() == 1" but I see
+        # nothing that guarantees a particular order here.  In the
+        # 'all_ordered_dicts' branch of PyPy (or with OrderedDict
+        # instead of sets), it consistently returns 5, but this test
+        # should not rely on this or any other order.
+        self.assert_(s.pop() in [5,43,2,1])
 
     def test_issue8750(self):
         empty = WithSet()
@@ -1010,8 +1015,9 @@
                                           c=3, e=5).items()), pairs)                # mixed input
 
         # make sure no positional args conflict with possible kwdargs
-        self.assertEqual(inspect.getargspec(OrderedDict.__dict__['__init__']).args,
-                         ['self'])
+        if '__init__' in OrderedDict.__dict__:   # absent in PyPy
+            self.assertEqual(inspect.getargspec(OrderedDict.__dict__['__init__']).args,
+                             ['self'])
 
         # Make sure that direct calls to __init__ do not clear previous contents
         d = OrderedDict([('a', 1), ('b', 2), ('c', 3), ('d', 44), ('e', 55)])
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -83,6 +83,37 @@
     def in_dll(self, dll, name):
         return self.from_address(dll._handle.getaddressindll(name))
 
+    def from_buffer(self, obj, offset=0):
+        size = self._sizeofinstances()
+        buf = buffer(obj, offset, size)
+        if len(buf) < size:
+            raise ValueError(
+                "Buffer size too small (%d instead of at least %d bytes)"
+                % (len(buf) + offset, size + offset))
+        raw_addr = buf._pypy_raw_address()
+        result = self.from_address(raw_addr)
+        result._ensure_objects()['ffffffff'] = obj
+        return result
+
+    def from_buffer_copy(self, obj, offset=0):
+        size = self._sizeofinstances()
+        buf = buffer(obj, offset, size)
+        if len(buf) < size:
+            raise ValueError(
+                "Buffer size too small (%d instead of at least %d bytes)"
+                % (len(buf) + offset, size + offset))
+        result = self()
+        dest = result._buffer.buffer
+        try:
+            raw_addr = buf._pypy_raw_address()
+        except ValueError:
+            _rawffi.rawstring2charp(dest, buf)
+        else:
+            from ctypes import memmove
+            memmove(dest, raw_addr, size)
+        return result
+
+
 class CArgObject(object):
     """ simple wrapper around buffer, just for the case of freeing
     it afterwards
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -1,3 +1,4 @@
+import sys
 import _rawffi
 from _ctypes.basics import _CData, _CDataMeta, keepalive_key,\
      store_reference, ensure_objects, CArgObject
@@ -178,6 +179,8 @@
         instance = StructOrUnion.__new__(self)
         if isinstance(address, _rawffi.StructureInstance):
             address = address.buffer
+        # fix the address: turn it into as unsigned, in case it is negative
+        address = address & (sys.maxint * 2 + 1)
         instance.__dict__['_buffer'] = self._ffistruct.fromaddress(address)
         return instance
 
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -6,3 +6,8 @@
 
 __version__ = "0.8.6"
 __version_info__ = (0, 8, 6)
+
+# The verifier module file names are based on the CRC32 of a string that
+# contains the following version number.  It may be older than __version__
+# if nothing is clearly incompatible.
+__version_verifier_modules__ = "0.8.6"
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -69,6 +69,7 @@
         self._function_caches = []
         self._libraries = []
         self._cdefsources = []
+        self._windows_unicode = None
         if hasattr(backend, 'set_ffi'):
             backend.set_ffi(self)
         for name in backend.__dict__:
@@ -77,6 +78,7 @@
         #
         with self._lock:
             self.BVoidP = self._get_cached_btype(model.voidp_type)
+            self.BCharA = self._get_cached_btype(model.char_array_type)
         if isinstance(backend, types.ModuleType):
             # _cffi_backend: attach these constants to the class
             if not hasattr(FFI, 'NULL'):
@@ -189,13 +191,16 @@
             cdecl = self._typeof(cdecl)
         return self._backend.alignof(cdecl)
 
-    def offsetof(self, cdecl, fieldname):
+    def offsetof(self, cdecl, *fields_or_indexes):
         """Return the offset of the named field inside the given
-        structure, which must be given as a C type name.
+        structure or array, which must be given as a C type name.  
+        You can give several field names in case of nested structures.
+        You can also give numeric values which correspond to array
+        items, in case of an array type.
         """
         if isinstance(cdecl, basestring):
             cdecl = self._typeof(cdecl)
-        return self._backend.typeoffsetof(cdecl, fieldname)[1]
+        return self._typeoffsetof(cdecl, *fields_or_indexes)[1]
 
     def new(self, cdecl, init=None):
         """Allocate an instance according to the specified C type and
@@ -264,6 +269,16 @@
         """
         return self._backend.buffer(cdata, size)
 
+    def from_buffer(self, python_buffer):
+        """Return a  that points to the data of the
+        given Python object, which must support the buffer interface.
+        Note that this is not meant to be used on the built-in types str,
+        unicode, or bytearray (you can build 'char[]' arrays explicitly)
+        but only on objects containing large quantities of raw data
+        in some other format, like 'array.array' or numpy arrays.
+        """
+        return self._backend.from_buffer(self.BCharA, python_buffer)
+
     def callback(self, cdecl, python_callable=None, error=None):
         """Return a callback object or a decorator making such a
         callback object.  'cdecl' must name a C function pointer type.
@@ -335,9 +350,23 @@
         which requires binary compatibility in the signatures.
         """
         from .verifier import Verifier, _caller_dir_pycache
+        #
+        # If set_unicode(True) was called, insert the UNICODE and
+        # _UNICODE macro declarations
+        if self._windows_unicode:
+            self._apply_windows_unicode(kwargs)
+        #
+        # Set the tmpdir here, and not in Verifier.__init__: it picks
+        # up the caller's directory, which we want to be the caller of
+        # ffi.verify(), as opposed to the caller of Veritier().
         tmpdir = tmpdir or _caller_dir_pycache()
+        #
+        # Make a Verifier() and use it to load the library.
         self.verifier = Verifier(self, source, tmpdir, **kwargs)
         lib = self.verifier.load_library()
+        #
+        # Save the loaded library for keep-alive purposes, even
+        # if the caller doesn't keep it alive itself (it should).
         self._libraries.append(lib)
         return lib
 
@@ -356,15 +385,29 @@
         with self._lock:
             return model.pointer_cache(self, ctype)
 
-    def addressof(self, cdata, field=None):
+    def addressof(self, cdata, *fields_or_indexes):
         """Return the address of a .
-        If 'field' is specified, return the address of this field.
+        If 'fields_or_indexes' are given, returns the address of that
+        field or array item in the structure or array, recursively in
+        case of nested structures.
         """
         ctype = self._backend.typeof(cdata)
-        ctype, offset = self._backend.typeoffsetof(ctype, field)
+        if fields_or_indexes:
+            ctype, offset = self._typeoffsetof(ctype, *fields_or_indexes)
+        else:
+            if ctype.kind == "pointer":
+                raise TypeError("addressof(pointer)")
+            offset = 0
         ctypeptr = self._pointer_to(ctype)
         return self._backend.rawaddressof(ctypeptr, cdata, offset)
 
+    def _typeoffsetof(self, ctype, field_or_index, *fields_or_indexes):
+        ctype, offset = self._backend.typeoffsetof(ctype, field_or_index)
+        for field1 in fields_or_indexes:
+            ctype, offset1 = self._backend.typeoffsetof(ctype, field1, 1)
+            offset += offset1
+        return ctype, offset
+
     def include(self, ffi_to_include):
         """Includes the typedefs, structs, unions and enums defined
         in another FFI instance.  Usage is similar to a #include in C,
@@ -387,6 +430,44 @@
     def from_handle(self, x):
         return self._backend.from_handle(x)
 
+    def set_unicode(self, enabled_flag):
+        """Windows: if 'enabled_flag' is True, enable the UNICODE and
+        _UNICODE defines in C, and declare the types like TCHAR and LPTCSTR
+        to be (pointers to) wchar_t.  If 'enabled_flag' is False,
+        declare these types to be (pointers to) plain 8-bit characters.
+        This is mostly for backward compatibility; you usually want True.
+        """
+        if self._windows_unicode is not None:
+            raise ValueError("set_unicode() can only be called once")
+        enabled_flag = bool(enabled_flag)
+        if enabled_flag:
+            self.cdef("typedef wchar_t TBYTE;"
+                      "typedef wchar_t TCHAR;"
+                      "typedef const wchar_t *LPCTSTR;"
+                      "typedef const wchar_t *PCTSTR;"
+                      "typedef wchar_t *LPTSTR;"
+                      "typedef wchar_t *PTSTR;"
+                      "typedef TBYTE *PTBYTE;"
+                      "typedef TCHAR *PTCHAR;")
+        else:
+            self.cdef("typedef char TBYTE;"
+                      "typedef char TCHAR;"
+                      "typedef const char *LPCTSTR;"
+                      "typedef const char *PCTSTR;"
+                      "typedef char *LPTSTR;"
+                      "typedef char *PTSTR;"
+                      "typedef TBYTE *PTBYTE;"
+                      "typedef TCHAR *PTCHAR;")
+        self._windows_unicode = enabled_flag
+
+    def _apply_windows_unicode(self, kwds):
+        defmacros = kwds.get('define_macros', ())
+        if not isinstance(defmacros, (list, tuple)):
+            raise TypeError("'define_macros' must be a list or tuple")
+        defmacros = list(defmacros) + [('UNICODE', '1'),
+                                       ('_UNICODE', '1')]
+        kwds['define_macros'] = defmacros
+
 
 def _load_backend_lib(backend, name, flags):
     if name is None:
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -169,6 +169,7 @@
 class CTypesGenericPtr(CTypesData):
     __slots__ = ['_address', '_as_ctype_ptr']
     _automatic_casts = False
+    kind = "pointer"
 
     @classmethod
     def _newp(cls, init):
@@ -370,10 +371,12 @@
                                 (CTypesPrimitive, type(source).__name__))
             return source
         #
+        kind1 = kind
         class CTypesPrimitive(CTypesGenericPrimitive):
             __slots__ = ['_value']
             _ctype = ctype
             _reftypename = '%s &' % name
+            kind = kind1
 
             def __init__(self, value):
                 self._value = value
@@ -703,12 +706,13 @@
         class struct_or_union(base_ctypes_class):
             pass
         struct_or_union.__name__ = '%s_%s' % (kind, name)
+        kind1 = kind
         #
         class CTypesStructOrUnion(CTypesBaseStructOrUnion):
             __slots__ = ['_blob']
             _ctype = struct_or_union
             _reftypename = '%s &' % (name,)
-            _kind = kind
+            _kind = kind = kind1
         #
         CTypesStructOrUnion._fix_class()
         return CTypesStructOrUnion
@@ -994,27 +998,42 @@
     def getcname(self, BType, replace_with):
         return BType._get_c_name(replace_with)
 
-    def typeoffsetof(self, BType, fieldname):
-        if fieldname is not None and issubclass(BType, CTypesGenericPtr):
-            BType = BType._BItem
-        if not issubclass(BType, CTypesBaseStructOrUnion):
-            raise TypeError("expected a struct or union ctype")
-        if fieldname is None:
-            return (BType, 0)
-        else:
+    def typeoffsetof(self, BType, fieldname, num=0):
+        if isinstance(fieldname, str):
+            if num == 0 and issubclass(BType, CTypesGenericPtr):
+                BType = BType._BItem
+            if not issubclass(BType, CTypesBaseStructOrUnion):
+                raise TypeError("expected a struct or union ctype")
             BField = BType._bfield_types[fieldname]
             if BField is Ellipsis:
                 raise TypeError("not supported for bitfields")
             return (BField, BType._offsetof(fieldname))
+        elif isinstance(fieldname, (int, long)):
+            if issubclass(BType, CTypesGenericArray):
+                BType = BType._CTPtr
+            if not issubclass(BType, CTypesGenericPtr):
+                raise TypeError("expected an array or ptr ctype")
+            BItem = BType._BItem
+            offset = BItem._get_size() * fieldname
+            if offset > sys.maxsize:
+                raise OverflowError
+            return (BItem, offset)
+        else:
+            raise TypeError(type(fieldname))
 
-    def rawaddressof(self, BTypePtr, cdata, offset):
+    def rawaddressof(self, BTypePtr, cdata, offset=None):
         if isinstance(cdata, CTypesBaseStructOrUnion):
             ptr = ctypes.pointer(type(cdata)._to_ctypes(cdata))
         elif isinstance(cdata, CTypesGenericPtr):
+            if offset is None or not issubclass(type(cdata)._BItem,
+                                                CTypesBaseStructOrUnion):
+                raise TypeError("unexpected cdata type")
+            ptr = type(cdata)._to_ctypes(cdata)
+        elif isinstance(cdata, CTypesGenericArray):
             ptr = type(cdata)._to_ctypes(cdata)
         else:
             raise TypeError("expected a ")
-        if offset != 0:
+        if offset:
             ptr = ctypes.cast(
                 ctypes.c_void_p(
                     ctypes.cast(ptr, ctypes.c_void_p).value + offset),
diff --git a/lib_pypy/cffi/commontypes.py b/lib_pypy/cffi/commontypes.py
--- a/lib_pypy/cffi/commontypes.py
+++ b/lib_pypy/cffi/commontypes.py
@@ -29,6 +29,9 @@
                 result = model.PointerType(resolve_common_type(result[:-2]))
         elif result in model.PrimitiveType.ALL_PRIMITIVE_TYPES:
             result = model.PrimitiveType(result)
+        elif result == 'set-unicode-needed':
+            raise api.FFIError("The Windows type %r is only available after "
+                               "you call ffi.set_unicode()" % (commontype,))
         else:
             if commontype == result:
                 raise api.FFIError("Unsupported type: %r.  Please file a bug "
@@ -86,8 +89,6 @@
         "ULONGLONG": "unsigned long long",
         "WCHAR": "wchar_t",
         "SHORT": "short",
-        "TBYTE": "WCHAR",
-        "TCHAR": "WCHAR",
         "UCHAR": "unsigned char",
         "UINT": "unsigned int",
         "UINT8": "unsigned char",
@@ -157,14 +158,12 @@
 
         "LPCVOID": model.const_voidp_type,
         "LPCWSTR": "const WCHAR *",
-        "LPCTSTR": "LPCWSTR",
         "LPDWORD": "DWORD *",
         "LPHANDLE": "HANDLE *",
         "LPINT": "int *",
         "LPLONG": "long *",
         "LPSTR": "CHAR *",
         "LPWSTR": "WCHAR *",
-        "LPTSTR": "LPWSTR",
         "LPVOID": model.voidp_type,
         "LPWORD": "WORD *",
         "LRESULT": "LONG_PTR",
@@ -173,7 +172,6 @@
         "PBYTE": "BYTE *",
         "PCHAR": "CHAR *",
         "PCSTR": "const CHAR *",
-        "PCTSTR": "LPCWSTR",
         "PCWSTR": "const WCHAR *",
         "PDWORD": "DWORD *",
         "PDWORDLONG": "DWORDLONG *",
@@ -200,9 +198,6 @@
         "PSIZE_T": "SIZE_T *",
         "PSSIZE_T": "SSIZE_T *",
         "PSTR": "CHAR *",
-        "PTBYTE": "TBYTE *",
-        "PTCHAR": "TCHAR *",
-        "PTSTR": "LPWSTR",
         "PUCHAR": "UCHAR *",
         "PUHALF_PTR": "UHALF_PTR *",
         "PUINT": "UINT *",
@@ -240,6 +235,15 @@
         "USN": "LONGLONG",
         "VOID": model.void_type,
         "WPARAM": "UINT_PTR",
+
+        "TBYTE": "set-unicode-needed",
+        "TCHAR": "set-unicode-needed",
+        "LPCTSTR": "set-unicode-needed",
+        "PCTSTR": "set-unicode-needed",
+        "LPTSTR": "set-unicode-needed",
+        "PTSTR": "set-unicode-needed",
+        "PTBYTE": "set-unicode-needed",
+        "PTCHAR": "set-unicode-needed",
         })
     return result
 
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -1,4 +1,3 @@
-
 from . import api, model
 from .commontypes import COMMON_TYPES, resolve_common_type
 try:
@@ -209,6 +208,8 @@
 
     def _add_constants(self, key, val):
         if key in self._int_constants:
+            if self._int_constants[key] == val:
+                return     # ignore identical double declarations
             raise api.FFIError(
                 "multiple declarations of constant: %s" % (key,))
         self._int_constants[key] = val
@@ -228,12 +229,18 @@
 
                 pyvalue = int(int_str, 0)
                 self._add_constants(key, pyvalue)
+                self._declare('macro ' + key, pyvalue)
             elif value == '...':
                 self._declare('macro ' + key, value)
             else:
-                raise api.CDefError('only supports the syntax "#define '
-                                    '%s ..." (literally) or "#define '
-                                    '%s 0x1FF" for now' % (key, key))
+                raise api.CDefError(
+                    'only supports one of the following syntax:\n'
+                    '  #define %s ...     (literally dot-dot-dot)\n'
+                    '  #define %s NUMBER  (with NUMBER an integer'
+                                    ' constant, decimal/hex/octal)\n'
+                    'got:\n'
+                    '  #define %s %s'
+                    % (key, key, key, value))
 
     def _parse_decl(self, decl):
         node = decl.type
@@ -460,6 +467,8 @@
             elif kind == 'union':
                 tp = model.UnionType(explicit_name, None, None, None)
             elif kind == 'enum':
+                if explicit_name == '__dotdotdot__':
+                    raise CDefError("Enums cannot be declared with ...")
                 tp = self._build_enum_type(explicit_name, type.values)
             else:
                 raise AssertionError("kind = %r" % (kind,))
@@ -532,9 +541,24 @@
 
     def _parse_constant(self, exprnode, partial_length_ok=False):
         # for now, limited to expressions that are an immediate number
-        # or negative number
+        # or positive/negative number
         if isinstance(exprnode, pycparser.c_ast.Constant):
-            return int(exprnode.value, 0)
+            s = exprnode.value
+            if s.startswith('0'):
+                if s.startswith('0x') or s.startswith('0X'):
+                    return int(s, 16)
+                return int(s, 8)
+            elif '1' <= s[0] <= '9':
+                return int(s, 10)
+            elif s[0] == "'" and s[-1] == "'" and (
+                    len(s) == 3 or (len(s) == 4 and s[1] == "\\")):
+                return ord(s[-2])
+            else:
+                raise api.CDefError("invalid constant %r" % (s,))
+        #
+        if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and
+                exprnode.op == '+'):
+            return self._parse_constant(exprnode.expr)
         #
         if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and
                 exprnode.op == '-'):
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -11,6 +11,9 @@
     """
 
 
+LIST_OF_FILE_NAMES = ['sources', 'include_dirs', 'library_dirs',
+                      'extra_objects', 'depends']
+
 def get_extension(srcfilename, modname, sources=(), **kwds):
     from distutils.core import Extension
     allsources = [srcfilename]
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -235,6 +235,8 @@
         BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist)
         return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length)
 
+char_array_type = ArrayType(PrimitiveType('char'), None)
+
 
 class StructOrUnionOrEnum(BaseTypeByIdentity):
     _attrs_ = ('name',)
@@ -478,7 +480,7 @@
     try:
         res = getattr(ffi._backend, funcname)(*args)
     except NotImplementedError as e:
-        raise NotImplementedError("%r: %s" % (srctype, e))
+        raise NotImplementedError("%s: %r: %s" % (funcname, srctype, e))
     # note that setdefault() on WeakValueDictionary is not atomic
     # and contains a rare bug (http://bugs.python.org/issue19542);
     # we have to use a lock and do it ourselves
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -65,7 +65,7 @@
         # The following two 'chained_list_constants' items contains
         # the head of these two chained lists, as a string that gives the
         # call to do, if any.
-        self._chained_list_constants = ['0', '0']
+        self._chained_list_constants = ['((void)lib,0)', '((void)lib,0)']
         #
         prnt = self._prnt
         # first paste some standard set of lines that are mostly '#define'
@@ -138,15 +138,22 @@
         prnt()
         prnt('#endif')
 
-    def load_library(self):
+    def load_library(self, flags=None):
         # XXX review all usages of 'self' here!
         # import it as a new extension module
+        if hasattr(sys, "getdlopenflags"):
+            previous_flags = sys.getdlopenflags()
         try:
+            if hasattr(sys, "setdlopenflags") and flags is not None:
+                sys.setdlopenflags(flags)
             module = imp.load_dynamic(self.verifier.get_module_name(),
                                       self.verifier.modulefilename)
         except ImportError as e:
             error = "importing %r: %s" % (self.verifier.modulefilename, e)
             raise ffiplatform.VerificationError(error)
+        finally:
+            if hasattr(sys, "setdlopenflags"):
+                sys.setdlopenflags(previous_flags)
         #
         # call loading_cpy_struct() to get the struct layout inferred by
         # the C compiler
@@ -228,7 +235,8 @@
                 converter = '_cffi_to_c_int'
                 extraarg = ', %s' % tp.name
             else:
-                converter = '_cffi_to_c_%s' % (tp.name.replace(' ', '_'),)
+                converter = '(%s)_cffi_to_c_%s' % (tp.get_c_name(''),
+                                                   tp.name.replace(' ', '_'))
             errvalue = '-1'
         #
         elif isinstance(tp, model.PointerType):
@@ -267,8 +275,8 @@
         self._prnt('  if (datasize != 0) {')
         self._prnt('    if (datasize < 0)')
         self._prnt('      %s;' % errcode)
-        self._prnt('    %s = alloca(datasize);' % (tovar,))
-        self._prnt('    memset((void *)%s, 0, datasize);' % (tovar,))
+        self._prnt('    %s = alloca((size_t)datasize);' % (tovar,))
+        self._prnt('    memset((void *)%s, 0, (size_t)datasize);' % (tovar,))
         self._prnt('    if (_cffi_convert_array_from_object('
                    '(char *)%s, _cffi_type(%d), %s) < 0)' % (
             tovar, self._gettypenum(tp), fromvar))
@@ -336,7 +344,7 @@
         prnt = self._prnt
         numargs = len(tp.args)
         if numargs == 0:
-            argname = 'no_arg'
+            argname = 'noarg'
         elif numargs == 1:
             argname = 'arg0'
         else:
@@ -386,6 +394,9 @@
         prnt('  Py_END_ALLOW_THREADS')
         prnt()
         #
+        prnt('  (void)self; /* unused */')
+        if numargs == 0:
+            prnt('  (void)noarg; /* unused */')
         if result_code:
             prnt('  return %s;' %
                  self._convert_expr_from_c(tp.result, 'result', 'result type'))
@@ -452,6 +463,7 @@
         prnt('static void %s(%s *p)' % (checkfuncname, cname))
         prnt('{')
         prnt('  /* only to generate compile-time warnings or errors */')
+        prnt('  (void)p;')
         for fname, ftype, fbitsize in tp.enumfields():
             if (isinstance(ftype, model.PrimitiveType)
                 and ftype.is_integer_type()) or fbitsize >= 0:
@@ -482,6 +494,8 @@
                 prnt('    sizeof(((%s *)0)->%s),' % (cname, fname))
         prnt('    -1')
         prnt('  };')
+        prnt('  (void)self; /* unused */')
+        prnt('  (void)noarg; /* unused */')
         prnt('  return _cffi_get_struct_layout(nums);')
         prnt('  /* the next line is not executed, but compiled */')
         prnt('  %s(0);' % (checkfuncname,))
@@ -578,7 +592,8 @@
     # constants, likely declared with '#define'
 
     def _generate_cpy_const(self, is_int, name, tp=None, category='const',
-                            vartp=None, delayed=True, size_too=False):
+                            vartp=None, delayed=True, size_too=False,
+                            check_value=None):
         prnt = self._prnt
         funcname = '_cffi_%s_%s' % (category, name)
         prnt('static int %s(PyObject *lib)' % funcname)
@@ -590,6 +605,9 @@
         else:
             assert category == 'const'
         #
+        if check_value is not None:
+            self._check_int_constant_value(name, check_value)
+        #
         if not is_int:
             if category == 'var':
                 realexpr = '&' + name
@@ -637,6 +655,27 @@
     # ----------
     # enums
 
+    def _check_int_constant_value(self, name, value, err_prefix=''):
+        prnt = self._prnt
+        if value <= 0:
+            prnt('  if ((%s) > 0 || (long)(%s) != %dL) {' % (
+                name, name, value))
+        else:
+            prnt('  if ((%s) <= 0 || (unsigned long)(%s) != %dUL) {' % (
+                name, name, value))
+        prnt('    char buf[64];')
+        prnt('    if ((%s) <= 0)' % name)
+        prnt('        snprintf(buf, 63, "%%ld", (long)(%s));' % name)
+        prnt('    else')
+        prnt('        snprintf(buf, 63, "%%lu", (unsigned long)(%s));' %
+             name)
+        prnt('    PyErr_Format(_cffi_VerificationError,')
+        prnt('                 "%s%s has the real value %s, not %s",')
+        prnt('                 "%s", "%s", buf, "%d");' % (
+            err_prefix, name, value))
+        prnt('    return -1;')
+        prnt('  }')
+
     def _enum_funcname(self, prefix, name):
         # "$enum_$1" => "___D_enum____D_1"
         name = name.replace('$', '___D_')
@@ -653,25 +692,8 @@
         prnt('static int %s(PyObject *lib)' % funcname)
         prnt('{')
         for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
-            if enumvalue < 0:
-                prnt('  if ((%s) >= 0 || (long)(%s) != %dL) {' % (
-                    enumerator, enumerator, enumvalue))
-            else:
-                prnt('  if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % (
-                    enumerator, enumerator, enumvalue))
-            prnt('    char buf[64];')
-            prnt('    if ((%s) < 0)' % enumerator)
-            prnt('        snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator)
-            prnt('    else')
-            prnt('        snprintf(buf, 63, "%%lu", (unsigned long)(%s));' %
-                 enumerator)
-            prnt('    PyErr_Format(_cffi_VerificationError,')
-            prnt('                 "enum %s: %s has the real value %s, '
-                 'not %s",')
-            prnt('                 "%s", "%s", buf, "%d");' % (
-                name, enumerator, enumvalue))
-            prnt('    return -1;')
-            prnt('  }')
+            self._check_int_constant_value(enumerator, enumvalue,
+                                           "enum %s: " % name)
         prnt('  return %s;' % self._chained_list_constants[True])
         self._chained_list_constants[True] = funcname + '(lib)'
         prnt('}')
@@ -695,8 +717,11 @@
     # macros: for now only for integers
 
     def _generate_cpy_macro_decl(self, tp, name):
-        assert tp == '...'
-        self._generate_cpy_const(True, name)
+        if tp == '...':
+            check_value = None
+        else:
+            check_value = tp     # an integer
+        self._generate_cpy_const(True, name, check_value=check_value)
 
     _generate_cpy_macro_collecttype = _generate_nothing
     _generate_cpy_macro_method = _generate_nothing
@@ -783,6 +808,24 @@
    typedef unsigned __int16 uint16_t;
    typedef unsigned __int32 uint32_t;
    typedef unsigned __int64 uint64_t;
+   typedef __int8 int_least8_t;
+   typedef __int16 int_least16_t;
+   typedef __int32 int_least32_t;
+   typedef __int64 int_least64_t;
+   typedef unsigned __int8 uint_least8_t;
+   typedef unsigned __int16 uint_least16_t;
+   typedef unsigned __int32 uint_least32_t;
+   typedef unsigned __int64 uint_least64_t;
+   typedef __int8 int_fast8_t;
+   typedef __int16 int_fast16_t;
+   typedef __int32 int_fast32_t;
+   typedef __int64 int_fast64_t;
+   typedef unsigned __int8 uint_fast8_t;
+   typedef unsigned __int16 uint_fast16_t;
+   typedef unsigned __int32 uint_fast32_t;
+   typedef unsigned __int64 uint_fast64_t;
+   typedef __int64 intmax_t;
+   typedef unsigned __int64 uintmax_t;
 # else
 #  include 
 # endif
@@ -828,12 +871,15 @@
             PyLong_FromLongLong((long long)(x)))
 
 #define _cffi_from_c_int(x, type)                                        \
-    (((type)-1) > 0 ?   /* unsigned */                                   \
-        (sizeof(type) < sizeof(long) ? PyInt_FromLong(x) :               \
-         sizeof(type) == sizeof(long) ? PyLong_FromUnsignedLong(x) :     \
-                                        PyLong_FromUnsignedLongLong(x))  \
-      : (sizeof(type) <= sizeof(long) ? PyInt_FromLong(x) :              \
-                                        PyLong_FromLongLong(x)))
+    (((type)-1) > 0 ? /* unsigned */                                     \
+        (sizeof(type) < sizeof(long) ?                                   \
+            PyInt_FromLong((long)x) :                                    \
+         sizeof(type) == sizeof(long) ?                                  \
+            PyLong_FromUnsignedLong((unsigned long)x) :                  \
+            PyLong_FromUnsignedLongLong((unsigned long long)x)) :        \
+        (sizeof(type) <= sizeof(long) ?                                  \
+            PyInt_FromLong((long)x) :                                    \
+            PyLong_FromLongLong((long long)x)))
 
 #define _cffi_to_c_int(o, type)                                          \
     (sizeof(type) == 1 ? (((type)-1) > 0 ? (type)_cffi_to_c_u8(o)        \
@@ -844,7 +890,7 @@
                                          : (type)_cffi_to_c_i32(o)) :    \
      sizeof(type) == 8 ? (((type)-1) > 0 ? (type)_cffi_to_c_u64(o)       \
                                          : (type)_cffi_to_c_i64(o)) :    \
-     (Py_FatalError("unsupported size for type " #type), 0))
+     (Py_FatalError("unsupported size for type " #type), (type)0))
 
 #define _cffi_to_c_i8                                                    \
                  ((int(*)(PyObject *))_cffi_exports[1])
@@ -907,6 +953,7 @@
 {
     PyObject *library;
     int was_alive = (_cffi_types != NULL);
+    (void)self; /* unused */
     if (!PyArg_ParseTuple(args, "OOO", &_cffi_types, &_cffi_VerificationError,
                                        &library))
         return NULL;
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -58,12 +58,12 @@
             modname = self.verifier.get_module_name()
             prnt("void %s%s(void) { }\n" % (prefix, modname))
 
-    def load_library(self):
+    def load_library(self, flags=0):
         # import it with the CFFI backend
         backend = self.ffi._backend
         # needs to make a path that contains '/', on Posix
         filename = os.path.join(os.curdir, self.verifier.modulefilename)
-        module = backend.load_library(filename)
+        module = backend.load_library(filename, flags)
         #
         # call loading_gen_struct() to get the struct layout inferred by
         # the C compiler
@@ -235,6 +235,7 @@
         prnt('static void %s(%s *p)' % (checkfuncname, cname))
         prnt('{')
         prnt('  /* only to generate compile-time warnings or errors */')
+        prnt('  (void)p;')
         for fname, ftype, fbitsize in tp.enumfields():
             if (isinstance(ftype, model.PrimitiveType)
                 and ftype.is_integer_type()) or fbitsize >= 0:
@@ -354,11 +355,20 @@
     # ----------
     # constants, likely declared with '#define'
 
-    def _generate_gen_const(self, is_int, name, tp=None, category='const'):
+    def _generate_gen_const(self, is_int, name, tp=None, category='const',
+                            check_value=None):
         prnt = self._prnt
         funcname = '_cffi_%s_%s' % (category, name)
         self.export_symbols.append(funcname)
-        if is_int:
+        if check_value is not None:
+            assert is_int
+            assert category == 'const'
+            prnt('int %s(char *out_error)' % funcname)
+            prnt('{')
+            self._check_int_constant_value(name, check_value)
+            prnt('  return 0;')
+            prnt('}')
+        elif is_int:
             assert category == 'const'
             prnt('int %s(long long *out_value)' % funcname)
             prnt('{')
@@ -367,6 +377,7 @@
             prnt('}')
         else:
             assert tp is not None
+            assert check_value is None
             prnt(tp.get_c_name(' %s(void)' % funcname, name),)
             prnt('{')
             if category == 'var':
@@ -383,9 +394,13 @@
 
     _loading_gen_constant = _loaded_noop
 
-    def _load_constant(self, is_int, tp, name, module):
+    def _load_constant(self, is_int, tp, name, module, check_value=None):
         funcname = '_cffi_const_%s' % name
-        if is_int:
+        if check_value is not None:
+            assert is_int
+            self._load_known_int_constant(module, funcname)
+            value = check_value
+        elif is_int:
             BType = self.ffi._typeof_locked("long long*")[0]
             BFunc = self.ffi._typeof_locked("int(*)(long long*)")[0]
             function = module.load_function(BFunc, funcname)
@@ -396,6 +411,7 @@
                 BLongLong = self.ffi._typeof_locked("long long")[0]
                 value += (1 << (8*self.ffi.sizeof(BLongLong)))
         else:
+            assert check_value is None
             BFunc = self.ffi._typeof_locked(tp.get_c_name('(*)(void)', name))[0]
             function = module.load_function(BFunc, funcname)
             value = function()
@@ -410,6 +426,36 @@
     # ----------
     # enums
 
+    def _check_int_constant_value(self, name, value):
+        prnt = self._prnt
+        if value <= 0:
+            prnt('  if ((%s) > 0 || (long)(%s) != %dL) {' % (
+                name, name, value))
+        else:
+            prnt('  if ((%s) <= 0 || (unsigned long)(%s) != %dUL) {' % (
+                name, name, value))
+        prnt('    char buf[64];')
+        prnt('    if ((%s) <= 0)' % name)
+        prnt('        sprintf(buf, "%%ld", (long)(%s));' % name)
+        prnt('    else')
+        prnt('        sprintf(buf, "%%lu", (unsigned long)(%s));' %
+             name)
+        prnt('    sprintf(out_error, "%s has the real value %s, not %s",')
+        prnt('            "%s", buf, "%d");' % (name[:100], value))
+        prnt('    return -1;')
+        prnt('  }')
+
+    def _load_known_int_constant(self, module, funcname):
+        BType = self.ffi._typeof_locked("char[]")[0]
+        BFunc = self.ffi._typeof_locked("int(*)(char*)")[0]
+        function = module.load_function(BFunc, funcname)
+        p = self.ffi.new(BType, 256)
+        if function(p) < 0:
+            error = self.ffi.string(p)
+            if sys.version_info >= (3,):
+                error = str(error, 'utf-8')
+            raise ffiplatform.VerificationError(error)
+
     def _enum_funcname(self, prefix, name):
         # "$enum_$1" => "___D_enum____D_1"
         name = name.replace('$', '___D_')
@@ -427,24 +473,7 @@
         prnt('int %s(char *out_error)' % funcname)
         prnt('{')
         for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
-            if enumvalue < 0:
-                prnt('  if ((%s) >= 0 || (long)(%s) != %dL) {' % (
-                    enumerator, enumerator, enumvalue))
-            else:
-                prnt('  if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % (
-                    enumerator, enumerator, enumvalue))
-            prnt('    char buf[64];')
-            prnt('    if ((%s) < 0)' % enumerator)
-            prnt('        sprintf(buf, "%%ld", (long)(%s));' % enumerator)
-            prnt('    else')
-            prnt('        sprintf(buf, "%%lu", (unsigned long)(%s));' %
-                 enumerator)
-            prnt('    sprintf(out_error,'
-                             ' "%s has the real value %s, not %s",')
-            prnt('            "%s", buf, "%d");' % (
-                enumerator[:100], enumvalue))
-            prnt('    return -1;')
-            prnt('  }')
+            self._check_int_constant_value(enumerator, enumvalue)
         prnt('  return 0;')
         prnt('}')
         prnt()
@@ -456,16 +485,8 @@
             tp.enumvalues = tuple(enumvalues)
             tp.partial_resolved = True
         else:
-            BType = self.ffi._typeof_locked("char[]")[0]
-            BFunc = self.ffi._typeof_locked("int(*)(char*)")[0]
             funcname = self._enum_funcname(prefix, name)
-            function = module.load_function(BFunc, funcname)
-            p = self.ffi.new(BType, 256)
-            if function(p) < 0:
-                error = self.ffi.string(p)
-                if sys.version_info >= (3,):
-                    error = str(error, 'utf-8')
-                raise ffiplatform.VerificationError(error)
+            self._load_known_int_constant(module, funcname)
 
     def _loaded_gen_enum(self, tp, name, module, library):
         for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
@@ -476,13 +497,21 @@
     # macros: for now only for integers
 
     def _generate_gen_macro_decl(self, tp, name):
-        assert tp == '...'
-        self._generate_gen_const(True, name)
+        if tp == '...':
+            check_value = None
+        else:
+            check_value = tp     # an integer
+        self._generate_gen_const(True, name, check_value=check_value)
 
     _loading_gen_macro = _loaded_noop
 
     def _loaded_gen_macro(self, tp, name, module, library):
-        value = self._load_constant(True, tp, name, module)
+        if tp == '...':
+            check_value = None
+        else:
+            check_value = tp     # an integer
+        value = self._load_constant(True, tp, name, module,
+                                    check_value=check_value)
         setattr(library, name, value)
         type(library)._cffi_dir.append(name)
 
@@ -565,6 +594,24 @@
    typedef unsigned __int16 uint16_t;
    typedef unsigned __int32 uint32_t;
    typedef unsigned __int64 uint64_t;
+   typedef __int8 int_least8_t;
+   typedef __int16 int_least16_t;
+   typedef __int32 int_least32_t;
+   typedef __int64 int_least64_t;
+   typedef unsigned __int8 uint_least8_t;
+   typedef unsigned __int16 uint_least16_t;
+   typedef unsigned __int32 uint_least32_t;
+   typedef unsigned __int64 uint_least64_t;
+   typedef __int8 int_fast8_t;
+   typedef __int16 int_fast16_t;
+   typedef __int32 int_fast32_t;
+   typedef __int64 int_fast64_t;
+   typedef unsigned __int8 uint_fast8_t;
+   typedef unsigned __int16 uint_fast16_t;
+   typedef unsigned __int32 uint_fast32_t;
+   typedef unsigned __int64 uint_fast64_t;
+   typedef __int64 intmax_t;
+   typedef unsigned __int64 uintmax_t;
 # else
 #  include 
 # endif
diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py
--- a/lib_pypy/cffi/verifier.py
+++ b/lib_pypy/cffi/verifier.py
@@ -1,12 +1,23 @@
-import sys, os, binascii, imp, shutil
-from . import __version__
+import sys, os, binascii, shutil
+from . import __version_verifier_modules__
 from . import ffiplatform
 
+if sys.version_info >= (3, 3):
+    import importlib.machinery
+    def _extension_suffixes():
+        return importlib.machinery.EXTENSION_SUFFIXES[:]
+else:
+    import imp
+    def _extension_suffixes():
+        return [suffix for suffix, _, type in imp.get_suffixes()
+                if type == imp.C_EXTENSION]
+
 
 class Verifier(object):
 
     def __init__(self, ffi, preamble, tmpdir=None, modulename=None,
-                 ext_package=None, tag='', force_generic_engine=False, **kwds):
+                 ext_package=None, tag='', force_generic_engine=False,
+                 source_extension='.c', flags=None, relative_to=None, **kwds):
         self.ffi = ffi
         self.preamble = preamble
         if not modulename:
@@ -14,14 +25,15 @@
         vengine_class = _locate_engine_class(ffi, force_generic_engine)
         self._vengine = vengine_class(self)
         self._vengine.patch_extension_kwds(kwds)
-        self.kwds = kwds
+        self.flags = flags
+        self.kwds = self.make_relative_to(kwds, relative_to)
         #
         if modulename:
             if tag:
                 raise TypeError("can't specify both 'modulename' and 'tag'")
         else:
-            key = '\x00'.join([sys.version[:3], __version__, preamble,
-                               flattened_kwds] +
+            key = '\x00'.join([sys.version[:3], __version_verifier_modules__,
+                               preamble, flattened_kwds] +
                               ffi._cdefsources)
             if sys.version_info >= (3,):
                 key = key.encode('utf-8')
@@ -33,7 +45,7 @@
                                               k1, k2)
         suffix = _get_so_suffixes()[0]
         self.tmpdir = tmpdir or _caller_dir_pycache()
-        self.sourcefilename = os.path.join(self.tmpdir, modulename + '.c')
+        self.sourcefilename = os.path.join(self.tmpdir, modulename + source_extension)
         self.modulefilename = os.path.join(self.tmpdir, modulename + suffix)
         self.ext_package = ext_package
         self._has_source = False
@@ -97,6 +109,20 @@
     def generates_python_module(self):
         return self._vengine._gen_python_module
 
+    def make_relative_to(self, kwds, relative_to):
+        if relative_to and os.path.dirname(relative_to):
+            dirname = os.path.dirname(relative_to)
+            kwds = kwds.copy()
+            for key in ffiplatform.LIST_OF_FILE_NAMES:
+                if key in kwds:
+                    lst = kwds[key]
+                    if not isinstance(lst, (list, tuple)):
+                        raise TypeError("keyword '%s' should be a list or tuple"
+                                        % (key,))
+                    lst = [os.path.join(dirname, fn) for fn in lst]
+                    kwds[key] = lst
+        return kwds
+
     # ----------
 
     def _locate_module(self):
@@ -148,7 +174,10 @@
 
     def _load_library(self):
         assert self._has_module
-        return self._vengine.load_library()
+        if self.flags is not None:
+            return self._vengine.load_library(self.flags)
+        else:
+            return self._vengine.load_library()
 
 # ____________________________________________________________
 
@@ -181,6 +210,9 @@
 def _caller_dir_pycache():
     if _TMPDIR:
         return _TMPDIR
+    result = os.environ.get('CFFI_TMPDIR')
+    if result:
+        return result
     filename = sys._getframe(2).f_code.co_filename
     return os.path.abspath(os.path.join(os.path.dirname(filename),
                            '__pycache__'))
@@ -222,11 +254,7 @@
             pass
 
 def _get_so_suffixes():
-    suffixes = []
-    for suffix, mode, type in imp.get_suffixes():
-        if type == imp.C_EXTENSION:
-            suffixes.append(suffix)
-
+    suffixes = _extension_suffixes()
     if not suffixes:
         # bah, no C_EXTENSION available.  Occurs on pypy without cpyext
         if sys.platform == 'win32':
diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
--- a/lib_pypy/greenlet.egg-info
+++ b/lib_pypy/greenlet.egg-info
@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: greenlet
-Version: 0.4.0
+Version: 0.4.5
 Summary: Lightweight in-process concurrent programming
 Home-page: https://github.com/python-greenlet/greenlet
 Author: Ralf Schmitt (for CPython), PyPy team
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -2,7 +2,7 @@
 import __pypy__
 import _continuation
 
-__version__ = "0.4.0"
+__version__ = "0.4.5"
 
 # ____________________________________________________________
 # Exceptions
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -47,6 +47,11 @@
 
 Install build-time dependencies
 -------------------------------
+(**Note**: for some hints on how to translate the Python interpreter under
+Windows, see the `windows document`_)
+
+.. _`windows document`: windows.html
+
 
 To build PyPy on Unix using the C translation backend, you need at least a C
 compiler and ``make`` installed. Further, some optional modules have additional
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -65,9 +65,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '2.4'
+version = '2.5'
 # The full version, including alpha/beta/rc tags.
-release = '2.4.0'
+release = '2.5.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -12,19 +12,19 @@
   Amaury Forgeot d'Arc
   Samuele Pedroni
   Alex Gaynor
+  Brian Kearns
+  Matti Picus
+  Philip Jenvey
   Michael Hudson
   David Schneider
-  Matti Picus
-  Brian Kearns
-  Philip Jenvey
   Holger Krekel
   Christian Tismer
   Hakan Ardo
   Benjamin Peterson
   Manuel Jacob
+  Ronan Lamy
   Anders Chrigstrom
   Eric van Riet Paap
-  Ronan Lamy
   Wim Lavrijsen
   Richard Emslie
   Alexander Schremmer
@@ -38,9 +38,9 @@
   Camillo Bruni
   Laura Creighton
   Toon Verwaest
+  Romain Guillebert
   Leonardo Santagada
   Seo Sanghyeon
-  Romain Guillebert
   Justin Peel
   Ronny Pfannschmidt
   David Edelsohn
@@ -61,15 +61,16 @@
   Michal Bendowski
   Jan de Mooij
   stian
+  Tyler Wade
   Michael Foord
   Stephan Diehl
-  Tyler Wade
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
   Patrick Maupin
   Bob Ippolito
   Bruno Gola
+  David Malcolm
   Jean-Paul Calderone
   Timo Paulssen
   Squeaky
@@ -78,18 +79,19 @@
   Marius Gedminas
   Martin Matusiak
   Konstantin Lopuhin
+  Wenzhu Man
   John Witulski
-  Wenzhu Man
+  Laurence Tratt
+  Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
   Mark Pearse
   Simon Cross
-  Ivan Sichmann Freitas
   Andreas Stührk
+  Stefano Rivera
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Stefano Rivera
   Paweł Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -99,7 +101,6 @@
   tav
   Taavi Burns
   Georg Brandl
-  Laurence Tratt
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -111,13 +112,12 @@
   Jeremy Thurgood
   Rami Chowdhury
   Tobias Pape
-  David Malcolm
   Eugene Oden
   Henry Mason
   Vasily Kuznetsov
   Preston Timmons
+  David Ripton
   Jeff Terrace
-  David Ripton
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
@@ -136,13 +136,16 @@
   Gintautas Miliauskas
   Michael Twomey
   Lucian Branescu Mihaila
+  Yichao Yu
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
   Karl Bartel
+  Wouter van Heyst
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  anatoly techtonik
   Stuart Williams
   Jasper Schulz
   Christian Hudon
@@ -152,12 +155,11 @@
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
-  anatoly techtonik
   Neil Shepperd
+  Stanislaw Halik
   Mikael Schönenberg
   Elmo M?ntynen
   Jonathan David Riehl
-  Stanislaw Halik
   Anders Qvist
   Corbin Simpson
   Chirag Jadwani
@@ -166,10 +168,13 @@
   Vincent Legoll
   Alan McIntyre
   Alexander Sedov
+  Attila Gobi
   Christopher Pope
   Christian Tismer 
   Marc Abramowitz
   Dan Stromberg
+  Arjun Naik
+  Valentina Mukhamedzhanova
   Stefano Parmesan
   Alexis Daboville
   Jens-Uwe Mager
@@ -183,8 +188,6 @@
   Sylvain Thenault
   Nathan Taylor
   Vladimir Kryachko
-  Arjun Naik
-  Attila Gobi
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -192,22 +195,23 @@
   Ryan Gonzalez
   Ian Foote
   Kristjan Valur Jonsson
+  David Lievens
   Neil Blakey-Milner
   Lutz Paelike
   Lucio Torre
   Lars Wassermann
-  Valentina Mukhamedzhanova
   Henrik Vendelbo
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
   Sergey Kishchenko
-  Yichao Yu
   Ignas Mikalajunas
   Christoph Gerum
   Martin Blais
   Lene Wagner
   Tomo Cocoa
+  Toni Mattis
+  Lucas Stadler
   roberto at goyle
   Yury V. Zaytsev
   Anna Katrina Dominguez
@@ -235,23 +239,30 @@
   Stephan Busemann
   Rafał Gałczyński
   Christian Muirhead
+  Berker Peksag
   James Lan
   shoma hosaka
-  Daniel Neuh?user
-  Matthew Miller
+  Daniel Neuhäuser
+  Ben Mather
+  halgari
+  Boglarka Vezer
+  Chris Pressey
   Buck Golemon
   Konrad Delong
   Dinu Gherman
   Chris Lambacher
   coolbutuseless at gmail.com
+  Jim Baker
   Rodrigo Araújo
-  Jim Baker
+  Nikolaos-Digenis Karagiannis
   James Robert
   Armin Ronacher
   Brett Cannon
+  Donald Stufft
   yrttyr
   aliceinwire
   OlivierBlanvillain
+  Dan Sanders
   Zooko Wilcox-O Hearn
   Tomer Chachamu
   Christopher Groskopf
@@ -265,6 +276,7 @@
   Markus Unterwaditzer
   Even Wiik Thomassen
   jbs
+  squeaky
   soareschen
   Kurt Griffiths
   Mike Bayer
@@ -276,5 +288,6 @@
   Anna Ravencroft
   Dan Crosta
   Julien Phalip
+  Roman Podoliaka
   Dan Loewenherz
 
diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
--- a/pypy/doc/cppyy.rst
+++ b/pypy/doc/cppyy.rst
@@ -74,7 +74,9 @@
 See the :doc:`backend documentation `.
 
 A standalone version of Reflex that also provides the dynamically loadable
-backend is available for `download`_.
+backend is available for `download`_. Note this is currently the only way to
+get the dynamically loadable backend, so use this first.
+
 That version, as well as any other distribution of Reflex (e.g. the one that
 comes with `ROOT`_, which may be part of your Linux distribution as part of
 the selection of scientific software) will also work for a build with the
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -30,12 +30,10 @@
 
    Initialize threads. Only need to be called if there are any threads involved
 
-.. function:: long pypy_setup_home(char* home, int verbose);
+.. function:: int pypy_setup_home(char* home, int verbose);
 
    This function searches the PyPy standard library starting from the given
-   "PyPy home directory".  It is not strictly necessary to execute it before
-   running Python code, but without it you will not be able to import any
-   non-builtin module from the standard library.  The arguments are:
+   "PyPy home directory".  The arguments are:
 
    * ``home``: NULL terminated path to an executable inside the pypy directory
      (can be a .so name, can be made up)
@@ -84,17 +82,22 @@
 
     const char source[] = "print 'hello from pypy'";
 
-    int main()
+    int main(void)
     {
-      int res;
+        int res;
 
-      rpython_startup_code();
-      // pypy_setup_home() is not needed in this trivial example
-      res = pypy_execute_source((char*)source);
-      if (res) {
-        printf("Error calling pypy_execute_source!\n");
-      }
-      return res;
+        rpython_startup_code();
+        res = pypy_setup_home("/opt/pypy/bin/libpypy-c.so", 1);
+        if (res) {
+            printf("Error setting pypy home!\n");
+            return 1;
+        }
+
+        res = pypy_execute_source((char*)source);
+        if (res) {
+            printf("Error calling pypy_execute_source!\n");
+        }
+        return res;
     }
 
 If we save it as ``x.c`` now, compile it and run it (on linux) with::
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -3,6 +3,13 @@
 
 .. contents::
 
+See also: `Frequently ask questions about RPython.`__
+
+.. __: http://rpython.readthedocs.org/en/latest/faq.html
+
+---------------------------
+
+
 What is PyPy?
 -------------
 
@@ -190,6 +197,55 @@
 (now-dead) object are still true about the new object.
 
 
+
+Would type annotations help PyPy's performance?
+-----------------------------------------------
+
+Two examples of type annotations that are being proposed for improved
+performance are `Cython types`__ and `PEP 484 - Type Hints`__.
+
+.. __: http://docs.cython.org/src/reference/language_basics.html#declaring-data-types
+.. __: https://www.python.org/dev/peps/pep-0484/
+
+**Cython types** are, by construction, similar to C declarations.  For
+example, a local variable or an instance attribute can be declared
+``"cdef int"`` to force a machine word to be used.  This changes the
+usual Python semantics (e.g. no overflow checks, and errors when
+trying to write other types of objects there).  It gives some extra
+performance, but the exact benefits are unclear: right now
+(January 2015) for example we are investigating a technique that would
+store machine-word integers directly on instances, giving part of the
+benefits without the user-supplied ``"cdef int"``.
+
+**PEP 484 - Type Hints,** on the other hand, is almost entirely
+useless if you're looking at performance.  First, as the name implies,
+they are *hints:* they must still be checked at runtime, like PEP 484
+says.  Or maybe you're fine with a mode in which you get very obscure
+crashes when the type annotations are wrong; but even in that case the
+speed benefits would be extremely minor.
+
+There are several reasons for why.  One of them is that annotations
+are at the wrong level (e.g. a PEP 484 "int" corresponds to Python 3's
+int type, which does not necessarily fits inside one machine word;
+even worse, an "int" annotation allows arbitrary int subclasses).
+Another is that a lot more information is needed to produce good code

From noreply at buildbot.pypy.org  Sun Feb  8 20:08:01 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  8 Feb 2015 20:08:01 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: avoid a race condition here
Message-ID: <20150208190801.25CD41C1278@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r75763:55403b25bb86
Date: 2015-02-06 11:29 +0200
http://bitbucket.org/pypy/pypy/changeset/55403b25bb86/

Log:	avoid a race condition here

diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
--- a/pypy/module/_vmprof/src/vmprof.c
+++ b/pypy/module/_vmprof/src/vmprof.c
@@ -289,9 +289,9 @@
 
 void vmprof_set_mainloop(void* func, ptrdiff_t sp_offset, 
                          vmprof_get_virtual_ip_t get_virtual_ip) {
-    vmprof_mainloop_func = func;
     mainloop_sp_offset = sp_offset;
     mainloop_get_virtual_ip = get_virtual_ip;
+    vmprof_mainloop_func = func;
 }
 
 int vmprof_enable(int fd, long period_usec, int write_header, char *s,

From noreply at buildbot.pypy.org  Sun Feb  8 20:08:02 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  8 Feb 2015 20:08:02 +0100 (CET)
Subject: [pypy-commit] pypy default: Try to make encode not call_may_force
Message-ID: <20150208190802.716031C1278@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r75764:1d3d9fecd11d
Date: 2015-02-08 20:58 +0200
http://bitbucket.org/pypy/pypy/changeset/1d3d9fecd11d/

Log:	Try to make encode not call_may_force

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -21,12 +21,7 @@
     # Fast version of the "strict" errors handler.
     def raise_unicode_exception_encode(errors, encoding, msg, u,
                                        startingpos, endingpos):
-        raise OperationError(space.w_UnicodeEncodeError,
-                             space.newtuple([space.wrap(encoding),
-                                             space.wrap(u),
-                                             space.wrap(startingpos),
-                                             space.wrap(endingpos),
-                                             space.wrap(msg)]))
+        raise UnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
     return raise_unicode_exception_encode
 
 # ____________________________________________________________
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -436,17 +436,26 @@
         w_encoder = space.sys.get_w_default_encoder()
     else:
         if errors is None or errors == 'strict':
-            if encoding == 'ascii':
-                u = space.unicode_w(w_object)
-                eh = unicodehelper.encode_error_handler(space)
-                return space.wrap(unicode_encode_ascii(
-                        u, len(u), None, errorhandler=eh))
-            if encoding == 'utf-8':
-                u = space.unicode_w(w_object)
-                eh = unicodehelper.encode_error_handler(space)
-                return space.wrap(unicode_encode_utf_8(
-                        u, len(u), None, errorhandler=eh,
-                        allow_surrogates=True))
+            try:
+                if encoding == 'ascii':
+                    u = space.unicode_w(w_object)
+                    eh = unicodehelper.encode_error_handler(space)
+                    return space.wrap(unicode_encode_ascii(
+                            u, len(u), None, errorhandler=eh))
+                if encoding == 'utf-8':
+                    u = space.unicode_w(w_object)
+                    eh = unicodehelper.encode_error_handler(space)
+                    return space.wrap(unicode_encode_utf_8(
+                            u, len(u), None, errorhandler=eh,
+                            allow_surrogates=True))
+            except UnicodeEncodeError, ue:
+                raise OperationError(space.w_UnicodeEncodeError,
+                                     space.newtuple([
+                    space.wrap(ue.encoding),
+                    space.wrap(ue.object),
+                    space.wrap(ue.start),
+                    space.wrap(ue.end),
+                    space.wrap(ue.reason)]))
         from pypy.module._codecs.interp_codecs import lookup_codec
         w_encoder = space.getitem(lookup_codec(space, encoding), space.wrap(0))
     if errors is None:

From noreply at buildbot.pypy.org  Sun Feb  8 20:08:03 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  8 Feb 2015 20:08:03 +0100 (CET)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20150208190803.C498F1C1278@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r75765:221df618cf77
Date: 2015-02-08 20:59 +0200
http://bitbucket.org/pypy/pypy/changeset/221df618cf77/

Log:	merge

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -235,6 +235,11 @@
 
         config.translation.suggest(check_str_without_nul=True)
         config.translation.suggest(shared=True)
+        if config.translation.shared:
+            if config.translation.output is not None:
+                raise Exception("Cannot use the --output option with PyPy "
+                                "when --shared is on (it is by default). "
+                                "See issue #1971.")
 
         if config.translation.thread:
             config.objspace.usemodules.thread = True
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -81,7 +81,10 @@
                 return w_object.descr_copy(space, w_order)
             elif not copy and (subok or type(w_object) is W_NDimArray):
                 return w_object
-        # we have a ndarray, but need to copy or change dtype or create W_NDimArray
+        if subok:
+            raise oefmt(space.w_NotImplementedError, 
+                "array(..., subok=True) only partially implemented")
+        # we have a ndarray, but need to copy or change dtype 
         if dtype is None:
             dtype = w_object.get_dtype()
         if dtype != w_object.get_dtype():
@@ -89,13 +92,12 @@
             copy = True
         if copy:
             shape = w_object.get_shape()
-            _elems_w = w_object.reshape(space, space.wrap(-1))
             elems_w = [None] * w_object.get_size()
-            for i in range(len(elems_w)):
-                elems_w[i] = _elems_w.descr_getitem(space, space.wrap(i))
-        elif subok:
-            raise oefmt(space.w_NotImplementedError, 
-                "array(...copy=False, subok=True) not implemented yet")
+            elsize = w_object.get_dtype().elsize
+            # TODO - use w_object.implementation without copying to a list
+            # unfortunately that causes a union error in translation
+            for i in range(w_object.get_size()):
+                elems_w[i] = w_object.implementation.getitem(i * elsize)
         else:
             sz = support.product(w_object.get_shape()) * dtype.elsize
             return W_NDimArray.from_shape_and_storage(space,
@@ -113,7 +115,7 @@
             dtype = descriptor.variable_dtype(space, dtype.char + '1')
 
     w_arr = W_NDimArray.from_shape(space, shape, dtype, order=order)
-    if len(elems_w) == 1:
+    if support.product(shape) == 1:
         w_arr.set_scalar_value(dtype.coerce(space, elems_w[0]))
     else:
         loop.assign(space, w_arr, elems_w)
diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py
--- a/pypy/module/micronumpy/test/test_subtype.py
+++ b/pypy/module/micronumpy/test/test_subtype.py
@@ -272,40 +272,103 @@
         import numpy as N
         # numpy's matrix class caused an infinite loop
         class matrix(N.ndarray):
-            getcnt = 0
             def __new__(subtype, data, dtype=None, copy=True):
+                print('matrix __new__')
+                if isinstance(data, matrix):
+                    dtype2 = data.dtype
+                    if (dtype is None):
+                        dtype = dtype2
+                    if (dtype2 == dtype) and (not copy):
+                        return data
+                    return data.astype(dtype)
+
+                if isinstance(data, N.ndarray):
+                    if dtype is None:
+                        intype = data.dtype
+                    else:
+                        intype = N.dtype(dtype)
+                    new = data.view(subtype)
+                    if intype != data.dtype:
+                        return new.astype(intype)
+                    if copy: return new.copy()
+                    else: return new
+
+                if isinstance(data, str):
+                    data = _convert_from_string(data)
+
+                # now convert data to an array
                 arr = N.array(data, dtype=dtype, copy=copy)
+                ndim = arr.ndim
                 shape = arr.shape
+                if (ndim > 2):
+                    raise ValueError("matrix must be 2-dimensional")
+                elif ndim == 0:
+                    shape = (1, 1)
+                elif ndim == 1:
+                    shape = (1, shape[0])
+
+                order = False
+                if (ndim == 2) and arr.flags.fortran:
+                    order = True
+
+                if not (order or arr.flags.contiguous):
+                    arr = arr.copy()
 
                 ret = N.ndarray.__new__(subtype, shape, arr.dtype,
                                         buffer=arr,
-                                        order=True)
+                                        order=order)
                 return ret
 
+            def __array_finalize__(self, obj):
+                print('matrix __array_finalize__')
+                self._getitem = False
+                if (isinstance(obj, matrix) and obj._getitem): return
+                ndim = self.ndim
+                if (ndim == 2):
+                    return
+                if (ndim > 2):
+                    newshape = tuple([x for x in self.shape if x > 1])
+                    ndim = len(newshape)
+                    if ndim == 2:
+                        self.shape = newshape
+                        return
+                    elif (ndim > 2):
+                        raise ValueError("shape too large to be a matrix.")
+                else:
+                    newshape = self.shape
+                if ndim == 0:
+                    self.shape = (1, 1)
+                elif ndim == 1:
+                    self.shape = (1, newshape[0])
+                return
+
             def __getitem__(self, index):
-                matrix.getcnt += 1
-                if matrix.getcnt > 10:
-                    # XXX strides.find_shape_and_elems is sensitive
-                    # to shape modification
-                    xxx
-                out = N.ndarray.__getitem__(self, index)
+                print('matrix __getitem__')
+                self._getitem = True
+
+                try:
+                    out = N.ndarray.__getitem__(self, index)
+                finally:
+                    self._getitem = False
 
                 if not isinstance(out, N.ndarray):
                     return out
+
+                if out.ndim == 0:
+                    return out[()]
+                if out.ndim == 1:
+                    sh = out.shape[0]
                     # Determine when we should have a column array
-                old_shape = out.shape
-                if out.ndim < 2:
-                    sh = out.shape[0]
                     try:
                         n = len(index)
                     except:
                         n = 0
-                    if n > 1:
+                    if n > 1 and isscalar(index[1]):
                         out.shape = (sh, 1)
                     else:
                         out.shape = (1, sh)
-                #print 'out, shape was',old_shape,'now',out.shape,'out',out
                 return out
+
         a = matrix([[1., 2.], [3., 4.]])
         b = N.array([a])
         assert (b == a).all()
@@ -318,6 +381,17 @@
         assert len(b.shape) == 2
         assert (b == a).all()
 
+        b = N.array(a, copy=True, dtype=int)
+        assert len(b.shape) == 2
+        assert (b == a).all()
+
+        c = matrix(a, copy=False)
+        assert c.base is not None
+        c[0, 0] = 100
+        assert a[0, 0] == 100
+        b = N.array(c, copy=True)
+        assert (b == a).all()
+
     def test_setstate_no_version(self):
         # Some subclasses of ndarray, like MaskedArray, do not use
         # version in __setstare__
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -151,9 +151,7 @@
                default=False, cmdline="--dont-write-c-files"),
     ArbitraryOption("instrumentctl", "internal",
                default=None),
-    StrOption("output", "Output file name (don't change for PyPy!"
-              " doesn't work with virtualenv+shared: issue 1971)",
-              cmdline="--really-force-output"),
+    StrOption("output", "Output file name", cmdline="--output"),
     StrOption("secondaryentrypoints",
             "Comma separated list of keys choosing secondary entrypoints",
             cmdline="--entrypoints", default="main"),
diff --git a/rpython/flowspace/generator.py b/rpython/flowspace/generator.py
--- a/rpython/flowspace/generator.py
+++ b/rpython/flowspace/generator.py
@@ -107,7 +107,7 @@
     # First, always run simplify_graph in order to reduce the number of
     # variables passed around
     simplify_graph(graph)
-    insert_empty_startblock(None, graph)
+    insert_empty_startblock(graph)
     _insert_reads(graph.startblock, Entry.varnames)
     Entry.block = graph.startblock
     #
@@ -130,7 +130,7 @@
             if hlop.opname == 'yield_':
                 [v_yielded_value] = hlop.args
                 del block.operations[index]
-                newlink = split_block(None, block, index)
+                newlink = split_block(block, index)
                 newblock = newlink.target
                 #
                 class Resume(AbstractPosition):
diff --git a/rpython/jit/backend/x86/arch.py b/rpython/jit/backend/x86/arch.py
--- a/rpython/jit/backend/x86/arch.py
+++ b/rpython/jit/backend/x86/arch.py
@@ -35,7 +35,9 @@
     PASS_ON_MY_FRAME = 15
     JITFRAME_FIXED_SIZE = 6 + 8 * 2 # 6 GPR + 8 XMM * 2 WORDS/float
     # 'threadlocal_addr' is passed as 2nd argument on the stack,
-    # and it can be left here for when it is needed
+    # and it can be left here for when it is needed.  As an additional hack,
+    # with asmgcc, it is made odd-valued to mean "already seen this frame
+    # during the previous minor collection".
     THREADLOCAL_OFS = (FRAME_FIXED_SIZE + 2) * WORD
 else:
     # rbp + rbx + r12 + r13 + r14 + r15 + threadlocal + 12 extra words = 19
@@ -43,7 +45,9 @@
     PASS_ON_MY_FRAME = 12
     JITFRAME_FIXED_SIZE = 28 # 13 GPR + 15 XMM
     # 'threadlocal_addr' is passed as 2nd argument in %esi,
-    # and is moved into this frame location
+    # and is moved into this frame location.  As an additional hack,
+    # with asmgcc, it is made odd-valued to mean "already seen this frame
+    # during the previous minor collection".
     THREADLOCAL_OFS = (FRAME_FIXED_SIZE - 1) * WORD
 
 assert PASS_ON_MY_FRAME >= 12       # asmgcc needs at least JIT_USE_WORDS + 3
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1980,6 +1980,23 @@
 
     def _call_assembler_emit_call(self, addr, argloc, _):
         threadlocal_loc = RawEspLoc(THREADLOCAL_OFS, INT)
+        if self._is_asmgcc():
+            # We need to remove the bit "already seen during the
+            # previous minor collection" instead of passing this
+            # value directly.
+            if IS_X86_64:
+                tmploc = esi    # already the correct place
+                if argloc is tmploc:
+                    self.mc.MOV_rr(esi.value, edi.value)
+                    argloc = edi
+            else:
+                tmploc = eax
+                if tmploc is argloc:
+                    tmploc = edx
+            self.mc.MOV(tmploc, threadlocal_loc)
+            self.mc.AND_ri(tmploc.value, ~1)
+            threadlocal_loc = tmploc
+        #
         self.simple_call(addr, [argloc, threadlocal_loc])
 
     def _call_assembler_emit_helper_call(self, addr, arglocs, result_loc):
@@ -2355,6 +2372,8 @@
         assert self.cpu.translate_support_code
         assert isinstance(resloc, RegLoc)
         self.mc.MOV_rs(resloc.value, THREADLOCAL_OFS)
+        if self._is_asmgcc():
+            self.mc.AND_ri(resloc.value, ~1)
         self.load_from_mem(resloc, addr_add_const(resloc, offset),
                            imm(size), imm(sign))
 
diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py
--- a/rpython/jit/backend/x86/callbuilder.py
+++ b/rpython/jit/backend/x86/callbuilder.py
@@ -167,6 +167,8 @@
                 self.tlofs_reg = r12
             self.mc.MOV_rs(self.tlofs_reg.value,
                            THREADLOCAL_OFS - self.current_esp)
+            if self.asm._is_asmgcc():
+                self.mc.AND_ri(self.tlofs_reg.value, ~1)
         return self.tlofs_reg
 
     def save_stack_position(self):
diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -304,13 +304,20 @@
 REX_B = 1
 
 @specialize.arg(2)
-def encode_rex(mc, rexbyte, basevalue, orbyte):
+def encode_rex(mc, rexbyte, w, orbyte):
     if mc.WORD == 8:
         assert 0 <= rexbyte < 8
-        if basevalue != 0 or rexbyte != 0:
-            if basevalue == 0:
-                basevalue = 0x40
-            mc.writechar(chr(basevalue | rexbyte))
+        mc.writechar(chr(0x40 | w | rexbyte))
+    else:
+        assert rexbyte == 0
+    return 0
+
+ at specialize.arg(2)
+def encode_rex_opt(mc, rexbyte, _, orbyte):
+    if mc.WORD == 8:
+        assert 0 <= rexbyte < 8
+        if rexbyte != 0:
+            mc.writechar(chr(0x40 | rexbyte))
     else:
         assert rexbyte == 0
     return 0
@@ -322,9 +329,9 @@
 # the REX prefix in all cases.  It is only useful on instructions which
 # have an 8-bit register argument, to force access to the "sil" or "dil"
 # registers (as opposed to "ah-dh").
-rex_w  = encode_rex, 0, (0x40 | REX_W), None      # a REX.W prefix
-rex_nw = encode_rex, 0, 0, None                   # an optional REX prefix
-rex_fw = encode_rex, 0, 0x40, None                # a forced REX prefix
+rex_w  = encode_rex, 0, REX_W, None       # a REX.W prefix
+rex_nw = encode_rex_opt, 0, 0, None       # an optional REX prefix
+rex_fw = encode_rex, 0, 0, None           # a forced REX prefix
 
 # ____________________________________________________________
 
diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -107,7 +107,7 @@
     """
     # split the block just before the jit_merge_point()
     if portalopindex > 0:
-        link = split_block(None, portalblock, portalopindex)
+        link = split_block(portalblock, portalopindex)
         portalblock = link.target
     portalop = portalblock.operations[0]
     # split again, this time enforcing the order of the live vars
@@ -115,7 +115,7 @@
     assert portalop.opname == 'jit_marker'
     assert portalop.args[0].value == 'jit_merge_point'
     greens_v, reds_v = decode_hp_hint_args(portalop)
-    link = split_block(None, portalblock, 0, greens_v + reds_v)
+    link = split_block(portalblock, 0, greens_v + reds_v)
     return link.target
 
 def sort_vars(args_v):
diff --git a/rpython/jit/metainterp/test/support.py b/rpython/jit/metainterp/test/support.py
--- a/rpython/jit/metainterp/test/support.py
+++ b/rpython/jit/metainterp/test/support.py
@@ -162,41 +162,67 @@
 class JitMixin:
     basic = True
 
+    # Basic terminology: the JIT produces "loops" and "bridges".
+    # Bridges are always attached to failing guards.  Every loop is
+    # the "trunk" of a tree of compiled code, which is formed by first
+    # compiling a loop and then incrementally adding some number of
+    # bridges to it.  Each loop and each bridge ends with either a
+    # FINISH or a JUMP instruction (the name "loop" is not really
+    # adapted any more).  The JUMP instruction jumps to any LABEL
+    # pseudo-instruction, which can be anywhere, within the same tree
+    # or another one.
+
     def check_resops(self, expected=None, **check):
+        """Check the instructions in all loops and bridges, ignoring
+        the ones that end in FINISH.  Either pass a dictionary (then
+        the check must match exactly), or some keyword arguments (then
+        the check is only about the instructions named)."""
         get_stats().check_resops(expected=expected, **check)
 
     def check_simple_loop(self, expected=None, **check):
+        """Useful in the simplest case when we have only one loop
+        ending with a jump back to itself and possibly a few bridges.
+        Only the operations within the loop formed by that single jump
+        will be counted; the bridges are all ignored.  If several loops
+        were compiled, complains."""
         get_stats().check_simple_loop(expected=expected, **check)
 
     def check_trace_count(self, count): # was check_loop_count
-        # The number of traces compiled
+        """Check the number of loops and bridges compiled."""
         assert get_stats().compiled_count == count
 
     def check_trace_count_at_most(self, count):
+        """Check the number of loops and bridges compiled."""
         assert get_stats().compiled_count <= count
 
     def check_jitcell_token_count(self, count): # was check_tree_loop_count
+        """This should check the number of independent trees of code.
+        (xxx it is not 100% clear that the count is correct)"""
         assert len(get_stats().jitcell_token_wrefs) == count
 
     def check_target_token_count(self, count):
+        """(xxx unknown)"""
         tokens = get_stats().get_all_jitcell_tokens()
         n = sum([len(t.target_tokens) for t in tokens])
         assert n == count
 
     def check_enter_count(self, count):
+        """Check the number of times pyjitpl ran.  (Every time, it
+        should have produced either one loop or one bridge, or aborted;
+        but it is not 100% clear that this is still correct in the
+        presence of unrolling.)"""
         assert get_stats().enter_count == count
 
     def check_enter_count_at_most(self, count):
+        """Check the number of times pyjitpl ran."""
         assert get_stats().enter_count <= count
 
-    def check_jumps(self, maxcount):
-        return # FIXME
-        assert get_stats().exec_jumps <= maxcount
-
     def check_aborted_count(self, count):
+        """Check the number of times pyjitpl was aborted."""
         assert get_stats().aborted_count == count
 
     def check_aborted_count_at_least(self, count):
+        """Check the number of times pyjitpl was aborted."""
         assert get_stats().aborted_count >= count
 
     def meta_interp(self, *args, **kwds):
diff --git a/rpython/jit/metainterp/test/test_send.py b/rpython/jit/metainterp/test/test_send.py
--- a/rpython/jit/metainterp/test/test_send.py
+++ b/rpython/jit/metainterp/test/test_send.py
@@ -202,7 +202,7 @@
         # the final one.
         self.check_trace_count(1)
         self.check_resops(guard_class=1, int_add=4, int_sub=4)
-        self.check_jumps(14)
+        #self.check_jumps(14)
 
     def test_oosend_guard_failure_2(self):
         # same as above, but using prebuilt objects 'w1' and 'w2'
@@ -244,7 +244,7 @@
         assert res == f(4, 28)
         self.check_trace_count(1)
         self.check_resops(guard_class=1, int_add=4, int_sub=4)
-        self.check_jumps(14)
+        #self.check_jumps(14)
 
     def test_oosend_different_initial_class(self):
         myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'w'])
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -375,6 +375,10 @@
         # the nursery.
         self.pinned_objects_in_nursery = 0
         #
+        # This flag is set if the previous minor collection found at least
+        # one pinned object alive.
+        self.any_pinned_object_kept = False
+        #
         # Keeps track of old objects pointing to pinned objects. These objects
         # must be traced every minor collection. Without tracing them the
         # referenced pinned object wouldn't be visited and therefore collected.
@@ -1489,7 +1493,9 @@
         # The following counter keeps track of alive and pinned young objects
         # inside the nursery. We reset it here and increace it in
         # '_trace_drag_out()'.
+        any_pinned_object_from_earlier = self.any_pinned_object_kept
         self.pinned_objects_in_nursery = 0
+        self.any_pinned_object_kept = False
         #
         # Before everything else, remove from 'old_objects_pointing_to_young'
         # the young arrays.
@@ -1513,7 +1519,7 @@
         # are copied out or flagged.  They are also added to the list
         # 'old_objects_pointing_to_young'.
         self.nursery_surviving_size = 0
-        self.collect_roots_in_nursery()
+        self.collect_roots_in_nursery(any_pinned_object_from_earlier)
         #
         # visit all objects that are known for pointing to pinned
         # objects. This way we populate 'surviving_pinned_objects'
@@ -1649,7 +1655,7 @@
     def _visit_old_objects_pointing_to_pinned(self, obj, ignore):
         self.trace(obj, self._trace_drag_out, obj)
 
-    def collect_roots_in_nursery(self):
+    def collect_roots_in_nursery(self, any_pinned_object_from_earlier):
         # we don't need to trace prebuilt GcStructs during a minor collect:
         # if a prebuilt GcStruct contains a pointer to a young object,
         # then the write_barrier must have ensured that the prebuilt
@@ -1659,10 +1665,19 @@
             callback = IncrementalMiniMarkGC._trace_drag_out1_marking_phase
         else:
             callback = IncrementalMiniMarkGC._trace_drag_out1
+        #
+        # Note a subtlety: if the nursery contains pinned objects "from
+        # earlier", i.e. created earlier than the previous minor
+        # collection, then we can't use the "is_minor=True" optimization.
+        # We really need to walk the complete stack to be sure we still
+        # see them.
+        use_jit_frame_stoppers = not any_pinned_object_from_earlier
+        #
         self.root_walker.walk_roots(
             callback,     # stack roots
             callback,     # static in prebuilt non-gc
-            None)         # static in prebuilt gc
+            None,         # static in prebuilt gc
+            is_minor=use_jit_frame_stoppers)
         debug_stop("gc-minor-walkroots")
 
     def collect_cardrefs_to_nursery(self):
@@ -1844,6 +1859,7 @@
             self.surviving_pinned_objects.append(
                 llarena.getfakearenaaddress(obj - size_gc_header))
             self.pinned_objects_in_nursery += 1
+            self.any_pinned_object_kept = True
             return
         else:
             # First visit to an object that has already a shadow.
diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -1322,7 +1322,8 @@
         self.root_walker.walk_roots(
             MiniMarkGC._trace_drag_out1,  # stack roots
             MiniMarkGC._trace_drag_out1,  # static in prebuilt non-gc
-            None)                         # static in prebuilt gc
+            None,                         # static in prebuilt gc
+            is_minor=True)
         debug_stop("gc-minor-walkroots")
 
     def collect_cardrefs_to_nursery(self):
diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py
--- a/rpython/memory/gc/test/test_direct.py
+++ b/rpython/memory/gc/test/test_direct.py
@@ -34,7 +34,8 @@
 
     def walk_roots(self, collect_stack_root,
                    collect_static_in_prebuilt_nongc,
-                   collect_static_in_prebuilt_gc):
+                   collect_static_in_prebuilt_gc,
+                   is_minor=False):
         gc = self.tester.gc
         layoutbuilder = self.tester.layoutbuilder
         if collect_static_in_prebuilt_gc:
diff --git a/rpython/memory/gctransform/asmgcroot.py b/rpython/memory/gctransform/asmgcroot.py
--- a/rpython/memory/gctransform/asmgcroot.py
+++ b/rpython/memory/gctransform/asmgcroot.py
@@ -340,9 +340,10 @@
         # called first, to initialize self.belongs_to_current_thread.
         assert not hasattr(self, 'gc_detach_callback_pieces_ptr')
 
-    def walk_stack_roots(self, collect_stack_root):
+    def walk_stack_roots(self, collect_stack_root, is_minor=False):
         gcdata = self.gcdata
         gcdata._gc_collect_stack_root = collect_stack_root
+        gcdata._gc_collect_is_minor = is_minor
         pypy_asm_stackwalk(llhelper(ASM_CALLBACK_PTR, self._asm_callback),
                            gcrootanchor)
 
@@ -468,6 +469,13 @@
             if gc.points_to_valid_gc_object(addr):
                 collect_stack_root(gc, addr)
         #
+        # small hack: the JIT reserves THREADLOCAL_OFS's last bit for
+        # us.  We use it to store an "already traced past this frame"
+        # flag.
+        if self._with_jit and self.gcdata._gc_collect_is_minor:
+            if self.mark_jit_frame_can_stop(callee):
+                return False
+        #
         # track where the caller_frame saved the registers from its own
         # caller
         #
@@ -548,6 +556,19 @@
         else:  # kind == LOC_EBP_MINUS:   at -N(%ebp)
             return ebp_in_caller - offset
 
+    def mark_jit_frame_can_stop(self, callee):
+        location = self._shape_decompressor.get_threadlocal_loc()
+        if location == LOC_NOWHERE:
+            return False
+        addr = self.getlocation(callee, llmemory.NULL, location)
+        #
+        x = addr.signed[0]
+        if x & 1:
+            return True            # this JIT stack frame is already marked!
+        else:
+            addr.signed[0] = x | 1    # otherwise, mark it but don't stop
+            return False
+
 
 LOC_REG       = 0
 LOC_ESP_PLUS  = 1
@@ -729,6 +750,17 @@
             llop.debug_fatalerror(lltype.Void, "asmgcroot: invalid index")
             return 0   # annotator fix
 
+    def get_threadlocal_loc(self):
+        index = self.jit_index
+        if index < 0:
+            return LOC_NOWHERE     # case "outside the jit"
+        else:
+            # case "in the jit"
+            from rpython.jit.backend.x86.arch import THREADLOCAL_OFS, WORD
+            return (LOC_ESP_PLUS |
+                    ((THREADLOCAL_OFS // WORD + self.extra_stack_depth) << 2))
+
+
 # ____________________________________________________________
 
 #
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -1462,7 +1462,8 @@
 
     def walk_roots(self, collect_stack_root,
                    collect_static_in_prebuilt_nongc,
-                   collect_static_in_prebuilt_gc):
+                   collect_static_in_prebuilt_gc,
+                   is_minor=False):
         gcdata = self.gcdata
         gc = self.gc
         if collect_static_in_prebuilt_nongc:
@@ -1482,7 +1483,7 @@
                     collect_static_in_prebuilt_gc(gc, result)
                 addr += sizeofaddr
         if collect_stack_root:
-            self.walk_stack_roots(collect_stack_root)     # abstract
+            self.walk_stack_roots(collect_stack_root, is_minor)     # abstract
 
     def finished_minor_collection(self):
         func = self.finished_minor_collection_func
diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py
--- a/rpython/memory/gctransform/shadowstack.py
+++ b/rpython/memory/gctransform/shadowstack.py
@@ -99,7 +99,7 @@
         self.shadow_stack_pool.initial_setup()
         BaseRootWalker.setup_root_walker(self)
 
-    def walk_stack_roots(self, collect_stack_root):
+    def walk_stack_roots(self, collect_stack_root, is_minor=False):
         gcdata = self.gcdata
         self.rootstackhook(collect_stack_root,
                            gcdata.root_stack_base, gcdata.root_stack_top)
diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py
--- a/rpython/memory/gctransform/transform.py
+++ b/rpython/memory/gctransform/transform.py
@@ -221,7 +221,7 @@
         # for sanity, we need an empty block at the start of the graph
         inserted_empty_startblock = False
         if not starts_with_empty_block(graph):
-            insert_empty_startblock(self.translator.annotator, graph)
+            insert_empty_startblock(graph)
             inserted_empty_startblock = True
         is_borrowed = self.compute_borrowed_vars(graph)
 
@@ -239,7 +239,7 @@
                 if link.prevblock.exitswitch is None:
                     link.prevblock.operations.extend(llops)
                 else:
-                    insert_empty_block(self.translator.annotator, link, llops)
+                    insert_empty_block(link, llops)
 
         # remove the empty block at the start of the graph, which should
         # still be empty (but let's check)
diff --git a/rpython/memory/gcwrapper.py b/rpython/memory/gcwrapper.py
--- a/rpython/memory/gcwrapper.py
+++ b/rpython/memory/gcwrapper.py
@@ -191,7 +191,8 @@
 
     def walk_roots(self, collect_stack_root,
                    collect_static_in_prebuilt_nongc,
-                   collect_static_in_prebuilt_gc):
+                   collect_static_in_prebuilt_gc,
+                   is_minor=False):
         gcheap = self.gcheap
         gc = gcheap.gc
         if collect_static_in_prebuilt_gc:
@@ -203,7 +204,7 @@
                 if self.gcheap.gc.points_to_valid_gc_object(addrofaddr):
                     collect_static_in_prebuilt_nongc(gc, addrofaddr)
         if collect_stack_root:
-            for addrofaddr in gcheap.llinterp.find_roots():
+            for addrofaddr in gcheap.llinterp.find_roots(is_minor):
                 if self.gcheap.gc.points_to_valid_gc_object(addrofaddr):
                     collect_stack_root(gc, addrofaddr)
 
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -146,13 +146,21 @@
                            }
             return self._tlobj
 
-    def find_roots(self):
+    def find_roots(self, is_minor=False):
         """Return a list of the addresses of the roots."""
         #log.findroots("starting")
         roots = []
-        for frame in self.frame_stack:
+        for frame in reversed(self.frame_stack):
             #log.findroots("graph", frame.graph.name)
             frame.find_roots(roots)
+            # If a call is done with 'is_minor=True', we can stop after the
+            # first frame in the stack that was already seen by the previous
+            # call with 'is_minor=True'.  (We still need to trace that frame,
+            # but not its callers.)
+            if is_minor:
+                if getattr(frame, '_find_roots_already_seen', False):
+                    break
+                frame._find_roots_already_seen = True
         return roots
 
     def find_exception(self, exc):
diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py
--- a/rpython/rtyper/rtyper.py
+++ b/rpython/rtyper/rtyper.py
@@ -371,9 +371,7 @@
                 assert 0 <= pos < len(newops) - 1
                 extraops = block.operations[pos+1:]
                 del block.operations[pos+1:]
-                extrablock = insert_empty_block(self.annotator,
-                                                noexclink,
-                                                newops = extraops)
+                extrablock = insert_empty_block(noexclink, newops=extraops)
 
         if extrablock is None:
             self.insert_link_conversions(block)
@@ -447,10 +445,9 @@
                     # cannot insert conversion operations around a single
                     # link, unless it is the only exit of this block.
                     # create a new block along the link...
-                    newblock = insert_empty_block(self.annotator,
-                                                  link,
+                    newblock = insert_empty_block(link,
                     # ...and store the conversions there.
-                                               newops=newops)
+                                                  newops=newops)
                     link = newblock.exits[0]
             for i, new_a1 in newlinkargs.items():
                 link.args[i] = new_a1
diff --git a/rpython/translator/backendopt/constfold.py b/rpython/translator/backendopt/constfold.py
--- a/rpython/translator/backendopt/constfold.py
+++ b/rpython/translator/backendopt/constfold.py
@@ -171,7 +171,7 @@
             v_result.concretetype = nextop.result.concretetype
             constants[nextop.result] = v_result
             callop = SpaceOperation('direct_call', callargs, v_result)
-            newblock = insert_empty_block(None, link, [callop])
+            newblock = insert_empty_block(link, [callop])
             [link] = newblock.exits
             assert link.target is block
             folded_count += 1
@@ -197,7 +197,7 @@
                 splitlink = block.exits[0]
             else:
                 # split the block at the given position
-                splitlink = split_block(None, block, position)
+                splitlink = split_block(block, position)
                 assert list(block.exits) == [splitlink]
             assert link.target is block
             assert splitlink.prevblock is block
diff --git a/rpython/translator/backendopt/inline.py b/rpython/translator/backendopt/inline.py
--- a/rpython/translator/backendopt/inline.py
+++ b/rpython/translator/backendopt/inline.py
@@ -396,7 +396,7 @@
         copiedexceptblock.recloseblock(Link(linkargs, blocks[0]))
 
     def do_inline(self, block, index_operation):
-        splitlink = split_block(None, block, index_operation)
+        splitlink = split_block(block, index_operation)
         afterblock = splitlink.target
         # these variables have to be passed along all the links in the inlined
         # graph because the original function needs them in the blocks after
diff --git a/rpython/translator/c/gcc/test/test_asmgcroot.py b/rpython/translator/c/gcc/test/test_asmgcroot.py
--- a/rpython/translator/c/gcc/test/test_asmgcroot.py
+++ b/rpython/translator/c/gcc/test/test_asmgcroot.py
@@ -251,13 +251,17 @@
     def define_callback_with_collect(cls):
         return lambda: 0
 
-class TestAsmGCRootWithSemiSpaceGC_Shared(TestAsmGCRootWithSemiSpaceGC):
-    @classmethod
-    def make_config(cls):
-        config = TestAsmGCRootWithSemiSpaceGC.make_config()
-        config.translation.shared = True
-        return config
+#class TestAsmGCRootWithSemiSpaceGC_Shared(TestAsmGCRootWithSemiSpaceGC):
+#    @classmethod
+#    def make_config(cls):
+#        config = TestAsmGCRootWithSemiSpaceGC.make_config()
+#        config.translation.shared = True
+#        return config
 
 class TestAsmGCRootWithHybridTagged(AbstractTestAsmGCRoot,
                                     test_newgc.TestHybridTaggedPointers):
     pass
+
+class TestAsmGCRootWithIncrementalMinimark(AbstractTestAsmGCRoot,
+                                    test_newgc.TestIncrementalMiniMarkGC):
+    pass
diff --git a/rpython/translator/c/support.py b/rpython/translator/c/support.py
--- a/rpython/translator/c/support.py
+++ b/rpython/translator/c/support.py
@@ -89,7 +89,8 @@
            ''')
 
 def _char_repr(c):
-    if c in '\\"': return '\\' + c
+    # escape with a '\' the characters '\', '"' or (for trigraphs) '?'
+    if c in '\\"?': return '\\' + c
     if ' ' <= c < '\x7F': return c
     return '\\%03o' % ord(c)
 
diff --git a/rpython/translator/exceptiontransform.py b/rpython/translator/exceptiontransform.py
--- a/rpython/translator/exceptiontransform.py
+++ b/rpython/translator/exceptiontransform.py
@@ -259,7 +259,7 @@
             if not self.raise_analyzer.can_raise(op):
                 continue
 
-            splitlink = split_block(None, block, i+1)
+            splitlink = split_block(block, i+1)
             afterblock = splitlink.target
             if lastblock is block:
                 lastblock = afterblock
@@ -432,7 +432,7 @@
 
         if insert_zeroing_op:
             if normalafterblock is None:
-                normalafterblock = insert_empty_block(None, l0)
+                normalafterblock = insert_empty_block(l0)
             v_result = spaceop.result
             if v_result in l0.args:
                 result_i = l0.args.index(v_result)
diff --git a/rpython/translator/goal/translate.py b/rpython/translator/goal/translate.py
--- a/rpython/translator/goal/translate.py
+++ b/rpython/translator/goal/translate.py
@@ -308,7 +308,9 @@
                 samefile = this_exe.samefile(exe_name)
                 assert not samefile, (
                     'Output file %s is the currently running '
-                    'interpreter (use --output=...)' % exe_name)
+                    'interpreter (please move the executable, and '
+                    'possibly its associated libpypy-c, somewhere else '
+                    'before you execute it)' % exe_name)
             except EnvironmentError:
                 pass
 
diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -1072,7 +1072,7 @@
                         link.target in stopblocks):
                         hints['exactlength'] = True
                     chints = Constant(hints)
-                    newblock = unsimplify.insert_empty_block(None, link)
+                    newblock = unsimplify.insert_empty_block(link)
                     index = link.args.index(vlist)
                     vlist2 = newblock.inputargs[index]
                     vlist3 = Variable(vlist2)
diff --git a/rpython/translator/test/test_unsimplify.py b/rpython/translator/test/test_unsimplify.py
--- a/rpython/translator/test/test_unsimplify.py
+++ b/rpython/translator/test/test_unsimplify.py
@@ -21,7 +21,7 @@
             w = x * y
             return z + w
         graph, t = translate(f, [int, int])
-        split_block(t.annotator, graph.startblock, i)
+        split_block(graph.startblock, i)
         checkgraph(graph)
         interp = LLInterpreter(t.rtyper)
         result = interp.eval_graph(graph, [1, 2])
@@ -35,7 +35,7 @@
             else:
                 return y + 2
         graph, t = translate(f, [int, int])
-        split_block(t.annotator, graph.startblock, i)
+        split_block(graph.startblock, i)
         checkgraph(graph)
         interp = LLInterpreter(t.rtyper)
         result = interp.eval_graph(graph, [-12, 2])
@@ -61,7 +61,7 @@
                 return 1
             return x
         graph, t = translate(catches, [int])
-        split_block(t.annotator, graph.startblock, i)
+        split_block(graph.startblock, i)
         checkgraph(graph)
         interp = LLInterpreter(t.rtyper)
         result = interp.eval_graph(graph, [0])
diff --git a/rpython/translator/unsimplify.py b/rpython/translator/unsimplify.py
--- a/rpython/translator/unsimplify.py
+++ b/rpython/translator/unsimplify.py
@@ -7,7 +7,7 @@
     var.concretetype = concretetype
     return var
 
-def insert_empty_block(annotator, link, newops=[]):
+def insert_empty_block(link, newops=[]):
     """Insert and return a new block along the given link."""
     vars = {}
     for v in link.args:
@@ -30,7 +30,7 @@
     link.target = newblock
     return newblock
 
-def insert_empty_startblock(annotator, graph):
+def insert_empty_startblock(graph):
     vars = [v.copy() for v in graph.startblock.inputargs]
     newblock = Block(vars)
     newblock.closeblock(Link(vars, graph.startblock))
@@ -41,7 +41,7 @@
             and graph.startblock.exitswitch is None
             and graph.startblock.exits[0].args == graph.getargs())
 
-def split_block(annotator, block, index, _forcelink=None):
+def split_block(block, index, _forcelink=None):
     """return a link where prevblock is the block leading up but excluding the
     index'th operation and target is a new block with the neccessary variables
     passed on.

From noreply at buildbot.pypy.org  Sun Feb  8 20:08:04 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  8 Feb 2015 20:08:04 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: merge
Message-ID: <20150208190804.DE6B61C1278@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r75766:63924af8af13
Date: 2015-02-08 21:07 +0200
http://bitbucket.org/pypy/pypy/changeset/63924af8af13/

Log:	merge


From noreply at buildbot.pypy.org  Sun Feb  8 20:15:35 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  8 Feb 2015 20:15:35 +0100 (CET)
Subject: [pypy-commit] pypy default: Details
Message-ID: <20150208191535.34D7F1C157E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r75767:9f3564782055
Date: 2015-02-08 20:15 +0100
http://bitbucket.org/pypy/pypy/changeset/9f3564782055/

Log:	Details

diff --git a/pypy/doc/interpreter-optimizations.rst b/pypy/doc/interpreter-optimizations.rst
--- a/pypy/doc/interpreter-optimizations.rst
+++ b/pypy/doc/interpreter-optimizations.rst
@@ -165,7 +165,7 @@
 We improved this by keeping method lookup separated from method call, unlike
 some other approaches, but using the value stack as a cache instead of building
 a temporary object.  We extended the bytecode compiler to (optionally) generate
-the following code for ``obj.meth(x)``::
+the following code for ``obj.meth(x, y)``::
 
     LOAD_GLOBAL     obj
     LOOKUP_METHOD   meth
@@ -181,7 +181,7 @@
 the attribute actually refers to a function object from the class; when this is
 not the case, ``LOOKUP_METHOD`` still pushes two values, but one *(im_func)* is
 simply the regular result that ``LOAD_ATTR`` would have returned, and the other
-*(im_self)* is a None placeholder.
+*(im_self)* is an interpreter-level None placeholder.
 
 After pushing the arguments, the layout of the stack in the above
 example is as follows (the stack grows upwards):

From noreply at buildbot.pypy.org  Sun Feb  8 20:32:39 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun,  8 Feb 2015 20:32:39 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Fix various tests
Message-ID: <20150208193239.D70FB1C1C43@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: py3k
Changeset: r75768:23b481782927
Date: 2015-02-08 20:32 +0100
http://bitbucket.org/pypy/pypy/changeset/23b481782927/

Log:	Fix various tests

diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -180,7 +180,7 @@
         if isinstance(address, _rawffi.StructureInstance):
             address = address.buffer
         # fix the address: turn it into as unsigned, in case it is negative
-        address = address & (sys.maxint * 2 + 1)
+        address = address & (sys.maxsize * 2 + 1)
         instance.__dict__['_buffer'] = self._ffistruct.fromaddress(address)
         return instance
 
diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py
--- a/pypy/module/micronumpy/app_numpy.py
+++ b/pypy/module/micronumpy/app_numpy.py
@@ -18,7 +18,7 @@
     length = int(length)
     arr = _numpypy.multiarray.empty(length, dtype=dtype)
     i = start
-    for j in xrange(arr.size):
+    for j in range(arr.size):
         arr[j] = i
         i += step
     return arr
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -258,17 +258,17 @@
     def test_reversed_dict(self):
         import __pypy__
         for d in [{}, {1: 2, 3: 4, 5: 6}, {"a": 5, "b": 2, "c": 6}]:
-            assert list(__pypy__.reversed_dict(d)) == d.keys()[::-1]
+            assert list(__pypy__.reversed_dict(d)) == list(d.keys())[::-1]
         raises(TypeError, __pypy__.reversed_dict, 42)
 
     def test_reversed_dict_runtimeerror(self):
         import __pypy__
         d = {1: 2, 3: 4, 5: 6}
         it = __pypy__.reversed_dict(d)
-        key = it.next()
+        key = next(it)
         assert key in [1, 3, 5]
         del d[key]
-        raises(RuntimeError, it.next)
+        raises(RuntimeError, next, it)
 
     def test_keys(self):
         d = {1: 2, 3: 4}

From noreply at buildbot.pypy.org  Sun Feb  8 20:33:30 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  8 Feb 2015 20:33:30 +0100 (CET)
Subject: [pypy-commit] pypy optimize-locks: A branch to try to optimize the
	locks
Message-ID: <20150208193330.D14021C1C43@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: optimize-locks
Changeset: r75769:56224378453f
Date: 2015-02-08 18:25 +0100
http://bitbucket.org/pypy/pypy/changeset/56224378453f/

Log:	A branch to try to optimize the locks


From noreply at buildbot.pypy.org  Sun Feb  8 20:33:32 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  8 Feb 2015 20:33:32 +0100 (CET)
Subject: [pypy-commit] pypy optimize-locks: Meh.  It's subtle code...
Message-ID: <20150208193332.1C7E21C1C43@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: optimize-locks
Changeset: r75770:a0ffa88963d1
Date: 2015-02-08 20:33 +0100
http://bitbucket.org/pypy/pypy/changeset/a0ffa88963d1/

Log:	Meh. It's subtle code...

diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -3,7 +3,7 @@
 from rpython.translator import cdir
 import py, sys
 from rpython.rlib import jit, rgc
-from rpython.rlib.debug import ll_assert
+from rpython.rlib.debug import ll_assert, fatalerror, debug_print
 from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.rlib.objectmodel import CDefinedIntSymbolic
 from rpython.rtyper.lltypesystem.lloperation import llop
@@ -62,7 +62,8 @@
                                   releasegil=True)    # release the GIL
 
 # another set of functions, this time in versions that don't cause the
-# GIL to be released.  To use to handle the GIL lock itself.
+# GIL to be released.  This was used to handle the GIL lock itself,
+# but nowadays it's done differently, so it's really only used in tests.
 c_thread_acquirelock_NOAUTO = llexternal('RPyThreadAcquireLock',
                                          [TLOCKP, rffi.INT], rffi.INT,
                                          _nowrapper=True)
@@ -128,27 +129,104 @@
     _immutable_fields_ = ["_lock"]
 
     def __init__(self, ll_lock):
+        # "_num_acquires" is the number of times we have started an
+        # "acquire" on this lock, minus the number of times we have
+        # done a "release", multiplied by two and with one added iff
+        # the low-level ll_lock is actually acquired.  As long as
+        # there is no contention, _num_acquires will only take the
+        # values 0 or 2, and the ll_lock will not be used (and the GIL
+        # will not be released because of operations on this lock).
         self._lock = ll_lock
+        self._num_acquires = 0
+
+    def _acquire_ll_lock(self, microseconds, intr_flag):
+        n = self._num_acquires
+        #debug_print("_really_acquire, n =", n)
+        if n == 2:
+            # if n is precisely 2, we must do a double acquire: one
+            # for really acquiring the lock, and another for blocking
+            # until it is released by someone else.  The first one
+            # should never block and we *must* not release the GIL
+            # around it.  Only afterwards will the lock be in a
+            # consistent state again: _num_acquires odd and _lock
+            # acquired.
+            flag = rffi.cast(rffi.INT, 1)
+            res = c_thread_acquirelock_NOAUTO(self._lock, flag)
+            res = rffi.cast(lltype.Signed, res)
+            if res != 1:
+                fatalerror("lock.acquire() failed unexpectedly")
+            n = 3
+        #
+        self._num_acquires = n + 2
+        res = c_thread_acquirelock_timed(self._lock, microseconds, intr_flag)
+        res = rffi.cast(lltype.Signed, res)
+        return res
+
+    @staticmethod
+    def _really_acquire(self):
+        res = self._acquire_ll_lock(-1, 0)
+        if res != 1:
+            fatalerror("lock.acquire() failed unexpectedly")
+        #
+        n = self._num_acquires - 2
+        assert n >= 0
+        self._num_acquires = n
+
+    @staticmethod
+    def _really_release(self):
+        #debug_print("_really_release, n =", self._num_acquires)
+        c_thread_releaselock(self._lock)
 
     def acquire(self, flag):
-        res = c_thread_acquirelock(self._lock, int(flag))
-        res = rffi.cast(lltype.Signed, res)
-        return bool(res)
+        if flag:
+            #debug_print("acquiring...")
+            # acquire(True): if _num_acquires >= 2, then there is contention
+            jit.conditional_call(self._num_acquires >= 2,
+                                 Lock._really_acquire, self)
+            #
+            # At this point, either _num_acquires was zero and we increment
+            # it here; or it was not zero, so we called _really_acquire(),
+            # which incremented it but decremented it again before
+            # returning --- so that we can increment it here *again*.  The
+            # point is that if the JIT compiles a small Python block of
+            # code which contains the release() soon afterwards, the
+            # following increment will be matched with the decrement and
+            # optimized away (as usual relying on the GIL).
+            self._num_acquires += 2
+            #debug_print("acquired")
+            return True
+        else:
+            # acquire(False): if n == 0, then we succeed; otherwise, we fail
+            if self._num_acquires == 0:
+                self._num_acquires = 2
+                return True
+            else:
+                return False
 
     def acquire_timed(self, timeout):
         """Timeout is in microseconds.  Returns 0 in case of failure,
         1 in case it works, 2 if interrupted by a signal."""
-        res = c_thread_acquirelock_timed(self._lock, timeout, 1)
-        res = rffi.cast(lltype.Signed, res)
+        n = self._num_acquires
+        # can't use jit.conditional_call() easily here, because of the
+        # return value.  Anyway in PyPy this code is not seen by the JIT.
+        if n == 0:
+            self._num_acquires = n + 2
+            res = 1      # no contention, worked
+        else:
+            res = self._acquire_ll_lock(timeout, 1)
         return res
 
     def release(self):
         # Sanity check: the lock must be locked
-        if self.acquire(False):
-            c_thread_releaselock(self._lock)
+        n = self._num_acquires
+        if n < 2:
             raise error("bad lock")
-        else:
-            c_thread_releaselock(self._lock)
+        n = n - 2
+        self._num_acquires = n
+        # the only case where we don't really want to release the ll_lock
+        # is when this release() makes _num_acquires go from 2 to 0.
+        jit.conditional_call(n != 0, Lock._really_release, self)
+        #debug_print("released")
 
     def __del__(self):
         if free_ll_lock is None:  # happens when tests are shutting down

From noreply at buildbot.pypy.org  Sun Feb  8 20:58:06 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  8 Feb 2015 20:58:06 +0100 (CET)
Subject: [pypy-commit] pypy default: don't use a standard exception here
Message-ID: <20150208195806.306CC1C03CF@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r75771:191745efb9be
Date: 2015-02-08 21:57 +0200
http://bitbucket.org/pypy/pypy/changeset/191745efb9be/

Log:	don't use a standard exception here

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -16,12 +16,20 @@
                                              space.wrap(msg)]))
     return raise_unicode_exception_decode
 
+class AppUnicodeEncodeError(Exception):
+    def __init__(self, encoding, object, start, end, reason):
+        self.encoding = encoding
+        self.object = object
+        self.start = start
+        self.end = end
+        self.reason = reason
+
 @specialize.memo()
 def encode_error_handler(space):
     # Fast version of the "strict" errors handler.
     def raise_unicode_exception_encode(errors, encoding, msg, u,
                                        startingpos, endingpos):
-        raise UnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
+        raise AppUnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
     return raise_unicode_exception_encode
 
 # ____________________________________________________________
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -448,7 +448,7 @@
                     return space.wrap(unicode_encode_utf_8(
                             u, len(u), None, errorhandler=eh,
                             allow_surrogates=True))
-            except UnicodeEncodeError, ue:
+            except unicodehelper.AppUnicodeEncodeError, ue:
                 raise OperationError(space.w_UnicodeEncodeError,
                                      space.newtuple([
                     space.wrap(ue.encoding),
diff --git a/rpython/annotator/classdef.py b/rpython/annotator/classdef.py
--- a/rpython/annotator/classdef.py
+++ b/rpython/annotator/classdef.py
@@ -3,7 +3,7 @@
 """
 from rpython.annotator.model import (
     SomePBC, s_ImpossibleValue, unionof, s_None, SomeInteger,
-    SomeTuple, SomeString, AnnotatorError)
+    SomeTuple, SomeString, AnnotatorError, SomeUnicodeString)
 from rpython.annotator import description
 
 

From noreply at buildbot.pypy.org  Sun Feb  8 20:58:07 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  8 Feb 2015 20:58:07 +0100 (CET)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20150208195807.513741C03CF@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r75772:3b020d7a1cff
Date: 2015-02-08 21:57 +0200
http://bitbucket.org/pypy/pypy/changeset/3b020d7a1cff/

Log:	merge

diff --git a/pypy/doc/interpreter-optimizations.rst b/pypy/doc/interpreter-optimizations.rst
--- a/pypy/doc/interpreter-optimizations.rst
+++ b/pypy/doc/interpreter-optimizations.rst
@@ -165,7 +165,7 @@
 We improved this by keeping method lookup separated from method call, unlike
 some other approaches, but using the value stack as a cache instead of building
 a temporary object.  We extended the bytecode compiler to (optionally) generate
-the following code for ``obj.meth(x)``::
+the following code for ``obj.meth(x, y)``::
 
     LOAD_GLOBAL     obj
     LOOKUP_METHOD   meth
@@ -181,7 +181,7 @@
 the attribute actually refers to a function object from the class; when this is
 not the case, ``LOOKUP_METHOD`` still pushes two values, but one *(im_func)* is
 simply the regular result that ``LOAD_ATTR`` would have returned, and the other
-*(im_self)* is a None placeholder.
+*(im_self)* is an interpreter-level None placeholder.
 
 After pushing the arguments, the layout of the stack in the above
 example is as follows (the stack grows upwards):

From noreply at buildbot.pypy.org  Sun Feb  8 20:59:49 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  8 Feb 2015 20:59:49 +0100 (CET)
Subject: [pypy-commit] pypy default: rename
Message-ID: <20150208195949.B5D741C03CF@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r75773:e1f05f94b491
Date: 2015-02-08 21:59 +0200
http://bitbucket.org/pypy/pypy/changeset/e1f05f94b491/

Log:	rename

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -16,7 +16,7 @@
                                              space.wrap(msg)]))
     return raise_unicode_exception_decode
 
-class AppUnicodeEncodeError(Exception):
+class RPyUnicodeEncodeError(Exception):
     def __init__(self, encoding, object, start, end, reason):
         self.encoding = encoding
         self.object = object
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -448,7 +448,7 @@
                     return space.wrap(unicode_encode_utf_8(
                             u, len(u), None, errorhandler=eh,
                             allow_surrogates=True))
-            except unicodehelper.AppUnicodeEncodeError, ue:
+            except unicodehelper.RPyUnicodeEncodeError, ue:
                 raise OperationError(space.w_UnicodeEncodeError,
                                      space.newtuple([
                     space.wrap(ue.encoding),

From noreply at buildbot.pypy.org  Sun Feb  8 21:00:26 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  8 Feb 2015 21:00:26 +0100 (CET)
Subject: [pypy-commit] pypy default: rename again
Message-ID: <20150208200026.B77A51C03CF@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r75774:458ebd4ba792
Date: 2015-02-08 22:00 +0200
http://bitbucket.org/pypy/pypy/changeset/458ebd4ba792/

Log:	rename again

diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -16,7 +16,7 @@
                                              space.wrap(msg)]))
     return raise_unicode_exception_decode
 
-class RPyUnicodeEncodeError(Exception):
+class RUnicodeEncodeError(Exception):
     def __init__(self, encoding, object, start, end, reason):
         self.encoding = encoding
         self.object = object
@@ -29,7 +29,7 @@
     # Fast version of the "strict" errors handler.
     def raise_unicode_exception_encode(errors, encoding, msg, u,
                                        startingpos, endingpos):
-        raise AppUnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
+        raise RUnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
     return raise_unicode_exception_encode
 
 # ____________________________________________________________
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -448,7 +448,7 @@
                     return space.wrap(unicode_encode_utf_8(
                             u, len(u), None, errorhandler=eh,
                             allow_surrogates=True))
-            except unicodehelper.RPyUnicodeEncodeError, ue:
+            except unicodehelper.RUnicodeEncodeError, ue:
                 raise OperationError(space.w_UnicodeEncodeError,
                                      space.newtuple([
                     space.wrap(ue.encoding),

From noreply at buildbot.pypy.org  Sun Feb  8 22:14:47 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  8 Feb 2015 22:14:47 +0100 (CET)
Subject: [pypy-commit] stmgc default: Subtle race condition
Message-ID: <20150208211447.E59E61C03CF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1611:005668d99755
Date: 2015-02-08 22:15 +0100
http://bitbucket.org/pypy/stmgc/changeset/005668d99755/

Log:	Subtle race condition

diff --git a/c7/stm/core.c b/c7/stm/core.c
--- a/c7/stm/core.c
+++ b/c7/stm/core.c
@@ -312,6 +312,11 @@
                                      FIRST_READMARKER_PAGE * 4096UL);
     dprintf(("reset_transaction_read_version: %p %ld\n", readmarkers,
              (long)(NB_READMARKER_PAGES * 4096UL)));
+
+    /* see hashtable.c for why we need the privatization lock here
+       (grep for reset_transaction_read_version)
+    */
+    acquire_privatization_lock();
     if (mmap(readmarkers, NB_READMARKER_PAGES * 4096UL,
              PROT_READ | PROT_WRITE,
              MAP_FIXED | MAP_PAGES_FLAGS, -1, 0) != readmarkers) {
@@ -322,6 +327,7 @@
         memset(readmarkers, 0, NB_READMARKER_PAGES * 4096UL);
     }
     STM_SEGMENT->transaction_read_version = 1;
+    release_privatization_lock();
 }
 
 static uint64_t _global_start_time = 0;
diff --git a/c7/stm/hashtable.c b/c7/stm/hashtable.c
--- a/c7/stm/hashtable.c
+++ b/c7/stm/hashtable.c
@@ -299,16 +299,28 @@
             */
 
             /* First fetch the read marker of 'hashtableobj' in all
-               segments, before allocate_outside_nursery_large()
-               which might trigger a GC */
+               segments, before allocate_outside_nursery_large() which
+               might trigger a GC.  Synchronization guarantee: if
+               stm_read(hobj) in stm_hashtable_list() has set the read
+               marker, then it did synchronize with us here by
+               acquiring and releasing this hashtable' lock.  However,
+               the interval of time between reading the readmarkers of
+               hobj and copying them to the new entry object might be
+               enough for the other threads to do anything, including
+               a reset_transaction_read_version(), so that we might in
+               theory write bogus read markers that are not valid any
+               more.  To prevent this, reset_transaction_read_version()
+               acquires the privatization_lock too.
+            */
             long j;
             uint8_t readmarkers[NB_SEGMENTS];
+
+            acquire_privatization_lock();
             for (j = 1; j <= NB_SEGMENTS; j++) {
                 readmarkers[j - 1] = get_read_marker(get_segment_base(j),
                                                      hashtableobj)->rm;
             }
 
-            acquire_privatization_lock();
             char *p = allocate_outside_nursery_large(
                           sizeof(stm_hashtable_entry_t));
             entry = (stm_hashtable_entry_t *)(p - stm_object_pages);
@@ -323,12 +335,12 @@
                 e->object = NULL;
             }
             hashtable->additions += 0x100;
-            release_privatization_lock();
 
             for (j = 1; j <= NB_SEGMENTS; j++) {
                 get_read_marker(get_segment_base(j), (object_t *)entry)->rm =
                     readmarkers[j - 1];
             }
+            release_privatization_lock();
         }
         write_fence();     /* make sure 'entry' is fully initialized here */
         table->items[i] = entry;

From noreply at buildbot.pypy.org  Sun Feb  8 22:15:56 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  8 Feb 2015 22:15:56 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: import stmgc/005668d99755
Message-ID: <20150208211556.22B0A1C03CF@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r75775:7dc78d95f4a2
Date: 2015-02-08 22:15 +0100
http://bitbucket.org/pypy/pypy/changeset/7dc78d95f4a2/

Log:	import stmgc/005668d99755

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-10c636ae449e
+005668d99755
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -313,6 +313,11 @@
                                      FIRST_READMARKER_PAGE * 4096UL);
     dprintf(("reset_transaction_read_version: %p %ld\n", readmarkers,
              (long)(NB_READMARKER_PAGES * 4096UL)));
+
+    /* see hashtable.c for why we need the privatization lock here
+       (grep for reset_transaction_read_version)
+    */
+    acquire_privatization_lock();
     if (mmap(readmarkers, NB_READMARKER_PAGES * 4096UL,
              PROT_READ | PROT_WRITE,
              MAP_FIXED | MAP_PAGES_FLAGS, -1, 0) != readmarkers) {
@@ -323,6 +328,7 @@
         memset(readmarkers, 0, NB_READMARKER_PAGES * 4096UL);
     }
     STM_SEGMENT->transaction_read_version = 1;
+    release_privatization_lock();
 }
 
 static uint64_t _global_start_time = 0;
diff --git a/rpython/translator/stm/src_stm/stm/hashtable.c b/rpython/translator/stm/src_stm/stm/hashtable.c
--- a/rpython/translator/stm/src_stm/stm/hashtable.c
+++ b/rpython/translator/stm/src_stm/stm/hashtable.c
@@ -300,16 +300,28 @@
             */
 
             /* First fetch the read marker of 'hashtableobj' in all
-               segments, before allocate_outside_nursery_large()
-               which might trigger a GC */
+               segments, before allocate_outside_nursery_large() which
+               might trigger a GC.  Synchronization guarantee: if
+               stm_read(hobj) in stm_hashtable_list() has set the read
+               marker, then it did synchronize with us here by
+               acquiring and releasing this hashtable' lock.  However,
+               the interval of time between reading the readmarkers of
+               hobj and copying them to the new entry object might be
+               enough for the other threads to do anything, including
+               a reset_transaction_read_version(), so that we might in
+               theory write bogus read markers that are not valid any
+               more.  To prevent this, reset_transaction_read_version()
+               acquires the privatization_lock too.
+            */
             long j;
             uint8_t readmarkers[NB_SEGMENTS];
+
+            acquire_privatization_lock();
             for (j = 1; j <= NB_SEGMENTS; j++) {
                 readmarkers[j - 1] = get_read_marker(get_segment_base(j),
                                                      hashtableobj)->rm;
             }
 
-            acquire_privatization_lock();
             char *p = allocate_outside_nursery_large(
                           sizeof(stm_hashtable_entry_t));
             entry = (stm_hashtable_entry_t *)(p - stm_object_pages);
@@ -324,12 +336,12 @@
                 e->object = NULL;
             }
             hashtable->additions += 0x100;
-            release_privatization_lock();
 
             for (j = 1; j <= NB_SEGMENTS; j++) {
                 get_read_marker(get_segment_base(j), (object_t *)entry)->rm =
                     readmarkers[j - 1];
             }
+            release_privatization_lock();
         }
         write_fence();     /* make sure 'entry' is fully initialized here */
         table->items[i] = entry;
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -26,7 +26,6 @@
 typedef TLPREFIX struct object_s object_t;
 typedef TLPREFIX struct stm_segment_info_s stm_segment_info_t;
 typedef TLPREFIX struct stm_read_marker_s stm_read_marker_t;
-typedef TLPREFIX struct stm_creation_marker_s stm_creation_marker_t;
 typedef TLPREFIX char stm_char;
 
 struct stm_read_marker_s {

From noreply at buildbot.pypy.org  Sun Feb  8 22:22:54 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun,  8 Feb 2015 22:22:54 +0100 (CET)
Subject: [pypy-commit] pypy default: fix for win32
Message-ID: <20150208212254.706E61C0183@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r75776:7b99e3307405
Date: 2015-02-08 23:23 +0200
http://bitbucket.org/pypy/pypy/changeset/7b99e3307405/

Log:	fix for win32

diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py
--- a/rpython/rlib/test/test_rsocket.py
+++ b/rpython/rlib/test/test_rsocket.py
@@ -574,6 +574,7 @@
 
 def test_translate_netdb_lock():
     def f():
+        rsocket_startup()
         gethostbyaddr("localhost")
         return 0
     fc = compile(f, [])
@@ -581,6 +582,7 @@
 
 def test_translate_netdb_lock_thread():
     def f():
+        rsocket_startup()
         gethostbyaddr("localhost")
         return 0
     fc = compile(f, [], thread=True)

From noreply at buildbot.pypy.org  Mon Feb  9 00:17:48 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  9 Feb 2015 00:17:48 +0100 (CET)
Subject: [pypy-commit] stmgc default: The hack to copy around the other
 segment's read markers doesn't
Message-ID: <20150208231748.2ABB81C0849@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1612:e0ff682f6615
Date: 2015-02-09 00:05 +0100
http://bitbucket.org/pypy/stmgc/changeset/e0ff682f6615/

Log:	The hack to copy around the other segment's read markers doesn't
	really seem to work at all. Change the approach.

diff --git a/c7/stm/core.c b/c7/stm/core.c
--- a/c7/stm/core.c
+++ b/c7/stm/core.c
@@ -373,6 +373,7 @@
 
     assert(list_is_empty(STM_PSEGMENT->modified_old_objects));
     assert(list_is_empty(STM_PSEGMENT->modified_old_objects_markers));
+    assert(list_is_empty(STM_PSEGMENT->modified_old_hashtables));
     assert(list_is_empty(STM_PSEGMENT->young_weakrefs));
     assert(tree_is_cleared(STM_PSEGMENT->young_outside_nursery));
     assert(tree_is_cleared(STM_PSEGMENT->nursery_objects_shadows));
@@ -437,29 +438,45 @@
             continue;    /* no need to check: is pending immediate abort */
 
         char *remote_base = get_segment_base(i);
+        object_t *conflicting_obj;
 
         LIST_FOREACH_R(
             STM_PSEGMENT->modified_old_objects,
             object_t * /*item*/,
             ({
                 if (was_read_remote(remote_base, item)) {
-                    /* A write-read conflict! */
-                    dprintf(("write-read conflict on %p, our seg: %d, other: %ld\n",
-                             item, STM_SEGMENT->segment_num, i));
-                    if (write_read_contention_management(i, item)) {
-                        /* If we reach this point, we didn't abort, but we
-                           had to wait for the other thread to commit.  If we
-                           did, then we have to restart committing from our call
-                           to synchronize_all_threads(). */
-                        return true;
-                    }
-                    /* we aborted the other transaction without waiting, so
-                       we can just break out of this loop on
-                       modified_old_objects and continue with the next
-                       segment */
-                    break;
+                    conflicting_obj = item;
+                    goto found_conflict;
                 }
             }));
+
+        LIST_FOREACH_R(
+            STM_PSEGMENT->modified_old_hashtables,
+            object_t * /*item*/,
+            ({
+                if (was_read_remote(remote_base, item)) {
+                    conflicting_obj = item;
+                    goto found_conflict;
+                }
+            }));
+
+        continue;
+
+     found_conflict:
+        /* A write-read conflict! */
+        dprintf(("write-read conflict on %p, our seg: %d, other: %ld\n",
+                 conflicting_obj, STM_SEGMENT->segment_num, i));
+        if (write_read_contention_management(i, conflicting_obj)) {
+            /* If we reach this point, we didn't abort, but we
+               had to wait for the other thread to commit.  If we
+               did, then we have to restart committing from our call
+               to synchronize_all_threads(). */
+            return true;
+        }
+        /* we aborted the other transaction without waiting, so we can
+           just ignore the rest of this (now aborted) segment.  Let's
+           move on to the next one. */
+        continue;
     }
 
     return false;
@@ -789,6 +806,7 @@
 
     list_clear(STM_PSEGMENT->modified_old_objects);
     list_clear(STM_PSEGMENT->modified_old_objects_markers);
+    list_clear(STM_PSEGMENT->modified_old_hashtables);
 }
 
 static void _finish_transaction(enum stm_event_e event)
@@ -950,6 +968,7 @@
 
     list_clear(pseg->modified_old_objects);
     list_clear(pseg->modified_old_objects_markers);
+    list_clear(pseg->modified_old_hashtables);
 }
 
 static void abort_data_structures_from_segment_num(int segment_num)
diff --git a/c7/stm/core.h b/c7/stm/core.h
--- a/c7/stm/core.h
+++ b/c7/stm/core.h
@@ -95,6 +95,14 @@
     struct list_s *modified_old_objects_markers;
     uintptr_t modified_old_objects_markers_num_old;
 
+    /* This list contains all old hashtables that have entries that we
+       modified.  Note that several transactions can all commit if
+       they have the same hashtable listed here.  The point of this
+       list is only that if another segment does a global "read" of
+       the hashtable (stm_hashtable_list), then it conflicts with this
+       segment if it has got any change to the hashtable. */
+    struct list_s *modified_old_hashtables;
+
     /* List of out-of-nursery objects that may contain pointers to
        nursery objects.  This is used to track the GC status: they are
        all objects outside the nursery on which an stm_write() occurred
@@ -281,17 +289,13 @@
 static stm_thread_local_t *abort_with_mutex_no_longjmp(void);
 static void abort_data_structures_from_segment_num(int segment_num);
 
-static inline struct stm_read_marker_s *get_read_marker(char *base,
-                                                        object_t *obj) {
-    return (struct stm_read_marker_s *)(base + (((uintptr_t)obj) >> 4));
-}
-
 static inline bool was_read_remote(char *base, object_t *obj)
 {
     uint8_t other_transaction_read_version =
         ((struct stm_segment_info_s *)REAL_ADDRESS(base, STM_PSEGMENT))
             ->transaction_read_version;
-    uint8_t rm = get_read_marker(base, obj)->rm;
+    uint8_t rm = ((struct stm_read_marker_s *)
+                  (base + (((uintptr_t)obj) >> 4)))->rm;
     assert(rm <= other_transaction_read_version);
     return rm == other_transaction_read_version;
 }
diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c
--- a/c7/stm/gcpage.c
+++ b/c7/stm/gcpage.c
@@ -475,13 +475,26 @@
     }
 }
 
-static void clean_up_segment_lists(void)
-{
 #pragma push_macro("STM_PSEGMENT")
 #pragma push_macro("STM_SEGMENT")
 #undef STM_PSEGMENT
 #undef STM_SEGMENT
 
+static void remove_objects_that_die(struct list_s *lst)
+{
+    if (lst != NULL) {
+        uintptr_t n = list_count(lst);
+        while (n > 0) {
+            object_t *obj = (object_t *)list_item(lst, --n);
+            if (!mark_visited_test(obj)) {
+                list_set_item(lst, n, list_pop_item(lst));
+            }
+        }
+    }
+}
+
+static void clean_up_segment_lists(void)
+{
     long i;
     for (i = 1; i <= NB_SEGMENTS; i++) {
         struct stm_priv_segment_info_s *pseg = get_priv_segment(i);
@@ -540,21 +553,14 @@
             }));
         list_clear(lst);
 
-        /* Remove from 'large_overflow_objects' all objects that die */
-        lst = pseg->large_overflow_objects;
-        if (lst != NULL) {
-            uintptr_t n = list_count(lst);
-            while (n > 0) {
-                object_t *obj = (object_t *)list_item(lst, --n);
-                if (!mark_visited_test(obj)) {
-                    list_set_item(lst, n, list_pop_item(lst));
-                }
-            }
-        }
+        /* Remove from 'large_overflow_objects' and 'modified_old_hashtables'
+           all objects that die */
+        remove_objects_that_die(pseg->large_overflow_objects);
+        remove_objects_that_die(pseg->modified_old_hashtables);
     }
+}
 #pragma pop_macro("STM_SEGMENT")
 #pragma pop_macro("STM_PSEGMENT")
-}
 
 static inline bool largemalloc_keep_object_at(char *data)
 {
diff --git a/c7/stm/hashtable.c b/c7/stm/hashtable.c
--- a/c7/stm/hashtable.c
+++ b/c7/stm/hashtable.c
@@ -297,34 +297,12 @@
                synchronization with other pieces of the code that may
                change.
             */
-
-            /* First fetch the read marker of 'hashtableobj' in all
-               segments, before allocate_outside_nursery_large() which
-               might trigger a GC.  Synchronization guarantee: if
-               stm_read(hobj) in stm_hashtable_list() has set the read
-               marker, then it did synchronize with us here by
-               acquiring and releasing this hashtable' lock.  However,
-               the interval of time between reading the readmarkers of
-               hobj and copying them to the new entry object might be
-               enough for the other threads to do anything, including
-               a reset_transaction_read_version(), so that we might in
-               theory write bogus read markers that are not valid any
-               more.  To prevent this, reset_transaction_read_version()
-               acquires the privatization_lock too.
-            */
-            long j;
-            uint8_t readmarkers[NB_SEGMENTS];
-
             acquire_privatization_lock();
-            for (j = 1; j <= NB_SEGMENTS; j++) {
-                readmarkers[j - 1] = get_read_marker(get_segment_base(j),
-                                                     hashtableobj)->rm;
-            }
-
             char *p = allocate_outside_nursery_large(
                           sizeof(stm_hashtable_entry_t));
             entry = (stm_hashtable_entry_t *)(p - stm_object_pages);
 
+            long j;
             for (j = 0; j <= NB_SEGMENTS; j++) {
                 struct stm_hashtable_entry_s *e;
                 e = (struct stm_hashtable_entry_s *)
@@ -335,11 +313,6 @@
                 e->object = NULL;
             }
             hashtable->additions += 0x100;
-
-            for (j = 1; j <= NB_SEGMENTS; j++) {
-                get_read_marker(get_segment_base(j), (object_t *)entry)->rm =
-                    readmarkers[j - 1];
-            }
             release_privatization_lock();
         }
         write_fence();     /* make sure 'entry' is fully initialized here */
@@ -369,15 +342,44 @@
     return e->object;
 }
 
+void stm_hashtable_write_entry(object_t *hobj, stm_hashtable_entry_t *entry,
+                               object_t *nvalue)
+{
+    if (stm_write((object_t *)entry)) {
+        uintptr_t i = list_count(STM_PSEGMENT->modified_old_objects);
+        if (i > 0 && list_item(STM_PSEGMENT->modified_old_objects, i - 1)
+                     == (uintptr_t)entry) {
+            /* 'modified_old_hashtables' is always obtained by taking
+               a subset of 'modified_old_objects' which contains only
+               stm_hashtable_entry_t objects, and then replacing the
+               stm_hashtable_entry_t objects with the hobj they come
+               from.  It's possible to have duplicates in
+               'modified_old_hashtables'; here we only try a bit to
+               avoid them --- at least the list should never be longer
+               than 'modified_old_objects'. */
+            i = list_count(STM_PSEGMENT->modified_old_hashtables);
+            if (i > 0 && list_item(STM_PSEGMENT->modified_old_hashtables, i - 1)
+                         == (uintptr_t)hobj) {
+                /* already added */
+            }
+            else {
+                LIST_APPEND(STM_PSEGMENT->modified_old_hashtables, hobj);
+            }
+        }
+    }
+    entry->object = nvalue;
+}
+
 void stm_hashtable_write(object_t *hobj, stm_hashtable_t *hashtable,
                          uintptr_t key, object_t *nvalue,
                          stm_thread_local_t *tl)
 {
     STM_PUSH_ROOT(*tl, nvalue);
+    STM_PUSH_ROOT(*tl, hobj);
     stm_hashtable_entry_t *e = stm_hashtable_lookup(hobj, hashtable, key);
-    stm_write((object_t *)e);
+    STM_POP_ROOT(*tl, hobj);
     STM_POP_ROOT(*tl, nvalue);
-    e->object = nvalue;
+    stm_hashtable_write_entry(hobj, e, nvalue);
 }
 
 long stm_hashtable_length_upper_bound(stm_hashtable_t *hashtable)
@@ -401,28 +403,15 @@
 long stm_hashtable_list(object_t *hobj, stm_hashtable_t *hashtable,
                         stm_hashtable_entry_t **results)
 {
-    stm_hashtable_table_t *table;
-    intptr_t rc;
-
     /* Set the read marker.  It will be left as long as we're running
        the same transaction.
     */
     stm_read(hobj);
 
-    /* Acquire and immediately release the lock.  We don't actually
-       need to do anything while we hold the lock, but the point is to
-       wait until the lock is available, and to synchronize other
-       threads with the stm_read() done above.
-     */
- restart:
-    table = VOLATILE_HASHTABLE(hashtable)->table;
-    rc = VOLATILE_TABLE(table)->resize_counter;
-    if (IS_EVEN(rc)) {
-        spin_loop();
-        goto restart;
-    }
-    if (!__sync_bool_compare_and_swap(&table->resize_counter, rc, rc))
-        goto restart;
+    /* Get the table.  No synchronization is needed: we may miss some
+       entries that are being added, but they would contain NULL in
+       this segment anyway. */
+    stm_hashtable_table_t *table = VOLATILE_HASHTABLE(hashtable)->table;
 
     /* Read all entries, check which ones are not NULL, count them,
        and optionally list them in 'results'.
diff --git a/c7/stm/setup.c b/c7/stm/setup.c
--- a/c7/stm/setup.c
+++ b/c7/stm/setup.c
@@ -122,6 +122,7 @@
         pr->large_overflow_objects = NULL;
         pr->modified_old_objects = list_create();
         pr->modified_old_objects_markers = list_create();
+        pr->modified_old_hashtables = list_create();
         pr->young_weakrefs = list_create();
         pr->old_weakrefs = list_create();
         pr->young_outside_nursery = tree_create();
@@ -166,6 +167,7 @@
         assert(pr->large_overflow_objects == NULL);
         list_free(pr->modified_old_objects);
         list_free(pr->modified_old_objects_markers);
+        list_free(pr->modified_old_hashtables);
         list_free(pr->young_weakrefs);
         list_free(pr->old_weakrefs);
         tree_free(pr->young_outside_nursery);
diff --git a/c7/stmgc.h b/c7/stmgc.h
--- a/c7/stmgc.h
+++ b/c7/stmgc.h
@@ -187,10 +187,13 @@
    necessary to call it immediately after stm_allocate().
 */
 __attribute__((always_inline))
-static inline void stm_write(object_t *obj)
+static inline int stm_write(object_t *obj)
 {
-    if (UNLIKELY((obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0))
+    if (UNLIKELY((obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0)) {
         _stm_write_slowpath(obj);
+        return 1;
+    }
+    return 0;
 }
 
 /* The following is a GC-optimized barrier that works on the granularity
@@ -544,6 +547,8 @@
 object_t *stm_hashtable_read(object_t *, stm_hashtable_t *, uintptr_t key);
 void stm_hashtable_write(object_t *, stm_hashtable_t *, uintptr_t key,
                          object_t *nvalue, stm_thread_local_t *);
+void stm_hashtable_write_entry(object_t *hobj, stm_hashtable_entry_t *entry,
+                               object_t *nvalue);
 long stm_hashtable_length_upper_bound(stm_hashtable_t *);
 long stm_hashtable_list(object_t *, stm_hashtable_t *,
                         stm_hashtable_entry_t **results);
diff --git a/c7/test/test_hashtable.py b/c7/test/test_hashtable.py
--- a/c7/test/test_hashtable.py
+++ b/c7/test/test_hashtable.py
@@ -284,6 +284,25 @@
             htset(h, 19 ^ i, stm_allocate(32), tl0)
         assert htlen(h) == 29
 
+    def test_len_conflicts_with_additions(self):
+        self.start_transaction()
+        h = self.allocate_hashtable()
+        self.push_root(h)
+        self.commit_transaction()
+        h = self.pop_root()
+        #
+        self.start_transaction()
+        assert htlen(h) == 0
+        #
+        self.switch(1)
+        self.start_transaction()
+        tl0 = self.tls[self.current_thread]
+        htset(h, 10, stm_allocate(32), tl0)
+        py.test.raises(Conflict, self.commit_transaction)
+        #
+        self.switch(0)
+        stm_major_collect()       # to get rid of the hashtable object
+
 
 class TestRandomHashtable(BaseTestHashtable):
 

From noreply at buildbot.pypy.org  Mon Feb  9 00:19:19 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  9 Feb 2015 00:19:19 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: update to stmgc/e0ff682f6615 and fix
	the RPython code
Message-ID: <20150208231919.A0DF01C0849@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r75777:0f6ff5506d41
Date: 2015-02-09 00:18 +0100
http://bitbucket.org/pypy/pypy/changeset/0f6ff5506d41/

Log:	update to stmgc/e0ff682f6615 and fix the RPython code

diff --git a/pypy/module/pypystm/hashtable.py b/pypy/module/pypystm/hashtable.py
--- a/pypy/module/pypystm/hashtable.py
+++ b/pypy/module/pypystm/hashtable.py
@@ -26,15 +26,14 @@
 
     @unwrap_spec(key=int)
     def setitem_w(self, key, w_value):
-        entry = self.h.lookup(key)
-        entry.object = cast_instance_to_gcref(w_value)
+        self.h.set(key, cast_instance_to_gcref(w_value))
 
     @unwrap_spec(key=int)
     def delitem_w(self, space, key):
         entry = self.h.lookup(key)
         if not entry.object:
             space.raise_key_error(space.wrap(key))
-        entry.object = rstm.NULL_GCREF
+        self.h.writeobj(entry, rstm.NULL_GCREF)
 
     @unwrap_spec(key=int)
     def contains_w(self, space, key):
@@ -53,7 +52,7 @@
         entry = self.h.lookup(key)
         gcref = entry.object
         if not gcref:
-            entry.object = cast_instance_to_gcref(w_default)
+            self.h.writeobj(entry, cast_instance_to_gcref(w_default))
             return w_default
         return cast_gcref_to_instance(W_Root, gcref)
 
diff --git a/pypy/module/pypystm/stmdict.py b/pypy/module/pypystm/stmdict.py
--- a/pypy/module/pypystm/stmdict.py
+++ b/pypy/module/pypystm/stmdict.py
@@ -43,7 +43,7 @@
         for i in range(length):
             dest[dest_start + i] = source[source_start + i]
 
-def pop_from_entry(entry, space, w_key):
+def pop_from_entry(h, entry, space, w_key):
     array = lltype.cast_opaque_ptr(PARRAY, entry.object)
     if not array:
         return None
@@ -59,7 +59,7 @@
         narray = lltype.malloc(ARRAY, L)
         ll_arraycopy(array, narray, 0, 0, i)
         ll_arraycopy(array, narray, i + 2, i, L - i)
-    entry.object = lltype.cast_opaque_ptr(llmemory.GCREF, narray)
+    h.writeobj(entry, lltype.cast_opaque_ptr(llmemory.GCREF, narray))
     return w_value
 
 
@@ -96,12 +96,12 @@
             L = 0
         narray[L] = cast_instance_to_gcref(w_key)
         narray[L + 1] = cast_instance_to_gcref(w_value)
-        entry.object = lltype.cast_opaque_ptr(llmemory.GCREF, narray)
+        self.h.writeobj(entry, lltype.cast_opaque_ptr(llmemory.GCREF, narray))
 
     def delitem_w(self, space, w_key):
         hkey = space.hash_w(w_key)
         entry = self.h.lookup(hkey)
-        if pop_from_entry(entry, space, w_key) is None:
+        if pop_from_entry(self.h, entry, space, w_key) is None:
             space.raise_key_error(w_key)
 
     def contains_w(self, space, w_key):
@@ -126,7 +126,7 @@
     def pop_w(self, space, w_key, w_default=None):
         hkey = space.hash_w(w_key)
         entry = self.h.lookup(hkey)
-        w_value = pop_from_entry(entry, space, w_key)
+        w_value = pop_from_entry(self.h, entry, space, w_key)
         if w_value is not None:
             return w_value
         elif w_default is not None:
@@ -152,7 +152,7 @@
             L = 0
         narray[L] = cast_instance_to_gcref(w_key)
         narray[L + 1] = cast_instance_to_gcref(w_default)
-        entry.object = lltype.cast_opaque_ptr(llmemory.GCREF, narray)
+        self.h.writeobj(entry, lltype.cast_opaque_ptr(llmemory.GCREF, narray))
         return w_default
 
     def get_length(self):
diff --git a/pypy/module/pypystm/stmset.py b/pypy/module/pypystm/stmset.py
--- a/pypy/module/pypystm/stmset.py
+++ b/pypy/module/pypystm/stmset.py
@@ -65,7 +65,7 @@
             narray = lltype.malloc(ARRAY, 1)
             L = 0
         narray[L] = cast_instance_to_gcref(w_key)
-        entry.object = lltype.cast_opaque_ptr(llmemory.GCREF, narray)
+        self.h.writeobj(entry, lltype.cast_opaque_ptr(llmemory.GCREF, narray))
 
     def try_remove(self, space, w_key):
         hkey = space.hash_w(w_key)
@@ -84,7 +84,7 @@
             narray = lltype.malloc(ARRAY, L)
             ll_arraycopy(array, narray, 0, 0, i)
             ll_arraycopy(array, narray, i + 1, i, L - i)
-        entry.object = lltype.cast_opaque_ptr(llmemory.GCREF, narray)
+        self.h.writeobj(entry, lltype.cast_opaque_ptr(llmemory.GCREF, narray))
         return True
 
     def remove_w(self, space, w_key):
diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -218,6 +218,10 @@
     return llop.stm_hashtable_lookup(_STM_HASHTABLE_ENTRY_P,
                                      h, h.ll_raw_hashtable, key)
 
+ at dont_look_inside
+def _ll_hashtable_writeobj(h, entry, value):
+    llop.stm_hashtable_write_entry(lltype.Void, h, entry, value)
+
 _HASHTABLE_OBJ = lltype.GcStruct('HASHTABLE_OBJ',
                                  ('ll_raw_hashtable', _STM_HASHTABLE_P),
                                  rtti=True,
@@ -226,7 +230,8 @@
                                            'len': _ll_hashtable_len,
                                           'list': _ll_hashtable_list,
                                       'freelist': _ll_hashtable_freelist,
-                                        'lookup': _ll_hashtable_lookup})
+                                        'lookup': _ll_hashtable_lookup,
+                                      'writeobj': _ll_hashtable_writeobj})
 NULL_HASHTABLE = lltype.nullptr(_HASHTABLE_OBJ)
 
 def _ll_hashtable_trace(gc, obj, callback, arg):
@@ -303,6 +308,10 @@
         assert type(key) is int
         return EntryObjectForTest(self, key)
 
+    def writeobj(self, entry, nvalue):
+        assert isinstance(entry, EntryObjectForTest)
+        self.set(entry.key, nvalue)
+
 class EntryObjectForTest(object):
     def __init__(self, hashtable, key):
         self.hashtable = hashtable
@@ -312,6 +321,7 @@
     def _getobj(self):
         return self.hashtable.get(self.key)
     def _setobj(self, nvalue):
-        self.hashtable.set(self.key, nvalue)
+        raise Exception("can't assign to the 'object' attribute:"
+                        " use h.writeobj() instead")
 
     object = property(_getobj, _setobj)
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -465,6 +465,7 @@
     'stm_hashtable_read':     LLOp(),
     'stm_hashtable_write':    LLOp(),
     'stm_hashtable_lookup':   LLOp(),
+    'stm_hashtable_write_entry':        LLOp(),
     'stm_hashtable_length_upper_bound': LLOp(),
     'stm_hashtable_list'  :   LLOp(),
     'stm_hashtable_tracefn':  LLOp(),
diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -312,6 +312,13 @@
     return ('stm_hashtable_write((object_t *)%s, %s, %s, (object_t *)%s, '
             '&stm_thread_local);' % (arg0, arg1, arg2, arg3))
 
+def stm_hashtable_write_entry(funcgen, op):
+    arg0 = funcgen.expr(op.args[0])
+    arg1 = funcgen.expr(op.args[1])
+    arg2 = funcgen.expr(op.args[2])
+    return ('stm_hashtable_write_entry((object_t *)%s, %s, (object_t *)%s);' % (
+        arg0, arg1, arg2))
+
 def stm_hashtable_lookup(funcgen, op):
     arg0 = funcgen.expr(op.args[0])
     arg1 = funcgen.expr(op.args[1])
diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-005668d99755
+e0ff682f6615
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -374,6 +374,7 @@
 
     assert(list_is_empty(STM_PSEGMENT->modified_old_objects));
     assert(list_is_empty(STM_PSEGMENT->modified_old_objects_markers));
+    assert(list_is_empty(STM_PSEGMENT->modified_old_hashtables));
     assert(list_is_empty(STM_PSEGMENT->young_weakrefs));
     assert(tree_is_cleared(STM_PSEGMENT->young_outside_nursery));
     assert(tree_is_cleared(STM_PSEGMENT->nursery_objects_shadows));
@@ -438,29 +439,45 @@
             continue;    /* no need to check: is pending immediate abort */
 
         char *remote_base = get_segment_base(i);
+        object_t *conflicting_obj;
 
         LIST_FOREACH_R(
             STM_PSEGMENT->modified_old_objects,
             object_t * /*item*/,
             ({
                 if (was_read_remote(remote_base, item)) {
-                    /* A write-read conflict! */
-                    dprintf(("write-read conflict on %p, our seg: %d, other: %ld\n",
-                             item, STM_SEGMENT->segment_num, i));
-                    if (write_read_contention_management(i, item)) {
-                        /* If we reach this point, we didn't abort, but we
-                           had to wait for the other thread to commit.  If we
-                           did, then we have to restart committing from our call
-                           to synchronize_all_threads(). */
-                        return true;
-                    }
-                    /* we aborted the other transaction without waiting, so
-                       we can just break out of this loop on
-                       modified_old_objects and continue with the next
-                       segment */
-                    break;
+                    conflicting_obj = item;
+                    goto found_conflict;
                 }
             }));
+
+        LIST_FOREACH_R(
+            STM_PSEGMENT->modified_old_hashtables,
+            object_t * /*item*/,
+            ({
+                if (was_read_remote(remote_base, item)) {
+                    conflicting_obj = item;
+                    goto found_conflict;
+                }
+            }));
+
+        continue;
+
+     found_conflict:
+        /* A write-read conflict! */
+        dprintf(("write-read conflict on %p, our seg: %d, other: %ld\n",
+                 conflicting_obj, STM_SEGMENT->segment_num, i));
+        if (write_read_contention_management(i, conflicting_obj)) {
+            /* If we reach this point, we didn't abort, but we
+               had to wait for the other thread to commit.  If we
+               did, then we have to restart committing from our call
+               to synchronize_all_threads(). */
+            return true;
+        }
+        /* we aborted the other transaction without waiting, so we can
+           just ignore the rest of this (now aborted) segment.  Let's
+           move on to the next one. */
+        continue;
     }
 
     return false;
@@ -790,6 +807,7 @@
 
     list_clear(STM_PSEGMENT->modified_old_objects);
     list_clear(STM_PSEGMENT->modified_old_objects_markers);
+    list_clear(STM_PSEGMENT->modified_old_hashtables);
 }
 
 static void _finish_transaction(enum stm_event_e event)
@@ -951,6 +969,7 @@
 
     list_clear(pseg->modified_old_objects);
     list_clear(pseg->modified_old_objects_markers);
+    list_clear(pseg->modified_old_hashtables);
 }
 
 static void abort_data_structures_from_segment_num(int segment_num)
diff --git a/rpython/translator/stm/src_stm/stm/core.h b/rpython/translator/stm/src_stm/stm/core.h
--- a/rpython/translator/stm/src_stm/stm/core.h
+++ b/rpython/translator/stm/src_stm/stm/core.h
@@ -96,6 +96,14 @@
     struct list_s *modified_old_objects_markers;
     uintptr_t modified_old_objects_markers_num_old;
 
+    /* This list contains all old hashtables that have entries that we
+       modified.  Note that several transactions can all commit if
+       they have the same hashtable listed here.  The point of this
+       list is only that if another segment does a global "read" of
+       the hashtable (stm_hashtable_list), then it conflicts with this
+       segment if it has got any change to the hashtable. */
+    struct list_s *modified_old_hashtables;
+
     /* List of out-of-nursery objects that may contain pointers to
        nursery objects.  This is used to track the GC status: they are
        all objects outside the nursery on which an stm_write() occurred
@@ -282,17 +290,13 @@
 static stm_thread_local_t *abort_with_mutex_no_longjmp(void);
 static void abort_data_structures_from_segment_num(int segment_num);
 
-static inline struct stm_read_marker_s *get_read_marker(char *base,
-                                                        object_t *obj) {
-    return (struct stm_read_marker_s *)(base + (((uintptr_t)obj) >> 4));
-}
-
 static inline bool was_read_remote(char *base, object_t *obj)
 {
     uint8_t other_transaction_read_version =
         ((struct stm_segment_info_s *)REAL_ADDRESS(base, STM_PSEGMENT))
             ->transaction_read_version;
-    uint8_t rm = get_read_marker(base, obj)->rm;
+    uint8_t rm = ((struct stm_read_marker_s *)
+                  (base + (((uintptr_t)obj) >> 4)))->rm;
     assert(rm <= other_transaction_read_version);
     return rm == other_transaction_read_version;
 }
diff --git a/rpython/translator/stm/src_stm/stm/gcpage.c b/rpython/translator/stm/src_stm/stm/gcpage.c
--- a/rpython/translator/stm/src_stm/stm/gcpage.c
+++ b/rpython/translator/stm/src_stm/stm/gcpage.c
@@ -476,13 +476,26 @@
     }
 }
 
-static void clean_up_segment_lists(void)
-{
 #pragma push_macro("STM_PSEGMENT")
 #pragma push_macro("STM_SEGMENT")
 #undef STM_PSEGMENT
 #undef STM_SEGMENT
 
+static void remove_objects_that_die(struct list_s *lst)
+{
+    if (lst != NULL) {
+        uintptr_t n = list_count(lst);
+        while (n > 0) {
+            object_t *obj = (object_t *)list_item(lst, --n);
+            if (!mark_visited_test(obj)) {
+                list_set_item(lst, n, list_pop_item(lst));
+            }
+        }
+    }
+}
+
+static void clean_up_segment_lists(void)
+{
     long i;
     for (i = 1; i <= NB_SEGMENTS; i++) {
         struct stm_priv_segment_info_s *pseg = get_priv_segment(i);
@@ -541,21 +554,14 @@
             }));
         list_clear(lst);
 
-        /* Remove from 'large_overflow_objects' all objects that die */
-        lst = pseg->large_overflow_objects;
-        if (lst != NULL) {
-            uintptr_t n = list_count(lst);
-            while (n > 0) {
-                object_t *obj = (object_t *)list_item(lst, --n);
-                if (!mark_visited_test(obj)) {
-                    list_set_item(lst, n, list_pop_item(lst));
-                }
-            }
-        }
+        /* Remove from 'large_overflow_objects' and 'modified_old_hashtables'
+           all objects that die */
+        remove_objects_that_die(pseg->large_overflow_objects);
+        remove_objects_that_die(pseg->modified_old_hashtables);
     }
+}
 #pragma pop_macro("STM_SEGMENT")
 #pragma pop_macro("STM_PSEGMENT")
-}
 
 static inline bool largemalloc_keep_object_at(char *data)
 {
diff --git a/rpython/translator/stm/src_stm/stm/hashtable.c b/rpython/translator/stm/src_stm/stm/hashtable.c
--- a/rpython/translator/stm/src_stm/stm/hashtable.c
+++ b/rpython/translator/stm/src_stm/stm/hashtable.c
@@ -298,34 +298,12 @@
                synchronization with other pieces of the code that may
                change.
             */
-
-            /* First fetch the read marker of 'hashtableobj' in all
-               segments, before allocate_outside_nursery_large() which
-               might trigger a GC.  Synchronization guarantee: if
-               stm_read(hobj) in stm_hashtable_list() has set the read
-               marker, then it did synchronize with us here by
-               acquiring and releasing this hashtable' lock.  However,
-               the interval of time between reading the readmarkers of
-               hobj and copying them to the new entry object might be
-               enough for the other threads to do anything, including
-               a reset_transaction_read_version(), so that we might in
-               theory write bogus read markers that are not valid any
-               more.  To prevent this, reset_transaction_read_version()
-               acquires the privatization_lock too.
-            */
-            long j;
-            uint8_t readmarkers[NB_SEGMENTS];
-
             acquire_privatization_lock();
-            for (j = 1; j <= NB_SEGMENTS; j++) {
-                readmarkers[j - 1] = get_read_marker(get_segment_base(j),
-                                                     hashtableobj)->rm;
-            }
-
             char *p = allocate_outside_nursery_large(
                           sizeof(stm_hashtable_entry_t));
             entry = (stm_hashtable_entry_t *)(p - stm_object_pages);
 
+            long j;
             for (j = 0; j <= NB_SEGMENTS; j++) {
                 struct stm_hashtable_entry_s *e;
                 e = (struct stm_hashtable_entry_s *)
@@ -336,11 +314,6 @@
                 e->object = NULL;
             }
             hashtable->additions += 0x100;
-
-            for (j = 1; j <= NB_SEGMENTS; j++) {
-                get_read_marker(get_segment_base(j), (object_t *)entry)->rm =
-                    readmarkers[j - 1];
-            }
             release_privatization_lock();
         }
         write_fence();     /* make sure 'entry' is fully initialized here */
@@ -370,15 +343,44 @@
     return e->object;
 }
 
+void stm_hashtable_write_entry(object_t *hobj, stm_hashtable_entry_t *entry,
+                               object_t *nvalue)
+{
+    if (stm_write((object_t *)entry)) {
+        uintptr_t i = list_count(STM_PSEGMENT->modified_old_objects);
+        if (i > 0 && list_item(STM_PSEGMENT->modified_old_objects, i - 1)
+                     == (uintptr_t)entry) {
+            /* 'modified_old_hashtables' is always obtained by taking
+               a subset of 'modified_old_objects' which contains only
+               stm_hashtable_entry_t objects, and then replacing the
+               stm_hashtable_entry_t objects with the hobj they come
+               from.  It's possible to have duplicates in
+               'modified_old_hashtables'; here we only try a bit to
+               avoid them --- at least the list should never be longer
+               than 'modified_old_objects'. */
+            i = list_count(STM_PSEGMENT->modified_old_hashtables);
+            if (i > 0 && list_item(STM_PSEGMENT->modified_old_hashtables, i - 1)
+                         == (uintptr_t)hobj) {
+                /* already added */
+            }
+            else {
+                LIST_APPEND(STM_PSEGMENT->modified_old_hashtables, hobj);
+            }
+        }
+    }
+    entry->object = nvalue;
+}
+
 void stm_hashtable_write(object_t *hobj, stm_hashtable_t *hashtable,
                          uintptr_t key, object_t *nvalue,
                          stm_thread_local_t *tl)
 {
     STM_PUSH_ROOT(*tl, nvalue);
+    STM_PUSH_ROOT(*tl, hobj);
     stm_hashtable_entry_t *e = stm_hashtable_lookup(hobj, hashtable, key);
-    stm_write((object_t *)e);
+    STM_POP_ROOT(*tl, hobj);
     STM_POP_ROOT(*tl, nvalue);
-    e->object = nvalue;
+    stm_hashtable_write_entry(hobj, e, nvalue);
 }
 
 long stm_hashtable_length_upper_bound(stm_hashtable_t *hashtable)
@@ -402,28 +404,15 @@
 long stm_hashtable_list(object_t *hobj, stm_hashtable_t *hashtable,
                         stm_hashtable_entry_t **results)
 {
-    stm_hashtable_table_t *table;
-    intptr_t rc;
-
     /* Set the read marker.  It will be left as long as we're running
        the same transaction.
     */
     stm_read(hobj);
 
-    /* Acquire and immediately release the lock.  We don't actually
-       need to do anything while we hold the lock, but the point is to
-       wait until the lock is available, and to synchronize other
-       threads with the stm_read() done above.
-     */
- restart:
-    table = VOLATILE_HASHTABLE(hashtable)->table;
-    rc = VOLATILE_TABLE(table)->resize_counter;
-    if (IS_EVEN(rc)) {
-        spin_loop();
-        goto restart;
-    }
-    if (!__sync_bool_compare_and_swap(&table->resize_counter, rc, rc))
-        goto restart;
+    /* Get the table.  No synchronization is needed: we may miss some
+       entries that are being added, but they would contain NULL in
+       this segment anyway. */
+    stm_hashtable_table_t *table = VOLATILE_HASHTABLE(hashtable)->table;
 
     /* Read all entries, check which ones are not NULL, count them,
        and optionally list them in 'results'.
diff --git a/rpython/translator/stm/src_stm/stm/setup.c b/rpython/translator/stm/src_stm/stm/setup.c
--- a/rpython/translator/stm/src_stm/stm/setup.c
+++ b/rpython/translator/stm/src_stm/stm/setup.c
@@ -123,6 +123,7 @@
         pr->large_overflow_objects = NULL;
         pr->modified_old_objects = list_create();
         pr->modified_old_objects_markers = list_create();
+        pr->modified_old_hashtables = list_create();
         pr->young_weakrefs = list_create();
         pr->old_weakrefs = list_create();
         pr->young_outside_nursery = tree_create();
@@ -167,6 +168,7 @@
         assert(pr->large_overflow_objects == NULL);
         list_free(pr->modified_old_objects);
         list_free(pr->modified_old_objects_markers);
+        list_free(pr->modified_old_hashtables);
         list_free(pr->young_weakrefs);
         list_free(pr->old_weakrefs);
         tree_free(pr->young_outside_nursery);
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -188,10 +188,13 @@
    necessary to call it immediately after stm_allocate().
 */
 __attribute__((always_inline))
-static inline void stm_write(object_t *obj)
+static inline int stm_write(object_t *obj)
 {
-    if (UNLIKELY((obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0))
+    if (UNLIKELY((obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0)) {
         _stm_write_slowpath(obj);
+        return 1;
+    }
+    return 0;
 }
 
 /* The following is a GC-optimized barrier that works on the granularity
@@ -545,6 +548,8 @@
 object_t *stm_hashtable_read(object_t *, stm_hashtable_t *, uintptr_t key);
 void stm_hashtable_write(object_t *, stm_hashtable_t *, uintptr_t key,
                          object_t *nvalue, stm_thread_local_t *);
+void stm_hashtable_write_entry(object_t *hobj, stm_hashtable_entry_t *entry,
+                               object_t *nvalue);
 long stm_hashtable_length_upper_bound(stm_hashtable_t *);
 long stm_hashtable_list(object_t *, stm_hashtable_t *,
                         stm_hashtable_entry_t **results);

From noreply at buildbot.pypy.org  Mon Feb  9 02:51:23 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  9 Feb 2015 02:51:23 +0100 (CET)
Subject: [pypy-commit] pypy default: It seems that in an obscure case,
 get_constant_class() is called on a
Message-ID: <20150209015123.A89551C0849@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r75778:97c2a0a39120
Date: 2015-02-09 02:51 +0100
http://bitbucket.org/pypy/pypy/changeset/97c2a0a39120/

Log:	It seems that in an obscure case, get_constant_class() is called on
	a box that is the constant null. Can't reproduce it with small
	tests though...

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -304,7 +304,7 @@
         level = self.getlevel()
         if level == LEVEL_KNOWNCLASS:
             return self.known_class
-        elif level == LEVEL_CONSTANT:
+        elif level == LEVEL_CONSTANT and not self.is_null():
             return cpu.ts.cls_of_box(self.box)
         else:
             return None

From noreply at buildbot.pypy.org  Mon Feb  9 10:22:02 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  9 Feb 2015 10:22:02 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: There are cases here where the list of
 pending setups is not empty after
Message-ID: <20150209092202.841C71C026B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r75779:c59b920f11dc
Date: 2015-02-09 10:21 +0100
http://bitbucket.org/pypy/pypy/changeset/c59b920f11dc/

Log:	There are cases here where the list of pending setups is not empty
	after specialize_block(). Empty it during the same transaction.
	Otherwise in rare cases some reprs stay non-setup()ed for a non-
	deterministic while and then other reprs' setup crash

diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py
--- a/rpython/rtyper/rtyper.py
+++ b/rpython/rtyper/rtyper.py
@@ -345,6 +345,17 @@
         return LowLevelOpList(self, block)
 
     def specialize_block(self, block):
+        lst = self._list_must_call_setup()
+        assert lst == []
+        self._specialize_block(block)
+        # There are cases here where the list of pending setups is not empty.
+        # Better empty it now during the same transaction.  Otherwise in rare
+        # cases some reprs stay non-setup()ed for a non-deterministic while
+        # and then other reprs' setup crash
+        while lst:
+            self.call_all_setups()
+
+    def _specialize_block(self, block):
         graph = self.annotator.annotated[block]
         if graph not in self.annotator.fixed_graphs:
             self.annotator.fixed_graphs.add(graph)

From noreply at buildbot.pypy.org  Mon Feb  9 11:46:49 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  9 Feb 2015 11:46:49 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: clean up the codemap in free()
Message-ID: <20150209104649.30F921C026B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r75780:c90b8022a288
Date: 2015-02-09 12:46 +0200
http://bitbucket.org/pypy/pypy/changeset/c90b8022a288/

Log:	clean up the codemap in free()

diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py
--- a/rpython/jit/backend/llsupport/asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/asmmemmgr.py
@@ -61,6 +61,10 @@
                              self.jit_addr_map[jit_adr_stop:])
         self.jit_frame_depth_map = (self.jit_frame_depth_map[:jit_adr_start] +
                                     self.jit_frame_depth_map[jit_adr_stop:])
+        # fix up codemap
+        codemap_adr = bisect_tuple(self.jit_codemap, start)
+        self.jit_codemap = (self.jit_codemap[:codemap_adr] +
+                            self.jit_codemap[codemap_adr + 1:])
 
     def open_malloc(self, minsize):
         """Allocate at least minsize bytes.  Returns (start, stop)."""

From noreply at buildbot.pypy.org  Mon Feb  9 12:20:37 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  9 Feb 2015 12:20:37 +0100 (CET)
Subject: [pypy-commit] stmgc default: can be compiled with
	"-DUSE_REMAP_FILE_PAGES=0"
Message-ID: <20150209112037.088731C0473@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1613:94964b956e2a
Date: 2015-02-09 12:20 +0100
http://bitbucket.org/pypy/stmgc/changeset/94964b956e2a/

Log:	can be compiled with "-DUSE_REMAP_FILE_PAGES=0"

diff --git a/c7/stm/pages.h b/c7/stm/pages.h
--- a/c7/stm/pages.h
+++ b/c7/stm/pages.h
@@ -19,7 +19,13 @@
 #define PAGE_FLAG_START   END_NURSERY_PAGE
 #define PAGE_FLAG_END     NB_PAGES
 
-#define USE_REMAP_FILE_PAGES
+/* can be compiled with "-DUSE_REMAP_FILE_PAGES=0" */
+#ifndef USE_REMAP_FILE_PAGES
+#  define USE_REMAP_FILE_PAGES   1
+#endif
+#if !USE_REMAP_FILE_PAGES
+#  undef USE_REMAP_FILE_PAGES
+#endif
 
 struct page_shared_s {
 #if NB_SEGMENTS <= 8

From noreply at buildbot.pypy.org  Mon Feb  9 15:57:15 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  9 Feb 2015 15:57:15 +0100 (CET)
Subject: [pypy-commit] stmgc default: Kill this,
	left around by mistake from e0ff682f6615
Message-ID: <20150209145715.1EC3F1C0103@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1614:a0ff85ec0506
Date: 2015-02-09 14:41 +0100
http://bitbucket.org/pypy/stmgc/changeset/a0ff85ec0506/

Log:	Kill this, left around by mistake from e0ff682f6615

diff --git a/c7/stm/core.c b/c7/stm/core.c
--- a/c7/stm/core.c
+++ b/c7/stm/core.c
@@ -313,10 +313,6 @@
     dprintf(("reset_transaction_read_version: %p %ld\n", readmarkers,
              (long)(NB_READMARKER_PAGES * 4096UL)));
 
-    /* see hashtable.c for why we need the privatization lock here
-       (grep for reset_transaction_read_version)
-    */
-    acquire_privatization_lock();
     if (mmap(readmarkers, NB_READMARKER_PAGES * 4096UL,
              PROT_READ | PROT_WRITE,
              MAP_FIXED | MAP_PAGES_FLAGS, -1, 0) != readmarkers) {
@@ -327,7 +323,6 @@
         memset(readmarkers, 0, NB_READMARKER_PAGES * 4096UL);
     }
     STM_SEGMENT->transaction_read_version = 1;
-    release_privatization_lock();
 }
 
 static uint64_t _global_start_time = 0;

From noreply at buildbot.pypy.org  Mon Feb  9 15:57:16 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  9 Feb 2015 15:57:16 +0100 (CET)
Subject: [pypy-commit] stmgc default: Simplify the forking logic by
 enforcing that stm_object_pages are
Message-ID: <20150209145716.446FC1C0103@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1615:ac07eddb02d5
Date: 2015-02-09 15:57 +0100
http://bitbucket.org/pypy/stmgc/changeset/ac07eddb02d5/

Log:	Simplify the forking logic by enforcing that stm_object_pages are
	private pages all the way to END_NURSERY_PAGE, and shared pages
	afterwards. Removes the need for page_is_null(), which seems one
	the main sources of fork slowness.

diff --git a/c7/stm/core.c b/c7/stm/core.c
--- a/c7/stm/core.c
+++ b/c7/stm/core.c
@@ -309,18 +309,19 @@
     /* force-reset all read markers to 0 */
 
     char *readmarkers = REAL_ADDRESS(STM_SEGMENT->segment_base,
-                                     FIRST_READMARKER_PAGE * 4096UL);
+                                     FIRST_OLD_RM_PAGE * 4096UL);
+    uintptr_t num_bytes = 4096UL *
+        (NB_READMARKER_PAGES - (FIRST_OLD_RM_PAGE - FIRST_READMARKER_PAGE));
+
     dprintf(("reset_transaction_read_version: %p %ld\n", readmarkers,
-             (long)(NB_READMARKER_PAGES * 4096UL)));
+             (long)num_bytes));
 
-    if (mmap(readmarkers, NB_READMARKER_PAGES * 4096UL,
+    if (mmap(readmarkers, num_bytes,
              PROT_READ | PROT_WRITE,
-             MAP_FIXED | MAP_PAGES_FLAGS, -1, 0) != readmarkers) {
-        /* fall-back */
-#if STM_TESTS
+             MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE,
+             -1, 0) != readmarkers) {
+        /* failed */
         stm_fatalerror("reset_transaction_read_version: %m");
-#endif
-        memset(readmarkers, 0, NB_READMARKER_PAGES * 4096UL);
     }
     STM_SEGMENT->transaction_read_version = 1;
 }
diff --git a/c7/stm/forksupport.c b/c7/stm/forksupport.c
--- a/c7/stm/forksupport.c
+++ b/c7/stm/forksupport.c
@@ -12,16 +12,6 @@
 static stm_thread_local_t *fork_this_tl;
 static bool fork_was_in_transaction;
 
-static bool page_is_null(char *p)
-{
-    long *q = (long *)p;
-    long i;
-    for (i = 0; i < 4096 / sizeof(long); i++)
-        if (q[i] != 0)
-            return false;
-    return true;
-}
-
 
 static void forksupport_prepare(void)
 {
@@ -71,27 +61,6 @@
     int big_copy_fd;
     char *big_copy = setup_mmap("stmgc's fork support", &big_copy_fd);
 
-    /* Copy each of the segment infos into the new mmap, nurseries,
-       and associated read markers
-     */
-    long i;
-    for (i = 1; i <= NB_SEGMENTS; i++) {
-        char *src, *dst;
-        struct stm_priv_segment_info_s *psrc = get_priv_segment(i);
-        dst = big_copy + (((char *)psrc) - stm_object_pages);
-        *(struct stm_priv_segment_info_s *)dst = *psrc;
-
-        src = get_segment_base(i) + FIRST_READMARKER_PAGE * 4096UL;
-        dst = big_copy + (src - stm_object_pages);
-        long j;
-        for (j = 0; j < END_NURSERY_PAGE - FIRST_READMARKER_PAGE; j++) {
-            if (!page_is_null(src))
-                pagecopy(dst, src);
-            src += 4096;
-            dst += 4096;
-        }
-    }
-
     /* Copy all the data from the two ranges of objects (large, small)
        into the new mmap
     */
@@ -201,16 +170,24 @@
        just release these locks early */
     s_mutex_unlock();
 
-    /* Move the copy of the mmap over the old one, overwriting it
-       and thus freeing the old mapping in this process
+    /* Move the copy of the mmap over the old one, overwriting it,
+       with "holes" for each segment's read markers (which are already
+       MAP_PRIVATE and shouldn't be overwritten).  Then free the copy.
     */
     assert(fork_big_copy != NULL);
     assert(stm_object_pages != NULL);
-    void *res = mremap(fork_big_copy, TOTAL_MEMORY, TOTAL_MEMORY,
-                       MREMAP_MAYMOVE | MREMAP_FIXED,
-                       stm_object_pages);
-    if (res != stm_object_pages)
-        stm_fatalerror("after fork: mremap failed: %m");
+
+    long j;
+    for (j = 0; j <= NB_SEGMENTS; j++) {
+        char *dst = get_segment_base(j) + END_NURSERY_PAGE * 4096UL;
+        char *src = fork_big_copy + (dst - stm_object_pages);
+        uintptr_t num_bytes = (NB_PAGES - END_NURSERY_PAGE) * 4096UL;
+        void *res = mremap(src, num_bytes, num_bytes,
+                           MREMAP_MAYMOVE | MREMAP_FIXED, dst);
+        if (res != dst)
+            stm_fatalerror("after fork: mremap failed: %m");
+    }
+    munmap(fork_big_copy, TOTAL_MEMORY);
     fork_big_copy = NULL;
     close_fd_mmap(stm_object_pages_fd);
     stm_object_pages_fd = fork_big_copy_fd;
diff --git a/c7/stm/setup.c b/c7/stm/setup.c
--- a/c7/stm/setup.c
+++ b/c7/stm/setup.c
@@ -99,9 +99,21 @@
 
     stm_object_pages = setup_mmap("initial stm_object_pages mmap()",
                                   &stm_object_pages_fd);
+
+    /* remap MAP_PRIVATE pages on the initial part of each segment
+       in stm_object_pages */
+    long i;
+    for (i = 0; i <= NB_SEGMENTS; i++) {
+        char *res;
+        res = mmap(get_segment_base(i), END_NURSERY_PAGE * 4096UL,
+                   PROT_READ | PROT_WRITE,
+                   MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE,
+                   -1, 0);
+        if (res == MAP_FAILED)
+            stm_fatalerror("mmap(private) failed: %m");
+    }
     setup_protection_settings();
 
-    long i;
     for (i = 1; i <= NB_SEGMENTS; i++) {
         char *segment_base = get_segment_base(i);
 
@@ -133,16 +145,11 @@
         pr->old_objects_with_light_finalizers = list_create();
         pr->overflow_number = GCFLAG_OVERFLOW_NUMBER_bit0 * i;
         highest_overflow_number = pr->overflow_number;
-        pr->pub.transaction_read_version = 0xff;
+        pr->pub.transaction_read_version = 0;
     }
 
     /* The pages are shared lazily, as remap_file_pages() takes a relatively
        long time for each page.
-
-       The read markers are initially zero, but we set anyway
-       transaction_read_version to 0xff in order to force the first
-       transaction to "clear" the read markers by mapping a different,
-       private range of addresses.
     */
 
     setup_sync();

From noreply at buildbot.pypy.org  Mon Feb  9 15:59:02 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  9 Feb 2015 15:59:02 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: update to stmgc/ac07eddb02d5
Message-ID: <20150209145902.E60071C0103@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r75781:b5a988e2bfab
Date: 2015-02-09 15:58 +0100
http://bitbucket.org/pypy/pypy/changeset/b5a988e2bfab/

Log:	update to stmgc/ac07eddb02d5

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-e0ff682f6615
+ac07eddb02d5
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -310,25 +310,21 @@
     /* force-reset all read markers to 0 */
 
     char *readmarkers = REAL_ADDRESS(STM_SEGMENT->segment_base,
-                                     FIRST_READMARKER_PAGE * 4096UL);
+                                     FIRST_OLD_RM_PAGE * 4096UL);
+    uintptr_t num_bytes = 4096UL *
+        (NB_READMARKER_PAGES - (FIRST_OLD_RM_PAGE - FIRST_READMARKER_PAGE));
+
     dprintf(("reset_transaction_read_version: %p %ld\n", readmarkers,
-             (long)(NB_READMARKER_PAGES * 4096UL)));
+             (long)num_bytes));
 
-    /* see hashtable.c for why we need the privatization lock here
-       (grep for reset_transaction_read_version)
-    */
-    acquire_privatization_lock();
-    if (mmap(readmarkers, NB_READMARKER_PAGES * 4096UL,
+    if (mmap(readmarkers, num_bytes,
              PROT_READ | PROT_WRITE,
-             MAP_FIXED | MAP_PAGES_FLAGS, -1, 0) != readmarkers) {
-        /* fall-back */
-#if STM_TESTS
+             MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE,
+             -1, 0) != readmarkers) {
+        /* failed */
         stm_fatalerror("reset_transaction_read_version: %m");
-#endif
-        memset(readmarkers, 0, NB_READMARKER_PAGES * 4096UL);
     }
     STM_SEGMENT->transaction_read_version = 1;
-    release_privatization_lock();
 }
 
 static uint64_t _global_start_time = 0;
diff --git a/rpython/translator/stm/src_stm/stm/forksupport.c b/rpython/translator/stm/src_stm/stm/forksupport.c
--- a/rpython/translator/stm/src_stm/stm/forksupport.c
+++ b/rpython/translator/stm/src_stm/stm/forksupport.c
@@ -13,16 +13,6 @@
 static stm_thread_local_t *fork_this_tl;
 static bool fork_was_in_transaction;
 
-static bool page_is_null(char *p)
-{
-    long *q = (long *)p;
-    long i;
-    for (i = 0; i < 4096 / sizeof(long); i++)
-        if (q[i] != 0)
-            return false;
-    return true;
-}
-
 
 static void forksupport_prepare(void)
 {
@@ -72,27 +62,6 @@
     int big_copy_fd;
     char *big_copy = setup_mmap("stmgc's fork support", &big_copy_fd);
 
-    /* Copy each of the segment infos into the new mmap, nurseries,
-       and associated read markers
-     */
-    long i;
-    for (i = 1; i <= NB_SEGMENTS; i++) {
-        char *src, *dst;
-        struct stm_priv_segment_info_s *psrc = get_priv_segment(i);
-        dst = big_copy + (((char *)psrc) - stm_object_pages);
-        *(struct stm_priv_segment_info_s *)dst = *psrc;
-
-        src = get_segment_base(i) + FIRST_READMARKER_PAGE * 4096UL;
-        dst = big_copy + (src - stm_object_pages);
-        long j;
-        for (j = 0; j < END_NURSERY_PAGE - FIRST_READMARKER_PAGE; j++) {
-            if (!page_is_null(src))
-                pagecopy(dst, src);
-            src += 4096;
-            dst += 4096;
-        }
-    }
-
     /* Copy all the data from the two ranges of objects (large, small)
        into the new mmap
     */
@@ -202,16 +171,24 @@
        just release these locks early */
     s_mutex_unlock();
 
-    /* Move the copy of the mmap over the old one, overwriting it
-       and thus freeing the old mapping in this process
+    /* Move the copy of the mmap over the old one, overwriting it,
+       with "holes" for each segment's read markers (which are already
+       MAP_PRIVATE and shouldn't be overwritten).  Then free the copy.
     */
     assert(fork_big_copy != NULL);
     assert(stm_object_pages != NULL);
-    void *res = mremap(fork_big_copy, TOTAL_MEMORY, TOTAL_MEMORY,
-                       MREMAP_MAYMOVE | MREMAP_FIXED,
-                       stm_object_pages);
-    if (res != stm_object_pages)
-        stm_fatalerror("after fork: mremap failed: %m");
+
+    long j;
+    for (j = 0; j <= NB_SEGMENTS; j++) {
+        char *dst = get_segment_base(j) + END_NURSERY_PAGE * 4096UL;
+        char *src = fork_big_copy + (dst - stm_object_pages);
+        uintptr_t num_bytes = (NB_PAGES - END_NURSERY_PAGE) * 4096UL;
+        void *res = mremap(src, num_bytes, num_bytes,
+                           MREMAP_MAYMOVE | MREMAP_FIXED, dst);
+        if (res != dst)
+            stm_fatalerror("after fork: mremap failed: %m");
+    }
+    munmap(fork_big_copy, TOTAL_MEMORY);
     fork_big_copy = NULL;
     close_fd_mmap(stm_object_pages_fd);
     stm_object_pages_fd = fork_big_copy_fd;
diff --git a/rpython/translator/stm/src_stm/stm/pages.h b/rpython/translator/stm/src_stm/stm/pages.h
--- a/rpython/translator/stm/src_stm/stm/pages.h
+++ b/rpython/translator/stm/src_stm/stm/pages.h
@@ -20,7 +20,13 @@
 #define PAGE_FLAG_START   END_NURSERY_PAGE
 #define PAGE_FLAG_END     NB_PAGES
 
-#define USE_REMAP_FILE_PAGES
+/* can be compiled with "-DUSE_REMAP_FILE_PAGES=0" */
+#ifndef USE_REMAP_FILE_PAGES
+#  define USE_REMAP_FILE_PAGES   1
+#endif
+#if !USE_REMAP_FILE_PAGES
+#  undef USE_REMAP_FILE_PAGES
+#endif
 
 struct page_shared_s {
 #if NB_SEGMENTS <= 8
diff --git a/rpython/translator/stm/src_stm/stm/setup.c b/rpython/translator/stm/src_stm/stm/setup.c
--- a/rpython/translator/stm/src_stm/stm/setup.c
+++ b/rpython/translator/stm/src_stm/stm/setup.c
@@ -100,9 +100,21 @@
 
     stm_object_pages = setup_mmap("initial stm_object_pages mmap()",
                                   &stm_object_pages_fd);
+
+    /* remap MAP_PRIVATE pages on the initial part of each segment
+       in stm_object_pages */
+    long i;
+    for (i = 0; i <= NB_SEGMENTS; i++) {
+        char *res;
+        res = mmap(get_segment_base(i), END_NURSERY_PAGE * 4096UL,
+                   PROT_READ | PROT_WRITE,
+                   MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE,
+                   -1, 0);
+        if (res == MAP_FAILED)
+            stm_fatalerror("mmap(private) failed: %m");
+    }
     setup_protection_settings();
 
-    long i;
     for (i = 1; i <= NB_SEGMENTS; i++) {
         char *segment_base = get_segment_base(i);
 
@@ -134,16 +146,11 @@
         pr->old_objects_with_light_finalizers = list_create();
         pr->overflow_number = GCFLAG_OVERFLOW_NUMBER_bit0 * i;
         highest_overflow_number = pr->overflow_number;
-        pr->pub.transaction_read_version = 0xff;
+        pr->pub.transaction_read_version = 0;
     }
 
     /* The pages are shared lazily, as remap_file_pages() takes a relatively
        long time for each page.
-
-       The read markers are initially zero, but we set anyway
-       transaction_read_version to 0xff in order to force the first
-       transaction to "clear" the read markers by mapping a different,
-       private range of addresses.
     */
 
     setup_sync();

From noreply at buildbot.pypy.org  Mon Feb  9 16:58:05 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  9 Feb 2015 16:58:05 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: call_all_setup() must run iteratively
 until there is no more setup to call.
Message-ID: <20150209155805.8174F1C026B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r75782:4c668d843284
Date: 2015-02-09 16:57 +0100
http://bitbucket.org/pypy/pypy/changeset/4c668d843284/

Log:	call_all_setup() must run iteratively until there is no more setup
	to call.

diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py
--- a/rpython/rtyper/rtyper.py
+++ b/rpython/rtyper/rtyper.py
@@ -303,21 +303,30 @@
 
     def call_all_setups(self, all_threads=False):
         # make sure all reprs so far have had their setup() called
-        must_setup_more = []
-        if all_threads:
-            lsts = self._all_lists_must_call_setup
-        else:
-            lsts = [self._list_must_call_setup()]
-        for lst in lsts:
-            while lst:
-                r = lst.pop()
-                if r.is_setup_delayed():
-                    pass    # will be re-added in set_setup_delayed(False)
+        while True:
+            if all_threads:
+                lsts = self._all_lists_must_call_setup
+                for lst in lsts:
+                    if lst:
+                        break
                 else:
-                    r.setup()
-                    must_setup_more.append(r)
-        for r in must_setup_more:
-            r.setup_final()
+                    return      # nothing to do
+            else:
+                lst = self._list_must_call_setup()
+                if not lst:
+                    return      # nothing to do
+                lsts = [lst]
+            must_setup_more = []
+            for lst in lsts:
+                while lst:
+                    r = lst.pop()
+                    if r.is_setup_delayed():
+                        pass    # will be re-added in set_setup_delayed(False)
+                    else:
+                        r.setup()
+                        must_setup_more.append(r)
+            for r in must_setup_more:
+                r.setup_final()
 
     def setconcretetype(self, v):
         assert isinstance(v, Variable)
@@ -352,8 +361,7 @@
         # Better empty it now during the same transaction.  Otherwise in rare
         # cases some reprs stay non-setup()ed for a non-deterministic while
         # and then other reprs' setup crash
-        while lst:
-            self.call_all_setups()
+        self.call_all_setups()
 
     def _specialize_block(self, block):
         graph = self.annotator.annotated[block]

From noreply at buildbot.pypy.org  Mon Feb  9 18:44:42 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  9 Feb 2015 18:44:42 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: fixes
Message-ID: <20150209174442.6D0971C0013@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r75783:6819c000c078
Date: 2015-02-09 18:24 +0100
http://bitbucket.org/pypy/pypy/changeset/6819c000c078/

Log:	fixes

diff --git a/rpython/rtyper/rlist.py b/rpython/rtyper/rlist.py
--- a/rpython/rtyper/rlist.py
+++ b/rpython/rtyper/rlist.py
@@ -57,7 +57,8 @@
                 return FixedSizeListRepr(rtyper, item_repr, listitem)
 
     def rtyper_makekey(self):
-        self.listdef.listitem.dont_change_any_more = True
+        if not self.listdef.listitem.dont_change_any_more:
+            self.listdef.listitem.dont_change_any_more = True
         return self.__class__, self.listdef.listitem
 
 
diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py
--- a/rpython/rtyper/rtyper.py
+++ b/rpython/rtyper/rtyper.py
@@ -48,6 +48,10 @@
         self.reprs = stmdict()
         self._seen_reprs_must_call_setup = stmset()
         self._all_lists_must_call_setup = []
+        try:
+            del annmodel.TLS._reprs_must_call_setup
+        except AttributeError:
+            pass
         self._dict_traits = {}
         self.rootclass_repr = RootClassRepr(self)
         self.rootclass_repr.setup()
@@ -306,10 +310,7 @@
         while True:
             if all_threads:
                 lsts = self._all_lists_must_call_setup
-                for lst in lsts:
-                    if lst:
-                        break
-                else:
+                if not any(lsts):
                     return      # nothing to do
             else:
                 lst = self._list_must_call_setup()

From noreply at buildbot.pypy.org  Mon Feb  9 18:44:43 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  9 Feb 2015 18:44:43 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Improve the tests. Seems for some
 reason that *only with the JIT* we're getting the wrong location reported
 for
Message-ID: <20150209174443.9210D1C0013@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r75784:a4514bd45065
Date: 2015-02-09 18:44 +0100
http://bitbucket.org/pypy/pypy/changeset/a4514bd45065/

Log:	Improve the tests. Seems for some reason that *only with the JIT*
	we're getting the wrong location reported for write-read conflicts .

diff --git a/pypy/module/pypystm/test_pypy_c/support.py b/pypy/module/pypystm/test_pypy_c/support.py
--- a/pypy/module/pypystm/test_pypy_c/support.py
+++ b/pypy/module/pypystm/test_pypy_c/support.py
@@ -101,8 +101,10 @@
             f.write(str(src) + '\n')
             f.write("print %s(%s)\n" % (funcname, arglist))
 
-    def _execute(self, import_site=False):
+    def _execute(self, import_site=False, jit=True):
         cmdline = [sys.executable]
+        if not jit:
+            cmdline.extend(['--jit', 'off'])
         if not import_site:
             cmdline.append('-S')
         cmdline.append(str(self.filepath))
@@ -125,22 +127,27 @@
         from pypy.stm.print_stm_log import StmLog
         return StmLog(str(self.logfile))
 
-    def _check_count_conflicts(self, func_or_src, args=[]):
+    def _check_count_conflicts(self, func_or_src, args=[], **kwds):
         self._write_source(func_or_src, args)
-        self._execute()
-        stmlog = self._parse_log()
-        count = stmlog.get_total_aborts_and_pauses()
+        self._execute(**kwds)
+        self.stmlog = self._parse_log()
+        self.stmlog.dump()
+        count = self.stmlog.get_total_aborts_and_pauses()
         print 'stmlog.get_total_aborts_and_pauses():', count
         return count
 
-    def check_almost_no_conflict(self, *args):
-        count = self._check_count_conflicts(*args)
+    def check_almost_no_conflict(self, *args, **kwds):
+        count = self._check_count_conflicts(*args, **kwds)
         assert count < 500
 
-    def check_MANY_conflicts(self, *args):
-        count = self._check_count_conflicts(*args)
+    def check_MANY_conflicts(self, *args, **kwds):
+        count = self._check_count_conflicts(*args, **kwds)
         assert count > 20000
 
-    def check_SOME_conflicts(self, *args):
-        count = self._check_count_conflicts(*args)
+    def check_SOME_conflicts(self, *args, **kwds):
+        count = self._check_count_conflicts(*args, **kwds)
         assert count > 1000
+
+    def check_conflict_location(self, text):
+        first_conflict = self.stmlog.get_conflicts()[0]
+        assert first_conflict.get_marker1().rstrip().endswith(text)
diff --git a/pypy/module/pypystm/test_pypy_c/test_conflict.py b/pypy/module/pypystm/test_pypy_c/test_conflict.py
--- a/pypy/module/pypystm/test_pypy_c/test_conflict.py
+++ b/pypy/module/pypystm/test_pypy_c/test_conflict.py
@@ -3,17 +3,21 @@
 
 class TestConflict(BaseTestSTM):
 
-    def test_obvious(self):
+    def test_obvious(self, jit=False):
         def f():
             class X(object):
                 pass
             x = X()     # shared
             x.a = 0
             def g():
-                x.a += 1
+                x.a += 1      #loc1
             run_in_threads(g)
         #
-        self.check_MANY_conflicts(f)
+        self.check_MANY_conflicts(f, jit=jit)
+        self.check_conflict_location("#loc1")
+
+    def test_obvious_with_jit(self):
+        self.test_obvious(jit=True)
 
     def test_plain_dict_access(self):
         def f():
@@ -44,3 +48,23 @@
             run_in_threads(g, arg_thread_num=True)
         #
         self.check_SOME_conflicts(f)
+
+    def test_plain_write_read(self, jit=False):
+        def f():
+            class X(object):
+                pass
+            x = X()
+            x.a = 0
+            def g(tnum):
+                if tnum == 0:
+                    x.a += 1      #loc2
+                else:
+                    if x.a < 0:
+                        raise AssertionError
+            run_in_threads(g, arg_thread_num=True)
+        #
+        self.check_SOME_conflicts(f, jit=jit)
+        self.check_conflict_location("#loc2")
+
+    def test_plain_write_read_with_jit(self):
+        self.test_plain_write_read(jit=True)
diff --git a/pypy/stm/print_stm_log.py b/pypy/stm/print_stm_log.py
--- a/pypy/stm/print_stm_log.py
+++ b/pypy/stm/print_stm_log.py
@@ -146,9 +146,19 @@
     def sortkey(self):
         return self.aborted_time + self.paused_time
 
+    def get_event_name(self):
+        return event_name[self.event]
+
+    def get_marker1(self):
+        return print_marker(self.marker1)
+
+    def get_marker2(self):
+        return print_marker(self.marker2)
+
     def __str__(self):
         s = '%.3fs lost in aborts, %.3fs paused (%dx %s)\n' % (
-            self.aborted_time, self.paused_time, self.num_events, event_name[self.event])
+            self.aborted_time, self.paused_time, self.num_events,
+            self.get_event_name())
         s += print_marker(self.marker1)
         if self.marker2:
             s += '\n%s' % print_marker(self.marker2)

From noreply at buildbot.pypy.org  Mon Feb  9 19:11:14 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  9 Feb 2015 19:11:14 +0100 (CET)
Subject: [pypy-commit] stmgc default: fix the read-write conflict location
 in case of a conflict between
Message-ID: <20150209181114.51D471C0013@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1616:65a89cd8a6c0
Date: 2015-02-09 19:11 +0100
http://bitbucket.org/pypy/stmgc/changeset/65a89cd8a6c0/

Log:	fix the read-write conflict location in case of a conflict between a
	stmdict.keys() and a write of an entry

diff --git a/c7/stm/core.c b/c7/stm/core.c
--- a/c7/stm/core.c
+++ b/c7/stm/core.c
@@ -435,26 +435,31 @@
 
         char *remote_base = get_segment_base(i);
         object_t *conflicting_obj;
+        uintptr_t j, limit;
+        struct list_s *lst;
 
-        LIST_FOREACH_R(
-            STM_PSEGMENT->modified_old_objects,
-            object_t * /*item*/,
-            ({
-                if (was_read_remote(remote_base, item)) {
-                    conflicting_obj = item;
-                    goto found_conflict;
-                }
-            }));
+        /* Look in forward order: this is an attempt to report the _first_
+           write that conflicts with another segment's reads
+        */
+        lst = STM_PSEGMENT->modified_old_objects;
+        limit = list_count(lst);
+        for (j = 0; j < limit; j++) {
+            object_t *obj = (object_t *)list_item(lst, j);
+            if (was_read_remote(remote_base, obj)) {
+                conflicting_obj = obj;
+                goto found_conflict;
+            }
+        }
 
-        LIST_FOREACH_R(
-            STM_PSEGMENT->modified_old_hashtables,
-            object_t * /*item*/,
-            ({
-                if (was_read_remote(remote_base, item)) {
-                    conflicting_obj = item;
-                    goto found_conflict;
-                }
-            }));
+        lst = STM_PSEGMENT->modified_old_hashtables;
+        limit = list_count(lst);
+        for (j = 0; j < limit; j += 2) {
+            object_t *hobj = (object_t *)list_item(lst, j);
+            if (was_read_remote(remote_base, hobj)) {
+                conflicting_obj = (object_t *)list_item(lst, j + 1);
+                goto found_conflict;
+            }
+        }
 
         continue;
 
diff --git a/c7/stm/core.h b/c7/stm/core.h
--- a/c7/stm/core.h
+++ b/c7/stm/core.h
@@ -96,7 +96,8 @@
     uintptr_t modified_old_objects_markers_num_old;
 
     /* This list contains all old hashtables that have entries that we
-       modified.  Note that several transactions can all commit if
+       modified.  It's actually a list of pairs hashtable/sample_entry.
+       Note that several transactions can all commit if
        they have the same hashtable listed here.  The point of this
        list is only that if another segment does a global "read" of
        the hashtable (stm_hashtable_list), then it conflicts with this
diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c
--- a/c7/stm/gcpage.c
+++ b/c7/stm/gcpage.c
@@ -475,26 +475,13 @@
     }
 }
 
+static void clean_up_segment_lists(void)
+{
 #pragma push_macro("STM_PSEGMENT")
 #pragma push_macro("STM_SEGMENT")
 #undef STM_PSEGMENT
 #undef STM_SEGMENT
 
-static void remove_objects_that_die(struct list_s *lst)
-{
-    if (lst != NULL) {
-        uintptr_t n = list_count(lst);
-        while (n > 0) {
-            object_t *obj = (object_t *)list_item(lst, --n);
-            if (!mark_visited_test(obj)) {
-                list_set_item(lst, n, list_pop_item(lst));
-            }
-        }
-    }
-}
-
-static void clean_up_segment_lists(void)
-{
     long i;
     for (i = 1; i <= NB_SEGMENTS; i++) {
         struct stm_priv_segment_info_s *pseg = get_priv_segment(i);
@@ -553,14 +540,39 @@
             }));
         list_clear(lst);
 
-        /* Remove from 'large_overflow_objects' and 'modified_old_hashtables'
-           all objects that die */
-        remove_objects_that_die(pseg->large_overflow_objects);
-        remove_objects_that_die(pseg->modified_old_hashtables);
+        /* Remove from 'large_overflow_objects' all objects that die */
+        lst = pseg->large_overflow_objects;
+        if (lst != NULL) {
+            uintptr_t n = list_count(lst);
+            while (n > 0) {
+                object_t *obj = (object_t *)list_item(lst, --n);
+                if (!mark_visited_test(obj))
+                    list_set_item(lst, n, list_pop_item(lst));
+            }
+        }
+
+        /* Remove from 'modified_old_hashtables' all old hashtables that
+           die, but keeping the order */
+        {
+            lst = pseg->modified_old_hashtables;
+            uintptr_t j, k = 0, limit = list_count(lst);
+            for (j = 0; j < limit; j += 2) {
+                object_t *hobj = (object_t *)list_item(lst, j);
+                if (mark_visited_test(hobj)) {
+                    /* hobj does not die */
+                    if (j != k) {
+                        list_set_item(lst, k, (uintptr_t)hobj);
+                        list_set_item(lst, k + 1, list_item(lst, j + 1));
+                    }
+                    k += 2;
+                }
+            }
+            lst->count = k;
+        }
     }
-}
 #pragma pop_macro("STM_SEGMENT")
 #pragma pop_macro("STM_PSEGMENT")
+}
 
 static inline bool largemalloc_keep_object_at(char *data)
 {
diff --git a/c7/stm/hashtable.c b/c7/stm/hashtable.c
--- a/c7/stm/hashtable.c
+++ b/c7/stm/hashtable.c
@@ -358,12 +358,13 @@
                avoid them --- at least the list should never be longer
                than 'modified_old_objects'. */
             i = list_count(STM_PSEGMENT->modified_old_hashtables);
-            if (i > 0 && list_item(STM_PSEGMENT->modified_old_hashtables, i - 1)
+            if (i > 0 && list_item(STM_PSEGMENT->modified_old_hashtables, i - 2)
                          == (uintptr_t)hobj) {
                 /* already added */
             }
             else {
-                LIST_APPEND(STM_PSEGMENT->modified_old_hashtables, hobj);
+                LIST_APPEND2(STM_PSEGMENT->modified_old_hashtables,
+                             hobj, entry);
             }
         }
     }
diff --git a/c7/stm/list.h b/c7/stm/list.h
--- a/c7/stm/list.h
+++ b/c7/stm/list.h
@@ -45,6 +45,9 @@
     return lst;
 }
 
+#define LIST_APPEND2(lst, e1, e2)   ((lst) = list_append2((lst),   \
+                                        (uintptr_t)(e1), (uintptr_t)(e2)))
+
 
 static inline void list_clear(struct list_s *lst)
 {

From noreply at buildbot.pypy.org  Mon Feb  9 22:42:25 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  9 Feb 2015 22:42:25 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Dump at least the bytecode position
 part of the stm_location to the logs
Message-ID: <20150209214225.6C91A1C0013@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r75785:95064c6140d9
Date: 2015-02-09 21:12 +0100
http://bitbucket.org/pypy/pypy/changeset/95064c6140d9/

Log:	Dump at least the bytecode position part of the stm_location to the
	logs

diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py
--- a/rpython/jit/metainterp/logger.py
+++ b/rpython/jit/metainterp/logger.py
@@ -132,11 +132,16 @@
             return '?'
 
     def repr_of_resop(self, op, ops_offset=None):
+        if op.stm_location is None or op.stm_location.num == 0:
+            stmloc = ''
+        else:
+            stmloc = ' {%d}' % op.stm_location.num
         if op.getopnum() == rop.DEBUG_MERGE_POINT:
             jd_sd = self.metainterp_sd.jitdrivers_sd[op.getarg(0).getint()]
             s = jd_sd.warmstate.get_location_str(op.getarglist()[3:])
             s = s.replace(',', '.') # we use comma for argument splitting
-            return "debug_merge_point(%d, %d, '%s')" % (op.getarg(1).getint(), op.getarg(2).getint(), s)
+            return "debug_merge_point(%d, %d, '%s')%s" % (
+                op.getarg(1).getint(), op.getarg(2).getint(), s, stmloc)
         if op.getopnum() == rop.JIT_DEBUG:
             args = op.getarglist()
             s = args[0]._get_str()
@@ -144,7 +149,7 @@
             s2 = ''
             for box in args[1:]:
                 s2 += ', %d' % box.getint()
-            return "jit_debug('%s'%s)" % (s, s2)
+            return "jit_debug('%s'%s)%s" % (s, s2, stmloc)
         if ops_offset is None:
             offset = -1
         else:
@@ -176,7 +181,8 @@
                                           for arg in op.getfailargs()]) + ']'
         else:
             fail_args = ''
-        return s_offset + res + op.getopname() + '(' + args + ')' + fail_args
+        return '%s%s%s(%s)%s%s' % (s_offset, res, op.getopname(),
+                                   args, fail_args, stmloc)
 
     def _log_inputarg_setup_ops(self, op):
         target_token = op.getdescr()

From noreply at buildbot.pypy.org  Mon Feb  9 22:42:27 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  9 Feb 2015 22:42:27 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: kill an old unused attribute
Message-ID: <20150209214227.B4AB61C0013@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r75787:95e31f4ec9dd
Date: 2015-02-09 21:15 +0100
http://bitbucket.org/pypy/pypy/changeset/95e31f4ec9dd/

Log:	kill an old unused attribute

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -46,7 +46,6 @@
     name = metainterp.staticdata.stats.name_for_new_loop()
     loop = TreeLoop(name_prefix + name)
     loop.call_pure_results = metainterp.call_pure_results
-    loop.stm_info = {}
     return loop
 
 
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -630,7 +630,6 @@
     inputargs = None
     operations = None
     call_pure_results = None
-    stm_info = None
     logops = None
     quasi_immutable_deps = None
 
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -390,7 +390,6 @@
         self.optearlyforce = None
         if loop is not None:
             self.call_pure_results = loop.call_pure_results
-            self.stm_info = loop.stm_info
 
         self.set_optimizations(optimizations)
         self.setup()
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py b/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_multilabel.py
@@ -32,7 +32,6 @@
                   if op.getopnum()==rop.LABEL]
         prv = 0
         last_label = []
-        stm_info = {}
         for nxt in labels + [len(loop.operations)]:
             assert prv != nxt
             operations = last_label + loop.operations[prv:nxt]
@@ -45,7 +44,7 @@
             part.operations = operations
 
             self.add_guard_future_condition(part)
-            self._do_optimize_loop(part, None, stm_info)
+            self._do_optimize_loop(part, None)
             if part.operations[-1].getopnum() == rop.LABEL:
                 last_label = [part.operations.pop()]
             else:
@@ -497,7 +496,7 @@
 
 class BaseTestOptimizerRenamingBoxes(BaseTestMultiLabel):
 
-    def _do_optimize_loop(self, loop, call_pure_results, stminfo):
+    def _do_optimize_loop(self, loop, call_pure_results):
         from rpython.jit.metainterp.optimizeopt.unroll import optimize_unroll
         from rpython.jit.metainterp.optimizeopt.util import args_dict
         from rpython.jit.metainterp.optimizeopt.pure import OptPure
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -104,7 +104,7 @@
         if loop.operations[-1].getopnum() == rop.JUMP:
             loop.operations[-1].setdescr(token)
         expected = convert_old_style_to_targets(self.parse(optops), jump=True)
-        self._do_optimize_loop(loop, call_pure_results, {})
+        self._do_optimize_loop(loop, call_pure_results)
         print '\n'.join([str(o) for o in loop.operations])
         self.assert_equal(loop, expected)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -391,14 +391,12 @@
                             expected.operations, False, remap, text_right,
                             expect_stm_locations_from_right=True)
 
-    def _do_optimize_loop(self, loop, call_pure_results,
-                          stm_info):
+    def _do_optimize_loop(self, loop, call_pure_results):
         from rpython.jit.metainterp.optimizeopt import optimize_trace
         from rpython.jit.metainterp.optimizeopt.util import args_dict
 
         self.loop = loop
         loop.call_pure_results = args_dict()
-        loop.stm_info = stm_info
         if call_pure_results is not None:
             for k, v in call_pure_results.items():
                 loop.call_pure_results[list(k)] = v
@@ -430,8 +428,7 @@
         preamble.operations = [ResOperation(rop.LABEL, inputargs, None, descr=TargetToken(token))] + \
                               operations +  \
                               [ResOperation(rop.LABEL, jump_args, None, descr=token)]
-        stm_info = {}
-        self._do_optimize_loop(preamble, call_pure_results, stm_info)
+        self._do_optimize_loop(preamble, call_pure_results)
 
         assert preamble.operations[-1].getopnum() == rop.LABEL
 
@@ -445,7 +442,7 @@
         assert loop.operations[0].getopnum() == rop.LABEL
         loop.inputargs = loop.operations[0].getarglist()
 
-        self._do_optimize_loop(loop, call_pure_results, stm_info)
+        self._do_optimize_loop(loop, call_pure_results)
         extra_same_as = []
         while loop.operations[0].getopnum() != rop.LABEL:
             extra_same_as.append(loop.operations[0])

From noreply at buildbot.pypy.org  Mon Feb  9 22:42:26 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  9 Feb 2015 22:42:26 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: tweaks
Message-ID: <20150209214226.91E971C0013@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r75786:fbaaff38d9d6
Date: 2015-02-09 21:12 +0100
http://bitbucket.org/pypy/pypy/changeset/fbaaff38d9d6/

Log:	tweaks

diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -159,6 +159,8 @@
                 iteritems.sort(key=lambda (x, y): x.sort_key())
             for ofs, value in iteritems:
                 subbox = value.force_box(optforce)
+                # STM note: recording stm_location is pointless, because
+                # it should not fail here: we just allocated 'box'
                 op = ResOperation(rop.SETFIELD_GC, [box, subbox], None,
                                   descr=ofs)
                 optforce.emit_operation(op)
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -87,6 +87,7 @@
         if descr is None:
             descr = self.getdescr()
         newop = ResOperation(opnum, args, result, descr)
+        newop.stm_location = self.stm_location
         return newop
 
     def clone(self):

From noreply at buildbot.pypy.org  Mon Feb  9 22:42:28 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  9 Feb 2015 22:42:28 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Add the original stm_location to
 resume's pending_setfields. Needed, 
Message-ID: <20150209214228.E33B21C0013@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r75788:03e15ee4e282
Date: 2015-02-09 22:42 +0100
http://bitbucket.org/pypy/pypy/changeset/03e15ee4e282/

Log:	Add the original stm_location to resume's pending_setfields.
	Needed, otherwise such setfields that occur at the start of a bridge
	have lost their origin and will be incorrectly reported.

diff --git a/TODO b/TODO
--- a/TODO
+++ b/TODO
@@ -146,6 +146,12 @@
 stm_read(p125)
 cond_call_gc_wb_array(p125...)    # don't need the stm_read maybe?
 
+------------------------------------------------------------
+
+we should fake the stm_location inside jit/metainterp, so that it
+is reported correctly even if we're (1) tracing, (2) blackholing,
+or (3) in ResumeDataDirectReader.change_stm_location
+
 
 
 ===============================================================================
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -23,8 +23,8 @@
 from rpython.jit.codewriter import longlong
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr,
-    ConstFloat, BoxInt, BoxFloat, INT, REF, FLOAT, TargetToken)
-from rpython.jit.metainterp.resoperation import rop, ResOperation, StmLocation
+    ConstFloat, BoxInt, BoxFloat, INT, REF, FLOAT, TargetToken, StmLocation)
+from rpython.jit.metainterp.resoperation import rop, ResOperation
 from rpython.rlib import rgc
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.rarithmetic import r_longlong, r_uint
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -776,6 +776,16 @@
         op.stm_location = self.stm_location
         self.operations.append(op)
 
+    def set_stm_location(self, num, ref):
+        old = self.stm_location
+        if old is None or old.num != num or old.ref != ref:
+            self.stm_location = StmLocation(num, ref)
+
+class StmLocation(object):
+    def __init__(self, num, ref):
+        self.num = num
+        self.ref = ref
+
 # ____________________________________________________________
 
 
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -428,7 +428,8 @@
                 else:
                     assert 0
                 pendingfields.append((op.getdescr(), value.box,
-                                      fieldvalue.get_key_box(), itemindex))
+                                      fieldvalue.get_key_box(), itemindex,
+                                      op.stm_location))
             else:
                 cf.force_lazy_setfield(self)
         return pendingfields
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -12,7 +12,7 @@
 from rpython.jit.metainterp.jitprof import EmptyProfiler
 from rpython.jit.metainterp.logger import Logger
 from rpython.jit.metainterp.optimizeopt.util import args_dict
-from rpython.jit.metainterp.resoperation import rop, StmLocation
+from rpython.jit.metainterp.resoperation import rop
 from rpython.rlib import nonconst, rstack
 from rpython.rlib.debug import debug_start, debug_stop, debug_print
 from rpython.rlib.debug import have_debug_prints, make_sure_not_resized
@@ -1165,8 +1165,7 @@
                 idx_num, idx_ref = report_location
                 num = greenkey[idx_num].getint()
                 ref = greenkey[idx_ref].getref_base()
-                location = StmLocation(num, ref)
-                self.metainterp.history.stm_location = location
+                self.metainterp.history.set_stm_location(num, ref)
         #
         self.metainterp.history.record(rop.DEBUG_MERGE_POINT, args, None)
 
@@ -2173,9 +2172,8 @@
             #
             if (self.staticdata.config.translation.stm and
                     isinstance(key, compile.ResumeGuardDescr)):
-                location = StmLocation(key.stm_location_int,
-                                       key.stm_location_ref)
-                self.history.stm_location = location
+                self.history.set_stm_location(key.stm_location_int,
+                                              key.stm_location_ref)
             #
             self.interpret()
         except SwitchToBlackhole, stb:
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -374,12 +374,6 @@
         self._args[i] = box
 
 
-class StmLocation(object):
-    def __init__(self, num, ref):
-        self.num = num
-        self.ref = ref
-
-
 # ____________________________________________________________
 
 _oplist = [
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -1,7 +1,7 @@
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.metainterp import jitprof
 from rpython.jit.metainterp.history import (Box, Const, ConstInt, getkind,
-    BoxInt, BoxPtr, BoxFloat, INT, REF, FLOAT, AbstractDescr)
+    BoxInt, BoxPtr, BoxFloat, INT, REF, FLOAT, AbstractDescr, StmLocation)
 from rpython.jit.metainterp.resoperation import rop
 from rpython.rlib import rarithmetic, rstack
 from rpython.rlib.objectmodel import (we_are_translated, specialize,
@@ -95,7 +95,9 @@
                                    ('lldescr', OBJECTPTR),
                                    ('num', rffi.SHORT),
                                    ('fieldnum', rffi.SHORT),
-                                   ('itemindex', rffi.INT))
+                                   ('itemindex', rffi.INT),
+                                   ('llstmloc', OBJECTPTR)) # xxx make this last
+                                                        # field optional on stm?
 PENDINGFIELDSP = lltype.Ptr(lltype.GcArray(PENDINGFIELDSTRUCT))
 
 TAGMASK = 3
@@ -378,7 +380,7 @@
                 value = optimizer.getvalue(box)
                 value.visitor_walk_recursive(self)
 
-        for _, box, fieldbox, _ in pending_setfields:
+        for _, box, fieldbox, _, _ in pending_setfields:
             self.register_box(box)
             self.register_box(fieldbox)
             value = optimizer.getvalue(fieldbox)
@@ -458,7 +460,7 @@
             n = len(pending_setfields)
             rd_pendingfields = lltype.malloc(PENDINGFIELDSP.TO, n)
             for i in range(n):
-                descr, box, fieldbox, itemindex = pending_setfields[i]
+                descr, box, fieldbox, itemindex, stmloc = pending_setfields[i]
                 lldescr = annlowlevel.cast_instance_to_base_ptr(descr)
                 num = self._gettagged(box)
                 fieldnum = self._gettagged(fieldbox)
@@ -471,6 +473,9 @@
                 rd_pendingfields[i].num = num
                 rd_pendingfields[i].fieldnum = fieldnum
                 rd_pendingfields[i].itemindex = itemindex
+                if stmloc is not None:
+                    llstmloc = annlowlevel.cast_instance_to_base_ptr(stmloc)
+                    rd_pendingfields[i].llstmloc = llstmloc
         self.storage.rd_pendingfields = rd_pendingfields
 
     def _gettagged(self, box):
@@ -880,14 +885,17 @@
                 num = pendingfields[i].num
                 fieldnum = pendingfields[i].fieldnum
                 itemindex = pendingfields[i].itemindex
+                llstmloc = pendingfields[i].llstmloc
                 descr = annlowlevel.cast_base_ptr_to_instance(AbstractDescr,
                                                               lldescr)
                 struct = self.decode_ref(num)
                 itemindex = rffi.cast(lltype.Signed, itemindex)
+                saved = self.change_stm_location(llstmloc)
                 if itemindex < 0:
                     self.setfield(struct, fieldnum, descr)
                 else:
                     self.setarrayitem(struct, itemindex, fieldnum, descr)
+                self.restore_stm_location(saved)
 
     def setarrayitem(self, array, index, fieldnum, arraydescr):
         if arraydescr.is_array_of_pointers():
@@ -918,6 +926,15 @@
         value = self.decode_float(self.cur_numb.nums[index])
         self.write_a_float(register_index, value)
 
+    def change_stm_location(self, ignored):
+        # xxx maybe we should really use this STM location in
+        # ResumeDataDirectReader, but later
+        pass
+
+    def restore_stm_location(self, ignored):
+        pass
+
+
 # ---------- when resuming for pyjitpl.py, make boxes ----------
 
 def rebuild_from_resumedata(metainterp, storage, deadframe,
@@ -1181,6 +1198,18 @@
         return self.metainterp.execute_and_record(rop.INT_ADD, None, intbox,
                                                   ConstInt(offset))
 
+    def change_stm_location(self, llstmloc):
+        if self.metainterp.jitdriver_sd.stm_report_location is None:
+            return None
+        saved_stm_location = self.metainterp.history.stm_location
+        stmloc = annlowlevel.cast_base_ptr_to_instance(StmLocation, llstmloc)
+        self.metainterp.history.stm_location = stmloc
+        return saved_stm_location
+
+    def restore_stm_location(self, stmloc):
+        self.metainterp.history.stm_location = stmloc
+
+
 # ---------- when resuming for blackholing, get direct values ----------
 
 def blackhole_from_resumedata(blackholeinterpbuilder, jitdriver_sd, storage,
diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -8,6 +8,7 @@
 from rpython.jit.metainterp.resume import *
 from rpython.jit.metainterp.history import BoxInt, BoxPtr, ConstInt
 from rpython.jit.metainterp.history import ConstPtr, ConstFloat
+from rpython.jit.metainterp.history import History, StmLocation
 from rpython.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin
 from rpython.jit.metainterp import executor
 from rpython.jit.codewriter import heaptracker, longlong
@@ -138,6 +139,9 @@
     _already_allocated_resume_virtuals = None
     callinfocollection = None
 
+    class jitdriver_sd:
+        stm_report_location = 'yes please'
+
     def __init__(self, cpu=None):
         if cpu is None:
             cpu = LLtypeMixin.cpu
@@ -145,6 +149,7 @@
         self.trace = []
         self.framestack = []
         self.resboxes = []
+        self.history = History()
 
     def newframe(self, jitcode):
         frame = FakeFrame(jitcode, -1)
@@ -157,6 +162,8 @@
                            list(argboxes),
                            resbox,
                            descr))
+        if self.history.stm_location is not None:
+            self.trace[-1] += (self.history.stm_location,)
         if resbox is not None:
             self.resboxes.append(resbox)
         return resbox
@@ -1378,7 +1385,9 @@
     liveboxes = []
     modifier._number_virtuals(liveboxes, FakeOptimizer(values), 0)
     assert liveboxes == [b2s, b4s] or liveboxes == [b4s, b2s]
-    modifier._add_pending_fields([(LLtypeMixin.nextdescr, b2s, b4s, -1)])
+    stmloc = StmLocation(43, 'foobar')
+    modifier._add_pending_fields([(LLtypeMixin.nextdescr, b2s, b4s, -1,
+                                   stmloc)])
     storage.rd_consts = memo.consts[:]
     storage.rd_numb = None
     # resume
@@ -1391,7 +1400,7 @@
     reader = ResumeDataFakeReader(storage, newboxes, metainterp)
     assert reader.virtuals_cache is None
     trace = metainterp.trace
-    b2set = (rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.nextdescr)
+    b2set = (rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.nextdescr, stmloc)
     expected = [b2set]
 
     for x, y in zip(expected, trace):
@@ -1414,7 +1423,7 @@
     modifier = ResumeDataVirtualAdder(storage, None)
     modifier.liveboxes_from_env = {42: rffi.cast(rffi.SHORT, 1042),
                                    61: rffi.cast(rffi.SHORT, 1061)}
-    modifier._add_pending_fields([(field_a, 42, 61, -1)])
+    modifier._add_pending_fields([(field_a, 42, 61, -1, None)])
     pf = storage.rd_pendingfields
     assert len(pf) == 1
     assert (annlowlevel.cast_base_ptr_to_instance(FieldDescr, pf[0].lldescr)
@@ -1430,8 +1439,8 @@
                                    61: rffi.cast(rffi.SHORT, 1061),
                                    62: rffi.cast(rffi.SHORT, 1062),
                                    63: rffi.cast(rffi.SHORT, 1063)}
-    modifier._add_pending_fields([(array_a, 42, 61, 0),
-                                  (array_a, 42, 62, 2147483647)])
+    modifier._add_pending_fields([(array_a, 42, 61, 0, None),
+                                  (array_a, 42, 62, 2147483647, None)])
     pf = storage.rd_pendingfields
     assert len(pf) == 2
     assert (annlowlevel.cast_base_ptr_to_instance(FieldDescr, pf[0].lldescr)
@@ -1446,7 +1455,7 @@
     assert rffi.cast(lltype.Signed, pf[1].itemindex) == 2147483647
     #
     py.test.raises(TagOverflow, modifier._add_pending_fields,
-                   [(array_a, 42, 63, 2147483648)])
+                   [(array_a, 42, 63, 2147483648, None)])
 
 def test_resume_reader_fields_and_arrayitems():
     class ResumeReader(AbstractResumeDataReader):
diff --git a/rpython/jit/tool/oparser.py b/rpython/jit/tool/oparser.py
--- a/rpython/jit/tool/oparser.py
+++ b/rpython/jit/tool/oparser.py
@@ -7,7 +7,8 @@
 from rpython.jit.tool.oparser_model import get_model
 from rpython.jit.metainterp.resoperation import rop, ResOperation, \
                                             ResOpWithDescr, N_aryOp, \
-                                            UnaryOp, PlainResOp, StmLocation
+                                            UnaryOp, PlainResOp
+from rpython.jit.metainterp.history import StmLocation
 
 r_skip_thread = re.compile(r'^(\d+#)?')
 

From noreply at buildbot.pypy.org  Mon Feb  9 23:34:31 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon,  9 Feb 2015 23:34:31 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: ssl: add win32-specific functions.
Message-ID: <20150209223431.67AB31C0103@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: stdlib-2.7.9
Changeset: r75789:13c3e6256e35
Date: 2015-02-09 23:34 +0100
http://bitbucket.org/pypy/pypy/changeset/13c3e6256e35/

Log:	ssl: add win32-specific functions. Completely blind translation,
	need to test on buildbot.

diff --git a/pypy/module/_ssl/__init__.py b/pypy/module/_ssl/__init__.py
--- a/pypy/module/_ssl/__init__.py
+++ b/pypy/module/_ssl/__init__.py
@@ -1,3 +1,4 @@
+import sys
 from rpython.rlib.rarithmetic import intmask
 from pypy.interpreter.mixedmodule import MixedModule
 from pypy.module._ssl import ssl_data
@@ -23,6 +24,10 @@
         '_SSLContext': 'interp_ssl._SSLContext',
     }
 
+    if sys.platform == 'win32':
+        interpleveldefs['enum_certificates'] = 'interp_win32.enum_certificates_w'
+        interpleveldefs['enum_crls'] = 'interp_win32.enum_crls_w'
+
     appleveldefs = {
     }
 
diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_ssl/interp_win32.py
@@ -0,0 +1,166 @@
+from rpython.rlib import rwin32
+from rpython.rtyper.lltypesystem import rffi
+from rpython.rtyper.tool import rffi_platform
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from pypy.interpreter.gateway import unwrap_spec
+from pypy.interpreter.error import wrap_windowserror
+
+eci = ExternalCompilationInfo(
+    includes = ['windows.h', 'stdio.h', 'stdlib.h'],
+    libraries = ['crypt32'],
+)
+
+class CConfig:
+    _compilation_info_ = eci
+
+    X509_ASN_ENCODING = rffi_platform.ConstantInteger('X509_ASN_ENCODING')
+    PKCS_7_ASN_ENCODING = rffi_platform.ConstantInteger('PKCS_7_ASN_ENCODING')
+
+    CERT_ENHKEY_USAGE = rffi_platform.Struct(
+        'CERT_ENHKEY_USAGE', [('cUsageIdentifier', rwin32.DWORD),
+                              ('rgpszUsageIdentifier', rffi.CCHARPP)])
+    CERT_CONTEXT = rffi_platform.Struct(
+        'CERT_CONTEXT', [('pbCertEncoded', rffi.CCHARP),
+                         ('cbCertEncoded', rwin32.DWORD),
+                         ('dwCertEncodingType', rwin32.DWORD)])
+    CRL_CONTEXT = rffi_platform.Struct(
+        'CRL_CONTEXT', [('pbCertEncoded', rffi.CCHARP),
+                        ('cbCertEncoded', rwin32.DWORD),
+                        ('dwCertEncodingType', rwin32.DWORD)])
+
+for k, v in rffi_platform.configure(CConfig).items():
+    globals()[k] = v
+
+PCERT_ENHKEY_USAGE = lltype.Ptr(CERT_ENHKEY_USAGE)
+PCCERT_CONTEXT = lltype.Ptr(CERT_CONTEXT)
+PCCRL_CONTEXT = lltype.Ptr(CRL_CONTEXT)
+
+def external(name, argtypes, restype, **kw):
+    kw['compilation_info'] = eci
+    kw['calling_conv'] = 'win'
+    return rffi.llexternal(
+        name, argtypes, restype, **kw)
+
+CertOpenSystemStore = external(
+    'CertOpenSystemStoreA', [rffi.VOIDP, rffi.CCHARP], rwin32.HANDLE)
+CertCloseStore = external(
+    'CertCloseStore', [rwin32.HANDLE, rwin32.DWORD], rwin32.BOOL)
+CertFreeCertificateContext = external(
+    'CertFreeCertificateContext', [PCCERT_CONTEXT], rwin32.BOOL)
+CertGetEnhancedKeyUsage = external(
+    'CertGetEnhancedKeyUsage', [PCCERT_CONTEXT, rwin32.DWORD,
+                                PCERT_ENHKEY_USAGE, rwin32.LPDWORD], rffi.BOOL)
+CertEnumCertificatesInStore = external(
+    'CertEnumCertificatesInStore',
+    [rffi.HANDLE, PCCERT_CONTEXT], PCCERT_CONTEXT)
+CertEnumCRLsInStore = external(
+    'CertEnumCRLsInStore',
+    [rffi.HANDLE, PCCRLT_CONTEXT], PCCRL_CONTEXT)
+
+def w_certEncodingType(space, encodingType):
+    if encodingType == X509_ASN_ENCODING:
+        return space.wrap("x509_asn")
+    elif encodingType == PKCS_7_ASN_ENCODING:
+        return space.wrap("pkcs_7_asn")
+    else:
+        return space.wrap(encodingType)
+
+def w_parseKeyUsage(space, pCertCtx, flags):
+    with rffi.scoped_alloc(rwin32.LPDWORD.TO, 1) as size_ptr:
+        if not CertGetEnhancedKeyUsage(pCertCtx, flags, None, size_ptr):
+            last_error = rwin32.lastSavedWindowsError()
+            if last_error == CRYPT_E_NOT_FOUND:
+                return space.w_True
+            raise wrap_windowserror(WindowsError(last_error))
+
+        size = rffi.widen(size_ptr[0])
+        with rffi.scoped_alloc(rffi.CHARP, size) as buf:
+            usage = rffi.cast(PCERT_ENHKEY_USAGE, buf)
+            # Now get the actual enhanced usage property
+            if not CertGetEnhancedKeyUsage(pCertCtx, flags, usage, size_ptr):
+                last_error = rwin32.lastSavedWindowsError()
+                if last_error == CRYPT_E_NOT_FOUND:
+                    return space.w_True
+                raise wrap_windowserror(WindowsError(last_error))
+
+            result_w = []
+            for i in range(usage.c_cUsageIdentifier):
+                if not usage.c_rgpszUsageIdentifier[i]:
+                    continue
+                result_w.append(
+                    space.wrap(rffi.charp2str(
+                        usage.c_rgpszUsageIdentifier[i])))
+            return space.newset(result_w)
+
+ at unwrap_spec(store_name=str)
+def enum_certificates_w(space, store_name):
+    """enum_certificates(store_name) -> []
+
+    Retrieve certificates from Windows' cert store. store_name may be one of
+    'CA', 'ROOT' or 'MY'. The system may provide more cert storages, too.
+    The function returns a list of (bytes, encoding_type, trust) tuples. The
+    encoding_type flag can be interpreted with X509_ASN_ENCODING or
+    PKCS_7_ASN_ENCODING. The trust setting is either a set of OIDs or the
+    boolean True."""
+
+    result_w = []
+    pCertCtx = lltype.nullptr(PCCERT_CONTEXT)
+    hStore = CertOpenSystemStore(None, store_name)
+    if not hStore:
+        raise wrap_windowserror(rwin32.lastSavedWindowsError())
+    try:
+        while True:
+            pCertCtx = CertEnumCertificatesInStore(hStore, pCertCtx)
+            if not pCertCtx:
+                break
+            w_cert = space.wrapbytes(
+                rffi.charpsize2str(pCertCtx.c_pbCertEncoded,
+                                   pCertCtx.c_cbCertEncoded))
+            w_enc = w_certEncodingType(space, pCertCtx.c_dwCertEncodingType)
+            w_keyusage = w_parseKeyUsage(
+                space, pCertCtx, CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG)
+            if space.is_w(w_keyusage, space.w_True):
+                w_keyusage = w_parseKeyUsage(
+                    space, pCertCtx, CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG)
+            result_w.append(space.newtuple([w_cert, w_enc, w_keyusage]))
+    finally:
+        if pCertCtx:
+            # loop ended with an error, need to clean up context manually
+            CertFreeCertificateContext(pCertCtx)
+        if not CertCloseStore(hStore, 0):
+            # This error case might shadow another exception.
+            raise wrap_windowserror(rwin32.lastSavedWindowsError())
+
+    return space.newlist(result_w)
+
+ at unwrap_spec(store_name=str)
+def enum_crls_w(space, store_name):
+    """enum_crls(store_name) -> []
+
+    Retrieve CRLs from Windows' cert store. store_name may be one of
+    'CA', 'ROOT' or 'MY'. The system may provide more cert storages, too.
+    The function returns a list of (bytes, encoding_type) tuples. The
+    encoding_type flag can be interpreted with X509_ASN_ENCODING or
+    PKCS_7_ASN_ENCODING."""
+    result_w = []
+    pCrlCtx = lltype.nullptr(PCCRL_CONTEXT)
+    hStore = CertOpenSystemStore(None, store_name)
+    if not hStore:
+        raise wrap_windowserror(rwin32.lastSavedWindowsError())
+    try:
+        while True:
+            pCrlCtx = CertEnumCRLsInStore(hStore, pCrlCtx)
+            if not pCrlCtx:
+                break
+            w_crl = space.wrapbytes(
+                rffi.charpsize2str(pCrlCtx.c_pbCrlEncoded,
+                                   pCrlCtx.c_cbCrlEncoded))
+            w_enc = w_certEncodingType(space, pCrlCtx.c_dwCertEncodingType)
+            result_w.append(space.newtuple([w_cert, w_enc]))
+    finally:
+        if pCrlCtx:
+            # loop ended with an error, need to clean up context manually
+            CertFreeCRLContext(pCrlCtx)
+        if not CertCloseStore(hStore, 0):
+            # This error case might shadow another exception.
+            raise wrap_windowserror(rwin32.lastSavedWindowsError())
diff --git a/pypy/module/_ssl/test/test_win32.py b/pypy/module/_ssl/test/test_win32.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_ssl/test/test_win32.py
@@ -0,0 +1,53 @@
+import sys
+
+if sys.platform != 'win32':
+    from py.test import skip
+    skip("ssl.enum_certificates is only available on Windows")
+
+class AppTestWin32:
+    spaceconfig = dict(usemodules=('_ssl',))
+
+    def setup_class(cls):
+        if sys.platform != 'win32':
+            py.test.skip("pexpect not found")
+
+    def test_enum_certificates(self):
+        import _ssl
+        assert _ssl.enum_certificates("CA")
+        assert _ssl.enum_certificates("ROOT")
+
+        raises(TypeError, _ssl.enum_certificates)
+        raises(WindowsError, _ssl.enum_certificates, "")
+
+        trust_oids = set()
+        for storename in ("CA", "ROOT"):
+            store = _ssl.enum_certificates(storename)
+            assert isinstance(store, list)
+            for element in store:
+                assert isinstance(element, tuple)
+                assert len(element) == 3
+                cert, enc, trust = element
+                assert isinstance(cert, bytes)
+                assert enc in {"x509_asn", "pkcs_7_asn"}
+                assert isinstance(trust, (set, bool))
+                if isinstance(trust, set):
+                    trust_oids.update(trust)
+
+        serverAuth = "1.3.6.1.5.5.7.3.1"
+        assert serverAuth in trust_oids
+
+    def test_enum_crls(self):
+        import _ssl
+        assert _ssl.enum_crls("CA")
+        raises(TypeError, _ssl.enum_crls)
+        raises(WindowsError, _ssl.enum_crls, "")
+
+        crls = _ssl.enum_crls("CA")
+        assert isinstance(crls, list)
+        for element in crls:
+            assert isinstance(element, tuple)
+            assert len(element) == 2
+            assert isinstance(element[0], bytes)
+            assert element[1] in {"x509_asn", "pkcs_7_asn"}
+
+

From noreply at buildbot.pypy.org  Mon Feb  9 23:51:01 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon,  9 Feb 2015 23:51:01 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: Try including wincrypt.h
Message-ID: <20150209225101.9279A1C0103@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: stdlib-2.7.9
Changeset: r75790:9373962c0444
Date: 2015-02-09 23:50 +0100
http://bitbucket.org/pypy/pypy/changeset/9373962c0444/

Log:	Try including wincrypt.h

diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py
--- a/pypy/module/_ssl/interp_win32.py
+++ b/pypy/module/_ssl/interp_win32.py
@@ -6,7 +6,7 @@
 from pypy.interpreter.error import wrap_windowserror
 
 eci = ExternalCompilationInfo(
-    includes = ['windows.h', 'stdio.h', 'stdlib.h'],
+    includes = ['windows.h', 'wincrypt.h'],
     libraries = ['crypt32'],
 )
 

From noreply at buildbot.pypy.org  Mon Feb  9 23:57:20 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  9 Feb 2015 23:57:20 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: import stmgc/65a89cd8a6c0
Message-ID: <20150209225720.691DF1C0103@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r75791:d9e24a0719bd
Date: 2015-02-09 22:46 +0100
http://bitbucket.org/pypy/pypy/changeset/d9e24a0719bd/

Log:	import stmgc/65a89cd8a6c0

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-ac07eddb02d5
+65a89cd8a6c0
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -436,26 +436,31 @@
 
         char *remote_base = get_segment_base(i);
         object_t *conflicting_obj;
+        uintptr_t j, limit;
+        struct list_s *lst;
 
-        LIST_FOREACH_R(
-            STM_PSEGMENT->modified_old_objects,
-            object_t * /*item*/,
-            ({
-                if (was_read_remote(remote_base, item)) {
-                    conflicting_obj = item;
-                    goto found_conflict;
-                }
-            }));
+        /* Look in forward order: this is an attempt to report the _first_
+           write that conflicts with another segment's reads
+        */
+        lst = STM_PSEGMENT->modified_old_objects;
+        limit = list_count(lst);
+        for (j = 0; j < limit; j++) {
+            object_t *obj = (object_t *)list_item(lst, j);
+            if (was_read_remote(remote_base, obj)) {
+                conflicting_obj = obj;
+                goto found_conflict;
+            }
+        }
 
-        LIST_FOREACH_R(
-            STM_PSEGMENT->modified_old_hashtables,
-            object_t * /*item*/,
-            ({
-                if (was_read_remote(remote_base, item)) {
-                    conflicting_obj = item;
-                    goto found_conflict;
-                }
-            }));
+        lst = STM_PSEGMENT->modified_old_hashtables;
+        limit = list_count(lst);
+        for (j = 0; j < limit; j += 2) {
+            object_t *hobj = (object_t *)list_item(lst, j);
+            if (was_read_remote(remote_base, hobj)) {
+                conflicting_obj = (object_t *)list_item(lst, j + 1);
+                goto found_conflict;
+            }
+        }
 
         continue;
 
diff --git a/rpython/translator/stm/src_stm/stm/core.h b/rpython/translator/stm/src_stm/stm/core.h
--- a/rpython/translator/stm/src_stm/stm/core.h
+++ b/rpython/translator/stm/src_stm/stm/core.h
@@ -97,7 +97,8 @@
     uintptr_t modified_old_objects_markers_num_old;
 
     /* This list contains all old hashtables that have entries that we
-       modified.  Note that several transactions can all commit if
+       modified.  It's actually a list of pairs hashtable/sample_entry.
+       Note that several transactions can all commit if
        they have the same hashtable listed here.  The point of this
        list is only that if another segment does a global "read" of
        the hashtable (stm_hashtable_list), then it conflicts with this
diff --git a/rpython/translator/stm/src_stm/stm/gcpage.c b/rpython/translator/stm/src_stm/stm/gcpage.c
--- a/rpython/translator/stm/src_stm/stm/gcpage.c
+++ b/rpython/translator/stm/src_stm/stm/gcpage.c
@@ -476,26 +476,13 @@
     }
 }
 
+static void clean_up_segment_lists(void)
+{
 #pragma push_macro("STM_PSEGMENT")
 #pragma push_macro("STM_SEGMENT")
 #undef STM_PSEGMENT
 #undef STM_SEGMENT
 
-static void remove_objects_that_die(struct list_s *lst)
-{
-    if (lst != NULL) {
-        uintptr_t n = list_count(lst);
-        while (n > 0) {
-            object_t *obj = (object_t *)list_item(lst, --n);
-            if (!mark_visited_test(obj)) {
-                list_set_item(lst, n, list_pop_item(lst));
-            }
-        }
-    }
-}
-
-static void clean_up_segment_lists(void)
-{
     long i;
     for (i = 1; i <= NB_SEGMENTS; i++) {
         struct stm_priv_segment_info_s *pseg = get_priv_segment(i);
@@ -554,14 +541,39 @@
             }));
         list_clear(lst);
 
-        /* Remove from 'large_overflow_objects' and 'modified_old_hashtables'
-           all objects that die */
-        remove_objects_that_die(pseg->large_overflow_objects);
-        remove_objects_that_die(pseg->modified_old_hashtables);
+        /* Remove from 'large_overflow_objects' all objects that die */
+        lst = pseg->large_overflow_objects;
+        if (lst != NULL) {
+            uintptr_t n = list_count(lst);
+            while (n > 0) {
+                object_t *obj = (object_t *)list_item(lst, --n);
+                if (!mark_visited_test(obj))
+                    list_set_item(lst, n, list_pop_item(lst));
+            }
+        }
+
+        /* Remove from 'modified_old_hashtables' all old hashtables that
+           die, but keeping the order */
+        {
+            lst = pseg->modified_old_hashtables;
+            uintptr_t j, k = 0, limit = list_count(lst);
+            for (j = 0; j < limit; j += 2) {
+                object_t *hobj = (object_t *)list_item(lst, j);
+                if (mark_visited_test(hobj)) {
+                    /* hobj does not die */
+                    if (j != k) {
+                        list_set_item(lst, k, (uintptr_t)hobj);
+                        list_set_item(lst, k + 1, list_item(lst, j + 1));
+                    }
+                    k += 2;
+                }
+            }
+            lst->count = k;
+        }
     }
-}
 #pragma pop_macro("STM_SEGMENT")
 #pragma pop_macro("STM_PSEGMENT")
+}
 
 static inline bool largemalloc_keep_object_at(char *data)
 {
diff --git a/rpython/translator/stm/src_stm/stm/hashtable.c b/rpython/translator/stm/src_stm/stm/hashtable.c
--- a/rpython/translator/stm/src_stm/stm/hashtable.c
+++ b/rpython/translator/stm/src_stm/stm/hashtable.c
@@ -359,12 +359,13 @@
                avoid them --- at least the list should never be longer
                than 'modified_old_objects'. */
             i = list_count(STM_PSEGMENT->modified_old_hashtables);
-            if (i > 0 && list_item(STM_PSEGMENT->modified_old_hashtables, i - 1)
+            if (i > 0 && list_item(STM_PSEGMENT->modified_old_hashtables, i - 2)
                          == (uintptr_t)hobj) {
                 /* already added */
             }
             else {
-                LIST_APPEND(STM_PSEGMENT->modified_old_hashtables, hobj);
+                LIST_APPEND2(STM_PSEGMENT->modified_old_hashtables,
+                             hobj, entry);
             }
         }
     }
diff --git a/rpython/translator/stm/src_stm/stm/list.h b/rpython/translator/stm/src_stm/stm/list.h
--- a/rpython/translator/stm/src_stm/stm/list.h
+++ b/rpython/translator/stm/src_stm/stm/list.h
@@ -46,6 +46,9 @@
     return lst;
 }
 
+#define LIST_APPEND2(lst, e1, e2)   ((lst) = list_append2((lst),   \
+                                        (uintptr_t)(e1), (uintptr_t)(e2)))
+
 
 static inline void list_clear(struct list_s *lst)
 {

From noreply at buildbot.pypy.org  Mon Feb  9 23:57:21 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  9 Feb 2015 23:57:21 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: The tests pass now.  Tweaks.
Message-ID: <20150209225721.9BF2D1C0103@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r75792:0bb524721029
Date: 2015-02-09 23:56 +0100
http://bitbucket.org/pypy/pypy/changeset/0bb524721029/

Log:	The tests pass now. Tweaks.

diff --git a/pypy/module/pypystm/test_pypy_c/support.py b/pypy/module/pypystm/test_pypy_c/support.py
--- a/pypy/module/pypystm/test_pypy_c/support.py
+++ b/pypy/module/pypystm/test_pypy_c/support.py
@@ -87,6 +87,7 @@
 
     def setup_method(self, meth):
         self.filepath = self.tmpdir.join(meth.im_func.func_name + '.py')
+        self.stmfile = self.filepath.new(ext='.stm')
         self.logfile = self.filepath.new(ext='.log')
 
     def _write_source(self, func_or_src, args=[]):
@@ -109,7 +110,8 @@
             cmdline.append('-S')
         cmdline.append(str(self.filepath))
         env = os.environ.copy()
-        env['PYPYSTM'] = str(self.logfile)
+        env['PYPYSTM'] = str(self.stmfile)
+        env['PYPYLOG'] = str(self.logfile)
         #
         pipe = subprocess.Popen(cmdline,
                                 env=env,
@@ -125,7 +127,7 @@
 
     def _parse_log(self):
         from pypy.stm.print_stm_log import StmLog
-        return StmLog(str(self.logfile))
+        return StmLog(str(self.stmfile))
 
     def _check_count_conflicts(self, func_or_src, args=[], **kwds):
         self._write_source(func_or_src, args)
diff --git a/pypy/module/pypystm/test_pypy_c/test_conflict.py b/pypy/module/pypystm/test_pypy_c/test_conflict.py
--- a/pypy/module/pypystm/test_pypy_c/test_conflict.py
+++ b/pypy/module/pypystm/test_pypy_c/test_conflict.py
@@ -10,7 +10,9 @@
             x = X()     # shared
             x.a = 0
             def g():
+                X()           #loc0
                 x.a += 1      #loc1
+                X()           #loc2
             run_in_threads(g)
         #
         self.check_MANY_conflicts(f, jit=jit)
@@ -57,7 +59,9 @@
             x.a = 0
             def g(tnum):
                 if tnum == 0:
+                    X()           #loc1
                     x.a += 1      #loc2
+                    X()           #loc3
                 else:
                     if x.a < 0:
                         raise AssertionError

From noreply at buildbot.pypy.org  Mon Feb  9 23:58:29 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon,  9 Feb 2015 23:58:29 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: Translation fix
Message-ID: <20150209225829.72B5E1C0103@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: stdlib-2.7.9
Changeset: r75793:fe2a4e798697
Date: 2015-02-09 23:58 +0100
http://bitbucket.org/pypy/pypy/changeset/fe2a4e798697/

Log:	Translation fix

diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py
--- a/pypy/module/_ssl/interp_win32.py
+++ b/pypy/module/_ssl/interp_win32.py
@@ -24,8 +24,8 @@
                          ('cbCertEncoded', rwin32.DWORD),
                          ('dwCertEncodingType', rwin32.DWORD)])
     CRL_CONTEXT = rffi_platform.Struct(
-        'CRL_CONTEXT', [('pbCertEncoded', rffi.CCHARP),
-                        ('cbCertEncoded', rwin32.DWORD),
+        'CRL_CONTEXT', [('pbCrlEncoded', rffi.CCHARP),
+                        ('cbCrlEncoded', rwin32.DWORD),
                         ('dwCertEncodingType', rwin32.DWORD)])
 
 for k, v in rffi_platform.configure(CConfig).items():

From noreply at buildbot.pypy.org  Tue Feb 10 00:06:18 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 10 Feb 2015 00:06:18 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: Translation fix
Message-ID: <20150209230618.257871C0103@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: stdlib-2.7.9
Changeset: r75794:ef5805046353
Date: 2015-02-10 00:06 +0100
http://bitbucket.org/pypy/pypy/changeset/ef5805046353/

Log:	Translation fix

diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py
--- a/pypy/module/_ssl/interp_win32.py
+++ b/pypy/module/_ssl/interp_win32.py
@@ -1,5 +1,5 @@
 from rpython.rlib import rwin32
-from rpython.rtyper.lltypesystem import rffi
+from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rtyper.tool import rffi_platform
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.interpreter.gateway import unwrap_spec

From noreply at buildbot.pypy.org  Tue Feb 10 09:29:49 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 10 Feb 2015 09:29:49 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: Translation fix
Message-ID: <20150210082949.C35911C0092@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: stdlib-2.7.9
Changeset: r75795:5ffa2d51f624
Date: 2015-02-10 09:29 +0100
http://bitbucket.org/pypy/pypy/changeset/5ffa2d51f624/

Log:	Translation fix

diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py
--- a/pypy/module/_ssl/interp_win32.py
+++ b/pypy/module/_ssl/interp_win32.py
@@ -48,8 +48,9 @@
 CertFreeCertificateContext = external(
     'CertFreeCertificateContext', [PCCERT_CONTEXT], rwin32.BOOL)
 CertGetEnhancedKeyUsage = external(
-    'CertGetEnhancedKeyUsage', [PCCERT_CONTEXT, rwin32.DWORD,
-                                PCERT_ENHKEY_USAGE, rwin32.LPDWORD], rffi.BOOL)
+    'CertGetEnhancedKeyUsage',
+    [PCCERT_CONTEXT, rwin32.DWORD, PCERT_ENHKEY_USAGE, rwin32.LPDWORD],
+    rwin32.BOOL)
 CertEnumCertificatesInStore = external(
     'CertEnumCertificatesInStore',
     [rffi.HANDLE, PCCERT_CONTEXT], PCCERT_CONTEXT)

From noreply at buildbot.pypy.org  Tue Feb 10 13:45:46 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 10 Feb 2015 13:45:46 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Marginally better
Message-ID: <20150210124546.0A3121C0884@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r75796:3324de0ff750
Date: 2015-02-10 11:27 +0100
http://bitbucket.org/pypy/pypy/changeset/3324de0ff750/

Log:	Marginally better

diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -74,7 +74,7 @@
     llop.stm_become_globally_unique_transaction(lltype.Void)
 
 def partial_commit_and_resume_other_threads():
-    pass    # for now
+    hint_commit_soon()    # for now
 
 @specialize.arg(0)
 def should_break_transaction(keep):

From noreply at buildbot.pypy.org  Tue Feb 10 13:45:47 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 10 Feb 2015 13:45:47 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: - Fix rbisect to match the expectations
	of asmmmemmgr.
Message-ID: <20150210124547.576451C0884@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: vmprof
Changeset: r75797:700eed63ab38
Date: 2015-02-10 13:44 +0100
http://bitbucket.org/pypy/pypy/changeset/700eed63ab38/

Log:	- Fix rbisect to match the expectations of asmmmemmgr.

	- Fix asmmemmgr to not kill a random jit_codemap entry every time
	free() is called on a block that does not contain code.

diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py
--- a/rpython/jit/backend/llsupport/asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/asmmemmgr.py
@@ -62,9 +62,12 @@
         self.jit_frame_depth_map = (self.jit_frame_depth_map[:jit_adr_start] +
                                     self.jit_frame_depth_map[jit_adr_stop:])
         # fix up codemap
-        codemap_adr = bisect_tuple(self.jit_codemap, start)
-        self.jit_codemap = (self.jit_codemap[:codemap_adr] +
-                            self.jit_codemap[codemap_adr + 1:])
+        # (there should only be zero or one codemap entry in that range,
+        # but still we use a range to distinguish between zero and one)
+        codemap_adr_start = bisect_tuple(self.jit_codemap, start)
+        codemap_adr_stop = bisect_tuple(self.jit_codemap, stop)
+        self.jit_codemap = (self.jit_codemap[:codemap_adr_start] +
+                            self.jit_codemap[codemap_adr_stop:])
 
     def open_malloc(self, minsize):
         """Allocate at least minsize bytes.  Returns (start, stop)."""
diff --git a/rpython/rlib/rbisect.py b/rpython/rlib/rbisect.py
--- a/rpython/rlib/rbisect.py
+++ b/rpython/rlib/rbisect.py
@@ -1,10 +1,12 @@
 
 def bisect(a, x):
+    """Return the index in the sorted list 'a' of 'x'.  If 'x' is not in 'a',
+    return the index where it can be inserted."""
     lo = 0
     hi = len(a)
     while lo < hi:
         mid = (lo+hi)//2
-        if x < a[mid]: hi = mid
+        if x <= a[mid]: hi = mid
         else: lo = mid+1
     return lo
 
@@ -14,6 +16,6 @@
     hi = len(a)
     while lo < hi:
         mid = (lo+hi)//2
-        if x < a[mid][0]: hi = mid
+        if x <= a[mid][0]: hi = mid
         else: lo = mid+1
     return lo
diff --git a/rpython/rlib/test/test_rbisect.py b/rpython/rlib/test/test_rbisect.py
--- a/rpython/rlib/test/test_rbisect.py
+++ b/rpython/rlib/test/test_rbisect.py
@@ -5,42 +5,42 @@
     cases = [
             ([], 1, 0),
             ([1], 0, 0),
-            ([1], 1, 1),
+            ([1], 1, 0),
             ([1], 2, 1),
             ([1, 1], 0, 0),
-            ([1, 1], 1, 2),
+            ([1, 1], 1, 0),
             ([1, 1], 2, 2),
             ([1, 1, 1], 0, 0),
-            ([1, 1, 1], 1, 3),
+            ([1, 1, 1], 1, 0),
             ([1, 1, 1], 2, 3),
             ([1, 1, 1, 1], 0, 0),
-            ([1, 1, 1, 1], 1, 4),
+            ([1, 1, 1, 1], 1, 0),
             ([1, 1, 1, 1], 2, 4),
             ([1, 2], 0, 0),
-            ([1, 2], 1, 1),
+            ([1, 2], 1, 0),
             ([1, 2], 1.5, 1),
-            ([1, 2], 2, 2),
+            ([1, 2], 2, 1),
             ([1, 2], 3, 2),
             ([1, 1, 2, 2], 0, 0),
-            ([1, 1, 2, 2], 1, 2),
+            ([1, 1, 2, 2], 1, 0),
             ([1, 1, 2, 2], 1.5, 2),
-            ([1, 1, 2, 2], 2, 4),
+            ([1, 1, 2, 2], 2, 2),
             ([1, 1, 2, 2], 3, 4),
             ([1, 2, 3], 0, 0),
-            ([1, 2, 3], 1, 1),
+            ([1, 2, 3], 1, 0),
             ([1, 2, 3], 1.5, 1),
-            ([1, 2, 3], 2, 2),
+            ([1, 2, 3], 2, 1),
             ([1, 2, 3], 2.5, 2),
-            ([1, 2, 3], 3, 3),
+            ([1, 2, 3], 3, 2),
             ([1, 2, 3], 4, 3),
             ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 0, 0),
-            ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 1, 1),
+            ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 1, 0),
             ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 1.5, 1),
-            ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 2, 3),
+            ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 2, 1),
             ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 2.5, 3),
-            ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 3, 6),
+            ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 3, 3),
             ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 3.5, 6),
-            ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 4, 10),
+            ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 4, 6),
             ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 5, 10),
         ]
     for lst, elem, exp in cases:

From noreply at buildbot.pypy.org  Tue Feb 10 13:52:16 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 10 Feb 2015 13:52:16 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: fix the tests
Message-ID: <20150210125216.D10E11C0884@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r75798:7af23468be1c
Date: 2015-02-10 14:51 +0200
http://bitbucket.org/pypy/pypy/changeset/7af23468be1c/

Log:	fix the tests

diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py
--- a/rpython/jit/backend/llsupport/codemap.py
+++ b/rpython/jit/backend/llsupport/codemap.py
@@ -21,10 +21,10 @@
 def stack_depth_at_loc(loc):
     _memmngr = asmmemmgr._memmngr
 
-    pos = bisect(_memmngr.jit_addr_map, loc)
+    pos = bisect(_memmngr.jit_addr_map, loc + 1)
     if pos == 0 or pos == len(_memmngr.jit_addr_map):
         return -1
-    return _memmngr.jit_frame_depth_map[pos-1]
+    return _memmngr.jit_frame_depth_map[pos - 1]
 
 @jit_entrypoint([], lltype.Signed, c_name='pypy_jit_start_addr')
 def jit_start_addr():
@@ -76,7 +76,8 @@
 
 def unpack_traceback(addr):
     codemap_pos = find_codemap_at_addr(addr)
-    assert codemap_pos >= 0
+    if codemap_pos == -1:
+        return [] # no codemap for that position
     storage = lltype.malloc(rffi.CArray(lltype.Signed), 1, flavor='raw')
     storage[0] = 0
     res = []
diff --git a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -3,6 +3,7 @@
 from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
 from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin
 from rpython.jit.backend.llsupport import asmmemmgr
+from rpython.jit.backend.llsupport.codemap import stack_depth_at_loc
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import debug
 
@@ -271,9 +272,9 @@
     mgr.register_frame_depth_map(30, [0, 5, 10], [4, 5, 6])
     mgr.register_frame_depth_map(0, [0, 5, 10], [7, 8, 9])
     asmmemmgr._memmngr = mgr
-    assert asmmemmgr.stack_depth_at_loc(13) == 1
-    assert asmmemmgr.stack_depth_at_loc(-3) == -1
-    assert asmmemmgr.stack_depth_at_loc(41) == -1
-    assert asmmemmgr.stack_depth_at_loc(5) == 8
-    assert asmmemmgr.stack_depth_at_loc(17) == 2
-    assert asmmemmgr.stack_depth_at_loc(38) == 5
+    assert stack_depth_at_loc(13) == 1
+    assert stack_depth_at_loc(-3) == -1
+    assert stack_depth_at_loc(41) == -1
+    assert stack_depth_at_loc(5) == 8
+    assert stack_depth_at_loc(17) == 2
+    assert stack_depth_at_loc(38) == 5
diff --git a/rpython/jit/backend/llsupport/test/test_gc.py b/rpython/jit/backend/llsupport/test/test_gc.py
--- a/rpython/jit/backend/llsupport/test/test_gc.py
+++ b/rpython/jit/backend/llsupport/test/test_gc.py
@@ -197,14 +197,6 @@
         assert is_valid_int(wbdescr.jit_wb_if_flag_byteofs)
         assert is_valid_int(wbdescr.jit_wb_if_flag_singlebyte)
 
-    def test_get_rid_of_debug_merge_point(self):
-        operations = [
-            ResOperation(rop.DEBUG_MERGE_POINT, ['dummy', 2], None),
-            ]
-        gc_ll_descr = self.gc_ll_descr
-        operations = gc_ll_descr.rewrite_assembler(None, operations, [])
-        assert len(operations) == 0
-
     def test_record_constptrs(self):
         class MyFakeCPU(object):
             def cast_adr_to_int(self, adr):

From noreply at buildbot.pypy.org  Tue Feb 10 14:38:49 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 10 Feb 2015 14:38:49 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: Tweaks,
 including one which is a fix for find_codemap_at_addr() I think
Message-ID: <20150210133849.6147E1C122C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: vmprof
Changeset: r75799:9d2347008c52
Date: 2015-02-10 14:38 +0100
http://bitbucket.org/pypy/pypy/changeset/9d2347008c52/

Log:	Tweaks, including one which is a fix for find_codemap_at_addr() I
	think

diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py
--- a/rpython/jit/backend/llsupport/asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/asmmemmgr.py
@@ -5,7 +5,7 @@
 from rpython.rlib.debug import debug_start, debug_print, debug_stop
 from rpython.rlib.debug import have_debug_prints
 from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib.rbisect import bisect, bisect_tuple
+from rpython.rlib.rbisect import bisect_left, bisect_left_tuple
 
 _memmngr = None # global reference so we can use @entrypoint :/
 
@@ -55,19 +55,16 @@
             self.total_mallocs -= r_uint(stop - start)
         self._add_free_block(start, stop)
         # fix up jit_addr_map
-        jit_adr_start = bisect(self.jit_addr_map, start)
-        jit_adr_stop = bisect(self.jit_addr_map, stop)
-        self.jit_addr_map = (self.jit_addr_map[:jit_adr_start] +
-                             self.jit_addr_map[jit_adr_stop:])
-        self.jit_frame_depth_map = (self.jit_frame_depth_map[:jit_adr_start] +
-                                    self.jit_frame_depth_map[jit_adr_stop:])
+        jit_adr_start = bisect_left(self.jit_addr_map, start)
+        jit_adr_stop = bisect_left(self.jit_addr_map, stop)
+        del self.jit_addr_map[jit_adr_start:jit_adr_stop]
+        del self.jit_frame_depth_map[jit_adr_start:jit_adr_stop]
         # fix up codemap
         # (there should only be zero or one codemap entry in that range,
         # but still we use a range to distinguish between zero and one)
-        codemap_adr_start = bisect_tuple(self.jit_codemap, start)
-        codemap_adr_stop = bisect_tuple(self.jit_codemap, stop)
-        self.jit_codemap = (self.jit_codemap[:codemap_adr_start] +
-                            self.jit_codemap[codemap_adr_stop:])
+        codemap_adr_start = bisect_left_tuple(self.jit_codemap, start)
+        codemap_adr_stop = bisect_left_tuple(self.jit_codemap, stop)
+        del self.jit_codemap[codemap_adr_start:codemap_adr_stop]
 
     def open_malloc(self, minsize):
         """Allocate at least minsize bytes.  Returns (start, stop)."""
@@ -183,7 +180,7 @@
             self.jit_addr_map += [0] * len(frame_positions)
             self.jit_frame_depth_map += [0] * len(frame_positions)
         else:
-            start = bisect(self.jit_addr_map, rawstart)
+            start = bisect_left(self.jit_addr_map, rawstart)
             self.jit_addr_map = (self.jit_addr_map[:start] +
                                  [0] * len(frame_positions) +
                                  self.jit_addr_map[start:])
@@ -196,12 +193,8 @@
 
     def register_codemap(self, codemap):
         start = codemap[0]
-        pos = bisect_tuple(self.jit_codemap, start)
-        if pos == len(self.jit_codemap): # common case
-            self.jit_codemap.append(codemap)
-        else:
-            self.jit_codemap = (self.jit_codemap[:pos] + [codemap] +
-                                self.jit_codemap[pos:])
+        pos = bisect_left_tuple(self.jit_codemap, start)
+        self.jit_codemap.insert(pos, codemap)
 
     def _delete(self):
         "NOT_RPYTHON"
diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py
--- a/rpython/jit/backend/llsupport/codemap.py
+++ b/rpython/jit/backend/llsupport/codemap.py
@@ -12,7 +12,7 @@
 from rpython.rlib import rgc
 from rpython.rlib.entrypoint import jit_entrypoint
 from rpython.jit.backend.llsupport import asmmemmgr
-from rpython.rlib.rbisect import bisect, bisect_tuple
+from rpython.rlib.rbisect import bisect_right, bisect_right_tuple
 from rpython.rtyper.lltypesystem import lltype, rffi
 
 @jit_entrypoint([lltype.Signed], lltype.Signed,
@@ -21,7 +21,7 @@
 def stack_depth_at_loc(loc):
     _memmngr = asmmemmgr._memmngr
 
-    pos = bisect(_memmngr.jit_addr_map, loc + 1)
+    pos = bisect_right(_memmngr.jit_addr_map, loc)
     if pos == 0 or pos == len(_memmngr.jit_addr_map):
         return -1
     return _memmngr.jit_frame_depth_map[pos - 1]
@@ -43,9 +43,7 @@
 def find_codemap_at_addr(addr):
     _memmngr = asmmemmgr._memmngr
 
-    res = bisect_tuple(_memmngr.jit_codemap, addr) - 1
-    if res == len(_memmngr.jit_codemap):
-        return -1
+    res = bisect_right_tuple(_memmngr.jit_codemap, addr) - 1
     return res
 
 @jit_entrypoint([lltype.Signed, lltype.Signed,
diff --git a/rpython/rlib/rbisect.py b/rpython/rlib/rbisect.py
--- a/rpython/rlib/rbisect.py
+++ b/rpython/rlib/rbisect.py
@@ -1,21 +1,39 @@
 
-def bisect(a, x):
+def bisect_left(a, x):
     """Return the index in the sorted list 'a' of 'x'.  If 'x' is not in 'a',
     return the index where it can be inserted."""
     lo = 0
     hi = len(a)
     while lo < hi:
         mid = (lo+hi)//2
-        if x <= a[mid]: hi = mid
+        if a[mid] < x: lo = mid+1
+        else: hi = mid
+    return lo
+
+def bisect_right(a, x):
+    lo = 0
+    hi = len(a)
+    while lo < hi:
+        mid = (lo+hi)//2
+        if x < a[mid]: hi = mid
         else: lo = mid+1
     return lo
 
 # a copy of the above, but compares the first item of a tuple only
-def bisect_tuple(a, x):
+def bisect_left_tuple(a, x):
     lo = 0
     hi = len(a)
     while lo < hi:
         mid = (lo+hi)//2
-        if x <= a[mid][0]: hi = mid
+        if a[mid][0] < x: lo = mid+1
+        else: hi = mid
+    return lo
+
+def bisect_right_tuple(a, x):
+    lo = 0
+    hi = len(a)
+    while lo < hi:
+        mid = (lo+hi)//2
+        if x < a[mid][0]: hi = mid
         else: lo = mid+1
     return lo
diff --git a/rpython/rlib/test/test_rbisect.py b/rpython/rlib/test/test_rbisect.py
--- a/rpython/rlib/test/test_rbisect.py
+++ b/rpython/rlib/test/test_rbisect.py
@@ -1,7 +1,7 @@
 
-from rpython.rlib.rbisect import bisect
+from rpython.rlib.rbisect import bisect_left, bisect_right
 
-def test_bisect():
+def test_bisect_left():
     cases = [
             ([], 1, 0),
             ([1], 0, 0),
@@ -44,4 +44,50 @@
             ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 5, 10),
         ]
     for lst, elem, exp in cases:
-        assert bisect(lst, elem) == exp
+        assert bisect_left(lst, elem) == exp
+
+def test_bisect_right():
+    cases = [
+
+            ([], 1, 0),
+            ([1], 0, 0),
+            ([1], 1, 1),
+            ([1], 2, 1),
+            ([1, 1], 0, 0),
+            ([1, 1], 1, 2),
+            ([1, 1], 2, 2),
+            ([1, 1, 1], 0, 0),
+            ([1, 1, 1], 1, 3),
+            ([1, 1, 1], 2, 3),
+            ([1, 1, 1, 1], 0, 0),
+            ([1, 1, 1, 1], 1, 4),
+            ([1, 1, 1, 1], 2, 4),
+            ([1, 2], 0, 0),
+            ([1, 2], 1, 1),
+            ([1, 2], 1.5, 1),
+            ([1, 2], 2, 2),
+            ([1, 2], 3, 2),
+            ([1, 1, 2, 2], 0, 0),
+            ([1, 1, 2, 2], 1, 2),
+            ([1, 1, 2, 2], 1.5, 2),
+            ([1, 1, 2, 2], 2, 4),
+            ([1, 1, 2, 2], 3, 4),
+            ([1, 2, 3], 0, 0),
+            ([1, 2, 3], 1, 1),
+            ([1, 2, 3], 1.5, 1),
+            ([1, 2, 3], 2, 2),
+            ([1, 2, 3], 2.5, 2),
+            ([1, 2, 3], 3, 3),
+            ([1, 2, 3], 4, 3),
+            ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 0, 0),
+            ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 1, 1),
+            ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 1.5, 1),
+            ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 2, 3),
+            ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 2.5, 3),
+            ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 3, 6),
+            ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 3.5, 6),
+            ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 4, 10),
+            ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 5, 10),
+        ]
+    for lst, elem, exp in cases:
+        assert bisect_right(lst, elem) == exp

From noreply at buildbot.pypy.org  Tue Feb 10 15:04:02 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 10 Feb 2015 15:04:02 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: Translation fix
Message-ID: <20150210140402.480BF1C1237@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: stdlib-2.7.9
Changeset: r75800:315890b5116a
Date: 2015-02-10 15:03 +0100
http://bitbucket.org/pypy/pypy/changeset/315890b5116a/

Log:	Translation fix

diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py
--- a/pypy/module/_ssl/interp_win32.py
+++ b/pypy/module/_ssl/interp_win32.py
@@ -53,10 +53,10 @@
     rwin32.BOOL)
 CertEnumCertificatesInStore = external(
     'CertEnumCertificatesInStore',
-    [rffi.HANDLE, PCCERT_CONTEXT], PCCERT_CONTEXT)
+    [rwin32.HANDLE, PCCERT_CONTEXT], PCCERT_CONTEXT)
 CertEnumCRLsInStore = external(
     'CertEnumCRLsInStore',
-    [rffi.HANDLE, PCCRLT_CONTEXT], PCCRL_CONTEXT)
+    [rwin32.HANDLE, PCCRLT_CONTEXT], PCCRL_CONTEXT)
 
 def w_certEncodingType(space, encodingType):
     if encodingType == X509_ASN_ENCODING:
@@ -75,7 +75,7 @@
             raise wrap_windowserror(WindowsError(last_error))
 
         size = rffi.widen(size_ptr[0])
-        with rffi.scoped_alloc(rffi.CHARP, size) as buf:
+        with rffi.scoped_alloc(rffi.CCHARP.TO, size) as buf:
             usage = rffi.cast(PCERT_ENHKEY_USAGE, buf)
             # Now get the actual enhanced usage property
             if not CertGetEnhancedKeyUsage(pCertCtx, flags, usage, size_ptr):

From noreply at buildbot.pypy.org  Tue Feb 10 15:22:25 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 10 Feb 2015 15:22:25 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: fix typo
Message-ID: <20150210142225.CEB681C03C6@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: stdlib-2.7.9
Changeset: r75801:268a268ecf64
Date: 2015-02-10 15:22 +0100
http://bitbucket.org/pypy/pypy/changeset/268a268ecf64/

Log:	fix typo

diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py
--- a/pypy/module/_ssl/interp_win32.py
+++ b/pypy/module/_ssl/interp_win32.py
@@ -56,7 +56,7 @@
     [rwin32.HANDLE, PCCERT_CONTEXT], PCCERT_CONTEXT)
 CertEnumCRLsInStore = external(
     'CertEnumCRLsInStore',
-    [rwin32.HANDLE, PCCRLT_CONTEXT], PCCRL_CONTEXT)
+    [rwin32.HANDLE, PCCRL_CONTEXT], PCCRL_CONTEXT)
 
 def w_certEncodingType(space, encodingType):
     if encodingType == X509_ASN_ENCODING:

From noreply at buildbot.pypy.org  Tue Feb 10 18:43:59 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 10 Feb 2015 18:43:59 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Fix Return.nomoreblocks()
Message-ID: <20150210174359.24D091C0092@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r75802:80fb13c783c8
Date: 2015-02-08 19:48 +0000
http://bitbucket.org/pypy/pypy/changeset/80fb13c783c8/

Log:	Fix Return.nomoreblocks()

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -1040,6 +1040,7 @@
         w_result = self.w_value
         link = Link([w_result], ctx.graph.returnblock)
         ctx.recorder.crnt_block.closeblock(link)
+        raise StopFlowing
 
     @property
     def args(self):

From noreply at buildbot.pypy.org  Tue Feb 10 18:44:00 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 10 Feb 2015 18:44:00 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Start new bc_blocks after
 instructions that modify the blockstack
Message-ID: <20150210174400.54DC91C0092@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r75803:591f5aefe17b
Date: 2015-02-09 16:33 +0000
http://bitbucket.org/pypy/pypy/changeset/591f5aefe17b/

Log:	Start new bc_blocks after instructions that modify the blockstack

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -545,6 +545,7 @@
     def bc_flow(self, reader):
         reader.curr_block.operations.append(self)
         self.target = reader.get_block_at(self.arg)
+        reader.end_block()
 
     def do_signals(self, reader):
         reader.blockstack.append(self.make_block(-1))
@@ -593,6 +594,10 @@
 
 @bc_reader.register_opcode
 class POP_BLOCK(BCInstruction):
+    def bc_flow(self, reader):
+        reader.curr_block.operations.append(self)
+        reader.end_block()
+
     def do_signals(self, reader):
         reader.blockstack.pop()
 

From noreply at buildbot.pypy.org  Tue Feb 10 18:44:01 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 10 Feb 2015 18:44:01 +0100 (CET)
Subject: [pypy-commit] pypy framestate: Directly call ctx.unroll() in some
	cases
Message-ID: <20150210174401.789A01C0092@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r75804:f4c41b6cf812
Date: 2015-02-10 01:17 +0000
http://bitbucket.org/pypy/pypy/changeset/f4c41b6cf812/

Log:	Directly call ctx.unroll() in some cases

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -529,7 +529,7 @@
 class BREAK_LOOP(BCInstruction):
     def eval(self, ctx):
         from rpython.flowspace.flowcontext import Break
-        raise Break
+        return ctx.unroll(Break())
 
 @bc_reader.register_opcode
 class CONTINUE_LOOP(BCInstruction):
@@ -539,7 +539,7 @@
 
     def eval(self, ctx):
         from rpython.flowspace.flowcontext import Continue
-        raise Continue(self.target)
+        return ctx.unroll(Continue(self.target))
 
 class SetupInstruction(BCInstruction):
     def bc_flow(self, reader):

From noreply at buildbot.pypy.org  Tue Feb 10 18:44:02 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Tue, 10 Feb 2015 18:44:02 +0100 (CET)
Subject: [pypy-commit] pypy framestate: find the actual target of break
 statements during analyze_signals() phase
Message-ID: <20150210174402.A15911C0092@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: framestate
Changeset: r75805:c984486d36dc
Date: 2015-02-10 06:39 +0000
http://bitbucket.org/pypy/pypy/changeset/c984486d36dc/

Log:	find the actual target of break statements during analyze_signals()
	phase

diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py
--- a/rpython/flowspace/bytecode.py
+++ b/rpython/flowspace/bytecode.py
@@ -146,7 +146,6 @@
             if self.offset in self.pending_blocks:
                 next_block = self.pending_blocks[self.offset]
                 if not self.curr_block.operations:
-                    import pdb; pdb.set_trace
                     self.blocks.pop()
                 self.enter_next_block(next_block)
             elif self.needs_new_block:
@@ -222,6 +221,12 @@
             for exit in block._exits:
                 exit.set_blockstack(self.blockstack)
 
+    def unroll(self, signal):
+        while self.blockstack:
+            block = self.blockstack.pop()
+            if isinstance(signal, block.handles):
+                return block
+
     def check_graph(self):
         for b in self.blocks:
             if not b._exits:
@@ -527,6 +532,15 @@
 
 @bc_reader.register_opcode
 class BREAK_LOOP(BCInstruction):
+    def bc_flow(self, reader):
+        reader.curr_block.operations.append(self)
+        reader.end_block()
+
+    def do_signals(self, reader):
+        from rpython.flowspace.flowcontext import Break
+        frameblock = reader.unroll(Break())
+        reader.curr_block.set_exits([frameblock.handler])
+
     def eval(self, ctx):
         from rpython.flowspace.flowcontext import Break
         return ctx.unroll(Break())

From noreply at buildbot.pypy.org  Tue Feb 10 19:15:23 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 10 Feb 2015 19:15:23 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: add missing function
Message-ID: <20150210181523.B6BE81C0884@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: stdlib-2.7.9
Changeset: r75806:0ab846ce79d9
Date: 2015-02-10 19:15 +0100
http://bitbucket.org/pypy/pypy/changeset/0ab846ce79d9/

Log:	add missing function

diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py
--- a/pypy/module/_ssl/interp_win32.py
+++ b/pypy/module/_ssl/interp_win32.py
@@ -45,8 +45,6 @@
     'CertOpenSystemStoreA', [rffi.VOIDP, rffi.CCHARP], rwin32.HANDLE)
 CertCloseStore = external(
     'CertCloseStore', [rwin32.HANDLE, rwin32.DWORD], rwin32.BOOL)
-CertFreeCertificateContext = external(
-    'CertFreeCertificateContext', [PCCERT_CONTEXT], rwin32.BOOL)
 CertGetEnhancedKeyUsage = external(
     'CertGetEnhancedKeyUsage',
     [PCCERT_CONTEXT, rwin32.DWORD, PCERT_ENHKEY_USAGE, rwin32.LPDWORD],
@@ -54,9 +52,13 @@
 CertEnumCertificatesInStore = external(
     'CertEnumCertificatesInStore',
     [rwin32.HANDLE, PCCERT_CONTEXT], PCCERT_CONTEXT)
+CertFreeCertificateContext = external(
+    'CertFreeCertificateContext', [PCCERT_CONTEXT], rwin32.BOOL)
 CertEnumCRLsInStore = external(
     'CertEnumCRLsInStore',
     [rwin32.HANDLE, PCCRL_CONTEXT], PCCRL_CONTEXT)
+CertFreeCRLContext = external(
+    'CertFreeCRLContext', [PCCRL_CONTEXT], rwin32.BOOL)
 
 def w_certEncodingType(space, encodingType):
     if encodingType == X509_ASN_ENCODING:

From noreply at buildbot.pypy.org  Tue Feb 10 19:18:30 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 10 Feb 2015 19:18:30 +0100 (CET)
Subject: [pypy-commit] pypy default: Document the result of "is" on NaNs.
Message-ID: <20150210181830.5F48D1C0884@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r75807:112c34045210
Date: 2015-02-10 19:18 +0100
http://bitbucket.org/pypy/pypy/changeset/112c34045210/

Log:	Document the result of "is" on NaNs.

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -300,6 +300,18 @@
 Notably missing from the list above are ``str`` and ``unicode``.  If your
 code relies on comparing strings with ``is``, then it might break in PyPy.
 
+Note that for floats there "``is``" only one object per "bit pattern"
+of the float.  So ``float('nan') is float('nan')`` is true on PyPy,
+but not on CPython because they are two objects; but ``0.0 is -0.0``
+is always False, as the bit patterns are different.  As usual,
+``float('nan') == float('nan')`` is always False.  When used in
+containers (as list items or in sets for example), the exact rule of
+equality used is "``if x is y or x == y``" (on both CPython and PyPy);
+as a consequence, because all ``nans`` are identical in PyPy, you
+cannot have several of them in a set, unlike in CPython.  (Issue `#1974`__)
+
+.. __: https://bitbucket.org/pypy/pypy/issue/1974/different-behaviour-for-collections-of
+
 
 Miscellaneous
 -------------

From noreply at buildbot.pypy.org  Tue Feb 10 19:29:23 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 10 Feb 2015 19:29:23 +0100 (CET)
Subject: [pypy-commit] pypy default: Issue #1975: copy and adapt the audioop
 module from the py3.3 branch.
Message-ID: <20150210182923.485D61C0884@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: 
Changeset: r75808:4e83debd33c2
Date: 2015-02-10 19:24 +0100
http://bitbucket.org/pypy/pypy/changeset/4e83debd33c2/

Log:	Issue #1975: copy and adapt the audioop module from the py3.3
	branch.

diff --git a/lib-python/2.7/test/test_audioop.py b/lib-python/2.7/test/test_audioop.py
--- a/lib-python/2.7/test/test_audioop.py
+++ b/lib-python/2.7/test/test_audioop.py
@@ -2,7 +2,7 @@
 import sys
 import unittest
 import struct
-from test.test_support import run_unittest, impl_detail
+from test.test_support import run_unittest
 
 
 formats = {
@@ -183,7 +183,6 @@
         self.assertEqual(audioop.lin2lin(datas[4], 4, 2),
             packs[2](0, 0x1234, 0x4567, -0x4568, 0x7fff, -0x8000, -1))
 
-    @impl_detail(pypy=False)
     def test_adpcm2lin(self):
         self.assertEqual(audioop.adpcm2lin(b'\x07\x7f\x7f', 1, None),
                          (b'\x00\x00\x00\xff\x00\xff', (-179, 40)))
@@ -198,7 +197,6 @@
             self.assertEqual(audioop.adpcm2lin(b'\0' * 5, w, None),
                              (b'\0' * w * 10, (0, 0)))
 
-    @impl_detail(pypy=False)
     def test_lin2adpcm(self):
         self.assertEqual(audioop.lin2adpcm(datas[1], 1, None),
                          (b'\x07\x7f\x7f', (-221, 39)))
@@ -212,7 +210,6 @@
             self.assertEqual(audioop.lin2adpcm(b'\0' * w * 10, w, None),
                              (b'\0' * 5, (0, 0)))
 
-    @impl_detail(pypy=False)
     def test_lin2alaw(self):
         self.assertEqual(audioop.lin2alaw(datas[1], 1),
                          b'\xd5\x87\xa4\x24\xaa\x2a\x5a')
@@ -221,7 +218,6 @@
         self.assertEqual(audioop.lin2alaw(datas[4], 4),
                          b'\xd5\x87\xa4\x24\xaa\x2a\x55')
 
-    @impl_detail(pypy=False)
     def test_alaw2lin(self):
         encoded = b'\x00\x03\x24\x2a\x51\x54\x55\x58\x6b\x71\x7f'\
                   b'\x80\x83\xa4\xaa\xd1\xd4\xd5\xd8\xeb\xf1\xff'
@@ -236,7 +232,6 @@
             decoded = audioop.alaw2lin(encoded, w)
             self.assertEqual(audioop.lin2alaw(decoded, w), encoded)
 
-    @impl_detail(pypy=False)
     def test_lin2ulaw(self):
         self.assertEqual(audioop.lin2ulaw(datas[1], 1),
                          b'\xff\xad\x8e\x0e\x80\x00\x67')
@@ -245,7 +240,6 @@
         self.assertEqual(audioop.lin2ulaw(datas[4], 4),
                          b'\xff\xad\x8e\x0e\x80\x00\x7e')
 
-    @impl_detail(pypy=False)
     def test_ulaw2lin(self):
         encoded = b'\x00\x0e\x28\x3f\x57\x6a\x76\x7c\x7e\x7f'\
                   b'\x80\x8e\xa8\xbf\xd7\xea\xf6\xfc\xfe\xff'
@@ -360,7 +354,6 @@
         self.assertRaises(audioop.error,
             audioop.findmax, ''.join( chr(x) for x in xrange(256)), -2392392)
 
-    @impl_detail(pypy=False)
     def test_issue7673(self):
         state = None
         for data, size in INVALID_DATA:
@@ -385,7 +378,6 @@
             self.assertRaises(audioop.error, audioop.lin2alaw, data, size)
             self.assertRaises(audioop.error, audioop.lin2adpcm, data, size, state)
 
-    @impl_detail(pypy=False)
     def test_wrongsize(self):
         data = b'abcdefgh'
         state = None
diff --git a/lib_pypy/audioop.py b/lib_pypy/audioop.py
--- a/lib_pypy/audioop.py
+++ b/lib_pypy/audioop.py
@@ -1,12 +1,11 @@
-from __future__ import division
 import __builtin__ as builtins
 import math
 import struct
 from fractions import gcd
-from ctypes import create_string_buffer
+from cffi import FFI
 
 
-_buffer = buffer
+_buffer = memoryview
 
 
 class error(Exception):
@@ -149,7 +148,7 @@
 def _sum2(cp1, cp2, length):
     size = 2
     return sum(getsample(cp1, size, i) * getsample(cp2, size, i)
-               for i in range(length))
+               for i in range(length)) + 0.0
 
 
 def findfit(cp1, cp2):
@@ -328,13 +327,14 @@
     _check_params(len(cp), size)
     clip = _get_clipfn(size)
 
-    result = create_string_buffer(len(cp))
+    rv = ffi.new("unsigned char[]", len(cp))
+    result = ffi.buffer(rv)
 
     for i, sample in enumerate(_get_samples(cp, size)):
         sample = clip(int(sample * factor))
         _put_sample(result, size, i, sample)
 
-    return result.raw
+    return result[:]
 
 
 def tomono(cp, size, fac1, fac2):
@@ -343,7 +343,8 @@
 
     sample_count = _sample_count(cp, size)
 
-    result = create_string_buffer(len(cp) // 2)
+    rv = ffi.new("unsigned char[]", len(cp) // 2)
+    result = ffi.buffer(rv)
 
     for i in range(0, sample_count, 2):
         l_sample = getsample(cp, size, i)
@@ -354,7 +355,7 @@
 
         _put_sample(result, size, i // 2, sample)
 
-    return result.raw
+    return result[:]
 
 
 def tostereo(cp, size, fac1, fac2):
@@ -362,7 +363,8 @@
 
     sample_count = _sample_count(cp, size)
 
-    result = create_string_buffer(len(cp) * 2)
+    rv = ffi.new("unsigned char[]", len(cp) * 2)
+    result = ffi.buffer(rv)
     clip = _get_clipfn(size)
 
     for i in range(sample_count):
@@ -374,7 +376,7 @@
         _put_sample(result, size, i * 2, l_sample)
         _put_sample(result, size, i * 2 + 1, r_sample)
 
-    return result.raw
+    return result[:]
 
 
 def add(cp1, cp2, size):
@@ -385,7 +387,8 @@
 
     clip = _get_clipfn(size)
     sample_count = _sample_count(cp1, size)
-    result = create_string_buffer(len(cp1))
+    rv = ffi.new("unsigned char[]", len(cp1))
+    result = ffi.buffer(rv)
 
     for i in range(sample_count):
         sample1 = getsample(cp1, size, i)
@@ -395,30 +398,32 @@
 
         _put_sample(result, size, i, sample)
 
-    return result.raw
+    return result[:]
 
 
 def bias(cp, size, bias):
     _check_params(len(cp), size)
 
-    result = create_string_buffer(len(cp))
+    rv = ffi.new("unsigned char[]", len(cp))
+    result = ffi.buffer(rv)
 
     for i, sample in enumerate(_get_samples(cp, size)):
         sample = _overflow(sample + bias, size)
         _put_sample(result, size, i, sample)
 
-    return result.raw
+    return result[:]
 
 
 def reverse(cp, size):
     _check_params(len(cp), size)
     sample_count = _sample_count(cp, size)
 
-    result = create_string_buffer(len(cp))
+    rv = ffi.new("unsigned char[]", len(cp))
+    result = ffi.buffer(rv)
     for i, sample in enumerate(_get_samples(cp, size)):
         _put_sample(result, size, sample_count - i - 1, sample)
 
-    return result.raw
+    return result[:]
 
 
 def lin2lin(cp, size, size2):
@@ -429,7 +434,8 @@
         return cp
 
     new_len = (len(cp) // size) * size2
-    result = create_string_buffer(new_len)
+    rv = ffi.new("unsigned char[]", new_len)
+    result = ffi.buffer(rv)
 
     for i in range(_sample_count(cp, size)):
         sample = _get_sample(cp, size, i)
@@ -444,7 +450,7 @@
         sample = _overflow(sample, size2)
         _put_sample(result, size2, i, sample)
 
-    return result.raw
+    return result[:]
 
 
 def ratecv(cp, size, nchannels, inrate, outrate, state, weightA=1, weightB=0):
@@ -489,7 +495,8 @@
     ceiling = (q + 1) * outrate
     nbytes = ceiling * bytes_per_frame
 
-    result = create_string_buffer(nbytes)
+    rv = ffi.new("unsigned char[]", nbytes)
+    result = ffi.buffer(rv)
 
     samples = _get_samples(cp, size)
     out_i = 0
@@ -497,7 +504,7 @@
         while d < 0:
             if frame_count == 0:
                 samps = zip(prev_i, cur_i)
-                retval = result.raw
+                retval = result[:]
 
                 # slice off extra bytes
                 trim_index = (out_i * bytes_per_frame) - len(retval)
@@ -528,25 +535,523 @@
                 d -= inrate
 
 
+ffi = FFI()
+ffi.cdef("""
+typedef short PyInt16;
+
+/* 2's complement (14-bit range) */
+unsigned char
+st_14linear2ulaw(PyInt16 pcm_val);
+PyInt16 st_ulaw2linear16(unsigned char);
+
+/* 2's complement (13-bit range) */
+unsigned char
+st_linear2alaw(PyInt16 pcm_val);
+PyInt16 st_alaw2linear16(unsigned char);
+
+
+void lin2adcpm(unsigned char* rv, unsigned char* cp, size_t len,
+               size_t size, int* state);
+void adcpm2lin(unsigned char* rv, unsigned char* cp, size_t len,
+               size_t size, int* state);
+""")
+
+# This code is directly copied from CPython file: Modules/audioop.c
+_AUDIOOP_C_MODULE = """
+typedef short PyInt16;
+typedef int Py_Int32;
+
+/* Code shamelessly stolen from sox, 12.17.7, g711.c
+** (c) Craig Reese, Joe Campbell and Jeff Poskanzer 1989 */
+
+/* From g711.c:
+ *
+ * December 30, 1994:
+ * Functions linear2alaw, linear2ulaw have been updated to correctly
+ * convert unquantized 16 bit values.
+ * Tables for direct u- to A-law and A- to u-law conversions have been
+ * corrected.
+ * Borge Lindberg, Center for PersonKommunikation, Aalborg University.
+ * bli at cpk.auc.dk
+ *
+ */
+#define BIAS 0x84   /* define the add-in bias for 16 bit samples */
+#define CLIP 32635
+#define SIGN_BIT        (0x80)          /* Sign bit for a A-law byte. */
+#define QUANT_MASK      (0xf)           /* Quantization field mask. */
+#define SEG_SHIFT       (4)             /* Left shift for segment number. */
+#define SEG_MASK        (0x70)          /* Segment field mask. */
+
+static PyInt16 seg_aend[8] = {0x1F, 0x3F, 0x7F, 0xFF,
+                              0x1FF, 0x3FF, 0x7FF, 0xFFF};
+static PyInt16 seg_uend[8] = {0x3F, 0x7F, 0xFF, 0x1FF,
+                              0x3FF, 0x7FF, 0xFFF, 0x1FFF};
+
+static PyInt16
+search(PyInt16 val, PyInt16 *table, int size)
+{
+    int i;
+
+    for (i = 0; i < size; i++) {
+        if (val <= *table++)
+            return (i);
+    }
+    return (size);
+}
+#define st_ulaw2linear16(uc) (_st_ulaw2linear16[uc])
+#define st_alaw2linear16(uc) (_st_alaw2linear16[uc])
+
+static PyInt16 _st_ulaw2linear16[256] = {
+    -32124,  -31100,  -30076,  -29052,  -28028,  -27004,  -25980,
+    -24956,  -23932,  -22908,  -21884,  -20860,  -19836,  -18812,
+    -17788,  -16764,  -15996,  -15484,  -14972,  -14460,  -13948,
+    -13436,  -12924,  -12412,  -11900,  -11388,  -10876,  -10364,
+     -9852,   -9340,   -8828,   -8316,   -7932,   -7676,   -7420,
+     -7164,   -6908,   -6652,   -6396,   -6140,   -5884,   -5628,
+     -5372,   -5116,   -4860,   -4604,   -4348,   -4092,   -3900,
+     -3772,   -3644,   -3516,   -3388,   -3260,   -3132,   -3004,
+     -2876,   -2748,   -2620,   -2492,   -2364,   -2236,   -2108,
+     -1980,   -1884,   -1820,   -1756,   -1692,   -1628,   -1564,
+     -1500,   -1436,   -1372,   -1308,   -1244,   -1180,   -1116,
+     -1052,    -988,    -924,    -876,    -844,    -812,    -780,
+      -748,    -716,    -684,    -652,    -620,    -588,    -556,
+      -524,    -492,    -460,    -428,    -396,    -372,    -356,
+      -340,    -324,    -308,    -292,    -276,    -260,    -244,
+      -228,    -212,    -196,    -180,    -164,    -148,    -132,
+      -120,    -112,    -104,     -96,     -88,     -80,     -72,
+       -64,     -56,     -48,     -40,     -32,     -24,     -16,
+    -8,       0,   32124,   31100,   30076,   29052,   28028,
+     27004,   25980,   24956,   23932,   22908,   21884,   20860,
+     19836,   18812,   17788,   16764,   15996,   15484,   14972,
+     14460,   13948,   13436,   12924,   12412,   11900,   11388,
+     10876,   10364,    9852,    9340,    8828,    8316,    7932,
+      7676,    7420,    7164,    6908,    6652,    6396,    6140,
+      5884,    5628,    5372,    5116,    4860,    4604,    4348,
+      4092,    3900,    3772,    3644,    3516,    3388,    3260,
+      3132,    3004,    2876,    2748,    2620,    2492,    2364,
+      2236,    2108,    1980,    1884,    1820,    1756,    1692,
+      1628,    1564,    1500,    1436,    1372,    1308,    1244,
+      1180,    1116,    1052,     988,     924,     876,     844,
+       812,     780,     748,     716,     684,     652,     620,
+       588,     556,     524,     492,     460,     428,     396,
+       372,     356,     340,     324,     308,     292,     276,
+       260,     244,     228,     212,     196,     180,     164,
+       148,     132,     120,     112,     104,      96,      88,
+    80,      72,      64,      56,      48,      40,      32,
+    24,      16,       8,       0
+};
+
+/*
+ * linear2ulaw() accepts a 14-bit signed integer and encodes it as u-law data
+ * stored in a unsigned char.  This function should only be called with
+ * the data shifted such that it only contains information in the lower
+ * 14-bits.
+ *
+ * In order to simplify the encoding process, the original linear magnitude
+ * is biased by adding 33 which shifts the encoding range from (0 - 8158) to
+ * (33 - 8191). The result can be seen in the following encoding table:
+ *
+ *      Biased Linear Input Code        Compressed Code
+ *      ------------------------        ---------------
+ *      00000001wxyza                   000wxyz
+ *      0000001wxyzab                   001wxyz
+ *      000001wxyzabc                   010wxyz
+ *      00001wxyzabcd                   011wxyz
+ *      0001wxyzabcde                   100wxyz
+ *      001wxyzabcdef                   101wxyz
+ *      01wxyzabcdefg                   110wxyz
+ *      1wxyzabcdefgh                   111wxyz
+ *
+ * Each biased linear code has a leading 1 which identifies the segment
+ * number. The value of the segment number is equal to 7 minus the number
+ * of leading 0's. The quantization interval is directly available as the
+ * four bits wxyz.  * The trailing bits (a - h) are ignored.
+ *
+ * Ordinarily the complement of the resulting code word is used for
+ * transmission, and so the code word is complemented before it is returned.
+ *
+ * For further information see John C. Bellamy's Digital Telephony, 1982,
+ * John Wiley & Sons, pps 98-111 and 472-476.
+ */
+static unsigned char
+st_14linear2ulaw(PyInt16 pcm_val)       /* 2's complement (14-bit range) */
+{
+    PyInt16         mask;
+    PyInt16         seg;
+    unsigned char   uval;
+
+    /* The original sox code does this in the calling function, not here */
+    pcm_val = pcm_val >> 2;
+
+    /* u-law inverts all bits */
+    /* Get the sign and the magnitude of the value. */
+    if (pcm_val < 0) {
+        pcm_val = -pcm_val;
+        mask = 0x7F;
+    } else {
+        mask = 0xFF;
+    }
+    if ( pcm_val > CLIP ) pcm_val = CLIP;           /* clip the magnitude */
+    pcm_val += (BIAS >> 2);
+
+    /* Convert the scaled magnitude to segment number. */
+    seg = search(pcm_val, seg_uend, 8);
+
+    /*
+     * Combine the sign, segment, quantization bits;
+     * and complement the code word.
+     */
+    if (seg >= 8)           /* out of range, return maximum value. */
+        return (unsigned char) (0x7F ^ mask);
+    else {
+        uval = (unsigned char) (seg << 4) | ((pcm_val >> (seg + 1)) & 0xF);
+        return (uval ^ mask);
+    }
+
+}
+
+static PyInt16 _st_alaw2linear16[256] = {
+     -5504,   -5248,   -6016,   -5760,   -4480,   -4224,   -4992,
+     -4736,   -7552,   -7296,   -8064,   -7808,   -6528,   -6272,
+     -7040,   -6784,   -2752,   -2624,   -3008,   -2880,   -2240,
+     -2112,   -2496,   -2368,   -3776,   -3648,   -4032,   -3904,
+     -3264,   -3136,   -3520,   -3392,  -22016,  -20992,  -24064,
+    -23040,  -17920,  -16896,  -19968,  -18944,  -30208,  -29184,
+    -32256,  -31232,  -26112,  -25088,  -28160,  -27136,  -11008,
+    -10496,  -12032,  -11520,   -8960,   -8448,   -9984,   -9472,
+    -15104,  -14592,  -16128,  -15616,  -13056,  -12544,  -14080,
+    -13568,    -344,    -328,    -376,    -360,    -280,    -264,
+      -312,    -296,    -472,    -456,    -504,    -488,    -408,
+      -392,    -440,    -424,     -88,     -72,    -120,    -104,
+       -24,      -8,     -56,     -40,    -216,    -200,    -248,
+      -232,    -152,    -136,    -184,    -168,   -1376,   -1312,
+     -1504,   -1440,   -1120,   -1056,   -1248,   -1184,   -1888,
+     -1824,   -2016,   -1952,   -1632,   -1568,   -1760,   -1696,
+      -688,    -656,    -752,    -720,    -560,    -528,    -624,
+      -592,    -944,    -912,   -1008,    -976,    -816,    -784,
+      -880,    -848,    5504,    5248,    6016,    5760,    4480,
+      4224,    4992,    4736,    7552,    7296,    8064,    7808,
+      6528,    6272,    7040,    6784,    2752,    2624,    3008,
+      2880,    2240,    2112,    2496,    2368,    3776,    3648,
+      4032,    3904,    3264,    3136,    3520,    3392,   22016,
+     20992,   24064,   23040,   17920,   16896,   19968,   18944,
+     30208,   29184,   32256,   31232,   26112,   25088,   28160,
+     27136,   11008,   10496,   12032,   11520,    8960,    8448,
+      9984,    9472,   15104,   14592,   16128,   15616,   13056,
+     12544,   14080,   13568,     344,     328,     376,     360,
+       280,     264,     312,     296,     472,     456,     504,
+       488,     408,     392,     440,     424,      88,      72,
+       120,     104,      24,       8,      56,      40,     216,
+       200,     248,     232,     152,     136,     184,     168,
+      1376,    1312,    1504,    1440,    1120,    1056,    1248,
+      1184,    1888,    1824,    2016,    1952,    1632,    1568,
+      1760,    1696,     688,     656,     752,     720,     560,
+       528,     624,     592,     944,     912,    1008,     976,
+       816,     784,     880,     848
+};
+
+/*
+ * linear2alaw() accepts an 13-bit signed integer and encodes it as A-law data
+ * stored in a unsigned char.  This function should only be called with
+ * the data shifted such that it only contains information in the lower
+ * 13-bits.
+ *
+ *              Linear Input Code       Compressed Code
+ *      ------------------------        ---------------
+ *      0000000wxyza                    000wxyz
+ *      0000001wxyza                    001wxyz
+ *      000001wxyzab                    010wxyz
+ *      00001wxyzabc                    011wxyz
+ *      0001wxyzabcd                    100wxyz
+ *      001wxyzabcde                    101wxyz
+ *      01wxyzabcdef                    110wxyz
+ *      1wxyzabcdefg                    111wxyz
+ *
+ * For further information see John C. Bellamy's Digital Telephony, 1982,
+ * John Wiley & Sons, pps 98-111 and 472-476.
+ */
+static unsigned char
+st_linear2alaw(PyInt16 pcm_val) /* 2's complement (13-bit range) */
+{
+    PyInt16         mask;
+    short           seg;
+    unsigned char   aval;
+
+    /* The original sox code does this in the calling function, not here */
+    pcm_val = pcm_val >> 3;
+
+    /* A-law using even bit inversion */
+    if (pcm_val >= 0) {
+        mask = 0xD5;            /* sign (7th) bit = 1 */
+    } else {
+        mask = 0x55;            /* sign bit = 0 */
+        pcm_val = -pcm_val - 1;
+    }
+
+    /* Convert the scaled magnitude to segment number. */
+    seg = search(pcm_val, seg_aend, 8);
+
+    /* Combine the sign, segment, and quantization bits. */
+
+    if (seg >= 8)           /* out of range, return maximum value. */
+        return (unsigned char) (0x7F ^ mask);
+    else {
+        aval = (unsigned char) seg << SEG_SHIFT;
+        if (seg < 2)
+            aval |= (pcm_val >> 1) & QUANT_MASK;
+        else
+            aval |= (pcm_val >> seg) & QUANT_MASK;
+        return (aval ^ mask);
+    }
+}
+/* End of code taken from sox */
+
+/* Intel ADPCM step variation table */
+static int indexTable[16] = {
+    -1, -1, -1, -1, 2, 4, 6, 8,
+    -1, -1, -1, -1, 2, 4, 6, 8,
+};
+
+static int stepsizeTable[89] = {
+    7, 8, 9, 10, 11, 12, 13, 14, 16, 17,
+    19, 21, 23, 25, 28, 31, 34, 37, 41, 45,
+    50, 55, 60, 66, 73, 80, 88, 97, 107, 118,
+    130, 143, 157, 173, 190, 209, 230, 253, 279, 307,
+    337, 371, 408, 449, 494, 544, 598, 658, 724, 796,
+    876, 963, 1060, 1166, 1282, 1411, 1552, 1707, 1878, 2066,
+    2272, 2499, 2749, 3024, 3327, 3660, 4026, 4428, 4871, 5358,
+    5894, 6484, 7132, 7845, 8630, 9493, 10442, 11487, 12635, 13899,
+    15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794, 32767
+};
+
+#define CHARP(cp, i) ((signed char *)(cp+i))
+#define SHORTP(cp, i) ((short *)(cp+i))
+#define LONGP(cp, i) ((Py_Int32 *)(cp+i))
+"""
+
+lib = ffi.verify(_AUDIOOP_C_MODULE + """
+void lin2adcpm(unsigned char* ncp, unsigned char* cp, size_t len,
+               size_t size, int* state)
+{
+    int step, outputbuffer = 0, bufferstep;
+    int val = 0;
+    int diff, vpdiff, sign, delta;
+    size_t i;
+    int valpred = state[0];
+    int index = state[1];
+
+    step = stepsizeTable[index];
+    bufferstep = 1;
+
+    for ( i=0; i < len; i += size ) {
+        if ( size == 1 )      val = ((int)*CHARP(cp, i)) << 8;
+        else if ( size == 2 ) val = (int)*SHORTP(cp, i);
+        else if ( size == 4 ) val = ((int)*LONGP(cp, i)) >> 16;
+
+        /* Step 1 - compute difference with previous value */
+        diff = val - valpred;
+        sign = (diff < 0) ? 8 : 0;
+        if ( sign ) diff = (-diff);
+
+        /* Step 2 - Divide and clamp */
+        /* Note:
+        ** This code *approximately* computes:
+        **    delta = diff*4/step;
+        **    vpdiff = (delta+0.5)*step/4;
+        ** but in shift step bits are dropped. The net result of this
+        ** is that even if you have fast mul/div hardware you cannot
+        ** put it to good use since the fixup would be too expensive.
+        */
+        delta = 0;
+        vpdiff = (step >> 3);
+
+        if ( diff >= step ) {
+            delta = 4;
+            diff -= step;
+            vpdiff += step;
+        }
+        step >>= 1;
+        if ( diff >= step  ) {
+            delta |= 2;
+            diff -= step;
+            vpdiff += step;
+        }
+        step >>= 1;
+        if ( diff >= step ) {
+            delta |= 1;
+            vpdiff += step;
+        }
+
+        /* Step 3 - Update previous value */
+        if ( sign )
+            valpred -= vpdiff;
+        else
+            valpred += vpdiff;
+
+        /* Step 4 - Clamp previous value to 16 bits */
+        if ( valpred > 32767 )
+            valpred = 32767;
+        else if ( valpred < -32768 )
+            valpred = -32768;
+
+        /* Step 5 - Assemble value, update index and step values */
+        delta |= sign;
+
+        index += indexTable[delta];
+        if ( index < 0 ) index = 0;
+        if ( index > 88 ) index = 88;
+        step = stepsizeTable[index];
+
+        /* Step 6 - Output value */
+        if ( bufferstep ) {
+            outputbuffer = (delta << 4) & 0xf0;
+        } else {
+            *ncp++ = (delta & 0x0f) | outputbuffer;
+        }
+        bufferstep = !bufferstep;
+    }
+    state[0] = valpred;
+    state[1] = index;
+}
+
+
+void adcpm2lin(unsigned char* ncp, unsigned char* cp, size_t len,
+               size_t size, int* state)
+{
+    int step, inputbuffer = 0, bufferstep;
+    int val = 0;
+    int diff, vpdiff, sign, delta;
+    size_t i;
+    int valpred = state[0];
+    int index = state[1];
+
+    step = stepsizeTable[index];
+    bufferstep = 0;
+
+    for ( i=0; i < len*size*2; i += size ) {
+        /* Step 1 - get the delta value and compute next index */
+        if ( bufferstep ) {
+            delta = inputbuffer & 0xf;
+        } else {
+            inputbuffer = *cp++;
+            delta = (inputbuffer >> 4) & 0xf;
+        }
+
+        bufferstep = !bufferstep;
+
+        /* Step 2 - Find new index value (for later) */
+        index += indexTable[delta];
+        if ( index < 0 ) index = 0;
+        if ( index > 88 ) index = 88;
+
+        /* Step 3 - Separate sign and magnitude */
+        sign = delta & 8;
+        delta = delta & 7;
+
+        /* Step 4 - Compute difference and new predicted value */
+        /*
+        ** Computes 'vpdiff = (delta+0.5)*step/4', but see comment
+        ** in adpcm_coder.
+        */
+        vpdiff = step >> 3;
+        if ( delta & 4 ) vpdiff += step;
+        if ( delta & 2 ) vpdiff += step>>1;
+        if ( delta & 1 ) vpdiff += step>>2;
+
+        if ( sign )
+            valpred -= vpdiff;
+        else
+            valpred += vpdiff;
+
+        /* Step 5 - clamp output value */
+        if ( valpred > 32767 )
+            valpred = 32767;
+        else if ( valpred < -32768 )
+            valpred = -32768;
+
+        /* Step 6 - Update step value */
+        step = stepsizeTable[index];
+
+        /* Step 6 - Output value */
+        if ( size == 1 ) *CHARP(ncp, i) = (signed char)(valpred >> 8);
+        else if ( size == 2 ) *SHORTP(ncp, i) = (short)(valpred);
+        else if ( size == 4 ) *LONGP(ncp, i) = (Py_Int32)(valpred<<16);
+    }
+    state[0] = valpred;
+    state[1] = index;
+}
+""")
+
+def _get_lin_samples(cp, size):
+    for sample in _get_samples(cp, size):
+        if size == 1:
+            yield sample << 8
+        elif size == 2:
+            yield sample
+        elif size == 4:
+            yield sample >> 16
+
+def _put_lin_sample(result, size, i, sample):
+    if size == 1:
+        sample >>= 8
+    elif size == 2:
+        pass
+    elif size == 4:
+        sample <<= 16
+    _put_sample(result, size, i, sample)
+
 def lin2ulaw(cp, size):
-    raise NotImplementedError()
+    _check_params(len(cp), size)
+    rv = ffi.new("unsigned char[]", _sample_count(cp, size))
+    for i, sample in enumerate(_get_lin_samples(cp, size)):
+        rv[i] = lib.st_14linear2ulaw(sample)
+    return ffi.buffer(rv)[:]
 
 
 def ulaw2lin(cp, size):
-    raise NotImplementedError()
+    _check_size(size)
+    rv = ffi.new("unsigned char[]", len(cp) * size)
+    result = ffi.buffer(rv)
+    for i, value in enumerate(cp):
+        sample = lib.st_ulaw2linear16(ord(value))
+        _put_lin_sample(result, size, i, sample)
+    return result[:]
 
 
 def lin2alaw(cp, size):
-    raise NotImplementedError()
+    _check_params(len(cp), size)
+    rv = ffi.new("unsigned char[]", _sample_count(cp, size))
+    for i, sample in enumerate(_get_lin_samples(cp, size)):
+        rv[i] = lib.st_linear2alaw(sample)
+    return ffi.buffer(rv)[:]
 
 
 def alaw2lin(cp, size):
-    raise NotImplementedError()
+    _check_size(size)
+    rv = ffi.new("unsigned char[]", len(cp) * size)
+    result = ffi.buffer(rv)
+    for i, value in enumerate(cp):
+        sample = lib.st_alaw2linear16(ord(value))
+        _put_lin_sample(result, size, i, sample)
+    return result[:]
 
 
 def lin2adpcm(cp, size, state):
-    raise NotImplementedError()
+    _check_params(len(cp), size)
+    if state is None:
+        state = (0, 0)
+    rv = ffi.new("unsigned char[]", len(cp) // size // 2)
+    state_ptr = ffi.new("int[]", state)
+    lib.lin2adcpm(rv, cp, len(cp), size, state_ptr)
+    return ffi.buffer(rv)[:], tuple(state_ptr)
 
 
 def adpcm2lin(cp, size, state):
-    raise NotImplementedError()
+    _check_size(size)
+    if state is None:
+        state = (0, 0)
+    rv = ffi.new("unsigned char[]", len(cp) * size * 2)
+    state_ptr = ffi.new("int[]", state)
+    lib.adcpm2lin(rv, cp, len(cp), size, state_ptr)
+    return ffi.buffer(rv)[:], tuple(state_ptr)
+

From noreply at buildbot.pypy.org  Tue Feb 10 19:32:22 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 10 Feb 2015 19:32:22 +0100 (CET)
Subject: [pypy-commit] pypy default: force compilation of ffi library in
	audioop.py
Message-ID: <20150210183222.4A81D1C0884@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: 
Changeset: r75809:91ccdfc264d5
Date: 2015-02-10 19:32 +0100
http://bitbucket.org/pypy/pypy/changeset/91ccdfc264d5/

Log:	force compilation of ffi library in audioop.py

diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -108,7 +108,7 @@
 '''
 
 def create_cffi_import_libraries(pypy_c, options):
-    modules = ['_sqlite3']
+    modules = ['_sqlite3', 'audioop']
     subprocess.check_call([str(pypy_c), '-c', 'import _sqlite3'])
     if not sys.platform == 'win32':
         modules += ['_curses', 'syslog', 'gdbm', '_sqlite3']

From noreply at buildbot.pypy.org  Tue Feb 10 19:56:13 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 10 Feb 2015 19:56:13 +0100 (CET)
Subject: [pypy-commit] pypy default: The float-list strategy needs to be
 slightly more careful about NaNs
Message-ID: <20150210185613.0870F1C0884@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r75810:690f33fb3b3e
Date: 2015-02-10 19:27 +0100
http://bitbucket.org/pypy/pypy/changeset/690f33fb3b3e/

Log:	The float-list strategy needs to be slightly more careful about NaNs

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1670,6 +1670,25 @@
     def getitems_float(self, w_list):
         return self.unerase(w_list.lstorage)
 
+    def _safe_find(self, w_list, obj, start, stop):
+        from rpython.rlib.rfloat import isnan
+        from rpython.rlib.longlong2float import float2longlong
+        #
+        l = self.unerase(w_list.lstorage)
+        stop = min(stop, len(l))
+        if not isnan(obj):
+            for i in range(start, stop):
+                val = l[i]
+                if val == obj:
+                    return i
+        else:
+            search = float2longlong(obj)
+            for i in range(start, stop):
+                val = l[i]
+                if float2longlong(val) == search:
+                    return i
+        raise ValueError
+
 
 class BytesListStrategy(ListStrategy):
     import_from_mixin(AbstractUnwrappedStrategy)
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -1565,6 +1565,18 @@
         assert l[::11] == [-sys.maxint, item11]
         assert item11 in l[::11]
 
+    def test_bug_list_of_nans(self):
+        N = float('nan')
+        L1 = [N, 'foo']       # general object strategy
+        assert N in L1
+        assert L1.index(N) == 0
+        assert L1 == [N, 'foo']
+        # our float list strategy needs to consider NaNs are equal!
+        L2 = [N, 0.0]         # float strategy
+        assert N in L2
+        assert L2.index(N) == 0
+        assert L2 == [N, -0.0]
+
 
 class AppTestListObjectWithRangeList(AppTestListObject):
     """Run the list object tests with range lists enabled. Tests should go in

From noreply at buildbot.pypy.org  Tue Feb 10 19:56:14 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 10 Feb 2015 19:56:14 +0100 (CET)
Subject: [pypy-commit] pypy default: Same issue with tuples containing NaNs
Message-ID: <20150210185614.2A2161C0884@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r75811:68255911a884
Date: 2015-02-10 19:34 +0100
http://bitbucket.org/pypy/pypy/changeset/68255911a884/

Log:	Same issue with tuples containing NaNs

diff --git a/pypy/objspace/std/specialisedtupleobject.py b/pypy/objspace/std/specialisedtupleobject.py
--- a/pypy/objspace/std/specialisedtupleobject.py
+++ b/pypy/objspace/std/specialisedtupleobject.py
@@ -5,6 +5,7 @@
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.tool.sourcetools import func_with_new_name
+from rpython.rlib.longlong2float import float2longlong
 
 
 class NotSpecialised(Exception):
@@ -97,6 +98,11 @@
                         return space.w_False
                 else:
                     if myval != otherval:
+                        if typetuple[i] == float:
+                            # issue with NaNs, which should be equal here
+                            if (float2longlong(myval) ==
+                                float2longlong(otherval)):
+                                continue
                         return space.w_False
             return space.w_True
 
diff --git a/pypy/objspace/std/test/test_specialisedtupleobject.py b/pypy/objspace/std/test/test_specialisedtupleobject.py
--- a/pypy/objspace/std/test/test_specialisedtupleobject.py
+++ b/pypy/objspace/std/test/test_specialisedtupleobject.py
@@ -222,6 +222,13 @@
         t = (F(42), F(43))
         assert type(t[0]) is F
 
+    def test_bug_tuples_of_nans(self):
+        N = float('nan')
+        T = (N, N)
+        assert N in T
+        assert T == (N, N)
+        assert (0.0, 0.0) == (-0.0, -0.0)
+
 
 class AppTestAll(test_tupleobject.AppTestW_TupleObject):
     spaceconfig = {"objspace.std.withspecialisedtuple": True}

From noreply at buildbot.pypy.org  Tue Feb 10 19:56:15 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 10 Feb 2015 19:56:15 +0100 (CET)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20150210185615.4F37A1C0884@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r75812:937407e63958
Date: 2015-02-10 19:37 +0100
http://bitbucket.org/pypy/pypy/changeset/937407e63958/

Log:	merge heads

diff --git a/lib-python/2.7/test/test_audioop.py b/lib-python/2.7/test/test_audioop.py
--- a/lib-python/2.7/test/test_audioop.py
+++ b/lib-python/2.7/test/test_audioop.py
@@ -2,7 +2,7 @@
 import sys
 import unittest
 import struct
-from test.test_support import run_unittest, impl_detail
+from test.test_support import run_unittest
 
 
 formats = {
@@ -183,7 +183,6 @@
         self.assertEqual(audioop.lin2lin(datas[4], 4, 2),
             packs[2](0, 0x1234, 0x4567, -0x4568, 0x7fff, -0x8000, -1))
 
-    @impl_detail(pypy=False)
     def test_adpcm2lin(self):
         self.assertEqual(audioop.adpcm2lin(b'\x07\x7f\x7f', 1, None),
                          (b'\x00\x00\x00\xff\x00\xff', (-179, 40)))
@@ -198,7 +197,6 @@
             self.assertEqual(audioop.adpcm2lin(b'\0' * 5, w, None),
                              (b'\0' * w * 10, (0, 0)))
 
-    @impl_detail(pypy=False)
     def test_lin2adpcm(self):
         self.assertEqual(audioop.lin2adpcm(datas[1], 1, None),
                          (b'\x07\x7f\x7f', (-221, 39)))
@@ -212,7 +210,6 @@
             self.assertEqual(audioop.lin2adpcm(b'\0' * w * 10, w, None),
                              (b'\0' * 5, (0, 0)))
 
-    @impl_detail(pypy=False)
     def test_lin2alaw(self):
         self.assertEqual(audioop.lin2alaw(datas[1], 1),
                          b'\xd5\x87\xa4\x24\xaa\x2a\x5a')
@@ -221,7 +218,6 @@
         self.assertEqual(audioop.lin2alaw(datas[4], 4),
                          b'\xd5\x87\xa4\x24\xaa\x2a\x55')
 
-    @impl_detail(pypy=False)
     def test_alaw2lin(self):
         encoded = b'\x00\x03\x24\x2a\x51\x54\x55\x58\x6b\x71\x7f'\
                   b'\x80\x83\xa4\xaa\xd1\xd4\xd5\xd8\xeb\xf1\xff'
@@ -236,7 +232,6 @@
             decoded = audioop.alaw2lin(encoded, w)
             self.assertEqual(audioop.lin2alaw(decoded, w), encoded)
 
-    @impl_detail(pypy=False)
     def test_lin2ulaw(self):
         self.assertEqual(audioop.lin2ulaw(datas[1], 1),
                          b'\xff\xad\x8e\x0e\x80\x00\x67')
@@ -245,7 +240,6 @@
         self.assertEqual(audioop.lin2ulaw(datas[4], 4),
                          b'\xff\xad\x8e\x0e\x80\x00\x7e')
 
-    @impl_detail(pypy=False)
     def test_ulaw2lin(self):
         encoded = b'\x00\x0e\x28\x3f\x57\x6a\x76\x7c\x7e\x7f'\
                   b'\x80\x8e\xa8\xbf\xd7\xea\xf6\xfc\xfe\xff'
@@ -360,7 +354,6 @@
         self.assertRaises(audioop.error,
             audioop.findmax, ''.join( chr(x) for x in xrange(256)), -2392392)
 
-    @impl_detail(pypy=False)
     def test_issue7673(self):
         state = None
         for data, size in INVALID_DATA:
@@ -385,7 +378,6 @@
             self.assertRaises(audioop.error, audioop.lin2alaw, data, size)
             self.assertRaises(audioop.error, audioop.lin2adpcm, data, size, state)
 
-    @impl_detail(pypy=False)
     def test_wrongsize(self):
         data = b'abcdefgh'
         state = None
diff --git a/lib_pypy/audioop.py b/lib_pypy/audioop.py
--- a/lib_pypy/audioop.py
+++ b/lib_pypy/audioop.py
@@ -1,12 +1,11 @@
-from __future__ import division
 import __builtin__ as builtins
 import math
 import struct
 from fractions import gcd
-from ctypes import create_string_buffer
+from cffi import FFI
 
 
-_buffer = buffer
+_buffer = memoryview
 
 
 class error(Exception):
@@ -149,7 +148,7 @@
 def _sum2(cp1, cp2, length):
     size = 2
     return sum(getsample(cp1, size, i) * getsample(cp2, size, i)
-               for i in range(length))
+               for i in range(length)) + 0.0
 
 
 def findfit(cp1, cp2):
@@ -328,13 +327,14 @@
     _check_params(len(cp), size)
     clip = _get_clipfn(size)
 
-    result = create_string_buffer(len(cp))
+    rv = ffi.new("unsigned char[]", len(cp))
+    result = ffi.buffer(rv)
 
     for i, sample in enumerate(_get_samples(cp, size)):
         sample = clip(int(sample * factor))
         _put_sample(result, size, i, sample)
 
-    return result.raw
+    return result[:]
 
 
 def tomono(cp, size, fac1, fac2):
@@ -343,7 +343,8 @@
 
     sample_count = _sample_count(cp, size)
 
-    result = create_string_buffer(len(cp) // 2)
+    rv = ffi.new("unsigned char[]", len(cp) // 2)
+    result = ffi.buffer(rv)
 
     for i in range(0, sample_count, 2):
         l_sample = getsample(cp, size, i)
@@ -354,7 +355,7 @@
 
         _put_sample(result, size, i // 2, sample)
 
-    return result.raw
+    return result[:]
 
 
 def tostereo(cp, size, fac1, fac2):
@@ -362,7 +363,8 @@
 
     sample_count = _sample_count(cp, size)
 
-    result = create_string_buffer(len(cp) * 2)
+    rv = ffi.new("unsigned char[]", len(cp) * 2)
+    result = ffi.buffer(rv)
     clip = _get_clipfn(size)
 
     for i in range(sample_count):
@@ -374,7 +376,7 @@
         _put_sample(result, size, i * 2, l_sample)
         _put_sample(result, size, i * 2 + 1, r_sample)
 
-    return result.raw
+    return result[:]
 
 
 def add(cp1, cp2, size):
@@ -385,7 +387,8 @@
 
     clip = _get_clipfn(size)
     sample_count = _sample_count(cp1, size)
-    result = create_string_buffer(len(cp1))
+    rv = ffi.new("unsigned char[]", len(cp1))
+    result = ffi.buffer(rv)
 
     for i in range(sample_count):
         sample1 = getsample(cp1, size, i)
@@ -395,30 +398,32 @@
 
         _put_sample(result, size, i, sample)
 
-    return result.raw
+    return result[:]
 
 
 def bias(cp, size, bias):
     _check_params(len(cp), size)
 
-    result = create_string_buffer(len(cp))
+    rv = ffi.new("unsigned char[]", len(cp))
+    result = ffi.buffer(rv)
 
     for i, sample in enumerate(_get_samples(cp, size)):
         sample = _overflow(sample + bias, size)
         _put_sample(result, size, i, sample)
 
-    return result.raw
+    return result[:]
 
 
 def reverse(cp, size):
     _check_params(len(cp), size)
     sample_count = _sample_count(cp, size)
 
-    result = create_string_buffer(len(cp))
+    rv = ffi.new("unsigned char[]", len(cp))
+    result = ffi.buffer(rv)
     for i, sample in enumerate(_get_samples(cp, size)):
         _put_sample(result, size, sample_count - i - 1, sample)
 
-    return result.raw
+    return result[:]
 
 
 def lin2lin(cp, size, size2):
@@ -429,7 +434,8 @@
         return cp
 
     new_len = (len(cp) // size) * size2
-    result = create_string_buffer(new_len)
+    rv = ffi.new("unsigned char[]", new_len)
+    result = ffi.buffer(rv)
 
     for i in range(_sample_count(cp, size)):
         sample = _get_sample(cp, size, i)
@@ -444,7 +450,7 @@
         sample = _overflow(sample, size2)
         _put_sample(result, size2, i, sample)
 
-    return result.raw
+    return result[:]
 
 
 def ratecv(cp, size, nchannels, inrate, outrate, state, weightA=1, weightB=0):
@@ -489,7 +495,8 @@
     ceiling = (q + 1) * outrate
     nbytes = ceiling * bytes_per_frame
 
-    result = create_string_buffer(nbytes)
+    rv = ffi.new("unsigned char[]", nbytes)
+    result = ffi.buffer(rv)
 
     samples = _get_samples(cp, size)
     out_i = 0
@@ -497,7 +504,7 @@
         while d < 0:
             if frame_count == 0:
                 samps = zip(prev_i, cur_i)
-                retval = result.raw
+                retval = result[:]
 
                 # slice off extra bytes
                 trim_index = (out_i * bytes_per_frame) - len(retval)
@@ -528,25 +535,523 @@
                 d -= inrate
 
 
+ffi = FFI()
+ffi.cdef("""
+typedef short PyInt16;
+
+/* 2's complement (14-bit range) */
+unsigned char
+st_14linear2ulaw(PyInt16 pcm_val);
+PyInt16 st_ulaw2linear16(unsigned char);
+
+/* 2's complement (13-bit range) */
+unsigned char
+st_linear2alaw(PyInt16 pcm_val);
+PyInt16 st_alaw2linear16(unsigned char);
+
+
+void lin2adcpm(unsigned char* rv, unsigned char* cp, size_t len,
+               size_t size, int* state);
+void adcpm2lin(unsigned char* rv, unsigned char* cp, size_t len,
+               size_t size, int* state);
+""")
+
+# This code is directly copied from CPython file: Modules/audioop.c
+_AUDIOOP_C_MODULE = """
+typedef short PyInt16;
+typedef int Py_Int32;
+
+/* Code shamelessly stolen from sox, 12.17.7, g711.c
+** (c) Craig Reese, Joe Campbell and Jeff Poskanzer 1989 */
+
+/* From g711.c:
+ *
+ * December 30, 1994:
+ * Functions linear2alaw, linear2ulaw have been updated to correctly
+ * convert unquantized 16 bit values.
+ * Tables for direct u- to A-law and A- to u-law conversions have been
+ * corrected.
+ * Borge Lindberg, Center for PersonKommunikation, Aalborg University.
+ * bli at cpk.auc.dk
+ *
+ */
+#define BIAS 0x84   /* define the add-in bias for 16 bit samples */
+#define CLIP 32635
+#define SIGN_BIT        (0x80)          /* Sign bit for a A-law byte. */
+#define QUANT_MASK      (0xf)           /* Quantization field mask. */
+#define SEG_SHIFT       (4)             /* Left shift for segment number. */
+#define SEG_MASK        (0x70)          /* Segment field mask. */
+
+static PyInt16 seg_aend[8] = {0x1F, 0x3F, 0x7F, 0xFF,
+                              0x1FF, 0x3FF, 0x7FF, 0xFFF};
+static PyInt16 seg_uend[8] = {0x3F, 0x7F, 0xFF, 0x1FF,
+                              0x3FF, 0x7FF, 0xFFF, 0x1FFF};
+
+static PyInt16
+search(PyInt16 val, PyInt16 *table, int size)
+{
+    int i;
+
+    for (i = 0; i < size; i++) {
+        if (val <= *table++)
+            return (i);
+    }
+    return (size);
+}
+#define st_ulaw2linear16(uc) (_st_ulaw2linear16[uc])
+#define st_alaw2linear16(uc) (_st_alaw2linear16[uc])
+
+static PyInt16 _st_ulaw2linear16[256] = {
+    -32124,  -31100,  -30076,  -29052,  -28028,  -27004,  -25980,
+    -24956,  -23932,  -22908,  -21884,  -20860,  -19836,  -18812,
+    -17788,  -16764,  -15996,  -15484,  -14972,  -14460,  -13948,
+    -13436,  -12924,  -12412,  -11900,  -11388,  -10876,  -10364,
+     -9852,   -9340,   -8828,   -8316,   -7932,   -7676,   -7420,
+     -7164,   -6908,   -6652,   -6396,   -6140,   -5884,   -5628,
+     -5372,   -5116,   -4860,   -4604,   -4348,   -4092,   -3900,
+     -3772,   -3644,   -3516,   -3388,   -3260,   -3132,   -3004,
+     -2876,   -2748,   -2620,   -2492,   -2364,   -2236,   -2108,
+     -1980,   -1884,   -1820,   -1756,   -1692,   -1628,   -1564,
+     -1500,   -1436,   -1372,   -1308,   -1244,   -1180,   -1116,
+     -1052,    -988,    -924,    -876,    -844,    -812,    -780,
+      -748,    -716,    -684,    -652,    -620,    -588,    -556,
+      -524,    -492,    -460,    -428,    -396,    -372,    -356,
+      -340,    -324,    -308,    -292,    -276,    -260,    -244,
+      -228,    -212,    -196,    -180,    -164,    -148,    -132,
+      -120,    -112,    -104,     -96,     -88,     -80,     -72,
+       -64,     -56,     -48,     -40,     -32,     -24,     -16,
+    -8,       0,   32124,   31100,   30076,   29052,   28028,
+     27004,   25980,   24956,   23932,   22908,   21884,   20860,
+     19836,   18812,   17788,   16764,   15996,   15484,   14972,
+     14460,   13948,   13436,   12924,   12412,   11900,   11388,
+     10876,   10364,    9852,    9340,    8828,    8316,    7932,
+      7676,    7420,    7164,    6908,    6652,    6396,    6140,
+      5884,    5628,    5372,    5116,    4860,    4604,    4348,
+      4092,    3900,    3772,    3644,    3516,    3388,    3260,
+      3132,    3004,    2876,    2748,    2620,    2492,    2364,
+      2236,    2108,    1980,    1884,    1820,    1756,    1692,
+      1628,    1564,    1500,    1436,    1372,    1308,    1244,
+      1180,    1116,    1052,     988,     924,     876,     844,
+       812,     780,     748,     716,     684,     652,     620,
+       588,     556,     524,     492,     460,     428,     396,
+       372,     356,     340,     324,     308,     292,     276,
+       260,     244,     228,     212,     196,     180,     164,
+       148,     132,     120,     112,     104,      96,      88,
+    80,      72,      64,      56,      48,      40,      32,
+    24,      16,       8,       0
+};
+
+/*
+ * linear2ulaw() accepts a 14-bit signed integer and encodes it as u-law data
+ * stored in a unsigned char.  This function should only be called with
+ * the data shifted such that it only contains information in the lower
+ * 14-bits.
+ *
+ * In order to simplify the encoding process, the original linear magnitude
+ * is biased by adding 33 which shifts the encoding range from (0 - 8158) to
+ * (33 - 8191). The result can be seen in the following encoding table:
+ *
+ *      Biased Linear Input Code        Compressed Code
+ *      ------------------------        ---------------
+ *      00000001wxyza                   000wxyz
+ *      0000001wxyzab                   001wxyz
+ *      000001wxyzabc                   010wxyz
+ *      00001wxyzabcd                   011wxyz
+ *      0001wxyzabcde                   100wxyz
+ *      001wxyzabcdef                   101wxyz
+ *      01wxyzabcdefg                   110wxyz
+ *      1wxyzabcdefgh                   111wxyz
+ *
+ * Each biased linear code has a leading 1 which identifies the segment
+ * number. The value of the segment number is equal to 7 minus the number
+ * of leading 0's. The quantization interval is directly available as the
+ * four bits wxyz.  * The trailing bits (a - h) are ignored.
+ *
+ * Ordinarily the complement of the resulting code word is used for
+ * transmission, and so the code word is complemented before it is returned.
+ *
+ * For further information see John C. Bellamy's Digital Telephony, 1982,
+ * John Wiley & Sons, pps 98-111 and 472-476.
+ */
+static unsigned char
+st_14linear2ulaw(PyInt16 pcm_val)       /* 2's complement (14-bit range) */
+{
+    PyInt16         mask;
+    PyInt16         seg;
+    unsigned char   uval;
+
+    /* The original sox code does this in the calling function, not here */
+    pcm_val = pcm_val >> 2;
+
+    /* u-law inverts all bits */
+    /* Get the sign and the magnitude of the value. */
+    if (pcm_val < 0) {
+        pcm_val = -pcm_val;
+        mask = 0x7F;
+    } else {
+        mask = 0xFF;
+    }
+    if ( pcm_val > CLIP ) pcm_val = CLIP;           /* clip the magnitude */
+    pcm_val += (BIAS >> 2);
+
+    /* Convert the scaled magnitude to segment number. */
+    seg = search(pcm_val, seg_uend, 8);
+
+    /*
+     * Combine the sign, segment, quantization bits;
+     * and complement the code word.
+     */
+    if (seg >= 8)           /* out of range, return maximum value. */
+        return (unsigned char) (0x7F ^ mask);
+    else {
+        uval = (unsigned char) (seg << 4) | ((pcm_val >> (seg + 1)) & 0xF);
+        return (uval ^ mask);
+    }
+
+}
+
+static PyInt16 _st_alaw2linear16[256] = {
+     -5504,   -5248,   -6016,   -5760,   -4480,   -4224,   -4992,
+     -4736,   -7552,   -7296,   -8064,   -7808,   -6528,   -6272,
+     -7040,   -6784,   -2752,   -2624,   -3008,   -2880,   -2240,
+     -2112,   -2496,   -2368,   -3776,   -3648,   -4032,   -3904,
+     -3264,   -3136,   -3520,   -3392,  -22016,  -20992,  -24064,
+    -23040,  -17920,  -16896,  -19968,  -18944,  -30208,  -29184,
+    -32256,  -31232,  -26112,  -25088,  -28160,  -27136,  -11008,
+    -10496,  -12032,  -11520,   -8960,   -8448,   -9984,   -9472,
+    -15104,  -14592,  -16128,  -15616,  -13056,  -12544,  -14080,
+    -13568,    -344,    -328,    -376,    -360,    -280,    -264,
+      -312,    -296,    -472,    -456,    -504,    -488,    -408,
+      -392,    -440,    -424,     -88,     -72,    -120,    -104,
+       -24,      -8,     -56,     -40,    -216,    -200,    -248,
+      -232,    -152,    -136,    -184,    -168,   -1376,   -1312,
+     -1504,   -1440,   -1120,   -1056,   -1248,   -1184,   -1888,
+     -1824,   -2016,   -1952,   -1632,   -1568,   -1760,   -1696,
+      -688,    -656,    -752,    -720,    -560,    -528,    -624,
+      -592,    -944,    -912,   -1008,    -976,    -816,    -784,
+      -880,    -848,    5504,    5248,    6016,    5760,    4480,
+      4224,    4992,    4736,    7552,    7296,    8064,    7808,
+      6528,    6272,    7040,    6784,    2752,    2624,    3008,
+      2880,    2240,    2112,    2496,    2368,    3776,    3648,
+      4032,    3904,    3264,    3136,    3520,    3392,   22016,
+     20992,   24064,   23040,   17920,   16896,   19968,   18944,
+     30208,   29184,   32256,   31232,   26112,   25088,   28160,
+     27136,   11008,   10496,   12032,   11520,    8960,    8448,
+      9984,    9472,   15104,   14592,   16128,   15616,   13056,
+     12544,   14080,   13568,     344,     328,     376,     360,
+       280,     264,     312,     296,     472,     456,     504,
+       488,     408,     392,     440,     424,      88,      72,
+       120,     104,      24,       8,      56,      40,     216,
+       200,     248,     232,     152,     136,     184,     168,
+      1376,    1312,    1504,    1440,    1120,    1056,    1248,
+      1184,    1888,    1824,    2016,    1952,    1632,    1568,
+      1760,    1696,     688,     656,     752,     720,     560,
+       528,     624,     592,     944,     912,    1008,     976,
+       816,     784,     880,     848
+};
+
+/*
+ * linear2alaw() accepts an 13-bit signed integer and encodes it as A-law data
+ * stored in a unsigned char.  This function should only be called with
+ * the data shifted such that it only contains information in the lower
+ * 13-bits.
+ *
+ *              Linear Input Code       Compressed Code
+ *      ------------------------        ---------------
+ *      0000000wxyza                    000wxyz
+ *      0000001wxyza                    001wxyz
+ *      000001wxyzab                    010wxyz
+ *      00001wxyzabc                    011wxyz
+ *      0001wxyzabcd                    100wxyz
+ *      001wxyzabcde                    101wxyz
+ *      01wxyzabcdef                    110wxyz
+ *      1wxyzabcdefg                    111wxyz
+ *
+ * For further information see John C. Bellamy's Digital Telephony, 1982,
+ * John Wiley & Sons, pps 98-111 and 472-476.
+ */
+static unsigned char
+st_linear2alaw(PyInt16 pcm_val) /* 2's complement (13-bit range) */
+{
+    PyInt16         mask;
+    short           seg;
+    unsigned char   aval;
+
+    /* The original sox code does this in the calling function, not here */
+    pcm_val = pcm_val >> 3;
+
+    /* A-law using even bit inversion */
+    if (pcm_val >= 0) {
+        mask = 0xD5;            /* sign (7th) bit = 1 */
+    } else {
+        mask = 0x55;            /* sign bit = 0 */
+        pcm_val = -pcm_val - 1;
+    }
+
+    /* Convert the scaled magnitude to segment number. */
+    seg = search(pcm_val, seg_aend, 8);
+
+    /* Combine the sign, segment, and quantization bits. */
+
+    if (seg >= 8)           /* out of range, return maximum value. */
+        return (unsigned char) (0x7F ^ mask);
+    else {
+        aval = (unsigned char) seg << SEG_SHIFT;
+        if (seg < 2)
+            aval |= (pcm_val >> 1) & QUANT_MASK;
+        else
+            aval |= (pcm_val >> seg) & QUANT_MASK;
+        return (aval ^ mask);
+    }
+}
+/* End of code taken from sox */
+
+/* Intel ADPCM step variation table */
+static int indexTable[16] = {
+    -1, -1, -1, -1, 2, 4, 6, 8,
+    -1, -1, -1, -1, 2, 4, 6, 8,
+};
+
+static int stepsizeTable[89] = {
+    7, 8, 9, 10, 11, 12, 13, 14, 16, 17,
+    19, 21, 23, 25, 28, 31, 34, 37, 41, 45,
+    50, 55, 60, 66, 73, 80, 88, 97, 107, 118,
+    130, 143, 157, 173, 190, 209, 230, 253, 279, 307,
+    337, 371, 408, 449, 494, 544, 598, 658, 724, 796,
+    876, 963, 1060, 1166, 1282, 1411, 1552, 1707, 1878, 2066,
+    2272, 2499, 2749, 3024, 3327, 3660, 4026, 4428, 4871, 5358,
+    5894, 6484, 7132, 7845, 8630, 9493, 10442, 11487, 12635, 13899,
+    15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794, 32767
+};
+
+#define CHARP(cp, i) ((signed char *)(cp+i))
+#define SHORTP(cp, i) ((short *)(cp+i))
+#define LONGP(cp, i) ((Py_Int32 *)(cp+i))
+"""
+
+lib = ffi.verify(_AUDIOOP_C_MODULE + """
+void lin2adcpm(unsigned char* ncp, unsigned char* cp, size_t len,
+               size_t size, int* state)
+{
+    int step, outputbuffer = 0, bufferstep;
+    int val = 0;
+    int diff, vpdiff, sign, delta;
+    size_t i;
+    int valpred = state[0];
+    int index = state[1];
+
+    step = stepsizeTable[index];
+    bufferstep = 1;
+
+    for ( i=0; i < len; i += size ) {
+        if ( size == 1 )      val = ((int)*CHARP(cp, i)) << 8;
+        else if ( size == 2 ) val = (int)*SHORTP(cp, i);
+        else if ( size == 4 ) val = ((int)*LONGP(cp, i)) >> 16;
+
+        /* Step 1 - compute difference with previous value */
+        diff = val - valpred;
+        sign = (diff < 0) ? 8 : 0;
+        if ( sign ) diff = (-diff);
+
+        /* Step 2 - Divide and clamp */
+        /* Note:
+        ** This code *approximately* computes:
+        **    delta = diff*4/step;
+        **    vpdiff = (delta+0.5)*step/4;
+        ** but in shift step bits are dropped. The net result of this
+        ** is that even if you have fast mul/div hardware you cannot
+        ** put it to good use since the fixup would be too expensive.
+        */
+        delta = 0;
+        vpdiff = (step >> 3);
+
+        if ( diff >= step ) {
+            delta = 4;
+            diff -= step;
+            vpdiff += step;
+        }
+        step >>= 1;
+        if ( diff >= step  ) {
+            delta |= 2;
+            diff -= step;
+            vpdiff += step;
+        }
+        step >>= 1;
+        if ( diff >= step ) {
+            delta |= 1;
+            vpdiff += step;
+        }
+
+        /* Step 3 - Update previous value */
+        if ( sign )
+            valpred -= vpdiff;
+        else
+            valpred += vpdiff;
+
+        /* Step 4 - Clamp previous value to 16 bits */
+        if ( valpred > 32767 )
+            valpred = 32767;
+        else if ( valpred < -32768 )
+            valpred = -32768;
+
+        /* Step 5 - Assemble value, update index and step values */
+        delta |= sign;
+
+        index += indexTable[delta];
+        if ( index < 0 ) index = 0;
+        if ( index > 88 ) index = 88;
+        step = stepsizeTable[index];
+
+        /* Step 6 - Output value */
+        if ( bufferstep ) {
+            outputbuffer = (delta << 4) & 0xf0;
+        } else {
+            *ncp++ = (delta & 0x0f) | outputbuffer;
+        }
+        bufferstep = !bufferstep;
+    }
+    state[0] = valpred;
+    state[1] = index;
+}
+
+
+void adcpm2lin(unsigned char* ncp, unsigned char* cp, size_t len,
+               size_t size, int* state)
+{
+    int step, inputbuffer = 0, bufferstep;
+    int val = 0;
+    int diff, vpdiff, sign, delta;
+    size_t i;
+    int valpred = state[0];
+    int index = state[1];
+
+    step = stepsizeTable[index];
+    bufferstep = 0;
+
+    for ( i=0; i < len*size*2; i += size ) {
+        /* Step 1 - get the delta value and compute next index */
+        if ( bufferstep ) {
+            delta = inputbuffer & 0xf;
+        } else {
+            inputbuffer = *cp++;
+            delta = (inputbuffer >> 4) & 0xf;
+        }
+
+        bufferstep = !bufferstep;
+
+        /* Step 2 - Find new index value (for later) */
+        index += indexTable[delta];
+        if ( index < 0 ) index = 0;
+        if ( index > 88 ) index = 88;
+
+        /* Step 3 - Separate sign and magnitude */
+        sign = delta & 8;
+        delta = delta & 7;
+
+        /* Step 4 - Compute difference and new predicted value */
+        /*
+        ** Computes 'vpdiff = (delta+0.5)*step/4', but see comment
+        ** in adpcm_coder.
+        */
+        vpdiff = step >> 3;
+        if ( delta & 4 ) vpdiff += step;
+        if ( delta & 2 ) vpdiff += step>>1;
+        if ( delta & 1 ) vpdiff += step>>2;
+
+        if ( sign )
+            valpred -= vpdiff;
+        else
+            valpred += vpdiff;
+
+        /* Step 5 - clamp output value */
+        if ( valpred > 32767 )
+            valpred = 32767;
+        else if ( valpred < -32768 )
+            valpred = -32768;
+
+        /* Step 6 - Update step value */
+        step = stepsizeTable[index];
+
+        /* Step 6 - Output value */
+        if ( size == 1 ) *CHARP(ncp, i) = (signed char)(valpred >> 8);
+        else if ( size == 2 ) *SHORTP(ncp, i) = (short)(valpred);
+        else if ( size == 4 ) *LONGP(ncp, i) = (Py_Int32)(valpred<<16);
+    }
+    state[0] = valpred;
+    state[1] = index;
+}
+""")
+
+def _get_lin_samples(cp, size):
+    for sample in _get_samples(cp, size):
+        if size == 1:
+            yield sample << 8
+        elif size == 2:
+            yield sample
+        elif size == 4:
+            yield sample >> 16
+
+def _put_lin_sample(result, size, i, sample):
+    if size == 1:
+        sample >>= 8
+    elif size == 2:
+        pass
+    elif size == 4:
+        sample <<= 16
+    _put_sample(result, size, i, sample)
+
 def lin2ulaw(cp, size):
-    raise NotImplementedError()
+    _check_params(len(cp), size)
+    rv = ffi.new("unsigned char[]", _sample_count(cp, size))
+    for i, sample in enumerate(_get_lin_samples(cp, size)):
+        rv[i] = lib.st_14linear2ulaw(sample)
+    return ffi.buffer(rv)[:]
 
 
 def ulaw2lin(cp, size):
-    raise NotImplementedError()
+    _check_size(size)
+    rv = ffi.new("unsigned char[]", len(cp) * size)
+    result = ffi.buffer(rv)
+    for i, value in enumerate(cp):
+        sample = lib.st_ulaw2linear16(ord(value))
+        _put_lin_sample(result, size, i, sample)
+    return result[:]
 
 
 def lin2alaw(cp, size):
-    raise NotImplementedError()
+    _check_params(len(cp), size)
+    rv = ffi.new("unsigned char[]", _sample_count(cp, size))
+    for i, sample in enumerate(_get_lin_samples(cp, size)):
+        rv[i] = lib.st_linear2alaw(sample)
+    return ffi.buffer(rv)[:]
 
 
 def alaw2lin(cp, size):
-    raise NotImplementedError()
+    _check_size(size)
+    rv = ffi.new("unsigned char[]", len(cp) * size)
+    result = ffi.buffer(rv)
+    for i, value in enumerate(cp):
+        sample = lib.st_alaw2linear16(ord(value))
+        _put_lin_sample(result, size, i, sample)
+    return result[:]
 
 
 def lin2adpcm(cp, size, state):
-    raise NotImplementedError()
+    _check_params(len(cp), size)
+    if state is None:
+        state = (0, 0)
+    rv = ffi.new("unsigned char[]", len(cp) // size // 2)
+    state_ptr = ffi.new("int[]", state)
+    lib.lin2adcpm(rv, cp, len(cp), size, state_ptr)
+    return ffi.buffer(rv)[:], tuple(state_ptr)
 
 
 def adpcm2lin(cp, size, state):
-    raise NotImplementedError()
+    _check_size(size)
+    if state is None:
+        state = (0, 0)
+    rv = ffi.new("unsigned char[]", len(cp) * size * 2)
+    state_ptr = ffi.new("int[]", state)
+    lib.adcpm2lin(rv, cp, len(cp), size, state_ptr)
+    return ffi.buffer(rv)[:], tuple(state_ptr)
+
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -108,7 +108,7 @@
 '''
 
 def create_cffi_import_libraries(pypy_c, options):
-    modules = ['_sqlite3']
+    modules = ['_sqlite3', 'audioop']
     subprocess.check_call([str(pypy_c), '-c', 'import _sqlite3'])
     if not sys.platform == 'win32':
         modules += ['_curses', 'syslog', 'gdbm', '_sqlite3']

From noreply at buildbot.pypy.org  Tue Feb 10 19:58:50 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 10 Feb 2015 19:58:50 +0100 (CET)
Subject: [pypy-commit] pypy default: Fix docstring
Message-ID: <20150210185850.83EF71C08B5@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r75813:821eb2210b1a
Date: 2015-02-10 19:58 +0100
http://bitbucket.org/pypy/pypy/changeset/821eb2210b1a/

Log:	Fix docstring

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -712,7 +712,7 @@
         return self.wrap(not self.is_true(w_obj))
 
     def eq_w(self, w_obj1, w_obj2):
-        """shortcut for space.is_true(space.eq(w_obj1, w_obj2))"""
+        """Implements equality with the double check 'x is y or x == y'."""
         return self.is_w(w_obj1, w_obj2) or self.is_true(self.eq(w_obj1, w_obj2))
 
     def is_(self, w_one, w_two):

From noreply at buildbot.pypy.org  Tue Feb 10 21:32:02 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 10 Feb 2015 21:32:02 +0100 (CET)
Subject: [pypy-commit] pypy default: do not modify caller's locals
Message-ID: <20150210203202.E99661C0471@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r75814:6ac9dab8648a
Date: 2015-02-10 22:32 +0200
http://bitbucket.org/pypy/pypy/changeset/6ac9dab8648a/

Log:	do not modify caller's locals

diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -196,7 +196,7 @@
         # So please be careful with the order of parameters! ;-)
         pdb_dir = oname.dirname
         if pdb_dir:
-                compile_args += ['/Fd%s\\' % (pdb_dir,)]
+                compile_args = compile_args + ['/Fd%s\\' % (pdb_dir,)]
         args = ['/nologo', '/c'] + compile_args + ['/Fo%s' % (oname,), str(cfile)]
         self._execute_c_compiler(cc, args, oname)
         return oname

From noreply at buildbot.pypy.org  Tue Feb 10 22:25:16 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 10 Feb 2015 22:25:16 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Fix a corner case where a key of
 module.__dict__ could be retrieved
Message-ID: <20150210212516.12FF01C0471@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: py3k
Changeset: r75815:8b50f6d3aebe
Date: 2015-02-10 22:20 +0100
http://bitbucket.org/pypy/pypy/changeset/8b50f6d3aebe/

Log:	Fix a corner case where a key of module.__dict__ could be retrieved
	as a bytes object.

	Probably fixes translation as well

diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py
--- a/pypy/objspace/std/celldict.py
+++ b/pypy/objspace/std/celldict.py
@@ -123,7 +123,7 @@
     def w_keys(self, w_dict):
         space = self.space
         l = self.unerase(w_dict.dstorage).keys()
-        return space.newlist_bytes(l)
+        return space.newlist_unicode(l)
 
     def values(self, w_dict):
         iterator = self.unerase(w_dict.dstorage).itervalues
diff --git a/pypy/objspace/std/test/test_celldict.py b/pypy/objspace/std/test/test_celldict.py
--- a/pypy/objspace/std/test/test_celldict.py
+++ b/pypy/objspace/std/test/test_celldict.py
@@ -106,6 +106,11 @@
         assert "s" not in d
         assert F() not in d
 
+    def test_reversed(self):
+        import __pypy__
+        name = next(__pypy__.reversed_dict(__pypy__.__dict__))
+        assert isinstance(name, str)
+
 
 class TestModuleDictImplementation(BaseTestRDictImplementation):
     StrategyClass = ModuleDictStrategy

From noreply at buildbot.pypy.org  Tue Feb 10 22:32:54 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 10 Feb 2015 22:32:54 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: typo
Message-ID: <20150210213254.B743C1C0471@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: stdlib-2.7.9
Changeset: r75816:cbe8c24dac40
Date: 2015-02-10 20:23 +0200
http://bitbucket.org/pypy/pypy/changeset/cbe8c24dac40/

Log:	typo

diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py
--- a/pypy/module/_ssl/interp_win32.py
+++ b/pypy/module/_ssl/interp_win32.py
@@ -159,7 +159,7 @@
                 rffi.charpsize2str(pCrlCtx.c_pbCrlEncoded,
                                    pCrlCtx.c_cbCrlEncoded))
             w_enc = w_certEncodingType(space, pCrlCtx.c_dwCertEncodingType)
-            result_w.append(space.newtuple([w_cert, w_enc]))
+            result_w.append(space.newtuple([w_crl, w_enc]))
     finally:
         if pCrlCtx:
             # loop ended with an error, need to clean up context manually

From noreply at buildbot.pypy.org  Tue Feb 10 22:32:55 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 10 Feb 2015 22:32:55 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: avoid modifying caller's args
Message-ID: <20150210213255.DCECF1C0471@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: stdlib-2.7.9
Changeset: r75817:6e439976f4a6
Date: 2015-02-10 20:55 +0200
http://bitbucket.org/pypy/pypy/changeset/6e439976f4a6/

Log:	avoid modifying caller's args

diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -196,7 +196,7 @@
         # So please be careful with the order of parameters! ;-)
         pdb_dir = oname.dirname
         if pdb_dir:
-                compile_args += ['/Fd%s\\' % (pdb_dir,)]
+                compile_args = compile_args + ['/Fd%s\\' % (pdb_dir,)]
         args = ['/nologo', '/c'] + compile_args + ['/Fo%s' % (oname,), str(cfile)]
         self._execute_c_compiler(cc, args, oname)
         return oname

From noreply at buildbot.pypy.org  Tue Feb 10 22:32:57 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 10 Feb 2015 22:32:57 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: translation fixes
Message-ID: <20150210213257.0AB691C0471@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: stdlib-2.7.9
Changeset: r75818:92a7192649f6
Date: 2015-02-10 23:27 +0200
http://bitbucket.org/pypy/pypy/changeset/92a7192649f6/

Log:	translation fixes

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -552,6 +552,9 @@
                 return _decode_certificate(space, self.peer_cert)
 
     def selected_npn_protocol(self, space):
+        if not HAS_NPN:
+            raise oefmt(space.w_NotImplementedError,
+                        "The NPN extension requires OpenSSL 1.0.1 or later.")
         with lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as out_ptr:
             with lltype.scoped_alloc(rffi.UINTP.TO, 1) as len_ptr:
                 libssl_SSL_get0_next_proto_negotiated(self.ssl,
diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py
--- a/pypy/module/_ssl/interp_win32.py
+++ b/pypy/module/_ssl/interp_win32.py
@@ -4,7 +4,7 @@
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.error import wrap_windowserror
-
+from rpython.rlib.rarithmetic import intmask, widen
 eci = ExternalCompilationInfo(
     includes = ['windows.h', 'wincrypt.h'],
     libraries = ['crypt32'],
@@ -15,6 +15,9 @@
 
     X509_ASN_ENCODING = rffi_platform.ConstantInteger('X509_ASN_ENCODING')
     PKCS_7_ASN_ENCODING = rffi_platform.ConstantInteger('PKCS_7_ASN_ENCODING')
+    CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG = rffi_platform.ConstantInteger('CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG')
+    CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG = rffi_platform.ConstantInteger('CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG')
+    CRYPT_E_NOT_FOUND = rffi_platform.ConstantInteger('CRYPT_E_NOT_FOUND')
 
     CERT_ENHKEY_USAGE = rffi_platform.Struct(
         'CERT_ENHKEY_USAGE', [('cUsageIdentifier', rwin32.DWORD),
@@ -38,6 +41,7 @@
 def external(name, argtypes, restype, **kw):
     kw['compilation_info'] = eci
     kw['calling_conv'] = 'win'
+    kw['save_err'] = rffi.RFFI_SAVE_LASTERROR
     return rffi.llexternal(
         name, argtypes, restype, **kw)
 
@@ -69,22 +73,22 @@
         return space.wrap(encodingType)
 
 def w_parseKeyUsage(space, pCertCtx, flags):
-    with rffi.scoped_alloc(rwin32.LPDWORD.TO, 1) as size_ptr:
+    with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as size_ptr:
         if not CertGetEnhancedKeyUsage(pCertCtx, flags, None, size_ptr):
             last_error = rwin32.lastSavedWindowsError()
-            if last_error == CRYPT_E_NOT_FOUND:
+            if last_error.errno == CRYPT_E_NOT_FOUND:
                 return space.w_True
-            raise wrap_windowserror(WindowsError(last_error))
+            raise wrap_windowserror(space, last_error)
 
-        size = rffi.widen(size_ptr[0])
-        with rffi.scoped_alloc(rffi.CCHARP.TO, size) as buf:
+        size = widen(size_ptr[0])
+        with lltype.scoped_alloc(rffi.CCHARP.TO, size) as buf:
             usage = rffi.cast(PCERT_ENHKEY_USAGE, buf)
             # Now get the actual enhanced usage property
             if not CertGetEnhancedKeyUsage(pCertCtx, flags, usage, size_ptr):
-                last_error = rwin32.lastSavedWindowsError()
-                if last_error == CRYPT_E_NOT_FOUND:
+                last_error= rwin32.lastSavedWindowsError()
+                if last_error.errno == CRYPT_E_NOT_FOUND:
                     return space.w_True
-                raise wrap_windowserror(WindowsError(last_error))
+                raise wrap_windowserror(space, last_error)
 
             result_w = []
             for i in range(usage.c_cUsageIdentifier):
@@ -93,7 +97,7 @@
                 result_w.append(
                     space.wrap(rffi.charp2str(
                         usage.c_rgpszUsageIdentifier[i])))
-            return space.newset(result_w)
+            return space.newlist(result_w) #space.newset(result_w)
 
 @unwrap_spec(store_name=str)
 def enum_certificates_w(space, store_name):
@@ -107,10 +111,10 @@
     boolean True."""
 
     result_w = []
-    pCertCtx = lltype.nullptr(PCCERT_CONTEXT)
+    pCertCtx = lltype.nullptr(CERT_CONTEXT)
     hStore = CertOpenSystemStore(None, store_name)
     if not hStore:
-        raise wrap_windowserror(rwin32.lastSavedWindowsError())
+        raise wrap_windowserror(space, rwin32.lastSavedWindowsError())
     try:
         while True:
             pCertCtx = CertEnumCertificatesInStore(hStore, pCertCtx)
@@ -118,7 +122,7 @@
                 break
             w_cert = space.wrapbytes(
                 rffi.charpsize2str(pCertCtx.c_pbCertEncoded,
-                                   pCertCtx.c_cbCertEncoded))
+                                   intmask(pCertCtx.c_cbCertEncoded)))
             w_enc = w_certEncodingType(space, pCertCtx.c_dwCertEncodingType)
             w_keyusage = w_parseKeyUsage(
                 space, pCertCtx, CERT_FIND_PROP_ONLY_ENHKEY_USAGE_FLAG)
@@ -126,13 +130,15 @@
                 w_keyusage = w_parseKeyUsage(
                     space, pCertCtx, CERT_FIND_EXT_ONLY_ENHKEY_USAGE_FLAG)
             result_w.append(space.newtuple([w_cert, w_enc, w_keyusage]))
+    except:
+        raise
     finally:
         if pCertCtx:
             # loop ended with an error, need to clean up context manually
             CertFreeCertificateContext(pCertCtx)
         if not CertCloseStore(hStore, 0):
             # This error case might shadow another exception.
-            raise wrap_windowserror(rwin32.lastSavedWindowsError())
+            raise wrap_windowserror(space, rwin32.lastSavedWindowsError())
 
     return space.newlist(result_w)
 
@@ -146,10 +152,11 @@
     encoding_type flag can be interpreted with X509_ASN_ENCODING or
     PKCS_7_ASN_ENCODING."""
     result_w = []
-    pCrlCtx = lltype.nullptr(PCCRL_CONTEXT)
+
+    pCrlCtx = lltype.nullptr(CRL_CONTEXT)
     hStore = CertOpenSystemStore(None, store_name)
     if not hStore:
-        raise wrap_windowserror(rwin32.lastSavedWindowsError())
+        raise wrap_windowserror(space, rwin32.lastSavedWindowsError())
     try:
         while True:
             pCrlCtx = CertEnumCRLsInStore(hStore, pCrlCtx)
@@ -157,13 +164,15 @@
                 break
             w_crl = space.wrapbytes(
                 rffi.charpsize2str(pCrlCtx.c_pbCrlEncoded,
-                                   pCrlCtx.c_cbCrlEncoded))
+                                   intmask(pCrlCtx.c_cbCrlEncoded)))
             w_enc = w_certEncodingType(space, pCrlCtx.c_dwCertEncodingType)
             result_w.append(space.newtuple([w_crl, w_enc]))
+    except:
+        raise
     finally:
         if pCrlCtx:
             # loop ended with an error, need to clean up context manually
             CertFreeCRLContext(pCrlCtx)
         if not CertCloseStore(hStore, 0):
             # This error case might shadow another exception.
-            raise wrap_windowserror(rwin32.lastSavedWindowsError())
+            raise wrap_windowserror(space, rwin32.lastSavedWindowsError())

From noreply at buildbot.pypy.org  Tue Feb 10 22:49:29 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 10 Feb 2015 22:49:29 +0100 (CET)
Subject: [pypy-commit] buildbot issue-1759: allow valid slave names in
 slaveinfo.py even if debugging
Message-ID: <20150210214929.226881C13CD@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: issue-1759
Changeset: r925:306d94b288c6
Date: 2015-02-09 20:05 +0200
http://bitbucket.org/pypy/buildbot/changeset/306d94b288c6/

Log:	allow valid slave names in slaveinfo.py even if debugging

diff --git a/master/master.cfg b/master/master.cfg
--- a/master/master.cfg
+++ b/master/master.cfg
@@ -23,5 +23,9 @@
 
 if we_are_debugging():
     for builderdict in BuildmasterConfig['builders']:
-        builderdict["slavenames"] = ['localhost']
+        valid_slaves = ['localhost']
+        for s in builderdict['slavenames']:
+            if s in slaveinfo.passwords:
+                valid_slaves.append(s)
+        builderdict["slavenames"] = valid_slaves
     BuildmasterConfig['buildbotURL'] = "http://localhost:%d/" % (httpPortNumber)

From noreply at buildbot.pypy.org  Tue Feb 10 22:49:30 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 10 Feb 2015 22:49:30 +0100 (CET)
Subject: [pypy-commit] buildbot issue-1759: unused import
Message-ID: <20150210214930.366DD1C13CD@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: issue-1759
Changeset: r926:6f2c2f52bd05
Date: 2015-02-09 20:06 +0200
http://bitbucket.org/pypy/buildbot/changeset/6f2c2f52bd05/

Log:	unused import

diff --git a/bot2/pypybuildbot/arm_master.py b/bot2/pypybuildbot/arm_master.py
--- a/bot2/pypybuildbot/arm_master.py
+++ b/bot2/pypybuildbot/arm_master.py
@@ -1,5 +1,5 @@
 from buildbot.scheduler import Nightly, Triggerable
-from pypybuildbot.util import we_are_debugging, load
+from pypybuildbot.util import load
 
 pypybuilds = load('pypybuildbot.builds')
 ARMCrossLock = pypybuilds.ARMCrossLock

From noreply at buildbot.pypy.org  Tue Feb 10 22:49:31 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 10 Feb 2015 22:49:31 +0100 (CET)
Subject: [pypy-commit] buildbot issue-1759: move numpy testing to a
 triggerable build, add win32 target as well
Message-ID: <20150210214931.3832A1C13CD@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: issue-1759
Changeset: r927:caa856444a44
Date: 2015-02-10 18:52 +0200
http://bitbucket.org/pypy/buildbot/changeset/caa856444a44/

Log:	move numpy testing to a triggerable build, add win32 target as well

diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -326,14 +326,14 @@
                 workdir=workdir,
                 logEnviron=False))
 
-def update_git(platform, factory, repourl, workdir, use_branch,
-              force_branch=None):
+def update_git(platform, factory, repourl, workdir, branch='master'):
     factory.addStep(
             Git(
                 repourl=repourl,
                 mode='full',
                 method='fresh',
                 workdir=workdir,
+                branch=branch,
                 logEnviron=False))
 
 def setup_steps(platform, factory, workdir=None,
@@ -473,7 +473,8 @@
                  interpreter='pypy',
                  lib_python=False,
                  pypyjit=False,
-                 prefix=None
+                 prefix=None,
+                 trigger=None,
                  ):
         factory.BuildFactory.__init__(self)
         if prefix is not None:
@@ -502,6 +503,9 @@
                                 workdir='.',
                                 blocksize=100 * 1024))
 
+        if trigger: # if provided trigger schedulers that depend on this one
+            self.addStep(Trigger(schedulerNames=[trigger]))
+
         add_translated_tests(self, prefix, platform, app_tests, lib_python, pypyjit)
 
 
@@ -882,6 +886,13 @@
             ))
 
         self.addStep(ShellCmd(
+            description="report version",
+            command=['install/bin/pypy', '--version'],
+            workdir='./',
+            haltOnFailure=True,
+            ))
+
+        self.addStep(ShellCmd(
             description="install nose",
             command=['install/bin/pip', 'install','nose'],
             workdir='./',
@@ -890,9 +901,7 @@
 
         # obtain a pypy-compatible branch of numpy
         numpy_url = 'https://www.bitbucket.org/pypy/numpy'
-        numpy_pypy_branch = 'pypy-compat'
-        update_git(platform, self, numpy_url, 'numpy_src', use_branch=True,
-              force_branch=numpy_pypy_branch)
+        update_git(platform, self, numpy_url, 'numpy_src', branch='master')
 
         self.addStep(ShellCmd(
             description="install numpy",
diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py
--- a/bot2/pypybuildbot/master.py
+++ b/bot2/pypybuildbot/master.py
@@ -1,6 +1,6 @@
 
 import os
-from buildbot.scheduler import Nightly
+from buildbot.scheduler import Nightly, Triggerable
 from buildbot.schedulers.forcesched import ForceScheduler
 from buildbot.schedulers.forcesched import ValidationError
 from buildbot.buildslave import BuildSlave
@@ -96,6 +96,7 @@
     pypyjit=True,
     app_tests=True,
     platform='linux64',
+    trigger='NUMPY64_scheduler',
     )
 
 pypyJITTranslatedTestFactoryIndiana = pypybuilds.Translated(
@@ -133,6 +134,7 @@
     lib_python=True,
     pypyjit=True,
     app_tests=True,
+    trigger='NUMPYWIN_scheduler',
     )
 
 pypyJITTranslatedTestFactoryFreeBSD = pypybuilds.Translated(
@@ -186,6 +188,7 @@
 JITBENCH64_NEW = 'jit-benchmark-linux-x86-64-single-run'
 CPYTHON_64 = "cpython-2-benchmark-x86-64"
 NUMPY_64 = "numpy-compatability-linux-x86-64"
+NUMPY_WIN = "numpy-compatability-win-x86-32"
 # buildbot builder
 PYPYBUILDBOT = 'pypy-buildbot'
 
@@ -216,8 +219,8 @@
             APPLVLLINUX64,             # on allegro64, uses 1 core
             # other platforms
             #MACOSX32,                 # on minime
-            JITWIN32,                  # on aurora, SalsaSalsa
-            WIN32,                     # on aurora, SalsaSalsa
+            JITWIN32,                  # on allegro_win32, SalsaSalsa
+            WIN32,                     # on allegro_win32, SalsaSalsa
             #JITFREEBSD764,            # on headless
             #JITFREEBSD864,            # on ananke
             JITFREEBSD964,             # on tavendo
@@ -227,14 +230,20 @@
             ], branch='default', hour=0, minute=0),
 
         Nightly("nightly-1-00", [
-            NUMPY_64,                  # on tannit64, uses 1 core, takes about 15min.
-                                       # XXX maybe use a trigger instead?
             JITBENCH,                  # on tannit32, uses 1 core (in part exclusively)
             JITBENCH64,                # on tannit64, uses 1 core (in part exclusively)
             JITBENCH64_NEW,            # on speed64, uses 1 core (in part exclusively)
 
         ], branch=None, hour=1, minute=0),
 
+        Triggerable("NUMPY64_scheduler", [
+            NUMPY_64,                  # on tannit64, uses 1 core, takes about 15min.
+        ]),
+
+        Triggerable("NUMPYWIN_scheduler", [
+            NUMPY_WIN,                  # on allegro_win32, SalsaSalsa
+        ]),
+
         Nightly("nightly-2-00-py3k", [
             LINUX64,                   # on allegro64, uses all cores
             APPLVLLINUX64,             # on allegro64, uses 1 core
@@ -279,6 +288,7 @@
                         JITBENCH64,
                         JITBENCH64_NEW,
                         NUMPY_64,
+                        NUMPY_WIN,
             ] + ARM.builderNames, properties=[]),
     ] + ARM.schedulers,
 
@@ -463,7 +473,14 @@
                    'factory': pypyNumpyCompatability,
                    'category': 'numpy',
                    'locks': [TannitCPU.access('counting')],
-                   },
+                  },
+                  {'name': NUMPY_WIN,
+                   'slavenames': ["allegro_win32", "SalsaSalsa"],
+                   'builddir': NUMPY_WIN,
+                   'factory': pypyNumpyCompatability,
+                   "locks": [WinSlaveLock.access('counting')],
+                   'category': 'numpy',
+                  },
                   {'name': PYPYBUILDBOT,
                    'slavenames': ['cobra'],
                    'builddir': PYPYBUILDBOT,

From noreply at buildbot.pypy.org  Tue Feb 10 22:49:32 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 10 Feb 2015 22:49:32 +0100 (CET)
Subject: [pypy-commit] buildbot issue-1759: platform and host attributes are
 not magic, this probably was a bug in the configuration
Message-ID: <20150210214932.3A3F11C13CD@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: issue-1759
Changeset: r928:766b7373be46
Date: 2015-02-10 20:47 +0200
http://bitbucket.org/pypy/buildbot/changeset/766b7373be46/

Log:	platform and host attributes are not magic, this probably was a bug
	in the configuration

diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py
--- a/bot2/pypybuildbot/master.py
+++ b/bot2/pypybuildbot/master.py
@@ -146,14 +146,17 @@
     app_tests=True,
     )
 
-pypyJITBenchmarkFactory_tannit = pypybuilds.JITBenchmark()
+pypyJITBenchmarkFactory_tannit = pypybuilds.JITBenchmark(host='tannit')
 pypyJITBenchmarkFactory64_tannit = pypybuilds.JITBenchmark(platform='linux64',
+                                                           host='tannit',
                                                            postfix='-64')
 pypyJITBenchmarkFactory64_speed = pypybuilds.JITBenchmarkSingleRun(
     platform='linux64',
+    host='speed_python',
     postfix='-64')
 
 pypyNumpyCompatability = pypybuilds.NativeNumpyTests(platform='linux64')
+pypyNumpyCompatabilityWin = pypybuilds.NativeNumpyTests(platform='win32')
 
 #
 
@@ -477,7 +480,7 @@
                   {'name': NUMPY_WIN,
                    'slavenames': ["allegro_win32", "SalsaSalsa"],
                    'builddir': NUMPY_WIN,
-                   'factory': pypyNumpyCompatability,
+                   'factory': pypyNumpyCompatabilityWin,
                    "locks": [WinSlaveLock.access('counting')],
                    'category': 'numpy',
                   },

From noreply at buildbot.pypy.org  Tue Feb 10 22:49:33 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 10 Feb 2015 22:49:33 +0100 (CET)
Subject: [pypy-commit] buildbot issue-1759: make dual numpy targets work
Message-ID: <20150210214933.3D1CB1C13CD@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: issue-1759
Changeset: r929:158e6b7e5482
Date: 2015-02-10 21:49 +0200
http://bitbucket.org/pypy/buildbot/changeset/158e6b7e5482/

Log:	make dual numpy targets work

diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -843,7 +843,6 @@
     '''
     def __init__(self, platform='linux',
                  app_tests=False,
-                 host = 'tannit',
                  lib_python=False,
                  pypyjit=True,
                  prefix=None,
@@ -854,9 +853,18 @@
         self.addStep(ParseRevision(hideStepIf=ParseRevision.hideStepIf,
                                   doStepIf=ParseRevision.doStepIf))
         # download corresponding nightly build
+        if platform == 'win32':
+            target = r'pypy-c\pypy.exe'
+            untar = ['unzip']
+            sep = '\\'
+        else:
+            target = r'pypy-c/bin/pypy'
+            untar = ['tar', '--strip-components=1', '--directory=.', '-xf']
+            sep = '/'
         self.addStep(ShellCmd(
-            description="Clear pypy-c",
-            command=['rm', '-rf', 'pypy-c'],
+            description="Clear",
+            # assume, as part of git, that windows has rm
+            command=['rm', '-rf', 'pypy-c', 'install'],
             workdir='.'))
         extension = get_extension(platform)
         name = build_name(platform, pypyjit, translationArgs, placeholder='%(final_file_name)s') + extension
@@ -867,12 +875,17 @@
             workdir='pypy-c'))
 
         # extract downloaded file
-        if platform.startswith('win'):
-            raise NotImplementedError
-        else:
+        self.addStep(ShellCmd(
+            description="decompress pypy-c",
+            command=untar + ['pypy_build'+ extension],
+            workdir='pypy-c',
+            haltOnFailure=True,
+            ))
+        
+        if platform == 'win32':
             self.addStep(ShellCmd(
-                description="decompress pypy-c",
-                command=['tar', '--extract', '--file=pypy_build'+ extension, '--strip-components=1', '--directory=.'],
+                description='move decomporessed dir',
+                command = ['mv', '*/*', '.'],
                 workdir='pypy-c',
                 haltOnFailure=True,
                 ))
@@ -880,21 +893,21 @@
         # virtualenv the download
         self.addStep(ShellCmd(
             description="create virtualenv",
-            command=['virtualenv','-p', 'pypy-c/bin/pypy', 'install'],
+            command=['virtualenv','-p', target, 'install'],
             workdir='./',
             haltOnFailure=True,
             ))
 
         self.addStep(ShellCmd(
             description="report version",
-            command=['install/bin/pypy', '--version'],
+            command=[sep.join(['install','bin','pypy'])] + ['--version'],
             workdir='./',
             haltOnFailure=True,
             ))
 
         self.addStep(ShellCmd(
             description="install nose",
-            command=['install/bin/pip', 'install','nose'],
+            command=[sep.join(['install','bin','pip'])] + ['install','nose'],
             workdir='./',
             haltOnFailure=True,
             ))
@@ -905,19 +918,20 @@
 
         self.addStep(ShellCmd(
             description="install numpy",
-            command=['../install/bin/python', 'setup.py','install'],
+            command=[sep.join(['..', 'install', 'bin', 'pypy'])] + ['setup.py','install'],
             workdir='numpy_src'))
 
         self.addStep(ShellCmd(
             description="test numpy",
-            command=['bin/nosetests', 'site-packages/numpy',
+            command=[sep.join(['bin', 'nosetests'])] + ['site-packages/numpy',
+                        # XXX enable '-with-doctest',
                     ],
             #logfiles={'pytestLog': 'pytest-numpy.log'},
             timeout=4000,
             workdir='install',
             #env={"PYTHONPATH": ['download']}, # shouldn't be needed, but what if it is set externally?
         ))
-        if host == 'tannit':
+        if platform != 'win32':
             self.addStep(ShellCmd(
                 description="install jinja2",
                 command=['install/bin/pip', 'install', 'jinja2'],

From noreply at buildbot.pypy.org  Tue Feb 10 23:12:48 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 10 Feb 2015 23:12:48 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: Add newset() to the FakeObjSpace,
 to fix test_ztranslation
Message-ID: <20150210221248.6527B1C0092@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: stdlib-2.7.9
Changeset: r75819:6e6e94392240
Date: 2015-02-10 23:11 +0100
http://bitbucket.org/pypy/pypy/changeset/6e6e94392240/

Log:	Add newset() to the FakeObjSpace, to fix test_ztranslation

diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -145,6 +145,12 @@
             is_root(w_x)
         return w_some_obj()
 
+    def newset(self, list_w=None):
+        if list_w is not None:
+            for w_x in list_w:
+                is_root(w_x)
+        return w_some_obj()
+
     def newlist(self, list_w):
         for w_x in list_w:
             is_root(w_x)

From noreply at buildbot.pypy.org  Tue Feb 10 23:12:49 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 10 Feb 2015 23:12:49 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: Try space.newset() again
Message-ID: <20150210221249.A3AED1C0092@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: stdlib-2.7.9
Changeset: r75820:30bb6fb168fa
Date: 2015-02-10 23:12 +0100
http://bitbucket.org/pypy/pypy/changeset/30bb6fb168fa/

Log:	Try space.newset() again

diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py
--- a/pypy/module/_ssl/interp_win32.py
+++ b/pypy/module/_ssl/interp_win32.py
@@ -97,7 +97,7 @@
                 result_w.append(
                     space.wrap(rffi.charp2str(
                         usage.c_rgpszUsageIdentifier[i])))
-            return space.newlist(result_w) #space.newset(result_w)
+            return space.newset(result_w)
 
 @unwrap_spec(store_name=str)
 def enum_certificates_w(space, store_name):

From noreply at buildbot.pypy.org  Tue Feb 10 23:19:57 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 10 Feb 2015 23:19:57 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: Looks like RPython WindowsError
 instances only have winerror.
Message-ID: <20150210221957.4A3421C0092@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: stdlib-2.7.9
Changeset: r75821:b2e29ad07409
Date: 2015-02-10 23:18 +0100
http://bitbucket.org/pypy/pypy/changeset/b2e29ad07409/

Log:	Looks like RPython WindowsError instances only have winerror.

diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py
--- a/pypy/module/_ssl/interp_win32.py
+++ b/pypy/module/_ssl/interp_win32.py
@@ -76,7 +76,7 @@
     with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as size_ptr:
         if not CertGetEnhancedKeyUsage(pCertCtx, flags, None, size_ptr):
             last_error = rwin32.lastSavedWindowsError()
-            if last_error.errno == CRYPT_E_NOT_FOUND:
+            if last_error.winerror == CRYPT_E_NOT_FOUND:
                 return space.w_True
             raise wrap_windowserror(space, last_error)
 
@@ -86,7 +86,7 @@
             # Now get the actual enhanced usage property
             if not CertGetEnhancedKeyUsage(pCertCtx, flags, usage, size_ptr):
                 last_error= rwin32.lastSavedWindowsError()
-                if last_error.errno == CRYPT_E_NOT_FOUND:
+                if last_error.winerror == CRYPT_E_NOT_FOUND:
                     return space.w_True
                 raise wrap_windowserror(space, last_error)
 

From noreply at buildbot.pypy.org  Tue Feb 10 23:19:58 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 10 Feb 2015 23:19:58 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: Use intmask() instead of widen(),
 the original is unsigned.
Message-ID: <20150210221958.7D1AC1C0092@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: stdlib-2.7.9
Changeset: r75822:43c604f7b364
Date: 2015-02-10 23:19 +0100
http://bitbucket.org/pypy/pypy/changeset/43c604f7b364/

Log:	Use intmask() instead of widen(), the original is unsigned.

diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py
--- a/pypy/module/_ssl/interp_win32.py
+++ b/pypy/module/_ssl/interp_win32.py
@@ -4,7 +4,7 @@
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.error import wrap_windowserror
-from rpython.rlib.rarithmetic import intmask, widen
+from rpython.rlib.rarithmetic import intmask
 eci = ExternalCompilationInfo(
     includes = ['windows.h', 'wincrypt.h'],
     libraries = ['crypt32'],
@@ -80,7 +80,7 @@
                 return space.w_True
             raise wrap_windowserror(space, last_error)
 
-        size = widen(size_ptr[0])
+        size = intmask(size_ptr[0])
         with lltype.scoped_alloc(rffi.CCHARP.TO, size) as buf:
             usage = rffi.cast(PCERT_ENHKEY_USAGE, buf)
             # Now get the actual enhanced usage property

From noreply at buildbot.pypy.org  Tue Feb 10 23:24:21 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 10 Feb 2015 23:24:21 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Need to decode keys before calling
	newlist_unicode.
Message-ID: <20150210222421.122181C0092@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: py3k
Changeset: r75823:b46622fb88b7
Date: 2015-02-10 23:00 +0100
http://bitbucket.org/pypy/pypy/changeset/b46622fb88b7/

Log:	Need to decode keys before calling newlist_unicode.

diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py
--- a/pypy/objspace/std/celldict.py
+++ b/pypy/objspace/std/celldict.py
@@ -122,8 +122,8 @@
 
     def w_keys(self, w_dict):
         space = self.space
-        l = self.unerase(w_dict.dstorage).keys()
-        return space.newlist_unicode(l)
+        keys = self.unerase(w_dict.dstorage).keys()
+        return space.newlist_unicode([key.decode('utf-8') for key in keys])
 
     def values(self, w_dict):
         iterator = self.unerase(w_dict.dstorage).itervalues
diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
--- a/pypy/objspace/std/kwargsdict.py
+++ b/pypy/objspace/std/kwargsdict.py
@@ -115,8 +115,9 @@
             return w_dict.getitem(w_key)
 
     def w_keys(self, w_dict):
-        l = self.unerase(w_dict.dstorage)[0]
-        return self.space.newlist_unicode(l[:])
+        space = self.space
+        keys = self.unerase(w_dict.dstorage)[0]
+        return space.newlist_unicode([key.decode('utf-8') for key in keys])
 
     def values(self, w_dict):
         return self.unerase(w_dict.dstorage)[1][:] # to make non-resizable

From noreply at buildbot.pypy.org  Tue Feb 10 23:41:45 2015
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 10 Feb 2015 23:41:45 +0100 (CET)
Subject: [pypy-commit] pypy stdlib-2.7.9: Avoid resizing the list passed to
	space.newset()
Message-ID: <20150210224145.A17F61C0471@cobra.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc 
Branch: stdlib-2.7.9
Changeset: r75824:ee0c5e8894ed
Date: 2015-02-10 23:40 +0100
http://bitbucket.org/pypy/pypy/changeset/ee0c5e8894ed/

Log:	Avoid resizing the list passed to space.newset()

diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py
--- a/pypy/module/_ssl/interp_win32.py
+++ b/pypy/module/_ssl/interp_win32.py
@@ -90,13 +90,12 @@
                     return space.w_True
                 raise wrap_windowserror(space, last_error)
 
-            result_w = []
+            result_w = [None] * usage.c_cUsageIdentifier
             for i in range(usage.c_cUsageIdentifier):
                 if not usage.c_rgpszUsageIdentifier[i]:
                     continue
-                result_w.append(
-                    space.wrap(rffi.charp2str(
-                        usage.c_rgpszUsageIdentifier[i])))
+                result_w[i] = space.wrap(rffi.charp2str(
+                    usage.c_rgpszUsageIdentifier[i]))
             return space.newset(result_w)
 
 @unwrap_spec(store_name=str)

From noreply at buildbot.pypy.org  Wed Feb 11 10:54:38 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 11 Feb 2015 10:54:38 +0100 (CET)
Subject: [pypy-commit] pypy default: issue #1978 resolved
Message-ID: <20150211095438.52EC21C12D0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r75825:2b2163d65ee4
Date: 2015-02-11 10:54 +0100
http://bitbucket.org/pypy/pypy/changeset/2b2163d65ee4/

Log:	issue #1978 resolved

diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py
--- a/lib_pypy/pyrepl/readline.py
+++ b/lib_pypy/pyrepl/readline.py
@@ -73,6 +73,7 @@
     assume_immutable_completions = False
     use_brackets = False
     sort_in_column = True
+    tab_insert_spaces_if_stem_is_empty = False
 
     def error(self, msg="none"):
         pass    # don't show error messages by default
@@ -86,6 +87,13 @@
         return ''.join(b[p+1:self.pos])
 
     def get_completions(self, stem):
+        if len(stem) == 0 and self.tab_insert_spaces_if_stem_is_empty:
+            b = self.buffer
+            p = self.pos
+            while p > 0 and b[p - 1] != '\n':
+                p -= 1
+            num_spaces = 4 - ((self.pos - p) % 4)
+            return [' ' * num_spaces]
         result = []
         function = self.config.readline_completer
         if function is not None:
@@ -204,14 +212,15 @@
         boolean value is true.
         """
         reader = self.get_reader()
-        saved = reader.more_lines
+        saved = reader.more_lines, reader.tab_insert_spaces_if_stem_is_empty
         try:
             reader.more_lines = more_lines
             reader.ps1 = reader.ps2 = ps1
             reader.ps3 = reader.ps4 = ps2
+            reader.tab_insert_spaces_if_stem_is_empty = True
             return reader.readline(returns_unicode=returns_unicode)
         finally:
-            reader.more_lines = saved
+            reader.more_lines, reader.tab_insert_spaces_if_stem_is_empty = saved
 
     def parse_and_bind(self, string):
         pass  # XXX we don't support parsing GNU-readline-style init files

From noreply at buildbot.pypy.org  Wed Feb 11 16:49:35 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 11 Feb 2015 16:49:35 +0100 (CET)
Subject: [pypy-commit] pypy default: issue #1976
Message-ID: <20150211154935.6B3DD1C02FD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r75826:adea6f0cdf60
Date: 2015-02-11 16:49 +0100
http://bitbucket.org/pypy/pypy/changeset/adea6f0cdf60/

Log:	issue #1976

	patch by David Naylor: better support for FreeBSD

diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -270,11 +270,14 @@
     _ffi.cdef("int sqlite3_enable_load_extension(sqlite3 *db, int onoff);")
 
 if sys.platform.startswith('freebsd'):
+    import os
+    import os.path
+    _localbase = os.environ.get('LOCALBASE', '/usr/local')
     _lib = _ffi.verify("""
     #include 
     """, libraries=['sqlite3'],
-         include_dirs=['/usr/local/include'],
-         library_dirs=['/usr/local/lib']
+         include_dirs=[os.path.join(_localbase, 'include')],
+         library_dirs=[os.path.join(_localbase, 'lib')]
     )
 else:
     _lib = _ffi.verify("""

From noreply at buildbot.pypy.org  Wed Feb 11 16:52:26 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 11 Feb 2015 16:52:26 +0100 (CET)
Subject: [pypy-commit] pypy default: issue #1977
Message-ID: <20150211155226.0B6AF1C115E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r75827:9249bb18f160
Date: 2015-02-11 16:52 +0100
http://bitbucket.org/pypy/pypy/changeset/9249bb18f160/

Log:	issue #1977

	patch by David Naylor: support FreeBSD

diff --git a/lib_pypy/gdbm.py b/lib_pypy/gdbm.py
--- a/lib_pypy/gdbm.py
+++ b/lib_pypy/gdbm.py
@@ -1,4 +1,4 @@
-import cffi, os
+import cffi, os, sys
 
 ffi = cffi.FFI()
 ffi.cdef('''
@@ -37,9 +37,19 @@
 ''')
 
 try:
-    lib = ffi.verify('''
-    #include "gdbm.h"
-    ''', libraries=['gdbm'])
+    if sys.platform.startswith('freebsd'):
+        import os.path
+        _localbase = os.environ.get('LOCALBASE', '/usr/local')
+        lib = ffi.verify('''
+        #include "gdbm.h"
+        ''', libraries=['gdbm'],
+             include_dirs=[os.path.join(_localbase, 'include')],
+             library_dirs=[os.path.join(_localbase, 'lib')]
+        )
+    else:
+        lib = ffi.verify('''
+        #include "gdbm.h"
+        ''', libraries=['gdbm'])
 except cffi.VerificationError as e:
     # distutils does not preserve the actual message,
     # but the verification is simple enough that the

From noreply at buildbot.pypy.org  Wed Feb 11 17:00:21 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 11 Feb 2015 17:00:21 +0100 (CET)
Subject: [pypy-commit] pypy.org extradoc: Complete the NumPy 2-paragraph
 instructions and link more clearly to
Message-ID: <20150211160021.78B171C1212@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: extradoc
Changeset: r566:0ef10b189227
Date: 2015-02-11 17:00 +0100
http://bitbucket.org/pypy/pypy.org/changeset/0ef10b189227/

Log:	Complete the NumPy 2-paragraph instructions and link more clearly to
	numpy/pypy.

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -213,10 +213,9 @@
 
      NumPy is an exception to the rule that most packages work without
 changes.  The “numpy” module needs to be installed from our own
 repository rather than from the official source.
      
-
      If you have pip (the command-line assumes that it finds the pip
-belonging to PyPy, not the one from CPython):
      
+
      If you have pip:
      
 
      -pip install git+https://bitbucket.org/pypy/numpy.git
+pypy -m pip install git+https://bitbucket.org/pypy/numpy.git
 
      
 
      Alternatively, the direct way:
      
 
      @@ -224,8 +223,13 @@
 cd numpy
 pypy setup.py install
 
      
+
      If you installed to a system directory, you need to also run this once:
      
+
      +sudo pypy -c 'import numpy'
+
      
 
      Note that NumPy support is still a work-in-progress, many things do not
-work and those that do may not be any faster than NumPy on CPython.
      
+work and those that do may not be any faster than NumPy on CPython.
+For further instructions see the pypy/numpy repository.

    Building from source

    diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -202,10 +202,9 @@ .. __: https://bitbucket.org/pypy/numpy -If you have pip (the command-line assumes that it finds the pip -belonging to PyPy, not the one from CPython):: +If you have pip:: - pip install git+https://bitbucket.org/pypy/numpy.git + pypy -m pip install git+https://bitbucket.org/pypy/numpy.git Alternatively, the direct way:: @@ -213,8 +212,16 @@ cd numpy pypy setup.py install +If you installed to a system directory, you need to also run this once:: + + sudo pypy -c 'import numpy' + Note that NumPy support is still a work-in-progress, many things do not work and those that do may not be any faster than NumPy on CPython. +For further instructions see `the pypy/numpy repository`__. + +.. __: https://bitbucket.org/pypy/numpy + .. _translate: From noreply at buildbot.pypy.org Wed Feb 11 18:55:41 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 11 Feb 2015 18:55:41 +0100 (CET) Subject: [pypy-commit] pypy default: Revert too much changes that were done in b782b42ddc67: we still need Message-ID: <20150211175541.F23211C02FD@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75828:70d63dd6d506 Date: 2015-02-11 18:55 +0100 http://bitbucket.org/pypy/pypy/changeset/70d63dd6d506/ Log: Revert too much changes that were done in b782b42ddc67: we still need to say "-Wl,--export-dynamic" in the Makefile, otherwise no symbol is exported if we're "--no-shared". diff --git a/rpython/translator/platform/__init__.py b/rpython/translator/platform/__init__.py --- a/rpython/translator/platform/__init__.py +++ b/rpython/translator/platform/__init__.py @@ -201,10 +201,14 @@ library_dirs = self._libdirs(library_dirs) libraries = self._libs(eci.libraries) link_files = self._linkfiles(eci.link_files) - return (library_dirs + list(self.link_flags) + + export_flags = self._exportsymbols_link_flags() + return (library_dirs + list(self.link_flags) + export_flags + link_files + list(eci.link_extra) + libraries + list(self.extra_libs)) + def _exportsymbols_link_flags(self): + return [] + def _finish_linking(self, ofiles, eci, outputfilename, standalone): if outputfilename is None: outputfilename = ofiles[0].purebasename diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py --- a/rpython/translator/platform/darwin.py +++ b/rpython/translator/platform/darwin.py @@ -36,6 +36,13 @@ include_dirs = self._includedirs(eci.include_dirs) return (args + frameworks + include_dirs) + def _exportsymbols_link_flags(self): + # XXX unsure if OS/X requires an option to the linker to tell + # "please export all RPY_EXPORTED symbols even in the case of + # making a binary and not a dynamically-linked library". + # It's not "-exported_symbols_list" but something close. + return [] + def gen_makefile(self, cfiles, eci, exe_name=None, path=None, shared=False, headers_to_precompile=[], no_precompile_cfiles = []): diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py --- a/rpython/translator/platform/posix.py +++ b/rpython/translator/platform/posix.py @@ -43,6 +43,12 @@ def _link_args_from_eci(self, eci, standalone): return Platform._link_args_from_eci(self, eci, standalone) + def _exportsymbols_link_flags(self): + if (self.cc == 'mingw32' or (self.cc== 'gcc' and os.name=='nt') + or sys.platform == 'cygwin'): + return ["-Wl,--export-all-symbols"] + return ["-Wl,--export-dynamic"] + def _link(self, cc, ofiles, link_args, standalone, exe_name): args = [str(ofile) for ofile in ofiles] + link_args args += ['-o', str(exe_name)] @@ -105,6 +111,8 @@ if shared: linkflags = self._args_for_shared(linkflags) + linkflags += self._exportsymbols_link_flags() + if shared: libname = exe_name.new(ext='').basename target_name = 'lib' + exe_name.new(ext=self.so_ext).basename diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -271,6 +271,7 @@ linkflags = list(self.link_flags) if shared: linkflags = self._args_for_shared(linkflags) + linkflags += self._exportsymbols_link_flags() # Make sure different functions end up at different addresses! # This is required for the JIT. linkflags.append('/opt:noicf') From noreply at buildbot.pypy.org Wed Feb 11 18:58:11 2015 From: noreply at buildbot.pypy.org (gutworth) Date: Wed, 11 Feb 2015 18:58:11 +0100 (CET) Subject: [pypy-commit] pypy default: remove trailing whitespace Message-ID: <20150211175811.8C2501C02FD@cobra.cs.uni-duesseldorf.de> Author: Benjamin Peterson Branch: Changeset: r75829:5066e0dc8fc9 Date: 2015-02-11 12:57 -0500 http://bitbucket.org/pypy/pypy/changeset/5066e0dc8fc9/ Log: remove trailing whitespace diff --git a/rpython/jit/backend/arm/instruction_builder.py b/rpython/jit/backend/arm/instruction_builder.py --- a/rpython/jit/backend/arm/instruction_builder.py +++ b/rpython/jit/backend/arm/instruction_builder.py @@ -379,7 +379,6 @@ | Q << 6 | M << 5 | (dm & 0xf)) - self.write32(instr) return f From noreply at buildbot.pypy.org Wed Feb 11 19:27:59 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 11 Feb 2015 19:27:59 +0100 (CET) Subject: [pypy-commit] pypy default: Add a test for 70d63dd6d506, which also fails because we have a few Message-ID: <20150211182759.7093C1C1048@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75830:b4bf2c4a705f Date: 2015-02-11 19:10 +0100 http://bitbucket.org/pypy/pypy/changeset/b4bf2c4a705f/ Log: Add a test for 70d63dd6d506, which also fails because we have a few functions not marked with any RPY_xxx macro diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -71,6 +71,32 @@ class TestStandalone(StandaloneTests): + def compile(self, *args, **kwds): + t, builder = StandaloneTests.compile(self, *args, **kwds) + # + # verify that the executable re-export symbols, but not too many + if sys.platform.startswith('linux'): + seen_main = False + g = os.popen("objdump -T '%s'" % builder.executable_name, 'r') + for line in g: + if not line.strip(): + continue + if '*UND*' in line: + continue + name = line.split()[-1] + if name.startswith('__'): + continue + if name == 'main': + seen_main = True + continue + if 'pypy' in name.lower() or 'rpy' in name.lower(): + raise Exception("seeing unexpected exported name %r" + % (name,)) + g.close() + assert seen_main, "did not see 'main' exported" + # + return t, builder + def test_hello_world(self): def entry_point(argv): os.write(1, "hello world\n") From noreply at buildbot.pypy.org Wed Feb 11 19:28:00 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 11 Feb 2015 19:28:00 +0100 (CET) Subject: [pypy-commit] pypy default: A sprinke of RPY_EXTERN here and there avoids some of the remaining Message-ID: <20150211182800.A92711C1048@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75831:af205bb7dad4 Date: 2015-02-11 19:27 +0100 http://bitbucket.org/pypy/pypy/changeset/af205bb7dad4/ Log: A sprinke of RPY_EXTERN here and there avoids some of the remaining unexpectedly-exported symbols. diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py --- a/rpython/rtyper/module/ll_os.py +++ b/rpython/rtyper/module/ll_os.py @@ -164,7 +164,7 @@ # we need an indirection via c functions to get macro calls working on llvm XXX still? if hasattr(os, 'WCOREDUMP'): decl_snippet = """ - %(ret_type)s pypy_macro_wrapper_%(name)s (int status); + RPY_EXTERN %(ret_type)s pypy_macro_wrapper_%(name)s (int status); """ def_snippet = """ %(ret_type)s pypy_macro_wrapper_%(name)s (int status) { diff --git a/rpython/translator/c/src/debug_print.c b/rpython/translator/c/src/debug_print.c --- a/rpython/translator/c/src/debug_print.c +++ b/rpython/translator/c/src/debug_print.c @@ -116,7 +116,7 @@ #ifndef _WIN32 - long long pypy_read_timestamp(void) + RPY_EXTERN long long pypy_read_timestamp(void) { # ifdef CLOCK_THREAD_CPUTIME_ID struct timespec tspec; diff --git a/rpython/translator/c/src/entrypoint.c b/rpython/translator/c/src/entrypoint.c --- a/rpython/translator/c/src/entrypoint.c +++ b/rpython/translator/c/src/entrypoint.c @@ -18,6 +18,7 @@ #ifdef __GNUC__ /* Hack to prevent this function from being inlined. Helps asmgcc because the main() function has often a different prologue/epilogue. */ +RPY_EXTERN int pypy_main_function(int argc, char *argv[]) __attribute__((__noinline__)); #endif @@ -26,6 +27,7 @@ # include "forwarddecl.h" # endif +RPY_EXTERN int pypy_main_function(int argc, char *argv[]) { char *errmsg; diff --git a/rpython/translator/c/src/mem.c b/rpython/translator/c/src/mem.c --- a/rpython/translator/c/src/mem.c +++ b/rpython/translator/c/src/mem.c @@ -15,6 +15,7 @@ static struct pypy_debug_alloc_s *pypy_debug_alloc_list = NULL; +RPY_EXTERN void pypy_debug_alloc_start(void *addr, const char *funcname) { struct pypy_debug_alloc_s *p = malloc(sizeof(struct pypy_debug_alloc_s)); @@ -25,6 +26,7 @@ pypy_debug_alloc_list = p; } +RPY_EXTERN void pypy_debug_alloc_stop(void *addr) { struct pypy_debug_alloc_s **p; @@ -40,6 +42,7 @@ RPyAssert(0, "free() of a never-malloc()ed object"); } +RPY_EXTERN void pypy_debug_alloc_results(void) { long count = 0; diff --git a/rpython/translator/c/src/rtyper.c b/rpython/translator/c/src/rtyper.c --- a/rpython/translator/c/src/rtyper.c +++ b/rpython/translator/c/src/rtyper.c @@ -9,7 +9,7 @@ #include #include -struct _RPyString_dump_t { +static struct _RPyString_dump_t { struct _RPyString_dump_t *next; char data[1]; } *_RPyString_dump = NULL; diff --git a/rpython/translator/c/src/support.c b/rpython/translator/c/src/support.c --- a/rpython/translator/c/src/support.c +++ b/rpython/translator/c/src/support.c @@ -10,6 +10,7 @@ /*** misc ***/ +RPY_EXTERN void RPyAssertFailed(const char* filename, long lineno, const char* function, const char *msg) { fprintf(stderr, @@ -19,8 +20,8 @@ abort(); } +RPY_EXTERN void RPyAbort(void) { fprintf(stderr, "Invalid RPython operation (NULL ptr or bad array index)\n"); abort(); } - diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -75,7 +75,7 @@ t, builder = StandaloneTests.compile(self, *args, **kwds) # # verify that the executable re-export symbols, but not too many - if sys.platform.startswith('linux'): + if sys.platform.startswith('linux') and not kwds.get('shared', False): seen_main = False g = os.popen("objdump -T '%s'" % builder.executable_name, 'r') for line in g: @@ -89,9 +89,13 @@ if name == 'main': seen_main = True continue + if name == 'pypy_debug_file': # ok to export this one + continue if 'pypy' in name.lower() or 'rpy' in name.lower(): - raise Exception("seeing unexpected exported name %r" - % (name,)) + raise Exception("Unexpected exported name %r. " + "What is likely missing is RPY_EXTERN before the " + "declaration of this C function or global variable" + % (name,)) g.close() assert seen_main, "did not see 'main' exported" # From noreply at buildbot.pypy.org Wed Feb 11 19:28:01 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 11 Feb 2015 19:28:01 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20150211182801.C93E21C1048@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75832:498cbdbc76f4 Date: 2015-02-11 19:27 +0100 http://bitbucket.org/pypy/pypy/changeset/498cbdbc76f4/ Log: merge heads diff --git a/rpython/jit/backend/arm/instruction_builder.py b/rpython/jit/backend/arm/instruction_builder.py --- a/rpython/jit/backend/arm/instruction_builder.py +++ b/rpython/jit/backend/arm/instruction_builder.py @@ -379,7 +379,6 @@ | Q << 6 | M << 5 | (dm & 0xf)) - self.write32(instr) return f From noreply at buildbot.pypy.org Wed Feb 11 20:47:51 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 11 Feb 2015 20:47:51 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: Remove explicit mention of --jit-backend=x86-without-sse2. Message-ID: <20150211194751.CFD631C0058@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r567:b04699f23493 Date: 2015-02-11 20:48 +0100 http://bitbucket.org/pypy/pypy.org/changeset/b04699f23493/ Log: Remove explicit mention of --jit-backend=x86-without-sse2. diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -161,8 +161,11 @@
  • All our downloads, including previous versions. We also have a mirror, but please use only if you have troubles accessing the links above
-

If your CPU is really old, it may not have SSE2. In this case, you need -to translate yourself with the option --jit-backend=x86-without-sse2.

+

If your CPU is really, really old, it may be a x86-32 without SSE2. +We could at some point make a PyPy with a JIT without SSE2 –ask us +on IRC if you really want to know more– but note that your machine +is probably low-spec enough that running CPython on it is a better +idea in the first place.

[1]: stating it again: the Linux binaries are provided for the distributions listed here. If your distribution is not exactly this one, it won't work, you will probably see: pypy: error while loading shared diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -129,8 +129,11 @@ .. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-2.4.0-src.zip .. __: https://bitbucket.org/pypy/pypy/downloads -If your CPU is really old, it may not have SSE2. In this case, you need -to translate_ yourself with the option ``--jit-backend=x86-without-sse2``. +If your CPU is really, really old, it may be a x86-32 without SSE2. +We could at some point make a PyPy with a JIT without SSE2 ---ask us +on IRC if you really want to know more--- but note that your machine +is probably low-spec enough that running CPython on it is a better +idea in the first place. ``[1]:`` stating it again: the Linux binaries are provided for the distributions listed here. **If your distribution is not exactly this From noreply at buildbot.pypy.org Thu Feb 12 00:29:16 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 12 Feb 2015 00:29:16 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.9: None is illegal in a llexternal call Message-ID: <20150211232916.ECB0D1C0058@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: stdlib-2.7.9 Changeset: r75833:4b28f7d27709 Date: 2015-02-12 01:29 +0200 http://bitbucket.org/pypy/pypy/changeset/4b28f7d27709/ Log: None is illegal in a llexternal call diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py --- a/pypy/module/_ssl/interp_win32.py +++ b/pypy/module/_ssl/interp_win32.py @@ -74,7 +74,8 @@ def w_parseKeyUsage(space, pCertCtx, flags): with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as size_ptr: - if not CertGetEnhancedKeyUsage(pCertCtx, flags, None, size_ptr): + if not CertGetEnhancedKeyUsage(pCertCtx, flags, + lltype.nullptr(CERT_ENHKEY_USAGE), size_ptr): last_error = rwin32.lastSavedWindowsError() if last_error.winerror == CRYPT_E_NOT_FOUND: return space.w_True From noreply at buildbot.pypy.org Thu Feb 12 12:11:18 2015 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 12 Feb 2015 12:11:18 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: this is no longer true Message-ID: <20150212111118.DA4FF1C02FD@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: extradoc Changeset: r568:da39381fe3f2 Date: 2015-02-12 13:11 +0200 http://bitbucket.org/pypy/pypy.org/changeset/da39381fe3f2/ Log: this is no longer true diff --git a/source/performance.txt b/source/performance.txt --- a/source/performance.txt +++ b/source/performance.txt @@ -290,10 +290,6 @@ Unrelated things that we know PyPy to be slow at (note that we're probably working on it): -* **Building very large dicts**: At present, this is an issue with our GCs. - Building large lists works much better; the random order of - dictionary elements is what hurts performance right now. - * **CPython C extension modules**: Any C extension module recompiled with PyPy takes a very large hit in performance. PyPy supports C extension modules solely to provide basic functionality. From noreply at buildbot.pypy.org Thu Feb 12 16:47:49 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 12 Feb 2015 16:47:49 +0100 (CET) Subject: [pypy-commit] cffi default: Move to version number "0.8.6+", to distinguish from the last Message-ID: <20150212154749.09E891C0507@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1648:0c02b23ebe2c Date: 2015-02-12 16:47 +0100 http://bitbucket.org/cffi/cffi/changeset/0c02b23ebe2c/ Log: Move to version number "0.8.6+", to distinguish from the last release 0.8.6. diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -5689,7 +5689,7 @@ if (v == NULL || PyModule_AddObject(m, "_C_API", v) < 0) INITERROR; - v = PyText_FromString("0.8.6"); + v = PyText_FromString("0.8.6+"); if (v == NULL || PyModule_AddObject(m, "__version__", v) < 0) INITERROR; diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -3252,4 +3252,4 @@ def test_version(): # this test is here mostly for PyPy - assert __version__ == "0.8.6" + assert __version__ == "0.8.6+" diff --git a/cffi/__init__.py b/cffi/__init__.py --- a/cffi/__init__.py +++ b/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.8.6" -__version_info__ = (0, 8, 6) +__version__ = "0.8.6+" +__version_info__ = (0, 8, 6, "plus") # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/doc/source/conf.py b/doc/source/conf.py --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -47,7 +47,7 @@ # The short X.Y version. version = '0.8' # The full version, including alpha/beta/rc tags. -release = '0.8.6' +release = '0.8.6+' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/testing/test_version.py b/testing/test_version.py --- a/testing/test_version.py +++ b/testing/test_version.py @@ -16,6 +16,7 @@ def test_version(): v = cffi.__version__ version_info = '.'.join(str(i) for i in cffi.__version_info__) + version_info = version_info.replace('.plus', '+') assert v == version_info #v = BACKEND_VERSIONS.get(v, v) assert v == _cffi_backend.__version__ @@ -31,7 +32,7 @@ def test_doc_version_file(): parent = os.path.dirname(os.path.dirname(__file__)) - v = cffi.__version__ + v = cffi.__version__.replace('+', '') p = os.path.join(parent, 'doc', 'source', 'index.rst') content = open(p).read() assert ("cffi/cffi-%s.tar.gz" % v) in content @@ -41,7 +42,7 @@ p = os.path.join(parent, 'setup.py') content = open(p).read() # - v = cffi.__version__ + v = cffi.__version__.replace('+', '') assert ("version='%s'" % v) in content def test_c_version(): From noreply at buildbot.pypy.org Thu Feb 12 16:49:52 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 12 Feb 2015 16:49:52 +0100 (CET) Subject: [pypy-commit] pypy default: update to cffi/0c02b23ebe2c Message-ID: <20150212154952.11F0E1C0507@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75834:07c31af7d104 Date: 2015-02-12 16:49 +0100 http://bitbucket.org/pypy/pypy/changeset/07c31af7d104/ Log: update to cffi/0c02b23ebe2c diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -8,7 +8,7 @@ appleveldefs = { } interpleveldefs = { - '__version__': 'space.wrap("0.8.6")', + '__version__': 'space.wrap("0.8.6+")', 'load_library': 'libraryobj.load_library', diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -3241,4 +3241,4 @@ def test_version(): # this test is here mostly for PyPy - assert __version__ == "0.8.6" + assert __version__ == "0.8.6+" From noreply at buildbot.pypy.org Thu Feb 12 16:56:08 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 12 Feb 2015 16:56:08 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: create html Message-ID: <20150212155608.813611C11C7@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: extradoc Changeset: r569:3c8780b7b72c Date: 2015-02-12 17:57 +0200 http://bitbucket.org/pypy/pypy.org/changeset/3c8780b7b72c/ Log: create html diff --git a/performance.html b/performance.html --- a/performance.html +++ b/performance.html @@ -315,9 +315,6 @@

Unrelated things that we know PyPy to be slow at (note that we're probably working on it):

    -
  • Building very large dicts: At present, this is an issue with our GCs. -Building large lists works much better; the random order of -dictionary elements is what hurts performance right now.
  • CPython C extension modules: Any C extension module recompiled with PyPy takes a very large hit in performance. PyPy supports C extension modules solely to provide basic functionality. From noreply at buildbot.pypy.org Thu Feb 12 18:10:15 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 12 Feb 2015 18:10:15 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: add participants Message-ID: <20150212171015.2ED0D1C11C7@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: extradoc Changeset: r5502:96e4aa687d80 Date: 2015-02-12 19:11 +0200 http://bitbucket.org/pypy/extradoc/changeset/96e4aa687d80/ Log: add participants diff --git a/sprintinfo/leysin-winter-2015/people.txt b/sprintinfo/leysin-winter-2015/people.txt --- a/sprintinfo/leysin-winter-2015/people.txt +++ b/sprintinfo/leysin-winter-2015/people.txt @@ -11,12 +11,13 @@ Name Arrive/Depart Accomodation ==================== ============== ======================= Armin Rigo private -Maciej Fijalkowski 21-28 Ermina +Maciej Fijalkowski 20-28 Ermina Remi Meier 21-28 Ermina Sebastian Pawlus 21-27 Ermina Manuel Jacob 21-28 Ermina Joan Massich 20-? Ermina Quim Sanchez 20-? Ermina +Francisco Fernandez 20-28 Ermina ==================== ============== ======================= From noreply at buildbot.pypy.org Thu Feb 12 18:57:18 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Thu, 12 Feb 2015 18:57:18 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.9: Fix socket.__repr__() on Windows. Message-ID: <20150212175718.426711C140E@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: stdlib-2.7.9 Changeset: r75835:304ca19782a2 Date: 2015-02-12 18:56 +0100 http://bitbucket.org/pypy/pypy/changeset/304ca19782a2/ Log: Fix socket.__repr__() on Windows. diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -143,9 +143,10 @@ return space.wrap(self.sock.family) def descr_repr(self, space): + fd = intmask(self.sock.fd) # Force to signed type even on Windows. return space.wrap("" % - (self.sock.fd, self.sock.family, + (fd, self.sock.family, self.sock.type, self.sock.proto)) def accept_w(self, space): From noreply at buildbot.pypy.org Thu Feb 12 19:03:31 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Thu, 12 Feb 2015 19:03:31 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.9: Actually return the list we have built. Message-ID: <20150212180331.86B931C14BD@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: stdlib-2.7.9 Changeset: r75836:62ee769c7ff5 Date: 2015-02-12 19:03 +0100 http://bitbucket.org/pypy/pypy/changeset/62ee769c7ff5/ Log: Actually return the list we have built. diff --git a/pypy/module/_ssl/interp_win32.py b/pypy/module/_ssl/interp_win32.py --- a/pypy/module/_ssl/interp_win32.py +++ b/pypy/module/_ssl/interp_win32.py @@ -176,3 +176,5 @@ if not CertCloseStore(hStore, 0): # This error case might shadow another exception. raise wrap_windowserror(space, rwin32.lastSavedWindowsError()) + + return space.newlist(result_w) From noreply at buildbot.pypy.org Thu Feb 12 20:12:48 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 12 Feb 2015 20:12:48 +0100 (CET) Subject: [pypy-commit] pypy default: fix arm cross-compile documentation for rpython/pypy split (Samureus) Message-ID: <20150212191248.1BD441C02FD@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r75837:de02541cbc09 Date: 2015-02-12 21:13 +0200 http://bitbucket.org/pypy/pypy/changeset/de02541cbc09/ Log: fix arm cross-compile documentation for rpython/pypy split (Samureus) diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst --- a/pypy/doc/build.rst +++ b/pypy/doc/build.rst @@ -48,9 +48,13 @@ Install build-time dependencies ------------------------------- (**Note**: for some hints on how to translate the Python interpreter under -Windows, see the `windows document`_) +Windows, see the `windows document`_ . For hints on how to cross-compile in +a chroot using scratchbox2, see the `arm document`_ in the +`RPython documentation`_) .. _`windows document`: windows.html +.. _`arm document`: http://rpython.readthedocs.org/en/latest/arm.html +.. _`RPython documentation`: http://rpython.readthedocs.org To build PyPy on Unix using the C translation backend, you need at least a C diff --git a/rpython/doc/arm.rst b/rpython/doc/arm.rst --- a/rpython/doc/arm.rst +++ b/rpython/doc/arm.rst @@ -38,7 +38,7 @@ - The dependencies above are in addition to the ones needed for a regular translation, `listed here`_. -.. _`listed here`: getting-started-python.html#translating-the-pypy-python-interpreter +.. _`listed here`: http://pypy.readthedocs.org/en/latest/build.html#install-build-time-dependencies Creating a Qemu based ARM chroot From noreply at buildbot.pypy.org Thu Feb 12 20:27:36 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Thu, 12 Feb 2015 20:27:36 +0100 (CET) Subject: [pypy-commit] pypy framestate: Kill WithBlock: it's exactly the same as FinallyBlock Message-ID: <20150212192736.EDAD21C11C7@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: framestate Changeset: r75838:98a783f80f0f Date: 2015-02-11 02:57 +0000 http://bitbucket.org/pypy/pypy/changeset/98a783f80f0f/ Log: Kill WithBlock: it's exactly the same as FinallyBlock diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py --- a/rpython/flowspace/bytecode.py +++ b/rpython/flowspace/bytecode.py @@ -590,8 +590,8 @@ @bc_reader.register_opcode class SETUP_WITH(SetupInstruction): def make_block(self, stackdepth): - from rpython.flowspace.flowcontext import WithBlock - return WithBlock(stackdepth, self.target) + from rpython.flowspace.flowcontext import FinallyBlock + return FinallyBlock(stackdepth, self.target) def eval(self, ctx): # A simpler version than the 'real' 2.7 one: diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py --- a/rpython/flowspace/flowcontext.py +++ b/rpython/flowspace/flowcontext.py @@ -1182,7 +1182,11 @@ return self.handler # jump to the handler class FinallyBlock(FrameBlock): - """A try:finally: block. Stores the position of the exception handler.""" + """A try:finally: or with: block + + A with: block begins with SETUP_WITH and its handler is just + 'WITH_CLEANUP END_FINALLY'. + """ handles = FlowSignal @@ -1192,9 +1196,3 @@ self.cleanupstack(ctx) ctx.pushvalue(unroller) return self.handler # jump to the handler - - -class WithBlock(FinallyBlock): - - def handle(self, ctx, unroller): - return FinallyBlock.handle(self, ctx, unroller) From noreply at buildbot.pypy.org Thu Feb 12 20:27:38 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Thu, 12 Feb 2015 20:27:38 +0100 (CET) Subject: [pypy-commit] pypy framestate: Replace CONTINUE_LOOP and BREAK_LOOP with jumps in simple cases Message-ID: <20150212192738.2F8A21C11C7@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: framestate Changeset: r75839:ed696843831a Date: 2015-02-12 05:10 +0000 http://bitbucket.org/pypy/pypy/changeset/ed696843831a/ Log: Replace CONTINUE_LOOP and BREAK_LOOP with jumps in simple cases diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py --- a/rpython/flowspace/bytecode.py +++ b/rpython/flowspace/bytecode.py @@ -221,12 +221,6 @@ for exit in block._exits: exit.set_blockstack(self.blockstack) - def unroll(self, signal): - while self.blockstack: - block = self.blockstack.pop() - if isinstance(signal, block.handles): - return block - def check_graph(self): for b in self.blocks: if not b._exits: @@ -537,9 +531,24 @@ reader.end_block() def do_signals(self, reader): - from rpython.flowspace.flowcontext import Break - frameblock = reader.unroll(Break()) - reader.curr_block.set_exits([frameblock.handler]) + block = reader.curr_block + assert block.operations[-1] is self + del block.operations[-1] + from rpython.flowspace.flowcontext import ExceptBlock, FinallyBlock + while reader.blockstack: + context = reader.blockstack.pop() + if isinstance(context, ExceptBlock): + block.operations.append(POP_BLOCK(-1, self.offset)) + elif isinstance(context, FinallyBlock): + block.operations.append(self) + block.set_exits([context.handler]) + return + else: # LoopBlock + block.operations.append(POP_BLOCK(-1, self.offset)) + block.set_exits([context.handler]) + return + raise BytecodeCorruption( + "A break statement should not escape from the function") def eval(self, ctx): from rpython.flowspace.flowcontext import Break @@ -550,6 +559,27 @@ def bc_flow(self, reader): reader.curr_block.operations.append(self) self.target = reader.get_block_at(self.arg) + reader.end_block() + + def do_signals(self, reader): + block = reader.curr_block + assert block.operations[-1] is self + del block.operations[-1] + from rpython.flowspace.flowcontext import ExceptBlock, FinallyBlock + while reader.blockstack: + context = reader.blockstack.pop() + if isinstance(context, ExceptBlock): + block.operations.append(POP_BLOCK(-1, self.offset)) + elif isinstance(context, FinallyBlock): + block.operations.append(self) + block.set_exits([context.handler]) + return + else: # LoopBlock + block.operations.append(POP_BLOCK(-1, self.offset)) + block.set_exits([self.target]) + return + raise BytecodeCorruption( + "A continue statement should not escape from the function") def eval(self, ctx): from rpython.flowspace.flowcontext import Continue From noreply at buildbot.pypy.org Thu Feb 12 20:27:39 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Thu, 12 Feb 2015 20:27:39 +0100 (CET) Subject: [pypy-commit] pypy framestate: move END_FINALLY to bytecode.py Message-ID: <20150212192739.591D41C11C7@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: framestate Changeset: r75840:8349857a3478 Date: 2015-02-12 06:58 +0000 http://bitbucket.org/pypy/pypy/changeset/8349857a3478/ Log: move END_FINALLY to bytecode.py diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py --- a/rpython/flowspace/bytecode.py +++ b/rpython/flowspace/bytecode.py @@ -585,6 +585,33 @@ from rpython.flowspace.flowcontext import Continue return ctx.unroll(Continue(self.target)) + at bc_reader.register_opcode +class END_FINALLY(BCInstruction): + def eval(self, ctx): + # unlike CPython, there are two statically distinct cases: the + # END_FINALLY might be closing an 'except' block or a 'finally' + # block. In the first case, the stack contains three items: + # [exception type we are now handling] + # [exception value we are now handling] + # [Raise] + # In the case of a finally: block, the stack contains only one + # item (unlike CPython which can have 1, 2 or 3 items): + # [subclass of FlowSignal] + from rpython.flowspace.flowcontext import FlowSignal + w_top = ctx.popvalue() + if w_top == const(None): + # finally: block with no unroller active + return + elif isinstance(w_top, FlowSignal): + # case of a finally: block + raise w_top + else: + # case of an except: block. We popped the exception type + ctx.popvalue() # Now we pop the exception value + signal = ctx.popvalue() + raise signal + + class SetupInstruction(BCInstruction): def bc_flow(self, reader): reader.curr_block.operations.append(self) diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py --- a/rpython/flowspace/flowcontext.py +++ b/rpython/flowspace/flowcontext.py @@ -610,29 +610,6 @@ w_returnvalue = self.popvalue() raise Return(w_returnvalue) - def END_FINALLY(self, oparg): - # unlike CPython, there are two statically distinct cases: the - # END_FINALLY might be closing an 'except' block or a 'finally' - # block. In the first case, the stack contains three items: - # [exception type we are now handling] - # [exception value we are now handling] - # [Raise] - # In the case of a finally: block, the stack contains only one - # item (unlike CPython which can have 1, 2 or 3 items): - # [subclass of FlowSignal] - w_top = self.popvalue() - if w_top == w_None: - # finally: block with no unroller active - return - elif isinstance(w_top, FlowSignal): - # case of a finally: block - raise w_top - else: - # case of an except: block. We popped the exception type - self.popvalue() # Now we pop the exception value - signal = self.popvalue() - raise signal - def YIELD_VALUE(self, _): assert self.pycode.is_generator w_result = self.popvalue() From noreply at buildbot.pypy.org Thu Feb 12 20:27:40 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Thu, 12 Feb 2015 20:27:40 +0100 (CET) Subject: [pypy-commit] pypy framestate: Record position of the matching END_FINALLY for except: and finally: blocks Message-ID: <20150212192740.82ACD1C11C7@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: framestate Changeset: r75841:933ada692ba6 Date: 2015-02-12 19:25 +0000 http://bitbucket.org/pypy/pypy/changeset/933ada692ba6/ Log: Record position of the matching END_FINALLY for except: and finally: blocks diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py --- a/rpython/flowspace/bytecode.py +++ b/rpython/flowspace/bytecode.py @@ -201,6 +201,7 @@ def build_flow(self, code): self.pending_blocks = {} + self.handlerstack = [] self.blocks = [SimpleBlock([])] self.curr_block = self.blocks[0] self.needs_new_block = False @@ -587,6 +588,11 @@ @bc_reader.register_opcode class END_FINALLY(BCInstruction): + def bc_flow(self, reader): + reader.curr_block.operations.append(self) + signal = reader.handlerstack.pop() + signal.handler_end = reader.curr_block + def eval(self, ctx): # unlike CPython, there are two statically distinct cases: the # END_FINALLY might be closing an 'except' block or a 'finally' @@ -613,42 +619,59 @@ class SetupInstruction(BCInstruction): + def __init__(self, arg, offset=-1): + BCInstruction.__init__(self, arg, offset=offset) + self.block = self.make_block(-1) + def bc_flow(self, reader): reader.curr_block.operations.append(self) self.target = reader.get_block_at(self.arg) + self.block.handler = self.target reader.end_block() def do_signals(self, reader): - reader.blockstack.append(self.make_block(-1)) + reader.blockstack.append(self.block) def eval(self, ctx): - block = self.make_block(ctx.stackdepth) - ctx.blockstack.append(block) + self.block.stackdepth = ctx.stackdepth + ctx.blockstack.append(self.block) @bc_reader.register_opcode class SETUP_EXCEPT(SetupInstruction): + def bc_flow(self, reader): + SetupInstruction.bc_flow(self, reader) + reader.handlerstack.append(self.block) + def make_block(self, stackdepth): from rpython.flowspace.flowcontext import ExceptBlock - return ExceptBlock(stackdepth, self.target) + return ExceptBlock(stackdepth, None) @bc_reader.register_opcode class SETUP_LOOP(SetupInstruction): def make_block(self, stackdepth): from rpython.flowspace.flowcontext import LoopBlock - return LoopBlock(stackdepth, self.target) + return LoopBlock(stackdepth, None) @bc_reader.register_opcode class SETUP_FINALLY(SetupInstruction): + def bc_flow(self, reader): + SetupInstruction.bc_flow(self, reader) + reader.handlerstack.append(self.block) + def make_block(self, stackdepth): from rpython.flowspace.flowcontext import FinallyBlock - return FinallyBlock(stackdepth, self.target) + return FinallyBlock(stackdepth, None) @bc_reader.register_opcode class SETUP_WITH(SetupInstruction): + def bc_flow(self, reader): + SetupInstruction.bc_flow(self, reader) + reader.handlerstack.append(self.block) + def make_block(self, stackdepth): from rpython.flowspace.flowcontext import FinallyBlock - return FinallyBlock(stackdepth, self.target) + return FinallyBlock(stackdepth, None) def eval(self, ctx): # A simpler version than the 'real' 2.7 one: @@ -659,8 +682,8 @@ ctx.settopvalue(w_exit) w_enter = op.getattr(w_manager, const('__enter__')).eval(ctx) w_result = op.simple_call(w_enter).eval(ctx) - block = self.make_block(ctx.stackdepth) - ctx.blockstack.append(block) + self.block.stackdepth = ctx.stackdepth + ctx.blockstack.append(self.block) ctx.pushvalue(w_result) @bc_reader.register_opcode From noreply at buildbot.pypy.org Thu Feb 12 20:27:50 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Thu, 12 Feb 2015 20:27:50 +0100 (CET) Subject: [pypy-commit] pypy framestate: hg merge default Message-ID: <20150212192750.899671C11C7@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: framestate Changeset: r75842:2a0e866552f1 Date: 2015-02-12 19:25 +0000 http://bitbucket.org/pypy/pypy/changeset/2a0e866552f1/ Log: hg merge default diff too long, truncating to 2000 out of 40693 lines diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -7,10 +7,7 @@ 9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm 20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0 -20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0 -0000000000000000000000000000000000000000 release-2.3.0 394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1 -32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 -0000000000000000000000000000000000000000 release-2.2=3.1 +10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -28,7 +28,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2014 +PyPy Copyright holders 2003-2015 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at @@ -42,19 +42,19 @@ Amaury Forgeot d'Arc Samuele Pedroni Alex Gaynor + Brian Kearns + Matti Picus + Philip Jenvey Michael Hudson David Schneider - Matti Picus - Brian Kearns - Philip Jenvey Holger Krekel Christian Tismer Hakan Ardo Benjamin Peterson Manuel Jacob + Ronan Lamy Anders Chrigstrom Eric van Riet Paap - Ronan Lamy Wim Lavrijsen Richard Emslie Alexander Schremmer @@ -68,9 +68,9 @@ Camillo Bruni Laura Creighton Toon Verwaest + Romain Guillebert Leonardo Santagada Seo Sanghyeon - Romain Guillebert Justin Peel Ronny Pfannschmidt David Edelsohn @@ -91,15 +91,16 @@ Michal Bendowski Jan de Mooij stian + Tyler Wade Michael Foord Stephan Diehl - Tyler Wade Stefan Schwarzer Valentino Volonghi Tomek Meka Patrick Maupin Bob Ippolito Bruno Gola + David Malcolm Jean-Paul Calderone Timo Paulssen Squeaky @@ -108,18 +109,19 @@ Marius Gedminas Martin Matusiak Konstantin Lopuhin + Wenzhu Man John Witulski - Wenzhu Man + Laurence Tratt + Ivan Sichmann Freitas Greg Price Dario Bertini Mark Pearse Simon Cross - Ivan Sichmann Freitas Andreas Stührk + Stefano Rivera Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov - Stefano Rivera Paweł Piotr Przeradowski Paul deGrandis Ilya Osadchiy @@ -129,7 +131,6 @@ tav Taavi Burns Georg Brandl - Laurence Tratt Bert Freudenberg Stian Andreassen Wanja Saatkamp @@ -141,13 +142,12 @@ Jeremy Thurgood Rami Chowdhury Tobias Pape - David Malcolm Eugene Oden Henry Mason Vasily Kuznetsov Preston Timmons + David Ripton Jeff Terrace - David Ripton Dusty Phillips Lukas Renggli Guenter Jantzen @@ -166,13 +166,16 @@ Gintautas Miliauskas Michael Twomey Lucian Branescu Mihaila + Yichao Yu Gabriel Lavoie Olivier Dormond Jared Grubb Karl Bartel + Wouter van Heyst Brian Dorsey Victor Stinner Andrews Medina + anatoly techtonik Stuart Williams Jasper Schulz Christian Hudon @@ -182,12 +185,11 @@ Michael Cheng Justas Sadzevicius Gasper Zejn - anatoly techtonik Neil Shepperd + Stanislaw Halik Mikael Schönenberg Elmo M?ntynen Jonathan David Riehl - Stanislaw Halik Anders Qvist Corbin Simpson Chirag Jadwani @@ -196,10 +198,13 @@ Vincent Legoll Alan McIntyre Alexander Sedov + Attila Gobi Christopher Pope Christian Tismer Marc Abramowitz Dan Stromberg + Arjun Naik + Valentina Mukhamedzhanova Stefano Parmesan Alexis Daboville Jens-Uwe Mager @@ -213,8 +218,6 @@ Sylvain Thenault Nathan Taylor Vladimir Kryachko - Arjun Naik - Attila Gobi Jacek Generowicz Alejandro J. Cura Jacob Oscarson @@ -222,22 +225,23 @@ Ryan Gonzalez Ian Foote Kristjan Valur Jonsson + David Lievens Neil Blakey-Milner Lutz Paelike Lucio Torre Lars Wassermann - Valentina Mukhamedzhanova Henrik Vendelbo Dan Buch Miguel de Val Borro Artur Lisiecki Sergey Kishchenko - Yichao Yu Ignas Mikalajunas Christoph Gerum Martin Blais Lene Wagner Tomo Cocoa + Toni Mattis + Lucas Stadler roberto at goyle Yury V. Zaytsev Anna Katrina Dominguez @@ -265,23 +269,30 @@ Stephan Busemann Rafał Gałczyński Christian Muirhead + Berker Peksag James Lan shoma hosaka - Daniel Neuh?user - Matthew Miller + Daniel Neuhäuser + Ben Mather + halgari + Boglarka Vezer + Chris Pressey Buck Golemon Konrad Delong Dinu Gherman Chris Lambacher coolbutuseless at gmail.com + Jim Baker Rodrigo Araújo - Jim Baker + Nikolaos-Digenis Karagiannis James Robert Armin Ronacher Brett Cannon + Donald Stufft yrttyr aliceinwire OlivierBlanvillain + Dan Sanders Zooko Wilcox-O Hearn Tomer Chachamu Christopher Groskopf @@ -295,6 +306,7 @@ Markus Unterwaditzer Even Wiik Thomassen jbs + squeaky soareschen Kurt Griffiths Mike Bayer @@ -306,6 +318,7 @@ Anna Ravencroft Dan Crosta Julien Phalip + Roman Podoliaka Dan Loewenherz Heinrich-Heine University, Germany diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py --- a/lib-python/2.7/collections.py +++ b/lib-python/2.7/collections.py @@ -17,6 +17,10 @@ except ImportError: assert '__pypy__' not in _sys.builtin_module_names newdict = lambda _ : {} +try: + from __pypy__ import reversed_dict +except ImportError: + reversed_dict = lambda d: reversed(d.keys()) try: from thread import get_ident as _get_ident @@ -29,142 +33,35 @@ ################################################################################ class OrderedDict(dict): - 'Dictionary that remembers insertion order' - # An inherited dict maps keys to values. - # The inherited dict provides __getitem__, __len__, __contains__, and get. - # The remaining methods are order-aware. - # Big-O running times for all methods are the same as regular dictionaries. + '''Dictionary that remembers insertion order. - # The internal self.__map dict maps keys to links in a doubly linked list. - # The circular doubly linked list starts and ends with a sentinel element. - # The sentinel element never gets deleted (this simplifies the algorithm). - # Each link is stored as a list of length three: [PREV, NEXT, KEY]. + In PyPy all dicts are ordered anyway. This is mostly useful as a + placeholder to mean "this dict must be ordered even on CPython". - def __init__(self, *args, **kwds): - '''Initialize an ordered dictionary. The signature is the same as - regular dictionaries, but keyword arguments are not recommended because - their insertion order is arbitrary. - - ''' - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__root - except AttributeError: - self.__root = root = [] # sentinel node - root[:] = [root, root, None] - self.__map = {} - self.__update(*args, **kwds) - - def __setitem__(self, key, value, dict_setitem=dict.__setitem__): - 'od.__setitem__(i, y) <==> od[i]=y' - # Setting a new item creates a new link at the end of the linked list, - # and the inherited dictionary is updated with the new key/value pair. - if key not in self: - root = self.__root - last = root[0] - last[1] = root[0] = self.__map[key] = [last, root, key] - return dict_setitem(self, key, value) - - def __delitem__(self, key, dict_delitem=dict.__delitem__): - 'od.__delitem__(y) <==> del od[y]' - # Deleting an existing item uses self.__map to find the link which gets - # removed by updating the links in the predecessor and successor nodes. - dict_delitem(self, key) - link_prev, link_next, _ = self.__map.pop(key) - link_prev[1] = link_next # update link_prev[NEXT] - link_next[0] = link_prev # update link_next[PREV] - - def __iter__(self): - 'od.__iter__() <==> iter(od)' - # Traverse the linked list in order. - root = self.__root - curr = root[1] # start at the first node - while curr is not root: - yield curr[2] # yield the curr[KEY] - curr = curr[1] # move to next node + Known difference: iterating over an OrderedDict which is being + concurrently modified raises RuntimeError in PyPy. In CPython + instead we get some behavior that appears reasonable in some + cases but is nonsensical in other cases. This is officially + forbidden by the CPython docs, so we forbid it explicitly for now. + ''' def __reversed__(self): - 'od.__reversed__() <==> reversed(od)' - # Traverse the linked list in reverse order. - root = self.__root - curr = root[0] # start at the last node - while curr is not root: - yield curr[2] # yield the curr[KEY] - curr = curr[0] # move to previous node - - def clear(self): - 'od.clear() -> None. Remove all items from od.' - root = self.__root - root[:] = [root, root, None] - self.__map.clear() - dict.clear(self) - - # -- the following methods do not depend on the internal structure -- - - def keys(self): - 'od.keys() -> list of keys in od' - return list(self) - - def values(self): - 'od.values() -> list of values in od' - return [self[key] for key in self] - - def items(self): - 'od.items() -> list of (key, value) pairs in od' - return [(key, self[key]) for key in self] - - def iterkeys(self): - 'od.iterkeys() -> an iterator over the keys in od' - return iter(self) - - def itervalues(self): - 'od.itervalues -> an iterator over the values in od' - for k in self: - yield self[k] - - def iteritems(self): - 'od.iteritems -> an iterator over the (key, value) pairs in od' - for k in self: - yield (k, self[k]) - - update = MutableMapping.update - - __update = update # let subclasses override update without breaking __init__ - - __marker = object() - - def pop(self, key, default=__marker): - '''od.pop(k[,d]) -> v, remove specified key and return the corresponding - value. If key is not found, d is returned if given, otherwise KeyError - is raised. - - ''' - if key in self: - result = self[key] - del self[key] - return result - if default is self.__marker: - raise KeyError(key) - return default - - def setdefault(self, key, default=None): - 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' - if key in self: - return self[key] - self[key] = default - return default + return reversed_dict(self) def popitem(self, last=True): '''od.popitem() -> (k, v), return and remove a (key, value) pair. Pairs are returned in LIFO order if last is true or FIFO order if false. ''' - if not self: - raise KeyError('dictionary is empty') - key = next(reversed(self) if last else iter(self)) - value = self.pop(key) - return key, value + if last: + return dict.popitem(self) + else: + it = dict.__iter__(self) + try: + k = it.next() + except StopIteration: + raise KeyError('dictionary is empty') + return (k, self.pop(k)) def __repr__(self, _repr_running={}): 'od.__repr__() <==> repr(od)' @@ -183,8 +80,6 @@ 'Return state information for pickling' items = [[k, self[k]] for k in self] inst_dict = vars(self).copy() - for k in vars(OrderedDict()): - inst_dict.pop(k, None) if inst_dict: return (self.__class__, (items,), inst_dict) return self.__class__, (items,) @@ -193,17 +88,6 @@ 'od.copy() -> a shallow copy of od' return self.__class__(self) - @classmethod - def fromkeys(cls, iterable, value=None): - '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S. - If not specified, the value defaults to None. - - ''' - self = cls() - for key in iterable: - self[key] = value - return self - def __eq__(self, other): '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive while comparison to a regular mapping is order-insensitive. diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py --- a/lib-python/2.7/ctypes/test/test_frombuffer.py +++ b/lib-python/2.7/ctypes/test/test_frombuffer.py @@ -2,7 +2,6 @@ import array import gc import unittest -from ctypes.test import xfail class X(Structure): _fields_ = [("c_int", c_int)] @@ -11,7 +10,6 @@ self._init_called = True class Test(unittest.TestCase): - @xfail def test_fom_buffer(self): a = array.array("i", range(16)) x = (c_int * 16).from_buffer(a) @@ -34,10 +32,9 @@ del a; gc.collect(); gc.collect(); gc.collect() self.assertEqual(x[:], expected) - self.assertRaises(TypeError, + self.assertRaises((TypeError, ValueError), (c_char * 16).from_buffer, "a" * 16) - @xfail def test_fom_buffer_with_offset(self): a = array.array("i", range(16)) x = (c_int * 15).from_buffer(a, sizeof(c_int)) @@ -46,7 +43,6 @@ self.assertRaises(ValueError, lambda: (c_int * 16).from_buffer(a, sizeof(c_int))) self.assertRaises(ValueError, lambda: (c_int * 1).from_buffer(a, 16 * sizeof(c_int))) - @xfail def test_from_buffer_copy(self): a = array.array("i", range(16)) x = (c_int * 16).from_buffer_copy(a) @@ -71,7 +67,6 @@ x = (c_char * 16).from_buffer_copy("a" * 16) self.assertEqual(x[:], "a" * 16) - @xfail def test_fom_buffer_copy_with_offset(self): a = array.array("i", range(16)) x = (c_int * 15).from_buffer_copy(a, sizeof(c_int)) diff --git a/lib-python/2.7/distutils/unixccompiler.py b/lib-python/2.7/distutils/unixccompiler.py --- a/lib-python/2.7/distutils/unixccompiler.py +++ b/lib-python/2.7/distutils/unixccompiler.py @@ -58,7 +58,7 @@ executables = {'preprocessor' : None, 'compiler' : ["cc"], 'compiler_so' : ["cc"], - 'compiler_cxx' : ["cc"], + 'compiler_cxx' : ["c++"], # pypy: changed, 'cc' is bogus 'linker_so' : ["cc", "-shared"], 'linker_exe' : ["cc"], 'archiver' : ["ar", "-cr"], diff --git a/lib-python/2.7/sqlite3/test/dbapi.py b/lib-python/2.7/sqlite3/test/dbapi.py --- a/lib-python/2.7/sqlite3/test/dbapi.py +++ b/lib-python/2.7/sqlite3/test/dbapi.py @@ -478,6 +478,29 @@ except TypeError: pass + def CheckCurDescription(self): + self.cu.execute("select * from test") + + actual = self.cu.description + expected = [ + ('id', None, None, None, None, None, None), + ('name', None, None, None, None, None, None), + ('income', None, None, None, None, None, None), + ] + self.assertEqual(expected, actual) + + def CheckCurDescriptionVoidStatement(self): + self.cu.execute("insert into test(name) values (?)", ("foo",)) + self.assertIsNone(self.cu.description) + + def CheckCurDescriptionWithoutStatement(self): + cu = self.cx.cursor() + try: + self.assertIsNone(cu.description) + finally: + cu.close() + + @unittest.skipUnless(threading, 'This test requires threading.') class ThreadTests(unittest.TestCase): def setUp(self): diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py --- a/lib-python/2.7/subprocess.py +++ b/lib-python/2.7/subprocess.py @@ -1589,7 +1589,7 @@ 'copyfile' in caller.f_globals): dest_dir = sys.pypy_resolvedirof(target_executable) src_dir = sys.pypy_resolvedirof(sys.executable) - for libname in ['libpypy-c.so']: + for libname in ['libpypy-c.so', 'libpypy-c.dylib']: dest_library = os.path.join(dest_dir, libname) src_library = os.path.join(src_dir, libname) if os.path.exists(src_library): diff --git a/lib-python/2.7/test/test_audioop.py b/lib-python/2.7/test/test_audioop.py --- a/lib-python/2.7/test/test_audioop.py +++ b/lib-python/2.7/test/test_audioop.py @@ -2,7 +2,7 @@ import sys import unittest import struct -from test.test_support import run_unittest, impl_detail +from test.test_support import run_unittest formats = { @@ -183,7 +183,6 @@ self.assertEqual(audioop.lin2lin(datas[4], 4, 2), packs[2](0, 0x1234, 0x4567, -0x4568, 0x7fff, -0x8000, -1)) - @impl_detail(pypy=False) def test_adpcm2lin(self): self.assertEqual(audioop.adpcm2lin(b'\x07\x7f\x7f', 1, None), (b'\x00\x00\x00\xff\x00\xff', (-179, 40))) @@ -198,7 +197,6 @@ self.assertEqual(audioop.adpcm2lin(b'\0' * 5, w, None), (b'\0' * w * 10, (0, 0))) - @impl_detail(pypy=False) def test_lin2adpcm(self): self.assertEqual(audioop.lin2adpcm(datas[1], 1, None), (b'\x07\x7f\x7f', (-221, 39))) @@ -212,7 +210,6 @@ self.assertEqual(audioop.lin2adpcm(b'\0' * w * 10, w, None), (b'\0' * 5, (0, 0))) - @impl_detail(pypy=False) def test_lin2alaw(self): self.assertEqual(audioop.lin2alaw(datas[1], 1), b'\xd5\x87\xa4\x24\xaa\x2a\x5a') @@ -221,7 +218,6 @@ self.assertEqual(audioop.lin2alaw(datas[4], 4), b'\xd5\x87\xa4\x24\xaa\x2a\x55') - @impl_detail(pypy=False) def test_alaw2lin(self): encoded = b'\x00\x03\x24\x2a\x51\x54\x55\x58\x6b\x71\x7f'\ b'\x80\x83\xa4\xaa\xd1\xd4\xd5\xd8\xeb\xf1\xff' @@ -236,7 +232,6 @@ decoded = audioop.alaw2lin(encoded, w) self.assertEqual(audioop.lin2alaw(decoded, w), encoded) - @impl_detail(pypy=False) def test_lin2ulaw(self): self.assertEqual(audioop.lin2ulaw(datas[1], 1), b'\xff\xad\x8e\x0e\x80\x00\x67') @@ -245,7 +240,6 @@ self.assertEqual(audioop.lin2ulaw(datas[4], 4), b'\xff\xad\x8e\x0e\x80\x00\x7e') - @impl_detail(pypy=False) def test_ulaw2lin(self): encoded = b'\x00\x0e\x28\x3f\x57\x6a\x76\x7c\x7e\x7f'\ b'\x80\x8e\xa8\xbf\xd7\xea\xf6\xfc\xfe\xff' @@ -360,7 +354,6 @@ self.assertRaises(audioop.error, audioop.findmax, ''.join( chr(x) for x in xrange(256)), -2392392) - @impl_detail(pypy=False) def test_issue7673(self): state = None for data, size in INVALID_DATA: @@ -385,7 +378,6 @@ self.assertRaises(audioop.error, audioop.lin2alaw, data, size) self.assertRaises(audioop.error, audioop.lin2adpcm, data, size, state) - @impl_detail(pypy=False) def test_wrongsize(self): data = b'abcdefgh' state = None diff --git a/lib-python/2.7/test/test_collections.py b/lib-python/2.7/test/test_collections.py --- a/lib-python/2.7/test/test_collections.py +++ b/lib-python/2.7/test/test_collections.py @@ -578,7 +578,12 @@ def __repr__(self): return "MySet(%s)" % repr(list(self)) s = MySet([5,43,2,1]) - self.assertEqual(s.pop(), 1) + # changed from CPython 2.7: it was "s.pop() == 1" but I see + # nothing that guarantees a particular order here. In the + # 'all_ordered_dicts' branch of PyPy (or with OrderedDict + # instead of sets), it consistently returns 5, but this test + # should not rely on this or any other order. + self.assert_(s.pop() in [5,43,2,1]) def test_issue8750(self): empty = WithSet() @@ -1010,8 +1015,9 @@ c=3, e=5).items()), pairs) # mixed input # make sure no positional args conflict with possible kwdargs - self.assertEqual(inspect.getargspec(OrderedDict.__dict__['__init__']).args, - ['self']) + if '__init__' in OrderedDict.__dict__: # absent in PyPy + self.assertEqual(inspect.getargspec(OrderedDict.__dict__['__init__']).args, + ['self']) # Make sure that direct calls to __init__ do not clear previous contents d = OrderedDict([('a', 1), ('b', 2), ('c', 3), ('d', 44), ('e', 55)]) @@ -1108,6 +1114,16 @@ od.popitem() self.assertEqual(len(od), 0) + def test_popitem_first(self): + pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] + shuffle(pairs) + od = OrderedDict(pairs) + while pairs: + self.assertEqual(od.popitem(last=False), pairs.pop(0)) + with self.assertRaises(KeyError): + od.popitem(last=False) + self.assertEqual(len(od), 0) + def test_pop(self): pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] shuffle(pairs) @@ -1179,7 +1195,11 @@ od = OrderedDict(pairs) # yaml.dump(od) --> # '!!python/object/apply:__main__.OrderedDict\n- - [a, 1]\n - [b, 2]\n' - self.assertTrue(all(type(pair)==list for pair in od.__reduce__()[1])) + + # PyPy bug fix: added [0] at the end of this line, because the + # test is really about the 2-tuples that need to be 2-lists + # inside the list of 6 of them + self.assertTrue(all(type(pair)==list for pair in od.__reduce__()[1][0])) def test_reduce_not_too_fat(self): # do not save instance dictionary if not needed @@ -1189,6 +1209,16 @@ od.x = 10 self.assertEqual(len(od.__reduce__()), 3) + def test_reduce_exact_output(self): + # PyPy: test that __reduce__() produces the exact same answer as + # CPython does, even though in the 'all_ordered_dicts' branch we + # have to emulate it. + pairs = [['c', 1], ['b', 2], ['d', 4]] + od = OrderedDict(pairs) + self.assertEqual(od.__reduce__(), (OrderedDict, (pairs,))) + od.x = 10 + self.assertEqual(od.__reduce__(), (OrderedDict, (pairs,), {'x': 10})) + def test_repr(self): od = OrderedDict([('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]) self.assertEqual(repr(od), diff --git a/lib-python/2.7/test/test_xml_etree.py b/lib-python/2.7/test/test_xml_etree.py --- a/lib-python/2.7/test/test_xml_etree.py +++ b/lib-python/2.7/test/test_xml_etree.py @@ -225,9 +225,9 @@ >>> element.remove(subelement) >>> serialize(element) # 5 '' - >>> element.remove(subelement) + >>> element.remove(subelement) # doctest: +ELLIPSIS Traceback (most recent call last): - ValueError: list.remove(x): x not in list + ValueError: list.remove(... >>> serialize(element) # 6 '' >>> element[0:0] = [subelement, subelement, subelement] diff --git a/lib-python/stdlib-upgrade.txt b/lib-python/stdlib-upgrade.txt --- a/lib-python/stdlib-upgrade.txt +++ b/lib-python/stdlib-upgrade.txt @@ -7,7 +7,7 @@ 1. check out the branch vendor/stdlib 2. upgrade the files there -3. update stdlib-versions.txt with the output of hg -id from the cpython repo +3. update stdlib-version.txt with the output of hg -id from the cpython repo 4. commit 5. update to default/py3k 6. create a integration branch for the new stdlib diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py --- a/lib_pypy/_ctypes/basics.py +++ b/lib_pypy/_ctypes/basics.py @@ -83,6 +83,37 @@ def in_dll(self, dll, name): return self.from_address(dll._handle.getaddressindll(name)) + def from_buffer(self, obj, offset=0): + size = self._sizeofinstances() + buf = buffer(obj, offset, size) + if len(buf) < size: + raise ValueError( + "Buffer size too small (%d instead of at least %d bytes)" + % (len(buf) + offset, size + offset)) + raw_addr = buf._pypy_raw_address() + result = self.from_address(raw_addr) + result._ensure_objects()['ffffffff'] = obj + return result + + def from_buffer_copy(self, obj, offset=0): + size = self._sizeofinstances() + buf = buffer(obj, offset, size) + if len(buf) < size: + raise ValueError( + "Buffer size too small (%d instead of at least %d bytes)" + % (len(buf) + offset, size + offset)) + result = self() + dest = result._buffer.buffer + try: + raw_addr = buf._pypy_raw_address() + except ValueError: + _rawffi.rawstring2charp(dest, buf) + else: + from ctypes import memmove + memmove(dest, raw_addr, size) + return result + + class CArgObject(object): """ simple wrapper around buffer, just for the case of freeing it afterwards diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -1,3 +1,4 @@ +import sys import _rawffi from _ctypes.basics import _CData, _CDataMeta, keepalive_key,\ store_reference, ensure_objects, CArgObject @@ -178,6 +179,8 @@ instance = StructOrUnion.__new__(self) if isinstance(address, _rawffi.StructureInstance): address = address.buffer + # fix the address: turn it into as unsigned, in case it is negative + address = address & (sys.maxint * 2 + 1) instance.__dict__['_buffer'] = self._ffistruct.fromaddress(address) return instance diff --git a/lib_pypy/_functools.py b/lib_pypy/_functools.py --- a/lib_pypy/_functools.py +++ b/lib_pypy/_functools.py @@ -9,7 +9,10 @@ of the given arguments and keywords. """ - def __init__(self, func, *args, **keywords): + def __init__(self, *args, **keywords): + if not args: + raise TypeError('__init__() takes at least 2 arguments (1 given)') + func, args = args[0], args[1:] if not callable(func): raise TypeError("the first argument must be callable") self._func = func diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -270,11 +270,14 @@ _ffi.cdef("int sqlite3_enable_load_extension(sqlite3 *db, int onoff);") if sys.platform.startswith('freebsd'): + import os + import os.path + _localbase = os.environ.get('LOCALBASE', '/usr/local') _lib = _ffi.verify(""" #include """, libraries=['sqlite3'], - include_dirs=['/usr/local/include'], - library_dirs=['/usr/local/lib'] + include_dirs=[os.path.join(_localbase, 'include')], + library_dirs=[os.path.join(_localbase, 'lib')] ) else: _lib = _ffi.verify(""" @@ -1175,8 +1178,9 @@ try: return self.__description except AttributeError: - self.__description = self.__statement._get_description() - return self.__description + if self.__statement: + self.__description = self.__statement._get_description() + return self.__description description = property(__get_description) def __get_lastrowid(self): diff --git a/lib_pypy/audioop.py b/lib_pypy/audioop.py --- a/lib_pypy/audioop.py +++ b/lib_pypy/audioop.py @@ -1,12 +1,11 @@ -from __future__ import division import __builtin__ as builtins import math import struct from fractions import gcd -from ctypes import create_string_buffer +from cffi import FFI -_buffer = buffer +_buffer = memoryview class error(Exception): @@ -149,7 +148,7 @@ def _sum2(cp1, cp2, length): size = 2 return sum(getsample(cp1, size, i) * getsample(cp2, size, i) - for i in range(length)) + for i in range(length)) + 0.0 def findfit(cp1, cp2): @@ -328,13 +327,14 @@ _check_params(len(cp), size) clip = _get_clipfn(size) - result = create_string_buffer(len(cp)) + rv = ffi.new("unsigned char[]", len(cp)) + result = ffi.buffer(rv) for i, sample in enumerate(_get_samples(cp, size)): sample = clip(int(sample * factor)) _put_sample(result, size, i, sample) - return result.raw + return result[:] def tomono(cp, size, fac1, fac2): @@ -343,7 +343,8 @@ sample_count = _sample_count(cp, size) - result = create_string_buffer(len(cp) // 2) + rv = ffi.new("unsigned char[]", len(cp) // 2) + result = ffi.buffer(rv) for i in range(0, sample_count, 2): l_sample = getsample(cp, size, i) @@ -354,7 +355,7 @@ _put_sample(result, size, i // 2, sample) - return result.raw + return result[:] def tostereo(cp, size, fac1, fac2): @@ -362,7 +363,8 @@ sample_count = _sample_count(cp, size) - result = create_string_buffer(len(cp) * 2) + rv = ffi.new("unsigned char[]", len(cp) * 2) + result = ffi.buffer(rv) clip = _get_clipfn(size) for i in range(sample_count): @@ -374,7 +376,7 @@ _put_sample(result, size, i * 2, l_sample) _put_sample(result, size, i * 2 + 1, r_sample) - return result.raw + return result[:] def add(cp1, cp2, size): @@ -385,7 +387,8 @@ clip = _get_clipfn(size) sample_count = _sample_count(cp1, size) - result = create_string_buffer(len(cp1)) + rv = ffi.new("unsigned char[]", len(cp1)) + result = ffi.buffer(rv) for i in range(sample_count): sample1 = getsample(cp1, size, i) @@ -395,30 +398,32 @@ _put_sample(result, size, i, sample) - return result.raw + return result[:] def bias(cp, size, bias): _check_params(len(cp), size) - result = create_string_buffer(len(cp)) + rv = ffi.new("unsigned char[]", len(cp)) + result = ffi.buffer(rv) for i, sample in enumerate(_get_samples(cp, size)): sample = _overflow(sample + bias, size) _put_sample(result, size, i, sample) - return result.raw + return result[:] def reverse(cp, size): _check_params(len(cp), size) sample_count = _sample_count(cp, size) - result = create_string_buffer(len(cp)) + rv = ffi.new("unsigned char[]", len(cp)) + result = ffi.buffer(rv) for i, sample in enumerate(_get_samples(cp, size)): _put_sample(result, size, sample_count - i - 1, sample) - return result.raw + return result[:] def lin2lin(cp, size, size2): @@ -429,7 +434,8 @@ return cp new_len = (len(cp) // size) * size2 - result = create_string_buffer(new_len) + rv = ffi.new("unsigned char[]", new_len) + result = ffi.buffer(rv) for i in range(_sample_count(cp, size)): sample = _get_sample(cp, size, i) @@ -444,7 +450,7 @@ sample = _overflow(sample, size2) _put_sample(result, size2, i, sample) - return result.raw + return result[:] def ratecv(cp, size, nchannels, inrate, outrate, state, weightA=1, weightB=0): @@ -489,7 +495,8 @@ ceiling = (q + 1) * outrate nbytes = ceiling * bytes_per_frame - result = create_string_buffer(nbytes) + rv = ffi.new("unsigned char[]", nbytes) + result = ffi.buffer(rv) samples = _get_samples(cp, size) out_i = 0 @@ -497,7 +504,7 @@ while d < 0: if frame_count == 0: samps = zip(prev_i, cur_i) - retval = result.raw + retval = result[:] # slice off extra bytes trim_index = (out_i * bytes_per_frame) - len(retval) @@ -528,25 +535,523 @@ d -= inrate +ffi = FFI() +ffi.cdef(""" +typedef short PyInt16; + +/* 2's complement (14-bit range) */ +unsigned char +st_14linear2ulaw(PyInt16 pcm_val); +PyInt16 st_ulaw2linear16(unsigned char); + +/* 2's complement (13-bit range) */ +unsigned char +st_linear2alaw(PyInt16 pcm_val); +PyInt16 st_alaw2linear16(unsigned char); + + +void lin2adcpm(unsigned char* rv, unsigned char* cp, size_t len, + size_t size, int* state); +void adcpm2lin(unsigned char* rv, unsigned char* cp, size_t len, + size_t size, int* state); +""") + +# This code is directly copied from CPython file: Modules/audioop.c +_AUDIOOP_C_MODULE = """ +typedef short PyInt16; +typedef int Py_Int32; + +/* Code shamelessly stolen from sox, 12.17.7, g711.c +** (c) Craig Reese, Joe Campbell and Jeff Poskanzer 1989 */ + +/* From g711.c: + * + * December 30, 1994: + * Functions linear2alaw, linear2ulaw have been updated to correctly + * convert unquantized 16 bit values. + * Tables for direct u- to A-law and A- to u-law conversions have been + * corrected. + * Borge Lindberg, Center for PersonKommunikation, Aalborg University. + * bli at cpk.auc.dk + * + */ +#define BIAS 0x84 /* define the add-in bias for 16 bit samples */ +#define CLIP 32635 +#define SIGN_BIT (0x80) /* Sign bit for a A-law byte. */ +#define QUANT_MASK (0xf) /* Quantization field mask. */ +#define SEG_SHIFT (4) /* Left shift for segment number. */ +#define SEG_MASK (0x70) /* Segment field mask. */ + +static PyInt16 seg_aend[8] = {0x1F, 0x3F, 0x7F, 0xFF, + 0x1FF, 0x3FF, 0x7FF, 0xFFF}; +static PyInt16 seg_uend[8] = {0x3F, 0x7F, 0xFF, 0x1FF, + 0x3FF, 0x7FF, 0xFFF, 0x1FFF}; + +static PyInt16 +search(PyInt16 val, PyInt16 *table, int size) +{ + int i; + + for (i = 0; i < size; i++) { + if (val <= *table++) + return (i); + } + return (size); +} +#define st_ulaw2linear16(uc) (_st_ulaw2linear16[uc]) +#define st_alaw2linear16(uc) (_st_alaw2linear16[uc]) + +static PyInt16 _st_ulaw2linear16[256] = { + -32124, -31100, -30076, -29052, -28028, -27004, -25980, + -24956, -23932, -22908, -21884, -20860, -19836, -18812, + -17788, -16764, -15996, -15484, -14972, -14460, -13948, + -13436, -12924, -12412, -11900, -11388, -10876, -10364, + -9852, -9340, -8828, -8316, -7932, -7676, -7420, + -7164, -6908, -6652, -6396, -6140, -5884, -5628, + -5372, -5116, -4860, -4604, -4348, -4092, -3900, + -3772, -3644, -3516, -3388, -3260, -3132, -3004, + -2876, -2748, -2620, -2492, -2364, -2236, -2108, + -1980, -1884, -1820, -1756, -1692, -1628, -1564, + -1500, -1436, -1372, -1308, -1244, -1180, -1116, + -1052, -988, -924, -876, -844, -812, -780, + -748, -716, -684, -652, -620, -588, -556, + -524, -492, -460, -428, -396, -372, -356, + -340, -324, -308, -292, -276, -260, -244, + -228, -212, -196, -180, -164, -148, -132, + -120, -112, -104, -96, -88, -80, -72, + -64, -56, -48, -40, -32, -24, -16, + -8, 0, 32124, 31100, 30076, 29052, 28028, + 27004, 25980, 24956, 23932, 22908, 21884, 20860, + 19836, 18812, 17788, 16764, 15996, 15484, 14972, + 14460, 13948, 13436, 12924, 12412, 11900, 11388, + 10876, 10364, 9852, 9340, 8828, 8316, 7932, + 7676, 7420, 7164, 6908, 6652, 6396, 6140, + 5884, 5628, 5372, 5116, 4860, 4604, 4348, + 4092, 3900, 3772, 3644, 3516, 3388, 3260, + 3132, 3004, 2876, 2748, 2620, 2492, 2364, + 2236, 2108, 1980, 1884, 1820, 1756, 1692, + 1628, 1564, 1500, 1436, 1372, 1308, 1244, + 1180, 1116, 1052, 988, 924, 876, 844, + 812, 780, 748, 716, 684, 652, 620, + 588, 556, 524, 492, 460, 428, 396, + 372, 356, 340, 324, 308, 292, 276, + 260, 244, 228, 212, 196, 180, 164, + 148, 132, 120, 112, 104, 96, 88, + 80, 72, 64, 56, 48, 40, 32, + 24, 16, 8, 0 +}; + +/* + * linear2ulaw() accepts a 14-bit signed integer and encodes it as u-law data + * stored in a unsigned char. This function should only be called with + * the data shifted such that it only contains information in the lower + * 14-bits. + * + * In order to simplify the encoding process, the original linear magnitude + * is biased by adding 33 which shifts the encoding range from (0 - 8158) to + * (33 - 8191). The result can be seen in the following encoding table: + * + * Biased Linear Input Code Compressed Code + * ------------------------ --------------- + * 00000001wxyza 000wxyz + * 0000001wxyzab 001wxyz + * 000001wxyzabc 010wxyz + * 00001wxyzabcd 011wxyz + * 0001wxyzabcde 100wxyz + * 001wxyzabcdef 101wxyz + * 01wxyzabcdefg 110wxyz + * 1wxyzabcdefgh 111wxyz + * + * Each biased linear code has a leading 1 which identifies the segment + * number. The value of the segment number is equal to 7 minus the number + * of leading 0's. The quantization interval is directly available as the + * four bits wxyz. * The trailing bits (a - h) are ignored. + * + * Ordinarily the complement of the resulting code word is used for + * transmission, and so the code word is complemented before it is returned. + * + * For further information see John C. Bellamy's Digital Telephony, 1982, + * John Wiley & Sons, pps 98-111 and 472-476. + */ +static unsigned char +st_14linear2ulaw(PyInt16 pcm_val) /* 2's complement (14-bit range) */ +{ + PyInt16 mask; + PyInt16 seg; + unsigned char uval; + + /* The original sox code does this in the calling function, not here */ + pcm_val = pcm_val >> 2; + + /* u-law inverts all bits */ + /* Get the sign and the magnitude of the value. */ + if (pcm_val < 0) { + pcm_val = -pcm_val; + mask = 0x7F; + } else { + mask = 0xFF; + } + if ( pcm_val > CLIP ) pcm_val = CLIP; /* clip the magnitude */ + pcm_val += (BIAS >> 2); + + /* Convert the scaled magnitude to segment number. */ + seg = search(pcm_val, seg_uend, 8); + + /* + * Combine the sign, segment, quantization bits; + * and complement the code word. + */ + if (seg >= 8) /* out of range, return maximum value. */ + return (unsigned char) (0x7F ^ mask); + else { + uval = (unsigned char) (seg << 4) | ((pcm_val >> (seg + 1)) & 0xF); + return (uval ^ mask); + } + +} + +static PyInt16 _st_alaw2linear16[256] = { + -5504, -5248, -6016, -5760, -4480, -4224, -4992, + -4736, -7552, -7296, -8064, -7808, -6528, -6272, + -7040, -6784, -2752, -2624, -3008, -2880, -2240, + -2112, -2496, -2368, -3776, -3648, -4032, -3904, + -3264, -3136, -3520, -3392, -22016, -20992, -24064, + -23040, -17920, -16896, -19968, -18944, -30208, -29184, + -32256, -31232, -26112, -25088, -28160, -27136, -11008, + -10496, -12032, -11520, -8960, -8448, -9984, -9472, + -15104, -14592, -16128, -15616, -13056, -12544, -14080, + -13568, -344, -328, -376, -360, -280, -264, + -312, -296, -472, -456, -504, -488, -408, + -392, -440, -424, -88, -72, -120, -104, + -24, -8, -56, -40, -216, -200, -248, + -232, -152, -136, -184, -168, -1376, -1312, + -1504, -1440, -1120, -1056, -1248, -1184, -1888, + -1824, -2016, -1952, -1632, -1568, -1760, -1696, + -688, -656, -752, -720, -560, -528, -624, + -592, -944, -912, -1008, -976, -816, -784, + -880, -848, 5504, 5248, 6016, 5760, 4480, + 4224, 4992, 4736, 7552, 7296, 8064, 7808, + 6528, 6272, 7040, 6784, 2752, 2624, 3008, + 2880, 2240, 2112, 2496, 2368, 3776, 3648, + 4032, 3904, 3264, 3136, 3520, 3392, 22016, + 20992, 24064, 23040, 17920, 16896, 19968, 18944, + 30208, 29184, 32256, 31232, 26112, 25088, 28160, + 27136, 11008, 10496, 12032, 11520, 8960, 8448, + 9984, 9472, 15104, 14592, 16128, 15616, 13056, + 12544, 14080, 13568, 344, 328, 376, 360, + 280, 264, 312, 296, 472, 456, 504, + 488, 408, 392, 440, 424, 88, 72, + 120, 104, 24, 8, 56, 40, 216, + 200, 248, 232, 152, 136, 184, 168, + 1376, 1312, 1504, 1440, 1120, 1056, 1248, + 1184, 1888, 1824, 2016, 1952, 1632, 1568, + 1760, 1696, 688, 656, 752, 720, 560, + 528, 624, 592, 944, 912, 1008, 976, + 816, 784, 880, 848 +}; + +/* + * linear2alaw() accepts an 13-bit signed integer and encodes it as A-law data + * stored in a unsigned char. This function should only be called with + * the data shifted such that it only contains information in the lower + * 13-bits. + * + * Linear Input Code Compressed Code + * ------------------------ --------------- + * 0000000wxyza 000wxyz + * 0000001wxyza 001wxyz + * 000001wxyzab 010wxyz + * 00001wxyzabc 011wxyz + * 0001wxyzabcd 100wxyz + * 001wxyzabcde 101wxyz + * 01wxyzabcdef 110wxyz + * 1wxyzabcdefg 111wxyz + * + * For further information see John C. Bellamy's Digital Telephony, 1982, + * John Wiley & Sons, pps 98-111 and 472-476. + */ +static unsigned char +st_linear2alaw(PyInt16 pcm_val) /* 2's complement (13-bit range) */ +{ + PyInt16 mask; + short seg; + unsigned char aval; + + /* The original sox code does this in the calling function, not here */ + pcm_val = pcm_val >> 3; + + /* A-law using even bit inversion */ + if (pcm_val >= 0) { + mask = 0xD5; /* sign (7th) bit = 1 */ + } else { + mask = 0x55; /* sign bit = 0 */ + pcm_val = -pcm_val - 1; + } + + /* Convert the scaled magnitude to segment number. */ + seg = search(pcm_val, seg_aend, 8); + + /* Combine the sign, segment, and quantization bits. */ + + if (seg >= 8) /* out of range, return maximum value. */ + return (unsigned char) (0x7F ^ mask); + else { + aval = (unsigned char) seg << SEG_SHIFT; + if (seg < 2) + aval |= (pcm_val >> 1) & QUANT_MASK; + else + aval |= (pcm_val >> seg) & QUANT_MASK; + return (aval ^ mask); + } +} +/* End of code taken from sox */ + +/* Intel ADPCM step variation table */ +static int indexTable[16] = { + -1, -1, -1, -1, 2, 4, 6, 8, + -1, -1, -1, -1, 2, 4, 6, 8, +}; + +static int stepsizeTable[89] = { + 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, + 19, 21, 23, 25, 28, 31, 34, 37, 41, 45, + 50, 55, 60, 66, 73, 80, 88, 97, 107, 118, + 130, 143, 157, 173, 190, 209, 230, 253, 279, 307, + 337, 371, 408, 449, 494, 544, 598, 658, 724, 796, + 876, 963, 1060, 1166, 1282, 1411, 1552, 1707, 1878, 2066, + 2272, 2499, 2749, 3024, 3327, 3660, 4026, 4428, 4871, 5358, + 5894, 6484, 7132, 7845, 8630, 9493, 10442, 11487, 12635, 13899, + 15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794, 32767 +}; + +#define CHARP(cp, i) ((signed char *)(cp+i)) +#define SHORTP(cp, i) ((short *)(cp+i)) +#define LONGP(cp, i) ((Py_Int32 *)(cp+i)) +""" + +lib = ffi.verify(_AUDIOOP_C_MODULE + """ +void lin2adcpm(unsigned char* ncp, unsigned char* cp, size_t len, + size_t size, int* state) +{ + int step, outputbuffer = 0, bufferstep; + int val = 0; + int diff, vpdiff, sign, delta; + size_t i; + int valpred = state[0]; + int index = state[1]; + + step = stepsizeTable[index]; + bufferstep = 1; + + for ( i=0; i < len; i += size ) { + if ( size == 1 ) val = ((int)*CHARP(cp, i)) << 8; + else if ( size == 2 ) val = (int)*SHORTP(cp, i); + else if ( size == 4 ) val = ((int)*LONGP(cp, i)) >> 16; + + /* Step 1 - compute difference with previous value */ + diff = val - valpred; + sign = (diff < 0) ? 8 : 0; + if ( sign ) diff = (-diff); + + /* Step 2 - Divide and clamp */ + /* Note: + ** This code *approximately* computes: + ** delta = diff*4/step; + ** vpdiff = (delta+0.5)*step/4; + ** but in shift step bits are dropped. The net result of this + ** is that even if you have fast mul/div hardware you cannot + ** put it to good use since the fixup would be too expensive. + */ + delta = 0; + vpdiff = (step >> 3); + + if ( diff >= step ) { + delta = 4; + diff -= step; + vpdiff += step; + } + step >>= 1; + if ( diff >= step ) { + delta |= 2; + diff -= step; + vpdiff += step; + } + step >>= 1; + if ( diff >= step ) { + delta |= 1; + vpdiff += step; + } + + /* Step 3 - Update previous value */ + if ( sign ) + valpred -= vpdiff; + else + valpred += vpdiff; + + /* Step 4 - Clamp previous value to 16 bits */ + if ( valpred > 32767 ) + valpred = 32767; + else if ( valpred < -32768 ) + valpred = -32768; + + /* Step 5 - Assemble value, update index and step values */ + delta |= sign; + + index += indexTable[delta]; + if ( index < 0 ) index = 0; + if ( index > 88 ) index = 88; + step = stepsizeTable[index]; + + /* Step 6 - Output value */ + if ( bufferstep ) { + outputbuffer = (delta << 4) & 0xf0; + } else { + *ncp++ = (delta & 0x0f) | outputbuffer; + } + bufferstep = !bufferstep; + } + state[0] = valpred; + state[1] = index; +} + + +void adcpm2lin(unsigned char* ncp, unsigned char* cp, size_t len, + size_t size, int* state) +{ + int step, inputbuffer = 0, bufferstep; + int val = 0; + int diff, vpdiff, sign, delta; + size_t i; + int valpred = state[0]; + int index = state[1]; + + step = stepsizeTable[index]; + bufferstep = 0; + + for ( i=0; i < len*size*2; i += size ) { + /* Step 1 - get the delta value and compute next index */ + if ( bufferstep ) { + delta = inputbuffer & 0xf; + } else { + inputbuffer = *cp++; + delta = (inputbuffer >> 4) & 0xf; + } + + bufferstep = !bufferstep; + + /* Step 2 - Find new index value (for later) */ + index += indexTable[delta]; + if ( index < 0 ) index = 0; + if ( index > 88 ) index = 88; + + /* Step 3 - Separate sign and magnitude */ + sign = delta & 8; + delta = delta & 7; + + /* Step 4 - Compute difference and new predicted value */ + /* + ** Computes 'vpdiff = (delta+0.5)*step/4', but see comment + ** in adpcm_coder. + */ + vpdiff = step >> 3; + if ( delta & 4 ) vpdiff += step; + if ( delta & 2 ) vpdiff += step>>1; + if ( delta & 1 ) vpdiff += step>>2; + + if ( sign ) + valpred -= vpdiff; + else + valpred += vpdiff; + + /* Step 5 - clamp output value */ + if ( valpred > 32767 ) + valpred = 32767; + else if ( valpred < -32768 ) + valpred = -32768; + + /* Step 6 - Update step value */ + step = stepsizeTable[index]; + + /* Step 6 - Output value */ + if ( size == 1 ) *CHARP(ncp, i) = (signed char)(valpred >> 8); + else if ( size == 2 ) *SHORTP(ncp, i) = (short)(valpred); + else if ( size == 4 ) *LONGP(ncp, i) = (Py_Int32)(valpred<<16); + } + state[0] = valpred; + state[1] = index; +} +""") + +def _get_lin_samples(cp, size): + for sample in _get_samples(cp, size): + if size == 1: + yield sample << 8 + elif size == 2: + yield sample + elif size == 4: + yield sample >> 16 + +def _put_lin_sample(result, size, i, sample): + if size == 1: + sample >>= 8 + elif size == 2: + pass + elif size == 4: + sample <<= 16 + _put_sample(result, size, i, sample) + def lin2ulaw(cp, size): - raise NotImplementedError() + _check_params(len(cp), size) + rv = ffi.new("unsigned char[]", _sample_count(cp, size)) + for i, sample in enumerate(_get_lin_samples(cp, size)): + rv[i] = lib.st_14linear2ulaw(sample) + return ffi.buffer(rv)[:] def ulaw2lin(cp, size): - raise NotImplementedError() + _check_size(size) + rv = ffi.new("unsigned char[]", len(cp) * size) + result = ffi.buffer(rv) + for i, value in enumerate(cp): + sample = lib.st_ulaw2linear16(ord(value)) + _put_lin_sample(result, size, i, sample) + return result[:] def lin2alaw(cp, size): - raise NotImplementedError() + _check_params(len(cp), size) + rv = ffi.new("unsigned char[]", _sample_count(cp, size)) + for i, sample in enumerate(_get_lin_samples(cp, size)): + rv[i] = lib.st_linear2alaw(sample) + return ffi.buffer(rv)[:] def alaw2lin(cp, size): - raise NotImplementedError() + _check_size(size) + rv = ffi.new("unsigned char[]", len(cp) * size) + result = ffi.buffer(rv) + for i, value in enumerate(cp): + sample = lib.st_alaw2linear16(ord(value)) + _put_lin_sample(result, size, i, sample) + return result[:] def lin2adpcm(cp, size, state): - raise NotImplementedError() + _check_params(len(cp), size) + if state is None: + state = (0, 0) + rv = ffi.new("unsigned char[]", len(cp) // size // 2) + state_ptr = ffi.new("int[]", state) + lib.lin2adcpm(rv, cp, len(cp), size, state_ptr) + return ffi.buffer(rv)[:], tuple(state_ptr) def adpcm2lin(cp, size, state): - raise NotImplementedError() + _check_size(size) + if state is None: + state = (0, 0) + rv = ffi.new("unsigned char[]", len(cp) * size * 2) + state_ptr = ffi.new("int[]", state) + lib.adcpm2lin(rv, cp, len(cp), size, state_ptr) + return ffi.buffer(rv)[:], tuple(state_ptr) + diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -6,3 +6,8 @@ __version__ = "0.8.6" __version_info__ = (0, 8, 6) + +# The verifier module file names are based on the CRC32 of a string that +# contains the following version number. It may be older than __version__ +# if nothing is clearly incompatible. +__version_verifier_modules__ = "0.8.6" diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -69,6 +69,7 @@ self._function_caches = [] self._libraries = [] self._cdefsources = [] + self._windows_unicode = None if hasattr(backend, 'set_ffi'): backend.set_ffi(self) for name in backend.__dict__: @@ -77,6 +78,7 @@ # with self._lock: self.BVoidP = self._get_cached_btype(model.voidp_type) + self.BCharA = self._get_cached_btype(model.char_array_type) if isinstance(backend, types.ModuleType): # _cffi_backend: attach these constants to the class if not hasattr(FFI, 'NULL'): @@ -189,13 +191,16 @@ cdecl = self._typeof(cdecl) return self._backend.alignof(cdecl) - def offsetof(self, cdecl, fieldname): + def offsetof(self, cdecl, *fields_or_indexes): """Return the offset of the named field inside the given - structure, which must be given as a C type name. + structure or array, which must be given as a C type name. + You can give several field names in case of nested structures. + You can also give numeric values which correspond to array + items, in case of an array type. """ if isinstance(cdecl, basestring): cdecl = self._typeof(cdecl) - return self._backend.typeoffsetof(cdecl, fieldname)[1] + return self._typeoffsetof(cdecl, *fields_or_indexes)[1] def new(self, cdecl, init=None): """Allocate an instance according to the specified C type and @@ -264,6 +269,16 @@ """ return self._backend.buffer(cdata, size) + def from_buffer(self, python_buffer): + """Return a that points to the data of the + given Python object, which must support the buffer interface. + Note that this is not meant to be used on the built-in types str, + unicode, or bytearray (you can build 'char[]' arrays explicitly) + but only on objects containing large quantities of raw data + in some other format, like 'array.array' or numpy arrays. + """ + return self._backend.from_buffer(self.BCharA, python_buffer) + def callback(self, cdecl, python_callable=None, error=None): """Return a callback object or a decorator making such a callback object. 'cdecl' must name a C function pointer type. @@ -335,9 +350,23 @@ which requires binary compatibility in the signatures. """ from .verifier import Verifier, _caller_dir_pycache + # + # If set_unicode(True) was called, insert the UNICODE and + # _UNICODE macro declarations + if self._windows_unicode: + self._apply_windows_unicode(kwargs) + # + # Set the tmpdir here, and not in Verifier.__init__: it picks + # up the caller's directory, which we want to be the caller of + # ffi.verify(), as opposed to the caller of Veritier(). tmpdir = tmpdir or _caller_dir_pycache() + # + # Make a Verifier() and use it to load the library. self.verifier = Verifier(self, source, tmpdir, **kwargs) lib = self.verifier.load_library() + # + # Save the loaded library for keep-alive purposes, even + # if the caller doesn't keep it alive itself (it should). self._libraries.append(lib) return lib @@ -356,15 +385,29 @@ with self._lock: return model.pointer_cache(self, ctype) - def addressof(self, cdata, field=None): + def addressof(self, cdata, *fields_or_indexes): """Return the address of a . - If 'field' is specified, return the address of this field. + If 'fields_or_indexes' are given, returns the address of that + field or array item in the structure or array, recursively in + case of nested structures. """ ctype = self._backend.typeof(cdata) - ctype, offset = self._backend.typeoffsetof(ctype, field) + if fields_or_indexes: + ctype, offset = self._typeoffsetof(ctype, *fields_or_indexes) + else: + if ctype.kind == "pointer": + raise TypeError("addressof(pointer)") + offset = 0 ctypeptr = self._pointer_to(ctype) return self._backend.rawaddressof(ctypeptr, cdata, offset) + def _typeoffsetof(self, ctype, field_or_index, *fields_or_indexes): + ctype, offset = self._backend.typeoffsetof(ctype, field_or_index) + for field1 in fields_or_indexes: + ctype, offset1 = self._backend.typeoffsetof(ctype, field1, 1) + offset += offset1 + return ctype, offset + def include(self, ffi_to_include): """Includes the typedefs, structs, unions and enums defined in another FFI instance. Usage is similar to a #include in C, @@ -387,6 +430,44 @@ def from_handle(self, x): return self._backend.from_handle(x) + def set_unicode(self, enabled_flag): + """Windows: if 'enabled_flag' is True, enable the UNICODE and + _UNICODE defines in C, and declare the types like TCHAR and LPTCSTR + to be (pointers to) wchar_t. If 'enabled_flag' is False, + declare these types to be (pointers to) plain 8-bit characters. + This is mostly for backward compatibility; you usually want True. + """ + if self._windows_unicode is not None: + raise ValueError("set_unicode() can only be called once") + enabled_flag = bool(enabled_flag) + if enabled_flag: + self.cdef("typedef wchar_t TBYTE;" + "typedef wchar_t TCHAR;" + "typedef const wchar_t *LPCTSTR;" + "typedef const wchar_t *PCTSTR;" + "typedef wchar_t *LPTSTR;" + "typedef wchar_t *PTSTR;" + "typedef TBYTE *PTBYTE;" + "typedef TCHAR *PTCHAR;") + else: + self.cdef("typedef char TBYTE;" + "typedef char TCHAR;" + "typedef const char *LPCTSTR;" + "typedef const char *PCTSTR;" + "typedef char *LPTSTR;" + "typedef char *PTSTR;" + "typedef TBYTE *PTBYTE;" + "typedef TCHAR *PTCHAR;") + self._windows_unicode = enabled_flag + + def _apply_windows_unicode(self, kwds): + defmacros = kwds.get('define_macros', ()) + if not isinstance(defmacros, (list, tuple)): + raise TypeError("'define_macros' must be a list or tuple") + defmacros = list(defmacros) + [('UNICODE', '1'), + ('_UNICODE', '1')] + kwds['define_macros'] = defmacros + def _load_backend_lib(backend, name, flags): if name is None: diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py --- a/lib_pypy/cffi/backend_ctypes.py +++ b/lib_pypy/cffi/backend_ctypes.py @@ -169,6 +169,7 @@ class CTypesGenericPtr(CTypesData): __slots__ = ['_address', '_as_ctype_ptr'] _automatic_casts = False + kind = "pointer" @classmethod def _newp(cls, init): @@ -370,10 +371,12 @@ (CTypesPrimitive, type(source).__name__)) return source # + kind1 = kind class CTypesPrimitive(CTypesGenericPrimitive): __slots__ = ['_value'] _ctype = ctype _reftypename = '%s &' % name + kind = kind1 def __init__(self, value): self._value = value @@ -703,12 +706,13 @@ class struct_or_union(base_ctypes_class): pass struct_or_union.__name__ = '%s_%s' % (kind, name) + kind1 = kind # class CTypesStructOrUnion(CTypesBaseStructOrUnion): __slots__ = ['_blob'] _ctype = struct_or_union _reftypename = '%s &' % (name,) - _kind = kind + _kind = kind = kind1 # CTypesStructOrUnion._fix_class() return CTypesStructOrUnion @@ -994,27 +998,42 @@ def getcname(self, BType, replace_with): return BType._get_c_name(replace_with) - def typeoffsetof(self, BType, fieldname): - if fieldname is not None and issubclass(BType, CTypesGenericPtr): - BType = BType._BItem - if not issubclass(BType, CTypesBaseStructOrUnion): - raise TypeError("expected a struct or union ctype") - if fieldname is None: - return (BType, 0) - else: + def typeoffsetof(self, BType, fieldname, num=0): + if isinstance(fieldname, str): + if num == 0 and issubclass(BType, CTypesGenericPtr): + BType = BType._BItem + if not issubclass(BType, CTypesBaseStructOrUnion): + raise TypeError("expected a struct or union ctype") BField = BType._bfield_types[fieldname] if BField is Ellipsis: raise TypeError("not supported for bitfields") return (BField, BType._offsetof(fieldname)) + elif isinstance(fieldname, (int, long)): + if issubclass(BType, CTypesGenericArray): + BType = BType._CTPtr + if not issubclass(BType, CTypesGenericPtr): + raise TypeError("expected an array or ptr ctype") + BItem = BType._BItem + offset = BItem._get_size() * fieldname + if offset > sys.maxsize: + raise OverflowError + return (BItem, offset) + else: + raise TypeError(type(fieldname)) - def rawaddressof(self, BTypePtr, cdata, offset): + def rawaddressof(self, BTypePtr, cdata, offset=None): if isinstance(cdata, CTypesBaseStructOrUnion): ptr = ctypes.pointer(type(cdata)._to_ctypes(cdata)) elif isinstance(cdata, CTypesGenericPtr): + if offset is None or not issubclass(type(cdata)._BItem, + CTypesBaseStructOrUnion): + raise TypeError("unexpected cdata type") + ptr = type(cdata)._to_ctypes(cdata) + elif isinstance(cdata, CTypesGenericArray): ptr = type(cdata)._to_ctypes(cdata) else: raise TypeError("expected a ") - if offset != 0: + if offset: ptr = ctypes.cast( ctypes.c_void_p( ctypes.cast(ptr, ctypes.c_void_p).value + offset), diff --git a/lib_pypy/cffi/commontypes.py b/lib_pypy/cffi/commontypes.py --- a/lib_pypy/cffi/commontypes.py +++ b/lib_pypy/cffi/commontypes.py @@ -29,6 +29,9 @@ result = model.PointerType(resolve_common_type(result[:-2])) elif result in model.PrimitiveType.ALL_PRIMITIVE_TYPES: result = model.PrimitiveType(result) + elif result == 'set-unicode-needed': + raise api.FFIError("The Windows type %r is only available after " + "you call ffi.set_unicode()" % (commontype,)) else: if commontype == result: raise api.FFIError("Unsupported type: %r. Please file a bug " @@ -86,8 +89,6 @@ "ULONGLONG": "unsigned long long", "WCHAR": "wchar_t", "SHORT": "short", - "TBYTE": "WCHAR", - "TCHAR": "WCHAR", "UCHAR": "unsigned char", "UINT": "unsigned int", "UINT8": "unsigned char", @@ -157,14 +158,12 @@ "LPCVOID": model.const_voidp_type, "LPCWSTR": "const WCHAR *", - "LPCTSTR": "LPCWSTR", "LPDWORD": "DWORD *", "LPHANDLE": "HANDLE *", "LPINT": "int *", "LPLONG": "long *", "LPSTR": "CHAR *", "LPWSTR": "WCHAR *", - "LPTSTR": "LPWSTR", "LPVOID": model.voidp_type, "LPWORD": "WORD *", "LRESULT": "LONG_PTR", @@ -173,7 +172,6 @@ "PBYTE": "BYTE *", "PCHAR": "CHAR *", "PCSTR": "const CHAR *", - "PCTSTR": "LPCWSTR", "PCWSTR": "const WCHAR *", "PDWORD": "DWORD *", "PDWORDLONG": "DWORDLONG *", @@ -200,9 +198,6 @@ "PSIZE_T": "SIZE_T *", "PSSIZE_T": "SSIZE_T *", "PSTR": "CHAR *", - "PTBYTE": "TBYTE *", - "PTCHAR": "TCHAR *", - "PTSTR": "LPWSTR", "PUCHAR": "UCHAR *", "PUHALF_PTR": "UHALF_PTR *", "PUINT": "UINT *", @@ -240,6 +235,15 @@ "USN": "LONGLONG", "VOID": model.void_type, "WPARAM": "UINT_PTR", + + "TBYTE": "set-unicode-needed", + "TCHAR": "set-unicode-needed", + "LPCTSTR": "set-unicode-needed", + "PCTSTR": "set-unicode-needed", + "LPTSTR": "set-unicode-needed", + "PTSTR": "set-unicode-needed", + "PTBYTE": "set-unicode-needed", + "PTCHAR": "set-unicode-needed", }) return result diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -1,4 +1,3 @@ - from . import api, model from .commontypes import COMMON_TYPES, resolve_common_type try: @@ -209,6 +208,8 @@ def _add_constants(self, key, val): if key in self._int_constants: + if self._int_constants[key] == val: + return # ignore identical double declarations raise api.FFIError( "multiple declarations of constant: %s" % (key,)) self._int_constants[key] = val @@ -228,12 +229,18 @@ pyvalue = int(int_str, 0) self._add_constants(key, pyvalue) + self._declare('macro ' + key, pyvalue) elif value == '...': self._declare('macro ' + key, value) else: - raise api.CDefError('only supports the syntax "#define ' - '%s ..." (literally) or "#define ' - '%s 0x1FF" for now' % (key, key)) + raise api.CDefError( + 'only supports one of the following syntax:\n' + ' #define %s ... (literally dot-dot-dot)\n' + ' #define %s NUMBER (with NUMBER an integer' + ' constant, decimal/hex/octal)\n' + 'got:\n' + ' #define %s %s' + % (key, key, key, value)) def _parse_decl(self, decl): node = decl.type @@ -460,6 +467,8 @@ elif kind == 'union': tp = model.UnionType(explicit_name, None, None, None) elif kind == 'enum': + if explicit_name == '__dotdotdot__': + raise CDefError("Enums cannot be declared with ...") tp = self._build_enum_type(explicit_name, type.values) else: raise AssertionError("kind = %r" % (kind,)) @@ -532,9 +541,24 @@ def _parse_constant(self, exprnode, partial_length_ok=False): # for now, limited to expressions that are an immediate number - # or negative number + # or positive/negative number if isinstance(exprnode, pycparser.c_ast.Constant): - return int(exprnode.value, 0) + s = exprnode.value + if s.startswith('0'): + if s.startswith('0x') or s.startswith('0X'): + return int(s, 16) + return int(s, 8) + elif '1' <= s[0] <= '9': + return int(s, 10) + elif s[0] == "'" and s[-1] == "'" and ( + len(s) == 3 or (len(s) == 4 and s[1] == "\\")): + return ord(s[-2]) + else: + raise api.CDefError("invalid constant %r" % (s,)) + # + if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and + exprnode.op == '+'): + return self._parse_constant(exprnode.expr) # if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and exprnode.op == '-'): diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py --- a/lib_pypy/cffi/ffiplatform.py +++ b/lib_pypy/cffi/ffiplatform.py @@ -11,6 +11,9 @@ """ +LIST_OF_FILE_NAMES = ['sources', 'include_dirs', 'library_dirs', + 'extra_objects', 'depends'] + def get_extension(srcfilename, modname, sources=(), **kwds): from distutils.core import Extension allsources = [srcfilename] diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py --- a/lib_pypy/cffi/model.py +++ b/lib_pypy/cffi/model.py @@ -235,6 +235,8 @@ BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist) return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length) +char_array_type = ArrayType(PrimitiveType('char'), None) + class StructOrUnionOrEnum(BaseTypeByIdentity): _attrs_ = ('name',) @@ -478,7 +480,7 @@ try: res = getattr(ffi._backend, funcname)(*args) except NotImplementedError as e: - raise NotImplementedError("%r: %s" % (srctype, e)) + raise NotImplementedError("%s: %r: %s" % (funcname, srctype, e)) # note that setdefault() on WeakValueDictionary is not atomic # and contains a rare bug (http://bugs.python.org/issue19542); # we have to use a lock and do it ourselves diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py --- a/lib_pypy/cffi/vengine_cpy.py +++ b/lib_pypy/cffi/vengine_cpy.py @@ -65,7 +65,7 @@ # The following two 'chained_list_constants' items contains # the head of these two chained lists, as a string that gives the # call to do, if any. - self._chained_list_constants = ['0', '0'] + self._chained_list_constants = ['((void)lib,0)', '((void)lib,0)'] # prnt = self._prnt # first paste some standard set of lines that are mostly '#define' @@ -138,15 +138,22 @@ From noreply at buildbot.pypy.org Thu Feb 12 21:12:11 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 12 Feb 2015 21:12:11 +0100 (CET) Subject: [pypy-commit] buildbot issue-1759: create and use tmp/pytest for all testing (not translation) Message-ID: <20150212201211.0C94A1C02FD@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: issue-1759 Changeset: r930:9efc03669b79 Date: 2015-02-12 02:32 +0200 http://bitbucket.org/pypy/buildbot/changeset/9efc03669b79/ Log: create and use tmp/pytest for all testing (not translation) diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -4,7 +4,7 @@ from buildbot.process import factory from buildbot.steps import shell, transfer from buildbot.steps.trigger import Trigger -from buildbot.process.properties import WithProperties +from buildbot.process.properties import WithProperties, Interpolate from buildbot import locks from pypybuildbot.util import symlink_force from buildbot.status.results import SKIPPED, SUCCESS @@ -378,6 +378,19 @@ return ".tar.bz2" def add_translated_tests(factory, prefix, platform, app_tests, lib_python, pypyjit): + factory.addStep(shell.SetPropertyFromCommand( + command=['python', '-c', "import tempfile, os ;print" + " tempfile.gettempdir() + os.path.sep"], + property="target_tmpdir")) + # If target_tmpdir is empty, crash. + tmp_or_crazy = '%(prop:target_tmpdir:-crazy/name/so/mkdir/fails/)s' + factory.addStep(PytestCmd( + description="mkdir for tests", + command=['python', '-c', Interpolate("import os; os.mkdir(r'" + \ + tmp_or_crazy + "pytest') if not os.path.exists(r'" + \ + tmp_or_crazy + "pytest') else True")], + haltOnFailure=True, + )) if app_tests: if app_tests == True: app_tests = [] @@ -392,7 +405,9 @@ ] + ["--config=%s" % cfg for cfg in app_tests], logfiles={'pytestLog': 'pytest-A.log'}, timeout=4000, - env={"PYTHONPATH": ['.']})) + env={"PYTHONPATH": ['.'], + "TMPDIR": Interpolate('%(prop:target_tmpdir)spytest'), + })) if lib_python: factory.addStep(PytestCmd( @@ -402,7 +417,9 @@ "--timeout=3600", "--resultlog=cpython.log", "lib-python"], timeout=4000, - logfiles={'pytestLog': 'cpython.log'})) + logfiles={'pytestLog': 'cpython.log'}, + env={"TMPDIR": Interpolate('%(prop:target_tmpdir)spytest'), + })) if pypyjit: # kill this step when the transition to test_pypy_c_new has been @@ -414,7 +431,9 @@ "--pypy=pypy/goal/pypy-c", "--resultlog=pypyjit.log", "pypy/module/pypyjit/test"], - logfiles={'pytestLog': 'pypyjit.log'})) + logfiles={'pytestLog': 'pypyjit.log'}, + env={"TMPDIR": Interpolate('%(prop:target_tmpdir)spytest'), + })) # # "new" test_pypy_c if platform == 'win32': @@ -426,7 +445,10 @@ command=prefix + [cmd, "pypy/test_all.py", "--resultlog=pypyjit_new.log", "pypy/module/pypyjit/test_pypy_c"], - logfiles={'pytestLog': 'pypyjit_new.log'})) + logfiles={'pytestLog': 'pypyjit_new.log'}, + env={"TMPDIR": Interpolate('%(prop:target_tmpdir)spytest'), + })) + # ---- @@ -438,6 +460,19 @@ setup_steps(platform, self) timeout=kwargs.get('timeout', 4000) + self.addStep(shell.SetPropertyFromCommand( + command=['python', '-c', "import tempfile, os ;print" + " tempfile.gettempdir() + os.path.sep"], + property="target_tmpdir")) + # If target_tmpdir is empty, crash. + tmp_or_crazy = '%(prop:target_tmpdir:-crazy/name/so/mkdir/fails/)s' + self.addStep(PytestCmd( + description="mkdir for tests", + command=['python', '-c', Interpolate("import os; os.mkdir(r'" + \ + tmp_or_crazy + "pytest') if not os.path.exists(r'" + \ + tmp_or_crazy + "pytest') else True")], + haltOnFailure=True, + )) self.addStep(PytestCmd( description="pytest pypy", command=["python", "testrunner/runner.py", @@ -449,7 +484,9 @@ logfiles={'pytestLog': 'testrun.log'}, timeout=timeout, env={"PYTHONPATH": ['.'], - "PYPYCHERRYPICK": cherrypick})) + "PYPYCHERRYPICK": cherrypick, + "TMPDIR": Interpolate('%(prop:target_tmpdir)spytest'), + })) self.addStep(PytestCmd( description="pytest rpython", @@ -462,7 +499,9 @@ logfiles={'pytestLog': 'testrun.log'}, timeout=timeout, env={"PYTHONPATH": ['.'], - "PYPYCHERRYPICK": cherrypick})) + "PYPYCHERRYPICK": cherrypick, + "TMPDIR": Interpolate('%(prop:target_tmpdir)spytest'), + })) class Translated(factory.BuildFactory): From noreply at buildbot.pypy.org Thu Feb 12 21:12:12 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 12 Feb 2015 21:12:12 +0100 (CET) Subject: [pypy-commit] buildbot issue-1759: add step to clean out old files from pypy-specific test directory Message-ID: <20150212201212.288631C02FD@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: issue-1759 Changeset: r931:bb9beee4185d Date: 2015-02-12 21:36 +0200 http://bitbucket.org/pypy/buildbot/changeset/bb9beee4185d/ Log: add step to clean out old files from pypy-specific test directory diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -384,13 +384,27 @@ property="target_tmpdir")) # If target_tmpdir is empty, crash. tmp_or_crazy = '%(prop:target_tmpdir:-crazy/name/so/mkdir/fails/)s' + pytest = "pytest" factory.addStep(PytestCmd( description="mkdir for tests", command=['python', '-c', Interpolate("import os; os.mkdir(r'" + \ - tmp_or_crazy + "pytest') if not os.path.exists(r'" + \ - tmp_or_crazy + "pytest') else True")], + tmp_or_crazy + pytest + "') if not os.path.exists(r'" + \ + tmp_or_crazy + pytest + "') else True")], haltOnFailure=True, )) + + nDays = '3' #str, not int + if platform == 'win32': + command = ['FORFILES', '/P', Interpolate(tmp_or_crazy + pytest), + '/D', '-' + nDays, '/c', "cmd /c rmdir /q /s @path"] + else: + command = ['find', Interpolate(tmp_or_crazy + pytest), '-mtime', + '+' + nDays, '-exec', 'rm', "'{}'", "';'"] + factory.addStep(PytestCmd( + description="cleanout old test files", + command = command, + )) + if app_tests: if app_tests == True: app_tests = [] @@ -406,7 +420,7 @@ logfiles={'pytestLog': 'pytest-A.log'}, timeout=4000, env={"PYTHONPATH": ['.'], - "TMPDIR": Interpolate('%(prop:target_tmpdir)spytest'), + "TMPDIR": Interpolate('%(prop:target_tmpdir)s' + pytest), })) if lib_python: @@ -418,7 +432,7 @@ "--resultlog=cpython.log", "lib-python"], timeout=4000, logfiles={'pytestLog': 'cpython.log'}, - env={"TMPDIR": Interpolate('%(prop:target_tmpdir)spytest'), + env={"TMPDIR": Interpolate('%(prop:target_tmpdir)s' + pytest), })) if pypyjit: @@ -432,7 +446,7 @@ "--resultlog=pypyjit.log", "pypy/module/pypyjit/test"], logfiles={'pytestLog': 'pypyjit.log'}, - env={"TMPDIR": Interpolate('%(prop:target_tmpdir)spytest'), + env={"TMPDIR": Interpolate('%(prop:target_tmpdir)s' + pytest), })) # # "new" test_pypy_c @@ -446,7 +460,7 @@ "--resultlog=pypyjit_new.log", "pypy/module/pypyjit/test_pypy_c"], logfiles={'pytestLog': 'pypyjit_new.log'}, - env={"TMPDIR": Interpolate('%(prop:target_tmpdir)spytest'), + env={"TMPDIR": Interpolate('%(prop:target_tmpdir)s' + pytest), })) @@ -460,19 +474,34 @@ setup_steps(platform, self) timeout=kwargs.get('timeout', 4000) + self.addStep(shell.SetPropertyFromCommand( command=['python', '-c', "import tempfile, os ;print" " tempfile.gettempdir() + os.path.sep"], property="target_tmpdir")) # If target_tmpdir is empty, crash. tmp_or_crazy = '%(prop:target_tmpdir:-crazy/name/so/mkdir/fails/)s' + pytest = "pytest" self.addStep(PytestCmd( description="mkdir for tests", command=['python', '-c', Interpolate("import os; os.mkdir(r'" + \ - tmp_or_crazy + "pytest') if not os.path.exists(r'" + \ - tmp_or_crazy + "pytest') else True")], + tmp_or_crazy + pytest + "') if not os.path.exists(r'" + \ + tmp_or_crazy + pytest + "') else True")], haltOnFailure=True, )) + + nDays = '3' #str, not int + if platform == 'win32': + command = ['FORFILES', '/P', Interpolate(tmp_or_crazy + pytest), + '/D', '-' + nDays, '/c', "cmd /c rmdir /q /s @path"] + else: + command = ['find', Interpolate(tmp_or_crazy + pytest), '-mtime', + '+' + nDays, '-exec', 'rm', "'{}'", "';'"] + self.addStep(PytestCmd( + description="cleanout old test files", + command = command, + )) + self.addStep(PytestCmd( description="pytest pypy", command=["python", "testrunner/runner.py", @@ -485,7 +514,7 @@ timeout=timeout, env={"PYTHONPATH": ['.'], "PYPYCHERRYPICK": cherrypick, - "TMPDIR": Interpolate('%(prop:target_tmpdir)spytest'), + "TMPDIR": Interpolate('%(prop:target_tmpdir)s' + pytest), })) self.addStep(PytestCmd( @@ -500,7 +529,7 @@ timeout=timeout, env={"PYTHONPATH": ['.'], "PYPYCHERRYPICK": cherrypick, - "TMPDIR": Interpolate('%(prop:target_tmpdir)spytest'), + "TMPDIR": Interpolate('%(prop:target_tmpdir)s' + pytest), })) From noreply at buildbot.pypy.org Thu Feb 12 21:12:13 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 12 Feb 2015 21:12:13 +0100 (CET) Subject: [pypy-commit] buildbot issue-1759: do not fail build if no old files found Message-ID: <20150212201213.3E6321C02FD@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: issue-1759 Changeset: r932:bec7d690f6ad Date: 2015-02-12 22:02 +0200 http://bitbucket.org/pypy/buildbot/changeset/bec7d690f6ad/ Log: do not fail build if no old files found diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -403,6 +403,8 @@ factory.addStep(PytestCmd( description="cleanout old test files", command = command, + flunkOnFailure=False, + haltOnFailure=False, )) if app_tests: @@ -500,6 +502,8 @@ self.addStep(PytestCmd( description="cleanout old test files", command = command, + flunkOnFailure=False, + haltOnFailure=False, )) self.addStep(PytestCmd( From noreply at buildbot.pypy.org Thu Feb 12 21:12:14 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 12 Feb 2015 21:12:14 +0100 (CET) Subject: [pypy-commit] buildbot issue-1759: fix for linux Message-ID: <20150212201214.53F031C02FD@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: issue-1759 Changeset: r933:ca41205fb092 Date: 2015-02-12 22:12 +0200 http://bitbucket.org/pypy/buildbot/changeset/ca41205fb092/ Log: fix for linux diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -399,7 +399,7 @@ '/D', '-' + nDays, '/c', "cmd /c rmdir /q /s @path"] else: command = ['find', Interpolate(tmp_or_crazy + pytest), '-mtime', - '+' + nDays, '-exec', 'rm', "'{}'", "';'"] + '+' + nDays, '-exec', 'rm', '{}', ';'] factory.addStep(PytestCmd( description="cleanout old test files", command = command, @@ -498,7 +498,7 @@ '/D', '-' + nDays, '/c', "cmd /c rmdir /q /s @path"] else: command = ['find', Interpolate(tmp_or_crazy + pytest), '-mtime', - '+' + nDays, '-exec', 'rm', "'{}'", "';'"] + '+' + nDays, '-exec', 'rm', '{}', ';'] self.addStep(PytestCmd( description="cleanout old test files", command = command, From noreply at buildbot.pypy.org Thu Feb 12 22:01:33 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 12 Feb 2015 22:01:33 +0100 (CET) Subject: [pypy-commit] pypy default: test, fix creating an array from a flatiter Message-ID: <20150212210133.75F811C02FD@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r75843:5bd4d9346f9a Date: 2015-02-12 23:01 +0200 http://bitbucket.org/pypy/pypy/changeset/5bd4d9346f9a/ Log: test, fix creating an array from a flatiter diff --git a/pypy/module/micronumpy/flatiter.py b/pypy/module/micronumpy/flatiter.py --- a/pypy/module/micronumpy/flatiter.py +++ b/pypy/module/micronumpy/flatiter.py @@ -15,6 +15,7 @@ self._base = base self.dtype = base.get_dtype() self.shape = [base.get_size()] + self.storage = self._base.implementation.storage def base(self): return self._base diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -2852,6 +2852,13 @@ c.flat = ['defgh', 'ijklmnop'] assert (c.flatten() == ['def', 'ijk']*5).all() + def test_flatiter_subtype(self): + from numpy import array + x = array([[1, 2], [3, 4]]).T + y = array(x.flat) + assert (x == [[1, 3], [2, 4]]).all() + + def test_slice_copy(self): from numpy import zeros a = zeros((10, 10)) From noreply at buildbot.pypy.org Fri Feb 13 00:10:58 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Fri, 13 Feb 2015 00:10:58 +0100 (CET) Subject: [pypy-commit] pypy default: Issue #1975: Port some audioop functions to cffi, Message-ID: <20150212231058.D19E11C02FD@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r75844:62a65dd49942 Date: 2015-02-13 00:08 +0100 http://bitbucket.org/pypy/pypy/changeset/62a65dd49942/ Log: Issue #1975: Port some audioop functions to cffi, this considerably improve the performance of the "pydub" example. pypy-no-jit is now ~30% slower than CPython. diff --git a/lib_pypy/audioop.py b/lib_pypy/audioop.py --- a/lib_pypy/audioop.py +++ b/lib_pypy/audioop.py @@ -364,19 +364,8 @@ sample_count = _sample_count(cp, size) rv = ffi.new("unsigned char[]", len(cp) * 2) - result = ffi.buffer(rv) - clip = _get_clipfn(size) - - for i in range(sample_count): - sample = _get_sample(cp, size, i) - - l_sample = clip(sample * fac1) - r_sample = clip(sample * fac2) - - _put_sample(result, size, i * 2, l_sample) - _put_sample(result, size, i * 2 + 1, r_sample) - - return result[:] + lib.tostereo(rv, cp, len(cp), size, fac1, fac2) + return ffi.buffer(rv)[:] def add(cp1, cp2, size): @@ -385,20 +374,9 @@ if len(cp1) != len(cp2): raise error("Lengths should be the same") - clip = _get_clipfn(size) - sample_count = _sample_count(cp1, size) rv = ffi.new("unsigned char[]", len(cp1)) - result = ffi.buffer(rv) - - for i in range(sample_count): - sample1 = getsample(cp1, size, i) - sample2 = getsample(cp2, size, i) - - sample = clip(sample1 + sample2) - - _put_sample(result, size, i, sample) - - return result[:] + lib.add(rv, cp1, cp2, len(cp1), size) + return ffi.buffer(rv)[:] def bias(cp, size, bias): @@ -477,11 +455,10 @@ inrate //= d outrate //= d - prev_i = [0] * nchannels - cur_i = [0] * nchannels - if state is None: d = -outrate + prev_i = ffi.new('int[]', nchannels) + cur_i = ffi.new('int[]', nchannels) else: d, samps = state @@ -489,56 +466,37 @@ raise error("illegal state argument") prev_i, cur_i = zip(*samps) - prev_i, cur_i = list(prev_i), list(cur_i) + prev_i = ffi.new('int[]', prev_i) + cur_i = ffi.new('int[]', cur_i) + state_d = ffi.new('int[]', (d,)) q = frame_count // inrate ceiling = (q + 1) * outrate nbytes = ceiling * bytes_per_frame rv = ffi.new("unsigned char[]", nbytes) - result = ffi.buffer(rv) - - samples = _get_samples(cp, size) - out_i = 0 - while True: - while d < 0: - if frame_count == 0: - samps = zip(prev_i, cur_i) - retval = result[:] - - # slice off extra bytes - trim_index = (out_i * bytes_per_frame) - len(retval) - retval = retval[:trim_index] - - return (retval, (d, tuple(samps))) - - for chan in range(nchannels): - prev_i[chan] = cur_i[chan] - cur_i[chan] = next(samples) - - cur_i[chan] = ( - (weightA * cur_i[chan] + weightB * prev_i[chan]) - // (weightA + weightB) - ) - - frame_count -= 1 - d += outrate - - while d >= 0: - for chan in range(nchannels): - cur_o = ( - (prev_i[chan] * d + cur_i[chan] * (outrate - d)) - // outrate - ) - _put_sample(result, size, out_i, _overflow(cur_o, size)) - out_i += 1 - d -= inrate + trim_index = lib.ratecv(rv, cp, frame_count, size, + nchannels, inrate, outrate, + state_d, prev_i, cur_i, + weightA, weightB) + result = ffi.buffer(rv)[:trim_index] + samps = zip(prev_i, cur_i) + return (result, (d, tuple(samps))) ffi = FFI() ffi.cdef(""" typedef short PyInt16; +int ratecv(char* rv, char* cp, size_t len, int size, + int nchannels, int inrate, int outrate, + int* state_d, int* prev_i, int* cur_i, + int weightA, int weightB); + +void tostereo(char* rv, char* cp, size_t len, int size, + double fac1, double fac2); +void add(char* rv, char* cp1, char* cp2, size_t len1, int size); + /* 2's complement (14-bit range) */ unsigned char st_14linear2ulaw(PyInt16 pcm_val); @@ -829,7 +787,166 @@ #define LONGP(cp, i) ((Py_Int32 *)(cp+i)) """ -lib = ffi.verify(_AUDIOOP_C_MODULE + """ +lib = ffi.verify(_AUDIOOP_C_MODULE + r""" +#include + +static const int maxvals[] = {0, 0x7F, 0x7FFF, 0x7FFFFF, 0x7FFFFFFF}; +/* -1 trick is needed on Windows to support -0x80000000 without a warning */ +static const int minvals[] = {0, -0x80, -0x8000, -0x800000, -0x7FFFFFFF-1}; + +static int +fbound(double val, double minval, double maxval) +{ + if (val > maxval) + val = maxval; + else if (val < minval + 1) + val = minval; + return val; +} + +static int +gcd(int a, int b) +{ + while (b > 0) { + int tmp = a % b; + a = b; + b = tmp; + } + return a; +} + +int ratecv(char* rv, char* cp, size_t len, int size, + int nchannels, int inrate, int outrate, + int* state_d, int* prev_i, int* cur_i, + int weightA, int weightB) +{ + char *ncp = rv; + int d, chan; + + /* divide inrate and outrate by their greatest common divisor */ + d = gcd(inrate, outrate); + inrate /= d; + outrate /= d; + /* divide weightA and weightB by their greatest common divisor */ + d = gcd(weightA, weightB); + weightA /= d; + weightA /= d; + + d = *state_d; + + for (;;) { + while (d < 0) { + if (len == 0) { + *state_d = d; + return ncp - rv; + } + for (chan = 0; chan < nchannels; chan++) { + prev_i[chan] = cur_i[chan]; + if (size == 1) + cur_i[chan] = ((int)*CHARP(cp, 0)) << 24; + else if (size == 2) + cur_i[chan] = ((int)*SHORTP(cp, 0)) << 16; + else if (size == 4) + cur_i[chan] = (int)*LONGP(cp, 0); + cp += size; + /* implements a simple digital filter */ + cur_i[chan] = (int)( + ((double)weightA * (double)cur_i[chan] + + (double)weightB * (double)prev_i[chan]) / + ((double)weightA + (double)weightB)); + } + len--; + d += outrate; + } + while (d >= 0) { + for (chan = 0; chan < nchannels; chan++) { + int cur_o; + cur_o = (int)(((double)prev_i[chan] * (double)d + + (double)cur_i[chan] * (double)(outrate - d)) / + (double)outrate); + if (size == 1) + *CHARP(ncp, 0) = (signed char)(cur_o >> 24); + else if (size == 2) + *SHORTP(ncp, 0) = (short)(cur_o >> 16); + else if (size == 4) + *LONGP(ncp, 0) = (Py_Int32)(cur_o); + ncp += size; + } + d -= inrate; + } + } +} + +void tostereo(char* rv, char* cp, size_t len, int size, + double fac1, double fac2) +{ + int val1, val2, val = 0; + double fval, maxval, minval; + char *ncp = rv; + int i; + + maxval = (double) maxvals[size]; + minval = (double) minvals[size]; + + for ( i=0; i < len; i += size ) { + if ( size == 1 ) val = (int)*CHARP(cp, i); + else if ( size == 2 ) val = (int)*SHORTP(cp, i); + else if ( size == 4 ) val = (int)*LONGP(cp, i); + + fval = (double)val*fac1; + val1 = (int)floor(fbound(fval, minval, maxval)); + + fval = (double)val*fac2; + val2 = (int)floor(fbound(fval, minval, maxval)); + + if ( size == 1 ) *CHARP(ncp, i*2) = (signed char)val1; + else if ( size == 2 ) *SHORTP(ncp, i*2) = (short)val1; + else if ( size == 4 ) *LONGP(ncp, i*2) = (Py_Int32)val1; + + if ( size == 1 ) *CHARP(ncp, i*2+1) = (signed char)val2; + else if ( size == 2 ) *SHORTP(ncp, i*2+2) = (short)val2; + else if ( size == 4 ) *LONGP(ncp, i*2+4) = (Py_Int32)val2; + } +} + +void add(char* rv, char* cp1, char* cp2, size_t len1, int size) +{ + int i; + int val1 = 0, val2 = 0, minval, maxval, newval; + char* ncp = rv; + + maxval = maxvals[size]; + minval = minvals[size]; + + for ( i=0; i < len1; i += size ) { + if ( size == 1 ) val1 = (int)*CHARP(cp1, i); + else if ( size == 2 ) val1 = (int)*SHORTP(cp1, i); + else if ( size == 4 ) val1 = (int)*LONGP(cp1, i); + + if ( size == 1 ) val2 = (int)*CHARP(cp2, i); + else if ( size == 2 ) val2 = (int)*SHORTP(cp2, i); + else if ( size == 4 ) val2 = (int)*LONGP(cp2, i); + + if (size < 4) { + newval = val1 + val2; + /* truncate in case of overflow */ + if (newval > maxval) + newval = maxval; + else if (newval < minval) + newval = minval; + } + else { + double fval = (double)val1 + (double)val2; + /* truncate in case of overflow */ + newval = (int)floor(fbound(fval, minval, maxval)); + } + + if ( size == 1 ) *CHARP(ncp, i) = (signed char)newval; + else if ( size == 2 ) *SHORTP(ncp, i) = (short)newval; + else if ( size == 4 ) *LONGP(ncp, i) = (Py_Int32)newval; + } +} + void lin2adcpm(unsigned char* ncp, unsigned char* cp, size_t len, size_t size, int* state) { From noreply at buildbot.pypy.org Fri Feb 13 08:03:20 2015 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 13 Feb 2015 08:03:20 +0100 (CET) Subject: [pypy-commit] pypy default: fix cffi versioning Message-ID: <20150213070320.4BDD51C11E2@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r75846:6e64507f2a09 Date: 2015-02-13 09:04 +0200 http://bitbucket.org/pypy/pypy/changeset/6e64507f2a09/ Log: fix cffi versioning diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,7 +4,7 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.8.6" +__version__ = "0.8.6+" __version_info__ = (0, 8, 6) # The verifier module file names are based on the CRC32 of a string that diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -55,7 +55,8 @@ # _cffi_backend.so compiled. import _cffi_backend as backend from . import __version__ - assert backend.__version__ == __version__ + assert backend.__version__ == __version__, \ + "version mismatch, %s != %s" % (backend.__version__, __version__) # (If you insist you can also try to pass the option # 'backend=backend_ctypes.CTypesBackend()', but don't # rely on it! It's probably not going to work well.) From noreply at buildbot.pypy.org Fri Feb 13 08:03:18 2015 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 13 Feb 2015 08:03:18 +0100 (CET) Subject: [pypy-commit] pypy default: remove old license cruft, make cffi import crashes non-fatal for buildbots Message-ID: <20150213070319.0A1B71C11E2@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r75845:dc805ab9b628 Date: 2015-02-13 09:03 +0200 http://bitbucket.org/pypy/pypy/changeset/dc805ab9b628/ Log: remove old license cruft, make cffi import crashes non-fatal for buildbots diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -50,68 +50,10 @@ os.system("chmod -R g-w %s" % dirname) -# -# Some crazy nonsense (imho) about including automatically the license -# of various libraries as they happen to be on this system. This is -# strange because most of these libraries are linked to dynamically, -# and so at runtime might end up with a different version. I (arigo) -# killed this logic and wrote some general info (which I hope is more -# sensible anyway) into our ../../../LICENSE file. -# -''' -sep_template = "\nThis copy of PyPy includes a copy of %s, which is licensed under the following terms:\n\n" - -def generate_license(basedir, options): - base_file = str(basedir.join('LICENSE')) - with open(base_file) as fid: - txt = fid.read() - if sys.platform == 'win32': - # shutil.copyfileobj(open("crtlicense.txt"), out) # We do not ship - # msvc runtime files, but otherwise we'd need this on Windows - searches = [("bzip2","bzip2-*", "LICENSE", ''), - ("openssl", "openssl-*", "LICENSE", '')] - else: - searches = [("bzip2","libbz2-*dev", "copyright", '---------'), - ("openssl", "openssl*", "copyright", 'LICENSE ISSUES')] - if not options.no_tk: - name = 'Tcl/Tk' - txt += "License for '%s'" %name - txt += '\n' + "="*(14 + len(name)) + '\n' - txt += sep_template % name - base_file = str(basedir.join('lib_pypy/_tkinter/license.terms')) - with open(base_file, 'r') as fid: - txt += fid.read() - for name, pat, fname, first_line in searches: - txt += "License for '" + name + "'" - txt += '\n' + "="*(14 + len(name)) + '\n' - txt += sep_template % name - dirs = glob.glob(options.license_base + "/" +pat) - if not dirs: - raise ValueError, "Could not find %s/%s" % (options.license_base, pat) - if len(dirs) > 1: - raise ValueError, "Multiple copies of %r: %r" % (pat, dirs) - dir = dirs[0] - with open(os.path.join(dir, fname)) as fid: - # Read up to the line dividing the packaging header from the actual copyright - for line in fid: - if first_line in line: - break - txt += line - for line in fid: - txt += line - if len(line.strip())<1: - txt += '\n' - txt += third_party_header - # Do something for gdbm, which is GPL - txt += gdbm_bit - return txt -''' - def create_cffi_import_libraries(pypy_c, options): modules = ['_sqlite3', 'audioop'] - subprocess.check_call([str(pypy_c), '-c', 'import _sqlite3']) if not sys.platform == 'win32': - modules += ['_curses', 'syslog', 'gdbm', '_sqlite3'] + modules += ['_curses', 'syslog', 'gdbm',] if not options.no_tk: modules.append('_tkinter') for module in modules: @@ -227,19 +169,12 @@ for file in ['_testcapimodule.c', '_ctypes_test.c']: shutil.copyfile(str(basedir.join('lib_pypy', file)), str(pypydir.join('lib_pypy', file))) - if 0: # disabled - license = generate_license(basedir, options) - with open(str(pypydir.join('LICENSE')), 'w') as LICENSE: - LICENSE.write(license) - else: - # Use original LICENCE file - #import traceback;traceback.print_exc() - base_file = str(basedir.join('LICENSE')) - with open(base_file) as fid: - license = fid.read() - with open(str(pypydir.join('LICENSE')), 'w') as LICENSE: - LICENSE.write(license) - #retval = -1 + # Use original LICENCE file + base_file = str(basedir.join('LICENSE')) + with open(base_file) as fid: + license = fid.read() + with open(str(pypydir.join('LICENSE')), 'w') as LICENSE: + LICENSE.write(license) # spdir = pypydir.ensure('site-packages', dir=True) shutil.copy(str(basedir.join('site-packages', 'README')), str(spdir)) @@ -308,16 +243,8 @@ argparse = imp.load_source('argparse', 'lib-python/2.7/argparse.py') if sys.platform == 'win32': pypy_exe = 'pypy.exe' - for p in [os.path.join(basedir, r'..\..\..\local'), #buildbot - os.path.join(basedir, r'..\local')]: # pypy/doc/windows.rst - if os.path.exists(p): - license_base = p - break - else: - license_base = 'unkown' else: pypy_exe = 'pypy' - license_base = '/usr/share/doc' parser = argparse.ArgumentParser() args = list(args) args[0] = str(args[0]) @@ -331,9 +258,6 @@ help='target executable name, defaults to "pypy"') parser.add_argument('--archive-name', dest='name', type=str, default='', help='pypy-VER-PLATFORM') - parser.add_argument('--license_base', type=str, default=license_base, - #help='where to start looking for third party upstream licensing info') - help='(ignored)') parser.add_argument('--builddir', type=str, default='', help='tmp dir for packaging') parser.add_argument('--targetdir', type=str, default='', From noreply at buildbot.pypy.org Fri Feb 13 09:04:48 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Feb 2015 09:04:48 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: update the values Message-ID: <20150213080448.A5E801C009F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r570:4879ebb61466 Date: 2015-02-13 09:05 +0100 http://bitbucket.org/pypy/pypy.org/changeset/4879ebb61466/ Log: update the values diff --git a/don4.html b/don4.html --- a/don4.html +++ b/don4.html @@ -9,7 +9,7 @@ @@ -17,7 +17,7 @@ 2nd call: - $21741 of $80000 (27.2%) + $22041 of $80000 (27.6%)
    From noreply at buildbot.pypy.org Fri Feb 13 09:09:39 2015 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 13 Feb 2015 09:09:39 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: whoops Message-ID: <20150213080939.4BF781C034E@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: extradoc Changeset: r5503:d9711dd77b27 Date: 2015-02-13 10:11 +0200 http://bitbucket.org/pypy/extradoc/changeset/d9711dd77b27/ Log: whoops diff --git a/sprintinfo/leysin-winter-2015/people.txt b/sprintinfo/leysin-winter-2015/people.txt --- a/sprintinfo/leysin-winter-2015/people.txt +++ b/sprintinfo/leysin-winter-2015/people.txt @@ -11,7 +11,7 @@ Name Arrive/Depart Accomodation ==================== ============== ======================= Armin Rigo private -Maciej Fijalkowski 20-28 Ermina +Maciej Fijalkowski 21-28 Ermina Remi Meier 21-28 Ermina Sebastian Pawlus 21-27 Ermina Manuel Jacob 21-28 Ermina From noreply at buildbot.pypy.org Fri Feb 13 10:43:55 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Feb 2015 10:43:55 +0100 (CET) Subject: [pypy-commit] cffi default: Copy from pypy/6e64507f2a09 Message-ID: <20150213094355.D36631C1308@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1649:7a9bab1c538a Date: 2015-02-13 10:44 +0100 http://bitbucket.org/cffi/cffi/changeset/7a9bab1c538a/ Log: Copy from pypy/6e64507f2a09 diff --git a/cffi/api.py b/cffi/api.py --- a/cffi/api.py +++ b/cffi/api.py @@ -55,7 +55,8 @@ # _cffi_backend.so compiled. import _cffi_backend as backend from . import __version__ - assert backend.__version__ == __version__ + assert backend.__version__ == __version__, \ + "version mismatch, %s != %s" % (backend.__version__, __version__) # (If you insist you can also try to pass the option # 'backend=backend_ctypes.CTypesBackend()', but don't # rely on it! It's probably not going to work well.) From noreply at buildbot.pypy.org Fri Feb 13 10:45:57 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Feb 2015 10:45:57 +0100 (CET) Subject: [pypy-commit] pypy default: pypy/tool/import_cffi.py Message-ID: <20150213094557.3BDDE1C131F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75847:2b94f0f584a7 Date: 2015-02-13 10:45 +0100 http://bitbucket.org/pypy/pypy/changeset/2b94f0f584a7/ Log: pypy/tool/import_cffi.py diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -5,7 +5,7 @@ from .ffiplatform import VerificationError, VerificationMissing __version__ = "0.8.6+" -__version_info__ = (0, 8, 6) +__version_info__ = (0, 8, 6, "plus") # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -56,7 +56,7 @@ import _cffi_backend as backend from . import __version__ assert backend.__version__ == __version__, \ - "version mismatch, %s != %s" % (backend.__version__, __version__) + "version mismatch, %s != %s" % (backend.__version__, __version__) # (If you insist you can also try to pass the option # 'backend=backend_ctypes.CTypesBackend()', but don't # rely on it! It's probably not going to work well.) diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py --- a/lib_pypy/cffi/backend_ctypes.py +++ b/lib_pypy/cffi/backend_ctypes.py @@ -2,11 +2,10 @@ from . import model if sys.version_info < (3,): - integer_types = (int, long) bytechr = chr else: unicode = str - integer_types = int + long = int xrange = range bytechr = lambda num: bytes([num]) @@ -181,7 +180,7 @@ address = 0 elif isinstance(source, CTypesData): address = source._cast_to_integer() - elif isinstance(source, integer_types): + elif isinstance(source, (int, long)): address = source else: raise TypeError("bad type for cast to %r: %r" % @@ -358,7 +357,7 @@ is_signed = (ctype(-1).value == -1) # def _cast_source_to_int(source): - if isinstance(source, (integer_types, float)): + if isinstance(source, (int, long, float)): source = int(source) elif isinstance(source, CTypesData): source = source._cast_to_integer() @@ -399,7 +398,7 @@ if kind == 'bool': @classmethod def _cast_from(cls, source): - if not isinstance(source, (integer_types, float)): + if not isinstance(source, (int, long, float)): source = _cast_source_to_int(source) return cls(bool(source)) def __int__(self): @@ -438,7 +437,7 @@ if kind == 'int' or kind == 'byte' or kind == 'bool': @staticmethod def _to_ctypes(x): - if not isinstance(x, integer_types): + if not isinstance(x, (int, long)): if isinstance(x, CTypesData): x = int(x) else: @@ -465,7 +464,7 @@ if kind == 'float': @staticmethod def _to_ctypes(x): - if not isinstance(x, (integer_types, float, CTypesData)): + if not isinstance(x, (int, long, float, CTypesData)): raise TypeError("float expected, got %s" % type(x).__name__) return ctype(x).value @@ -529,14 +528,14 @@ self._own = True def __add__(self, other): - if isinstance(other, integer_types): + if isinstance(other, (int, long)): return self._new_pointer_at(self._address + other * self._bitem_size) else: return NotImplemented def __sub__(self, other): - if isinstance(other, integer_types): + if isinstance(other, (int, long)): return self._new_pointer_at(self._address - other * self._bitem_size) elif type(self) is type(other): @@ -611,7 +610,7 @@ def __init__(self, init): if length is None: - if isinstance(init, integer_types): + if isinstance(init, (int, long)): len1 = init init = None elif kind == 'char' and isinstance(init, bytes): @@ -686,7 +685,7 @@ return CTypesPtr._arg_to_ctypes(value) def __add__(self, other): - if isinstance(other, integer_types): + if isinstance(other, (int, long)): return CTypesPtr._new_pointer_at( ctypes.addressof(self._blob) + other * ctypes.sizeof(BItem._ctype)) diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py --- a/lib_pypy/cffi/verifier.py +++ b/lib_pypy/cffi/verifier.py @@ -1,4 +1,4 @@ -import sys, os, binascii, shutil +import sys, os, binascii, shutil, io from . import __version_verifier_modules__ from . import ffiplatform @@ -13,6 +13,16 @@ if type == imp.C_EXTENSION] +if sys.version_info >= (3,): + NativeIO = io.StringIO +else: + class NativeIO(io.BytesIO): + def write(self, s): + if isinstance(s, unicode): + s = s.encode('ascii') + super(NativeIO, self).write(s) + + class Verifier(object): def __init__(self, ffi, preamble, tmpdir=None, modulename=None, @@ -144,19 +154,36 @@ self._vengine.collect_types() self._has_module = True - def _write_source(self, file=None): - must_close = (file is None) - if must_close: - _ensure_dir(self.sourcefilename) - file = open(self.sourcefilename, 'w') + def _write_source_to(self, file): self._vengine._f = file try: self._vengine.write_source_to_f() finally: del self._vengine._f - if must_close: - file.close() - if must_close: + + def _write_source(self, file=None): + if file is not None: + self._write_source_to(file) + else: + # Write our source file to an in memory file. + f = NativeIO() + self._write_source_to(f) + source_data = f.getvalue() + + # Determine if this matches the current file + if os.path.exists(self.sourcefilename): + with open(self.sourcefilename, "r") as fp: + needs_written = not (fp.read() == source_data) + else: + needs_written = True + + # Actually write the file out if it doesn't match + if needs_written: + _ensure_dir(self.sourcefilename) + with open(self.sourcefilename, "w") as fp: + fp.write(source_data) + + # Set this flag self._has_source = True def _compile_module(self): diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_ownlib.py b/pypy/module/test_lib_pypy/cffi_tests/test_ownlib.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_ownlib.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_ownlib.py @@ -8,34 +8,137 @@ SOURCE = """\ #include -int test_getting_errno(void) { +#ifdef _WIN32 +#define EXPORT __declspec(dllexport) +#else +#define EXPORT +#endif + +EXPORT int test_getting_errno(void) { errno = 123; return -1; } -int test_setting_errno(void) { +EXPORT int test_setting_errno(void) { return errno; +}; + +typedef struct { + long x; + long y; +} POINT; + +typedef struct { + long left; + long top; + long right; + long bottom; +} RECT; + + +EXPORT int PointInRect(RECT *prc, POINT pt) +{ + if (pt.x < prc->left) + return 0; + if (pt.x > prc->right) + return 0; + if (pt.y < prc->top) + return 0; + if (pt.y > prc->bottom) + return 0; + return 1; +}; + +EXPORT long left = 10; +EXPORT long top = 20; +EXPORT long right = 30; +EXPORT long bottom = 40; + +EXPORT RECT ReturnRect(int i, RECT ar, RECT* br, POINT cp, RECT dr, + RECT *er, POINT fp, RECT gr) +{ + /*Check input */ + if (ar.left + br->left + dr.left + er->left + gr.left != left * 5) + { + ar.left = 100; + return ar; + } + if (ar.right + br->right + dr.right + er->right + gr.right != right * 5) + { + ar.right = 100; + return ar; + } + if (cp.x != fp.x) + { + ar.left = -100; + } + if (cp.y != fp.y) + { + ar.left = -200; + } + switch(i) + { + case 0: + return ar; + break; + case 1: + return dr; + break; + case 2: + return gr; + break; + + } + return ar; } -int my_array[7] = {0, 1, 2, 3, 4, 5, 6}; +EXPORT int my_array[7] = {0, 1, 2, 3, 4, 5, 6}; """ class TestOwnLib(object): Backend = CTypesBackend def setup_class(cls): - if sys.platform == 'win32': - return + cls.module = None from pypy.module.test_lib_pypy.cffi_tests.udir import udir udir.join('testownlib.c').write(SOURCE) - subprocess.check_call( - 'gcc testownlib.c -shared -fPIC -o testownlib.so', - cwd=str(udir), shell=True) - cls.module = str(udir.join('testownlib.so')) + if sys.platform == 'win32': + import os + # did we already build it? + if os.path.exists(str(udir.join('testownlib.dll'))): + cls.module = str(udir.join('testownlib.dll')) + return + # try (not too hard) to find the version used to compile this python + # no mingw + from distutils.msvc9compiler import get_build_version + version = get_build_version() + toolskey = "VS%0.f0COMNTOOLS" % version + toolsdir = os.environ.get(toolskey, None) + if toolsdir is None: + return + productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC") + productdir = os.path.abspath(productdir) + vcvarsall = os.path.join(productdir, "vcvarsall.bat") + # 64? + arch = 'x86' + if sys.maxsize > 2**32: + arch = 'amd64' + if os.path.isfile(vcvarsall): + cmd = '"%s" %s' % (vcvarsall, arch) + ' & cl.exe testownlib.c ' \ + ' /LD /Fetestownlib.dll' + subprocess.check_call(cmd, cwd = str(udir), shell=True) + cls.module = str(udir.join('testownlib.dll')) + else: + subprocess.check_call( + 'gcc testownlib.c -shared -fPIC -o testownlib.so', + cwd=str(udir), shell=True) + cls.module = str(udir.join('testownlib.so')) def test_getting_errno(self): + if self.module is None: + py.test.skip("fix the auto-generation of the tiny test lib") if sys.platform == 'win32': - py.test.skip("fix the auto-generation of the tiny test lib") + py.test.skip("fails, errno at multiple addresses") ffi = FFI(backend=self.Backend()) ffi.cdef(""" int test_getting_errno(void); @@ -46,8 +149,10 @@ assert ffi.errno == 123 def test_setting_errno(self): + if self.module is None: + py.test.skip("fix the auto-generation of the tiny test lib") if sys.platform == 'win32': - py.test.skip("fix the auto-generation of the tiny test lib") + py.test.skip("fails, errno at multiple addresses") if self.Backend is CTypesBackend and '__pypy__' in sys.modules: py.test.skip("XXX errno issue with ctypes on pypy?") ffi = FFI(backend=self.Backend()) @@ -61,7 +166,7 @@ assert ffi.errno == 42 def test_my_array_7(self): - if sys.platform == 'win32': + if self.module is None: py.test.skip("fix the auto-generation of the tiny test lib") ffi = FFI(backend=self.Backend()) ffi.cdef(""" @@ -81,7 +186,7 @@ assert ownlib.my_array[i] == i def test_my_array_no_length(self): - if sys.platform == 'win32': + if self.module is None: py.test.skip("fix the auto-generation of the tiny test lib") if self.Backend is CTypesBackend: py.test.skip("not supported by the ctypes backend") @@ -101,7 +206,7 @@ assert ownlib.my_array[i] == i def test_keepalive_lib(self): - if sys.platform == 'win32': + if self.module is None: py.test.skip("fix the auto-generation of the tiny test lib") ffi = FFI(backend=self.Backend()) ffi.cdef(""" @@ -119,7 +224,7 @@ assert res == -1 def test_keepalive_ffi(self): - if sys.platform == 'win32': + if self.module is None: py.test.skip("fix the auto-generation of the tiny test lib") ffi = FFI(backend=self.Backend()) ffi.cdef(""" @@ -135,4 +240,46 @@ assert ownlib_r() is not None # kept alive by ffi res = func() assert res == -1 - assert ffi.errno == 123 + if sys.platform != 'win32': # else, errno at multiple addresses + assert ffi.errno == 123 + + def test_struct_by_value(self): + if self.module is None: + py.test.skip("fix the auto-generation of the tiny test lib") + ffi = FFI(backend=self.Backend()) + ffi.cdef(""" + typedef struct { + long x; + long y; + } POINT; + + typedef struct { + long left; + long top; + long right; + long bottom; + } RECT; + + long left, top, right, bottom; + + RECT ReturnRect(int i, RECT ar, RECT* br, POINT cp, RECT dr, + RECT *er, POINT fp, RECT gr); + """) + ownlib = ffi.dlopen(self.module) + + rect = ffi.new('RECT[1]') + pt = ffi.new('POINT[1]') + pt[0].x = 15 + pt[0].y = 25 + rect[0].left = ownlib.left + rect[0].right = ownlib.right + rect[0].top = ownlib.top + rect[0].bottom = ownlib.bottom + + for i in range(4): + ret = ownlib.ReturnRect(i, rect[0], rect, pt[0], rect[0], + rect, pt[0], rect[0]) + assert ret.left == ownlib.left + assert ret.right == ownlib.right + assert ret.top == ownlib.top + assert ret.bottom == ownlib.bottom diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/test_verify.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_verify.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_verify.py @@ -1368,7 +1368,7 @@ ffi = FFI() ffi.cdef("int foo(int);") f = open(os.path.join(tmpdir, 'foo.h'), 'w') - print >> f, "int foo(int a) { return a + 42; }" + f.write("int foo(int a) { return a + 42; }\n") f.close() lib = ffi.verify('#include "foo.h"', include_dirs=['.'], @@ -2129,7 +2129,7 @@ n = lib.GetModuleFileName(ffi.NULL, outbuf, 500) assert 0 < n < 500 for i in range(n): - print repr(outbuf[i]) + #print repr(outbuf[i]) assert ord(outbuf[i]) != 0 assert ord(outbuf[n]) == 0 assert ord(outbuf[0]) < 128 # should be a letter, or '\' diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_version.py b/pypy/module/test_lib_pypy/cffi_tests/test_version.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_version.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_version.py @@ -17,6 +17,7 @@ def test_version(): v = cffi.__version__ version_info = '.'.join(str(i) for i in cffi.__version_info__) + version_info = version_info.replace('.plus', '+') assert v == version_info #v = BACKEND_VERSIONS.get(v, v) assert v == _cffi_backend.__version__ @@ -32,7 +33,7 @@ def test_doc_version_file(): parent = os.path.dirname(os.path.dirname(__file__)) - v = cffi.__version__ + v = cffi.__version__.replace('+', '') p = os.path.join(parent, 'doc', 'source', 'index.rst') content = open(p).read() assert ("cffi/cffi-%s.tar.gz" % v) in content @@ -42,7 +43,7 @@ p = os.path.join(parent, 'setup.py') content = open(p).read() # - v = cffi.__version__ + v = cffi.__version__.replace('+', '') assert ("version='%s'" % v) in content def test_c_version(): From noreply at buildbot.pypy.org Fri Feb 13 10:54:36 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Feb 2015 10:54:36 +0100 (CET) Subject: [pypy-commit] pypy default: Replace explicit xfails with skips in the hope that these are more Message-ID: <20150213095436.99E6A1C13D4@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75848:aec97d922bdb Date: 2015-02-13 10:54 +0100 http://bitbucket.org/pypy/pypy/changeset/aec97d922bdb/ Log: Replace explicit xfails with skips in the hope that these are more supported by our automated test runner infrastructure. diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py --- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py +++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py @@ -27,7 +27,7 @@ log = self.run(main, [libm_name]) pow_addr, res = log.result assert res == 8.0 * 300 - py.test.xfail() # XXX re-optimize _ffi for the JIT? + py.test.skip("XXX re-optimize _ffi for the JIT?") loop, = log.loops_by_filename(self.filepath) if 'ConstClass(pow)' in repr(loop): # e.g. OS/X pow_addr = 'ConstClass(pow)' @@ -134,7 +134,7 @@ ops = loop.allops() opnames = log.opnames(ops) assert opnames.count('new_with_vtable') == 1 # only the virtualref - py.test.xfail() # XXX re-optimize _ffi for the JIT? + py.test.skip("XXX re-optimize _ffi for the JIT?") assert opnames.count('call_release_gil') == 1 idx = opnames.index('call_release_gil') call = ops[idx] @@ -159,7 +159,7 @@ return struct.getfield('x') # log = self.run(main, []) - py.test.xfail() # XXX re-optimize _ffi for the JIT? + py.test.skip("XXX re-optimize _ffi for the JIT?") loop, = log.loops_by_filename(self.filepath) assert loop.match_by_id('getfield', """ guard_not_invalidated(descr=...) From noreply at buildbot.pypy.org Fri Feb 13 11:17:34 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Feb 2015 11:17:34 +0100 (CET) Subject: [pypy-commit] pypy default: issue #1982 Message-ID: <20150213101734.44E8A1C034E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75849:55c0915afc7a Date: 2015-02-13 11:17 +0100 http://bitbucket.org/pypy/pypy/changeset/55c0915afc7a/ Log: issue #1982 quick fix: rename _names to _pyctypes__names. diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -60,7 +60,7 @@ is_bitfield = (len(field) == 3) startpos = self._ffistruct.fieldoffset(name) if name in anonymous_fields: - for subname in value._names: + for subname in value._pyctypes__names: resnames.append(subname) subfield = getattr(value, subname) relpos = startpos + subfield.offset @@ -71,7 +71,7 @@ else: resnames.append(name) names = resnames - self._names = names + self._pyctypes__names = names for name, field in fields.items(): setattr(self, name, field) @@ -230,9 +230,9 @@ def __init__(self, *args, **kwds): type(self)._make_final() - if len(args) > len(self._names): + if len(args) > len(self._pyctypes__names): raise TypeError("too many initializers") - for name, arg in zip(self._names, args): + for name, arg in zip(self._pyctypes__names, args): if name in kwds: raise TypeError("duplicate value for argument %r" % ( name,)) From noreply at buildbot.pypy.org Fri Feb 13 12:13:30 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Feb 2015 12:13:30 +0100 (CET) Subject: [pypy-commit] pypy default: fix test Message-ID: <20150213111330.BD3941C0976@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75850:3e2087f324da Date: 2015-02-13 12:13 +0100 http://bitbucket.org/pypy/pypy/changeset/3e2087f324da/ Log: fix test diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_structures.py b/pypy/module/test_lib_pypy/ctypes_tests/test_structures.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_structures.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_structures.py @@ -23,7 +23,7 @@ assert Y._fields_ == [("b", c_int)] assert Z._fields_ == [("a", c_int)] - assert Y._names == ['a', 'b'] + assert Y._pyctypes__names == ['a', 'b'] def test_subclass_delayed(self): class X(Structure): From noreply at buildbot.pypy.org Fri Feb 13 13:48:42 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Feb 2015 13:48:42 +0100 (CET) Subject: [pypy-commit] pypy default: Redo parts of the very old swap-of-arguments optimization that was Message-ID: <20150213124842.2385F1C009F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75851:8bc24ee62997 Date: 2015-02-13 13:48 +0100 http://bitbucket.org/pypy/pypy/changeset/8bc24ee62997/ Log: Redo parts of the very old swap-of-arguments optimization that was disabled in the x86 backend. diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -934,7 +934,7 @@ getattr(self.mc, asmop)(arglocs[0]) return genop_unary - def _binaryop(asmop, can_swap=False): + def _binaryop(asmop): def genop_binary(self, op, arglocs, result_loc): getattr(self.mc, asmop)(arglocs[0], arglocs[1]) return genop_binary @@ -1078,18 +1078,18 @@ genop_int_neg = _unaryop("NEG") genop_int_invert = _unaryop("NOT") - genop_int_add = _binaryop_or_lea("ADD", True) - genop_int_sub = _binaryop_or_lea("SUB", False) - genop_int_mul = _binaryop("IMUL", True) - genop_int_and = _binaryop("AND", True) - genop_int_or = _binaryop("OR", True) - genop_int_xor = _binaryop("XOR", True) + genop_int_add = _binaryop_or_lea("ADD", is_add=True) + genop_int_sub = _binaryop_or_lea("SUB", is_add=False) + genop_int_mul = _binaryop("IMUL") + genop_int_and = _binaryop("AND") + genop_int_or = _binaryop("OR") + genop_int_xor = _binaryop("XOR") genop_int_lshift = _binaryop("SHL") genop_int_rshift = _binaryop("SAR") genop_uint_rshift = _binaryop("SHR") - genop_float_add = _binaryop("ADDSD", True) + genop_float_add = _binaryop("ADDSD") genop_float_sub = _binaryop('SUBSD') - genop_float_mul = _binaryop('MULSD', True) + genop_float_mul = _binaryop('MULSD') genop_float_truediv = _binaryop('DIVSD') genop_int_lt = _cmpop("L", "G") @@ -1273,11 +1273,11 @@ self.mc.XOR_rr(edx.value, edx.value) self.mc.DIV_r(ecx.value) - genop_llong_add = _binaryop("PADDQ", True) + genop_llong_add = _binaryop("PADDQ") genop_llong_sub = _binaryop("PSUBQ") - genop_llong_and = _binaryop("PAND", True) - genop_llong_or = _binaryop("POR", True) - genop_llong_xor = _binaryop("PXOR", True) + genop_llong_and = _binaryop("PAND") + genop_llong_or = _binaryop("POR") + genop_llong_xor = _binaryop("PXOR") def genop_llong_to_int(self, op, arglocs, resloc): loc = arglocs[0] diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -421,9 +421,19 @@ consider_guard_nonnull_class = consider_guard_class - def _consider_binop_part(self, op): + def _consider_binop_part(self, op, symm=False): x = op.getarg(0) - argloc = self.loc(op.getarg(1)) + y = op.getarg(1) + argloc = self.loc(y) + # + # For symmetrical operations, if 'y' is already in a register + # and won't be used after the current operation finishes, + # then swap the role of 'x' and 'y' + if (symm and isinstance(argloc, RegLoc) and + self.rm.longevity[y][1] == self.rm.position): + x, y = y, x + argloc = self.loc(y) + # args = op.getarglist() loc = self.rm.force_result_in_reg(op.result, x, args) return loc, argloc @@ -432,6 +442,10 @@ loc, argloc = self._consider_binop_part(op) self.perform(op, [loc, argloc], loc) + def _consider_binop_symm(self, op): + loc, argloc = self._consider_binop_part(op, symm=True) + self.perform(op, [loc, argloc], loc) + def _consider_lea(self, op, loc): argloc = self.loc(op.getarg(1)) resloc = self.force_allocate_reg(op.result) @@ -444,7 +458,7 @@ isinstance(y, ConstInt) and rx86.fits_in_32bits(y.value)): self._consider_lea(op, loc) else: - self._consider_binop(op) + self._consider_binop_symm(op) def consider_int_sub(self, op): loc = self.loc(op.getarg(0)) @@ -455,18 +469,22 @@ else: self._consider_binop(op) - consider_int_mul = _consider_binop - consider_int_and = _consider_binop - consider_int_or = _consider_binop - consider_int_xor = _consider_binop + consider_int_mul = _consider_binop_symm + consider_int_and = _consider_binop_symm + consider_int_or = _consider_binop_symm + consider_int_xor = _consider_binop_symm def _consider_binop_with_guard(self, op, guard_op): loc, argloc = self._consider_binop_part(op) self.perform_with_guard(op, guard_op, [loc, argloc], loc) - consider_int_mul_ovf = _consider_binop_with_guard + def _consider_binop_with_guard_symm(self, op, guard_op): + loc, argloc = self._consider_binop_part(op, symm=True) + self.perform_with_guard(op, guard_op, [loc, argloc], loc) + + consider_int_mul_ovf = _consider_binop_with_guard_symm consider_int_sub_ovf = _consider_binop_with_guard - consider_int_add_ovf = _consider_binop_with_guard + consider_int_add_ovf = _consider_binop_with_guard_symm def consider_int_neg(self, op): res = self.rm.force_result_in_reg(op.result, op.getarg(0)) @@ -551,9 +569,9 @@ loc0 = self.xrm.force_result_in_reg(op.result, op.getarg(0), args) self.perform(op, [loc0, loc1], loc0) - consider_float_add = _consider_float_op + consider_float_add = _consider_float_op # xxx could be _symm consider_float_sub = _consider_float_op - consider_float_mul = _consider_float_op + consider_float_mul = _consider_float_op # xxx could be _symm consider_float_truediv = _consider_float_op def _consider_float_cmp(self, op, guard_op): @@ -632,6 +650,7 @@ # must force both arguments into xmm registers, because we don't # know if they will be suitably aligned. Exception: if the second # argument is a constant, we can ask it to be aligned to 16 bytes. + # xxx some of these operations could be '_symm'. args = [op.getarg(1), op.getarg(2)] loc1 = self.load_xmm_aligned_16_bytes(args[1]) loc0 = self.xrm.force_result_in_reg(op.result, args[0], args) From noreply at buildbot.pypy.org Fri Feb 13 14:27:29 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 13 Feb 2015 14:27:29 +0100 (CET) Subject: [pypy-commit] pypy framestate: kill dead method Message-ID: <20150213132729.22B3D1C034E@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: framestate Changeset: r75853:ede92a81f6be Date: 2015-02-13 09:40 +0000 http://bitbucket.org/pypy/pypy/changeset/ede92a81f6be/ Log: kill dead method diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py --- a/rpython/flowspace/flowcontext.py +++ b/rpython/flowspace/flowcontext.py @@ -1123,21 +1123,8 @@ class LoopBlock(FrameBlock): """A loop block. Stores the end-of-loop pointer in case of 'break'.""" - handles = (Break, Continue) - def handle(self, ctx, unroller): - if isinstance(unroller, Continue): - # re-push the loop block without cleaning up the value stack, - # and jump to the beginning of the loop, stored in the - # exception's argument - ctx.blockstack.append(self) - return unroller.jump_to - else: - # jump to the end of the loop - self.cleanupstack(ctx) - return self.handler - class ExceptBlock(FrameBlock): """An try:except: block. Stores the position of the exception handler.""" From noreply at buildbot.pypy.org Fri Feb 13 14:27:27 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 13 Feb 2015 14:27:27 +0100 (CET) Subject: [pypy-commit] pypy framestate: Remove all CONTINUE_LOOP/BREAK_LOOP instrs before FlowContext.build_flow() phase Message-ID: <20150213132727.E4ED91C034E@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: framestate Changeset: r75852:26a661c1f513 Date: 2015-02-13 06:44 +0000 http://bitbucket.org/pypy/pypy/changeset/26a661c1f513/ Log: Remove all CONTINUE_LOOP/BREAK_LOOP instrs before FlowContext.build_flow() phase diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py --- a/rpython/flowspace/bytecode.py +++ b/rpython/flowspace/bytecode.py @@ -202,25 +202,54 @@ def build_flow(self, code): self.pending_blocks = {} self.handlerstack = [] - self.blocks = [SimpleBlock([])] - self.curr_block = self.blocks[0] + self.all_handlers = [] + start_block = self.new_block() + self.blocks = [start_block] + self.curr_block = start_block self.needs_new_block = False - self.graph = graph = BytecodeGraph(self.blocks[0]) + self.graph = graph = BytecodeGraph(start_block) for instr in self._iter_instr(code): instr.bc_flow(self) + self.analyze_contexts(graph) self.analyze_signals(graph) self.check_graph() return graph + def analyze_contexts(self, graph): + start = graph.entry._exits[0] + self.pending = [start] + start.set_blockstack([]) + done = set() + while self.pending: + block = self.pending.pop(0) + if block in done: + continue + self.blockstack = block.blockstack[:] + for instr in block: + instr.context_effect(self) + for child in block._exits: + child.set_blockstack(self.blockstack) + self.pending.append(child) + done.add(block) + def analyze_signals(self, graph): for block in graph.iterblocks(): self.curr_block = block - block.init_blockstack() self.blockstack = block.blockstack[:] for instr in block: instr.do_signals(self) - for exit in block._exits: - exit.set_blockstack(self.blockstack) + + def splice_finally_handler(self, block, context): + def copy_block(handler): + b = handler.copy() + if handler is context.handler_end: + instr = b.operations.pop() + assert isinstance(instr, END_FINALLY) + else: + b.set_exits([copy_block(child) for child in handler._exits]) + self.blocks.append(b) + return b + block.set_exits([copy_block(context.handler)]) def check_graph(self): for b in self.blocks: @@ -315,7 +344,7 @@ def set_blockstack(self, blockstack): if self.blockstack is None: - self.blockstack = blockstack + self.blockstack = blockstack[:] else: assert self.blockstack == blockstack @@ -342,6 +371,13 @@ if exit: self.set_exits([exit]) + def copy(self): + block = SimpleBlock(self.operations[:]) + block.set_exits(self._exits) + if self.blockstack is not None: + block.blockstack = self.blockstack[:] + return block + OPNAMES = host_bytecode_spec.method_names NO_ARG = -1 @@ -361,6 +397,9 @@ reader.end_block() reader.get_block_at(self.arg) + def context_effect(self, reader): + pass + def do_signals(self, reader): pass @@ -538,23 +577,18 @@ from rpython.flowspace.flowcontext import ExceptBlock, FinallyBlock while reader.blockstack: context = reader.blockstack.pop() + block.operations.append(POP_BLOCK(-1, self.offset)) if isinstance(context, ExceptBlock): - block.operations.append(POP_BLOCK(-1, self.offset)) + pass elif isinstance(context, FinallyBlock): - block.operations.append(self) - block.set_exits([context.handler]) - return + reader.splice_finally_handler(block, context) + block = context.handler_end else: # LoopBlock - block.operations.append(POP_BLOCK(-1, self.offset)) block.set_exits([context.handler]) return raise BytecodeCorruption( "A break statement should not escape from the function") - def eval(self, ctx): - from rpython.flowspace.flowcontext import Break - return ctx.unroll(Break()) - @bc_reader.register_opcode class CONTINUE_LOOP(BCInstruction): def bc_flow(self, reader): @@ -572,26 +606,23 @@ if isinstance(context, ExceptBlock): block.operations.append(POP_BLOCK(-1, self.offset)) elif isinstance(context, FinallyBlock): - block.operations.append(self) - block.set_exits([context.handler]) - return + block.operations.append(POP_BLOCK(-1, self.offset)) + reader.splice_finally_handler(block, context) + block = context.handler_end else: # LoopBlock - block.operations.append(POP_BLOCK(-1, self.offset)) + reader.blockstack.append(context) block.set_exits([self.target]) return raise BytecodeCorruption( "A continue statement should not escape from the function") - def eval(self, ctx): - from rpython.flowspace.flowcontext import Continue - return ctx.unroll(Continue(self.target)) - @bc_reader.register_opcode class END_FINALLY(BCInstruction): def bc_flow(self, reader): reader.curr_block.operations.append(self) signal = reader.handlerstack.pop() signal.handler_end = reader.curr_block + reader.end_block() def eval(self, ctx): # unlike CPython, there are two statically distinct cases: the @@ -629,7 +660,9 @@ self.block.handler = self.target reader.end_block() - def do_signals(self, reader): + def context_effect(self, reader): + self.target.set_blockstack(reader.blockstack) + reader.pending.append(self.target) reader.blockstack.append(self.block) def eval(self, ctx): @@ -642,6 +675,7 @@ def bc_flow(self, reader): SetupInstruction.bc_flow(self, reader) reader.handlerstack.append(self.block) + reader.all_handlers.append(self.block) def make_block(self, stackdepth): from rpython.flowspace.flowcontext import ExceptBlock @@ -658,6 +692,7 @@ def bc_flow(self, reader): SetupInstruction.bc_flow(self, reader) reader.handlerstack.append(self.block) + reader.all_handlers.append(self.block) def make_block(self, stackdepth): from rpython.flowspace.flowcontext import FinallyBlock @@ -668,6 +703,7 @@ def bc_flow(self, reader): SetupInstruction.bc_flow(self, reader) reader.handlerstack.append(self.block) + reader.all_handlers.append(self.block) def make_block(self, stackdepth): from rpython.flowspace.flowcontext import FinallyBlock @@ -692,7 +728,7 @@ reader.curr_block.operations.append(self) reader.end_block() - def do_signals(self, reader): + def context_effect(self, reader): reader.blockstack.pop() def eval(self, ctx): From noreply at buildbot.pypy.org Fri Feb 13 14:27:30 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 13 Feb 2015 14:27:30 +0100 (CET) Subject: [pypy-commit] pypy framestate: Add failing test for the case where 'break' is hidden away inside a handler Message-ID: <20150213132730.392321C034E@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: framestate Changeset: r75854:7f1ca2ab5456 Date: 2015-02-13 12:51 +0000 http://bitbucket.org/pypy/pypy/changeset/7f1ca2ab5456/ Log: Add failing test for the case where 'break' is hidden away inside a handler diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py --- a/rpython/flowspace/test/test_objspace.py +++ b/rpython/flowspace/test/test_objspace.py @@ -206,6 +206,22 @@ def test_break_continue(self): x = self.codetest(self.break_continue) + def test_break_from_handler(self): + def f(x): + while True: + try: + x() + except TypeError: + if x: + raise + break + assert f(0) is None + graph = self.codetest(f) + simplify_graph(graph) + entrymap = mkentrymap(graph) + links = entrymap[graph.returnblock] + assert len(links) == 1 + #__________________________________________________________ def unpack_tuple(lst): a, b, c = lst From noreply at buildbot.pypy.org Fri Feb 13 14:33:24 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Fri, 13 Feb 2015 14:33:24 +0100 (CET) Subject: [pypy-commit] pypy framestate: follow links from SETUP_ instrs to their targets in graph.iterblocks() Message-ID: <20150213133324.C58E51C034E@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: framestate Changeset: r75855:452634b78aa1 Date: 2015-02-13 13:33 +0000 http://bitbucket.org/pypy/pypy/changeset/452634b78aa1/ Log: follow links from SETUP_ instrs to their targets in graph.iterblocks() diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py --- a/rpython/flowspace/bytecode.py +++ b/rpython/flowspace/bytecode.py @@ -217,20 +217,13 @@ def analyze_contexts(self, graph): start = graph.entry._exits[0] - self.pending = [start] start.set_blockstack([]) - done = set() - while self.pending: - block = self.pending.pop(0) - if block in done: - continue + for block in graph.iterblocks(): self.blockstack = block.blockstack[:] for instr in block: instr.context_effect(self) for child in block._exits: child.set_blockstack(self.blockstack) - self.pending.append(child) - done.add(block) def analyze_signals(self, graph): for block in graph.iterblocks(): @@ -294,10 +287,13 @@ stack = block._exits[:] while stack: block = stack.pop() - if block not in seen: - yield block - seen.add(block) - stack.extend(block._exits[:]) + if block in seen: + continue + yield block + seen.add(block) + stack.extend(block._exits[:]) + if block.special_exit: + stack.append(block.special_exit) def all_blocks(self): return list(self.iterblocks()) @@ -368,6 +364,7 @@ def __init__(self, operations, exit=None): BytecodeBlock.__init__(self) self.operations = operations + self.special_exit = None if exit: self.set_exits([exit]) @@ -655,14 +652,15 @@ self.block = self.make_block(-1) def bc_flow(self, reader): - reader.curr_block.operations.append(self) + block = reader.curr_block + block.operations.append(self) self.target = reader.get_block_at(self.arg) self.block.handler = self.target + block.special_exit = self.target reader.end_block() def context_effect(self, reader): self.target.set_blockstack(reader.blockstack) - reader.pending.append(self.target) reader.blockstack.append(self.block) def eval(self, ctx): From noreply at buildbot.pypy.org Fri Feb 13 16:05:51 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Feb 2015 16:05:51 +0100 (CET) Subject: [pypy-commit] stmgc default: Officialize the hack done by hashtable.c to invent "pre-existing" Message-ID: <20150213150551.3ABCE1C0976@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1617:6d9efeaf307b Date: 2015-02-13 16:06 +0100 http://bitbucket.org/pypy/stmgc/changeset/6d9efeaf307b/ Log: Officialize the hack done by hashtable.c to invent "pre-existing" objects. diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c --- a/c7/stm/gcpage.c +++ b/c7/stm/gcpage.c @@ -127,6 +127,27 @@ return o; } +object_t *stm_allocate_preexisting(ssize_t size_rounded_up, + struct object_s *initial_data) +{ + initial_data->stm_flags = GCFLAG_WRITE_BARRIER; + + acquire_privatization_lock(); + + char *p = allocate_outside_nursery_large(size_rounded_up); + uintptr_t nobj = p - stm_object_pages; + long j; + for (j = 0; j <= NB_SEGMENTS; j++) { + char *dest = get_segment_base(j) + nobj; + memcpy(dest, initial_data, size_rounded_up); + } + + release_privatization_lock(); + + write_fence(); /* make sure 'nobj' is fully initialized from + all threads here */ + return (object_t *)nobj; +} /************************************************************/ diff --git a/c7/stm/hashtable.c b/c7/stm/hashtable.c --- a/c7/stm/hashtable.c +++ b/c7/stm/hashtable.c @@ -297,25 +297,16 @@ synchronization with other pieces of the code that may change. */ - acquire_privatization_lock(); - char *p = allocate_outside_nursery_large( - sizeof(stm_hashtable_entry_t)); - entry = (stm_hashtable_entry_t *)(p - stm_object_pages); - - long j; - for (j = 0; j <= NB_SEGMENTS; j++) { - struct stm_hashtable_entry_s *e; - e = (struct stm_hashtable_entry_s *) - REAL_ADDRESS(get_segment_base(j), entry); - e->header.stm_flags = GCFLAG_WRITE_BARRIER; - e->userdata = stm_hashtable_entry_userdata; - e->index = index; - e->object = NULL; - } + struct stm_hashtable_entry_s initial = { + .userdata = stm_hashtable_entry_userdata, + .index = index, + .object = NULL + }; + entry = (stm_hashtable_entry_t *) + stm_allocate_preexisting(sizeof(stm_hashtable_entry_t), + &initial.header); hashtable->additions += 0x100; - release_privatization_lock(); } - write_fence(); /* make sure 'entry' is fully initialized here */ table->items[i] = entry; write_fence(); /* make sure 'table->items' is written here */ VOLATILE_TABLE(table)->resize_counter = rc - 6; /* unlock */ diff --git a/c7/stmgc.h b/c7/stmgc.h --- a/c7/stmgc.h +++ b/c7/stmgc.h @@ -562,6 +562,16 @@ object_t *object; }; +/* Make an object "that always existed". Can be used for cases where + the object pointer could also be seen by other running transactions + even if the current one did not commit yet (by means outside the + normal scope of stmgc). Must provide the initial content of the + object seen in all threads; afterwards, the running transaction can + use stm_write() and make local changes. +*/ +object_t *stm_allocate_preexisting(ssize_t size_rounded_up, + struct object_s *initial_data); + /* ==================== END ==================== */ #endif From noreply at buildbot.pypy.org Fri Feb 13 16:18:16 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Feb 2015 16:18:16 +0100 (CET) Subject: [pypy-commit] stmgc default: Test for 6d9efeaf307b Message-ID: <20150213151816.273911C1204@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1618:e5d94f869b43 Date: 2015-02-13 16:18 +0100 http://bitbucket.org/pypy/stmgc/changeset/e5d94f869b43/ Log: Test for 6d9efeaf307b diff --git a/c7/test/support.py b/c7/test/support.py --- a/c7/test/support.py +++ b/c7/test/support.py @@ -184,6 +184,8 @@ stm_hashtable_t *_get_hashtable(object_t *obj); uintptr_t _get_entry_index(stm_hashtable_entry_t *entry); object_t *_get_entry_object(stm_hashtable_entry_t *entry); + +object_t *stm_allocate_preexisting(ssize_t size_rounded_up, struct object_s *); """) diff --git a/c7/test/test_gcpage.py b/c7/test/test_gcpage.py --- a/c7/test/test_gcpage.py +++ b/c7/test/test_gcpage.py @@ -247,3 +247,16 @@ p1 = self.pop_root() assert stm_get_char(p1) == 'o' assert stm_get_char(p2) == 't' + + def test_allocate_preexisting(self): + self.start_transaction() + p1 = stm_allocate(32) + stm_set_char(p1, 'X') + rawaddr = stm_get_real_address(p1) + p2 = lib.stm_allocate_preexisting(32, rawaddr) + # + assert stm_get_char(p2) == 'X' + # + self.switch(1) + self.start_transaction() + assert stm_get_char(p2) == 'X' From noreply at buildbot.pypy.org Fri Feb 13 16:38:12 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Feb 2015 16:38:12 +0100 (CET) Subject: [pypy-commit] stmgc default: tweak Message-ID: <20150213153812.052EF1C1213@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1619:19bb1dde7aca Date: 2015-02-13 16:38 +0100 http://bitbucket.org/pypy/stmgc/changeset/19bb1dde7aca/ Log: tweak diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c --- a/c7/stm/gcpage.c +++ b/c7/stm/gcpage.c @@ -128,10 +128,8 @@ } object_t *stm_allocate_preexisting(ssize_t size_rounded_up, - struct object_s *initial_data) + const char *initial_data) { - initial_data->stm_flags = GCFLAG_WRITE_BARRIER; - acquire_privatization_lock(); char *p = allocate_outside_nursery_large(size_rounded_up); @@ -140,6 +138,7 @@ for (j = 0; j <= NB_SEGMENTS; j++) { char *dest = get_segment_base(j) + nobj; memcpy(dest, initial_data, size_rounded_up); + ((struct object_s *)dest)->stm_flags = GCFLAG_WRITE_BARRIER; } release_privatization_lock(); diff --git a/c7/stm/hashtable.c b/c7/stm/hashtable.c --- a/c7/stm/hashtable.c +++ b/c7/stm/hashtable.c @@ -304,7 +304,7 @@ }; entry = (stm_hashtable_entry_t *) stm_allocate_preexisting(sizeof(stm_hashtable_entry_t), - &initial.header); + (char *)&initial.header); hashtable->additions += 0x100; } table->items[i] = entry; diff --git a/c7/stmgc.h b/c7/stmgc.h --- a/c7/stmgc.h +++ b/c7/stmgc.h @@ -570,7 +570,7 @@ use stm_write() and make local changes. */ object_t *stm_allocate_preexisting(ssize_t size_rounded_up, - struct object_s *initial_data); + const char *initial_data); /* ==================== END ==================== */ diff --git a/c7/test/support.py b/c7/test/support.py --- a/c7/test/support.py +++ b/c7/test/support.py @@ -185,7 +185,7 @@ uintptr_t _get_entry_index(stm_hashtable_entry_t *entry); object_t *_get_entry_object(stm_hashtable_entry_t *entry); -object_t *stm_allocate_preexisting(ssize_t size_rounded_up, struct object_s *); +object_t *stm_allocate_preexisting(ssize_t size_rounded_up, const char *); """) From noreply at buildbot.pypy.org Fri Feb 13 16:39:09 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Feb 2015 16:39:09 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: import stmgc/19bb1dde7aca Message-ID: <20150213153909.B77B71C034E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r75856:3a99d6cba477 Date: 2015-02-13 16:38 +0100 http://bitbucket.org/pypy/pypy/changeset/3a99d6cba477/ Log: import stmgc/19bb1dde7aca diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -65a89cd8a6c0 +19bb1dde7aca diff --git a/rpython/translator/stm/src_stm/stm/gcpage.c b/rpython/translator/stm/src_stm/stm/gcpage.c --- a/rpython/translator/stm/src_stm/stm/gcpage.c +++ b/rpython/translator/stm/src_stm/stm/gcpage.c @@ -128,6 +128,26 @@ return o; } +object_t *stm_allocate_preexisting(ssize_t size_rounded_up, + const char *initial_data) +{ + acquire_privatization_lock(); + + char *p = allocate_outside_nursery_large(size_rounded_up); + uintptr_t nobj = p - stm_object_pages; + long j; + for (j = 0; j <= NB_SEGMENTS; j++) { + char *dest = get_segment_base(j) + nobj; + memcpy(dest, initial_data, size_rounded_up); + ((struct object_s *)dest)->stm_flags = GCFLAG_WRITE_BARRIER; + } + + release_privatization_lock(); + + write_fence(); /* make sure 'nobj' is fully initialized from + all threads here */ + return (object_t *)nobj; +} /************************************************************/ diff --git a/rpython/translator/stm/src_stm/stm/hashtable.c b/rpython/translator/stm/src_stm/stm/hashtable.c --- a/rpython/translator/stm/src_stm/stm/hashtable.c +++ b/rpython/translator/stm/src_stm/stm/hashtable.c @@ -298,25 +298,16 @@ synchronization with other pieces of the code that may change. */ - acquire_privatization_lock(); - char *p = allocate_outside_nursery_large( - sizeof(stm_hashtable_entry_t)); - entry = (stm_hashtable_entry_t *)(p - stm_object_pages); - - long j; - for (j = 0; j <= NB_SEGMENTS; j++) { - struct stm_hashtable_entry_s *e; - e = (struct stm_hashtable_entry_s *) - REAL_ADDRESS(get_segment_base(j), entry); - e->header.stm_flags = GCFLAG_WRITE_BARRIER; - e->userdata = stm_hashtable_entry_userdata; - e->index = index; - e->object = NULL; - } + struct stm_hashtable_entry_s initial = { + .userdata = stm_hashtable_entry_userdata, + .index = index, + .object = NULL + }; + entry = (stm_hashtable_entry_t *) + stm_allocate_preexisting(sizeof(stm_hashtable_entry_t), + (char *)&initial.header); hashtable->additions += 0x100; - release_privatization_lock(); } - write_fence(); /* make sure 'entry' is fully initialized here */ table->items[i] = entry; write_fence(); /* make sure 'table->items' is written here */ VOLATILE_TABLE(table)->resize_counter = rc - 6; /* unlock */ diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h --- a/rpython/translator/stm/src_stm/stmgc.h +++ b/rpython/translator/stm/src_stm/stmgc.h @@ -563,6 +563,16 @@ object_t *object; }; +/* Make an object "that always existed". Can be used for cases where + the object pointer could also be seen by other running transactions + even if the current one did not commit yet (by means outside the + normal scope of stmgc). Must provide the initial content of the + object seen in all threads; afterwards, the running transaction can + use stm_write() and make local changes. +*/ +object_t *stm_allocate_preexisting(ssize_t size_rounded_up, + const char *initial_data); + /* ==================== END ==================== */ #endif From noreply at buildbot.pypy.org Fri Feb 13 16:39:10 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Feb 2015 16:39:10 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Add rstm.allocate_preexisting() Message-ID: <20150213153910.E22C31C034E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r75857:79ec4673db42 Date: 2015-02-13 16:39 +0100 http://bitbucket.org/pypy/pypy/changeset/79ec4673db42/ Log: Add rstm.allocate_preexisting() diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py --- a/rpython/rlib/rstm.py +++ b/rpython/rlib/rstm.py @@ -162,6 +162,12 @@ def stm_count(): return llop.stm_count(lltype.Signed) + at specialize.ll() +def allocate_preexisting(p): + TP = lltype.typeOf(p) + size = llmemory.sizeof(TP.TO) + return llop.stm_allocate_preexisting(TP, size, p) + # ____________________________________________________________ class _Entry(ExtRegistryEntry): diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -418,6 +418,7 @@ 'stm_allocate_weakref': LLOp(sideeffects=False, canmallocgc=True), 'stm_allocate_finalizer': LLOp(sideeffects=False, canmallocgc=True), 'stm_allocate_f_light': LLOp(sideeffects=False, canmallocgc=True), + 'stm_allocate_preexisting':LLOp(sideeffects=False, canmallocgc=True), 'stm_get_from_obj': LLOp(sideeffects=False), 'stm_get_from_obj_const': LLOp(canfold=True), 'stm_set_into_obj': LLOp(), diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py --- a/rpython/translator/stm/funcgen.py +++ b/rpython/translator/stm/funcgen.py @@ -108,6 +108,15 @@ '((rpyobj_t *)%s)->tid = %s;\n' % (result, arg_type_id) + 'stm_enable_light_finalizer((object_t *)%s);' % (result,)) +def stm_allocate_preexisting(funcgen, op): + arg_size = funcgen.expr(op.args[0]) + arg_idata = funcgen.expr(op.args[1]) + result = funcgen.expr(op.result) + resulttype = cdecl(funcgen.lltypename(op.result), '') + return ('%s = (%s)stm_allocate_preexisting(%s,' + ' _stm_real_address((object_t *)%s));' % ( + result, resulttype, arg_size, arg_idata)) + def stm_get_from_obj(funcgen, op): assert op.args[0].concretetype == llmemory.GCREF arg_obj = funcgen.expr(op.args[0]) diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py --- a/rpython/translator/stm/test/test_ztranslated.py +++ b/rpython/translator/stm/test/test_ztranslated.py @@ -603,3 +603,20 @@ t, cbuilder = self.compile(main, backendopt=True) data = cbuilder.cmdexec('') assert 'ok!\n' in data + + def test_allocate_preexisting(self): + S = lltype.GcStruct('S', ('n', lltype.Signed)) + + def main(argv): + s1 = lltype.malloc(S) + s1.n = 42 + s2 = rstm.allocate_preexisting(s1) + s1.n += 1 + assert s2.n == 42 + # + print "ok!" + return 0 + + t, cbuilder = self.compile(main) + data = cbuilder.cmdexec('') + assert 'ok!\n' in data From noreply at buildbot.pypy.org Fri Feb 13 17:19:50 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Fri, 13 Feb 2015 17:19:50 +0100 (CET) Subject: [pypy-commit] pypy default: Issue #1982 Avoid name clash and rename cls._names to cls._names_ Message-ID: <20150213161950.E23CD1C00A8@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r75858:2a0f777f5f5b Date: 2015-02-13 09:42 +0100 http://bitbucket.org/pypy/pypy/changeset/2a0f777f5f5b/ Log: Issue #1982 Avoid name clash and rename cls._names to cls._names_ diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -60,7 +60,7 @@ is_bitfield = (len(field) == 3) startpos = self._ffistruct.fieldoffset(name) if name in anonymous_fields: - for subname in value._pyctypes__names: + for subname in value._names_: resnames.append(subname) subfield = getattr(value, subname) relpos = startpos + subfield.offset @@ -71,7 +71,7 @@ else: resnames.append(name) names = resnames - self._pyctypes__names = names + self._names_ = names for name, field in fields.items(): setattr(self, name, field) @@ -230,9 +230,9 @@ def __init__(self, *args, **kwds): type(self)._make_final() - if len(args) > len(self._pyctypes__names): + if len(args) > len(self._names_): raise TypeError("too many initializers") - for name, arg in zip(self._pyctypes__names, args): + for name, arg in zip(self._names_, args): if name in kwds: raise TypeError("duplicate value for argument %r" % ( name,)) diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_anon.py b/pypy/module/test_lib_pypy/ctypes_tests/test_anon.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_anon.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_anon.py @@ -56,3 +56,4 @@ assert Y._.offset == sizeof(c_int) assert Y.y.offset == sizeof(c_int) * 2 + assert Y._names_ == ['x', 'a', 'b', 'y'] diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_structures.py b/pypy/module/test_lib_pypy/ctypes_tests/test_structures.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_structures.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_structures.py @@ -23,7 +23,7 @@ assert Y._fields_ == [("b", c_int)] assert Z._fields_ == [("a", c_int)] - assert Y._pyctypes__names == ['a', 'b'] + assert Y._names_ == ['a', 'b'] def test_subclass_delayed(self): class X(Structure): From noreply at buildbot.pypy.org Fri Feb 13 17:19:52 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Fri, 13 Feb 2015 17:19:52 +0100 (CET) Subject: [pypy-commit] pypy default: Rename more class attributes to follow ctypes convention for _private_ names. Message-ID: <20150213161952.36B931C00A8@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r75859:7d23e48e92d6 Date: 2015-02-13 16:49 +0100 http://bitbucket.org/pypy/pypy/changeset/7d23e48e92d6/ Log: Rename more class attributes to follow ctypes convention for _private_ names. diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py --- a/lib_pypy/_ctypes/array.py +++ b/lib_pypy/_ctypes/array.py @@ -9,7 +9,7 @@ def __new__(self, name, cls, typedict): res = type.__new__(self, name, cls, typedict) if '_type_' in typedict: - ffiarray = _rawffi.Array(typedict['_type_']._ffishape) + ffiarray = _rawffi.Array(typedict['_type_']._ffishape_) res._ffiarray = ffiarray subletter = getattr(typedict['_type_'], '_type_', None) if subletter == 'c': @@ -58,8 +58,8 @@ res.value = property(getvalue, setvalue) if '_length_' in typedict: - res._ffishape = (ffiarray, typedict['_length_']) - res._fficompositesize = res._sizeofinstances() + res._ffishape_ = (ffiarray, typedict['_length_']) + res._fficompositesize_ = res._sizeofinstances() else: res._ffiarray = None return res @@ -156,7 +156,7 @@ class Array(_CData): __metaclass__ = ArrayMeta - _ffiargshape = 'P' + _ffiargshape_ = 'P' def __init__(self, *args): if not hasattr(self, '_buffer'): @@ -191,13 +191,13 @@ if ensure_objects(cobj) is not None: store_reference(self, index, cobj._objects) arg = cobj._get_buffer_value() - if self._type_._fficompositesize is None: + if self._type_._fficompositesize_ is None: self._buffer[index] = arg # something more sophisticated, cannot set field directly else: from ctypes import memmove dest = self._buffer.itemaddress(index) - memmove(dest, arg, self._type_._fficompositesize) + memmove(dest, arg, self._type_._fficompositesize_) def __getitem__(self, index): if isinstance(index, slice): diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py --- a/lib_pypy/_ctypes/basics.py +++ b/lib_pypy/_ctypes/basics.py @@ -52,7 +52,7 @@ def get_ffi_argtype(self): if self._ffiargtype: return self._ffiargtype - self._ffiargtype = _shape_to_ffi_type(self._ffiargshape) + self._ffiargtype = _shape_to_ffi_type(self._ffiargshape_) return self._ffiargtype def _CData_output(self, resbuffer, base=None, index=-1): diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py --- a/lib_pypy/_ctypes/function.py +++ b/lib_pypy/_ctypes/function.py @@ -65,9 +65,9 @@ _restype_ = None _errcheck_ = None _flags_ = 0 - _ffiargshape = 'P' - _ffishape = 'P' - _fficompositesize = None + _ffiargshape_ = 'P' + _ffishape_ = 'P' + _fficompositesize_ = None _ffiarray = _rawffi.Array('P') _needs_free = False callable = None @@ -98,7 +98,7 @@ argtypes = property(_getargtypes, _setargtypes) def _check_argtypes_for_fastpath(self): - if all([hasattr(argtype, '_ffiargshape') for argtype in self._argtypes_]): + if all([hasattr(argtype, '_ffiargshape_') for argtype in self._argtypes_]): fastpath_cls = make_fastpath_subclass(self.__class__) fastpath_cls.enable_fastpath_maybe(self) @@ -135,7 +135,7 @@ _flag = flag & PARAMFLAG_COMBINED if _flag == PARAMFLAG_FOUT: typ = self._argtypes_[idx] - if getattr(typ, '_ffiargshape', None) not in ('P', 'z', 'Z'): + if getattr(typ, '_ffiargshape_', None) not in ('P', 'z', 'Z'): raise TypeError( "'out' parameter %d must be a pointer type, not %s" % (idx+1, type(typ).__name__) @@ -182,11 +182,11 @@ def _ffishapes(self, args, restype): if args is None: args = [] - argtypes = [arg._ffiargshape for arg in args] + argtypes = [arg._ffiargshape_ for arg in args] if restype is not None: if not isinstance(restype, SimpleType): raise TypeError("invalid result type for callback function") - restype = restype._ffiargshape + restype = restype._ffiargshape_ else: restype = 'O' # void return argtypes, restype @@ -599,7 +599,7 @@ if self._is_primitive(restype) and not restype._is_pointer_like(): return result # - shape = restype._ffishape + shape = restype._ffishape_ if is_struct_shape(shape): buf = result else: diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py --- a/lib_pypy/_ctypes/pointer.py +++ b/lib_pypy/_ctypes/pointer.py @@ -21,9 +21,9 @@ size = _rawffi.sizeof('P'), align = _rawffi.alignment('P'), length = 1, - _ffiargshape = 'P', - _ffishape = 'P', - _fficompositesize = None, + _ffiargshape_ = 'P', + _ffishape_ = 'P', + _fficompositesize_ = None, ) # XXX check if typedict['_type_'] is any sane # XXX remember about paramfunc diff --git a/lib_pypy/_ctypes/primitive.py b/lib_pypy/_ctypes/primitive.py --- a/lib_pypy/_ctypes/primitive.py +++ b/lib_pypy/_ctypes/primitive.py @@ -117,9 +117,9 @@ default = TP_TO_DEFAULT[tp] ffiarray = _rawffi.Array(tp) result = type.__new__(self, name, bases, dct) - result._ffiargshape = tp - result._ffishape = tp - result._fficompositesize = None + result._ffiargshape_ = tp + result._ffishape_ = tp + result._fficompositesize_ = None result._ffiarray = ffiarray if tp == 'z': # c_char_p diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -36,9 +36,9 @@ rawfields = [] for f in all_fields: if len(f) > 2: - rawfields.append((f[0], f[1]._ffishape, f[2])) + rawfields.append((f[0], f[1]._ffishape_, f[2])) else: - rawfields.append((f[0], f[1]._ffishape)) + rawfields.append((f[0], f[1]._ffishape_)) _set_shape(self, rawfields, self._is_union) @@ -48,8 +48,8 @@ value = field[1] is_bitfield = (len(field) == 3) fields[name] = Field(name, - self._ffistruct.fieldoffset(name), - self._ffistruct.fieldsize(name), + self._ffistruct_.fieldoffset(name), + self._ffistruct_.fieldsize(name), value, i, is_bitfield) if anonymous_fields: @@ -58,7 +58,7 @@ name = field[0] value = field[1] is_bitfield = (len(field) == 3) - startpos = self._ffistruct.fieldoffset(name) + startpos = self._ffistruct_.fieldoffset(name) if name in anonymous_fields: for subname in value._names_: resnames.append(subname) @@ -114,19 +114,19 @@ elif ensure_objects(cobj) is not None: store_reference(obj, key, cobj._objects) arg = cobj._get_buffer_value() - if fieldtype._fficompositesize is not None: + if fieldtype._fficompositesize_ is not None: from ctypes import memmove dest = obj._buffer.fieldaddress(self.name) - memmove(dest, arg, fieldtype._fficompositesize) + memmove(dest, arg, fieldtype._fficompositesize_) else: obj._buffer.__setattr__(self.name, arg) def _set_shape(tp, rawfields, is_union=False): - tp._ffistruct = _rawffi.Structure(rawfields, is_union, + tp._ffistruct_ = _rawffi.Structure(rawfields, is_union, getattr(tp, '_pack_', 0)) - tp._ffiargshape = tp._ffishape = (tp._ffistruct, 1) - tp._fficompositesize = tp._ffistruct.size + tp._ffiargshape_ = tp._ffishape_ = (tp._ffistruct_, 1) + tp._fficompositesize_ = tp._ffistruct_.size def struct_setattr(self, name, value): @@ -181,16 +181,16 @@ address = address.buffer # fix the address: turn it into as unsigned, in case it is negative address = address & (sys.maxint * 2 + 1) - instance.__dict__['_buffer'] = self._ffistruct.fromaddress(address) + instance.__dict__['_buffer'] = self._ffistruct_.fromaddress(address) return instance def _sizeofinstances(self): - if not hasattr(self, '_ffistruct'): + if not hasattr(self, '_ffistruct_'): return 0 - return self._ffistruct.size + return self._ffistruct_.size def _alignmentofinstances(self): - return self._ffistruct.alignment + return self._ffistruct_.alignment def from_param(self, value): if isinstance(value, tuple): @@ -203,7 +203,7 @@ def _CData_output(self, resarray, base=None, index=-1): res = StructOrUnion.__new__(self) - ffistruct = self._ffistruct.fromaddress(resarray.buffer) + ffistruct = self._ffistruct_.fromaddress(resarray.buffer) res.__dict__['_buffer'] = ffistruct res.__dict__['_base'] = base res.__dict__['_index'] = index @@ -224,8 +224,8 @@ self = super(_CData, cls).__new__(cls, *args, **kwds) if '_abstract_' in cls.__dict__: raise TypeError("abstract class") - if hasattr(cls, '_ffistruct'): - self.__dict__['_buffer'] = self._ffistruct(autofree=True) + if hasattr(cls, '_ffistruct_'): + self.__dict__['_buffer'] = self._ffistruct_(autofree=True) return self def __init__(self, *args, **kwds): @@ -244,7 +244,7 @@ """Return a _rawffi array of length 1 whose address is the same as the address of the field 'name' of self.""" address = self._buffer.fieldaddress(name) - A = _rawffi.Array(fieldtype._ffishape) + A = _rawffi.Array(fieldtype._ffishape_) return A.fromaddress(address, 1) def _get_buffer_for_param(self): From noreply at buildbot.pypy.org Fri Feb 13 18:36:01 2015 From: noreply at buildbot.pypy.org (Alexander Schremmer) Date: Fri, 13 Feb 2015 18:36:01 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: Added myself to sprintinfo (21-23). Message-ID: <20150213173601.ED0081C1213@cobra.cs.uni-duesseldorf.de> Author: Alexander Schremmer Branch: extradoc Changeset: r5504:0d55ce307244 Date: 2015-02-13 18:36 +0100 http://bitbucket.org/pypy/extradoc/changeset/0d55ce307244/ Log: Added myself to sprintinfo (21-23). diff --git a/sprintinfo/leysin-winter-2015/people.txt b/sprintinfo/leysin-winter-2015/people.txt --- a/sprintinfo/leysin-winter-2015/people.txt +++ b/sprintinfo/leysin-winter-2015/people.txt @@ -18,6 +18,7 @@ Joan Massich 20-? Ermina Quim Sanchez 20-? Ermina Francisco Fernandez 20-28 Ermina +Alexander Schremmer 21-23 Ermina ==================== ============== ======================= @@ -56,7 +57,6 @@ Richard Emslie ? ? Johan Hahn ? ? Stephan Diehl ? ? -Alexander Schremmer ? ? Anders Chrigstroem ? ? Eric van Riet Paap ? ? Holger Krekel ? ? From noreply at buildbot.pypy.org Fri Feb 13 18:40:16 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Feb 2015 18:40:16 +0100 (CET) Subject: [pypy-commit] stmgc default: Found out how to cancel a become_globally_unique_transaction() while Message-ID: <20150213174016.9D01C1C009F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1620:756a54dd920b Date: 2015-02-13 18:40 +0100 http://bitbucket.org/pypy/stmgc/changeset/756a54dd920b/ Log: Found out how to cancel a become_globally_unique_transaction() while still keeping the current transaction inevitable --- by making it the next inevitable transaction atomically. diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -330,10 +330,11 @@ static void _stm_start_transaction(stm_thread_local_t *tl) { - assert(!_stm_in_transaction(tl)); - - while (!acquire_thread_segment(tl)) - ; + if (!will_start_inevitable) { + assert(!_stm_in_transaction(tl)); + while (!acquire_thread_segment(tl)) + ; + } /* GS invalid before this point! */ assert(STM_PSEGMENT->safe_point == SP_NO_TRANSACTION); @@ -350,10 +351,12 @@ STM_PSEGMENT->shadowstack_at_start_of_transaction = tl->shadowstack; STM_PSEGMENT->threadlocal_at_start_of_transaction = tl->thread_local_obj; - enter_safe_point_if_requested(); - dprintf(("start_transaction\n")); + if (!will_start_inevitable) { + enter_safe_point_if_requested(); + dprintf(("start_transaction\n")); - s_mutex_unlock(); + s_mutex_unlock(); + } /* Now running the SP_RUNNING start. We can set our 'transaction_read_version' after releasing the mutex, @@ -827,7 +830,8 @@ stm_thread_local_t *tl = STM_SEGMENT->running_thread; timing_event(tl, event); - release_thread_segment(tl); + if (!will_start_inevitable) + release_thread_segment(tl); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ } @@ -847,6 +851,11 @@ s_mutex_lock(); + if (will_start_inevitable) { + assert(will_start_inevitable == 1); + assert(will_start_inevitable = 2); /* 1 -> 2, only if !NDEBUG */ + } + restart: /* force all other threads to be paused. They will unpause automatically when we are done here, i.e. at mutex_unlock(). @@ -905,6 +914,9 @@ _finish_transaction(STM_TRANSACTION_COMMIT); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ + if (will_start_inevitable) + return; /* hack: return with the mutex still held */ + s_mutex_unlock(); invoke_general_finalizers(tl); @@ -1160,3 +1172,29 @@ synchronize_all_threads(STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE); s_mutex_unlock(); } + +void stm_commit_and_start_inevitable(stm_thread_local_t *tl, const char *msg) +{ + stm_become_inevitable(tl, msg); /* may still abort */ + + /* cannot abort any more from here */ + will_start_inevitable = 1; + + /* as long as 'will_start_inevitable' is true, we cannot release the + mutex_lock or do a cond_wait. We must go through uninterrupted + with all the steps below. + */ + stm_commit_transaction(); + assert(will_start_inevitable == 2); + assert(_has_mutex()); + + _stm_start_transaction(tl); + assert(_has_mutex()); + + /* this line should be the only step from _stm_become_inevitable() we + must do in this case */ + STM_PSEGMENT->transaction_state = TS_INEVITABLE; + + will_start_inevitable = 0; + s_mutex_unlock(); +} diff --git a/c7/stm/sync.c b/c7/stm/sync.c --- a/c7/stm/sync.c +++ b/c7/stm/sync.c @@ -91,6 +91,7 @@ static inline void s_mutex_unlock(void) { assert(_has_mutex_here); + assert(!will_start_inevitable); if (UNLIKELY(pthread_mutex_unlock(&sync_ctl.global_mutex) != 0)) stm_fatalerror("pthread_mutex_unlock: %m"); assert((_has_mutex_here = false, 1)); @@ -103,6 +104,7 @@ #endif assert(_has_mutex_here); + assert(will_start_inevitable < 2); if (UNLIKELY(pthread_cond_wait(&sync_ctl.cond[ctype], &sync_ctl.global_mutex) != 0)) stm_fatalerror("pthread_cond_wait/%d: %m", (int)ctype); diff --git a/c7/stm/sync.h b/c7/stm/sync.h --- a/c7/stm/sync.h +++ b/c7/stm/sync.h @@ -38,5 +38,6 @@ static void committed_globally_unique_transaction(void); static bool pause_signalled, globally_unique_transaction; +static uint8_t will_start_inevitable; void signal_other_to_commit_soon(struct stm_priv_segment_info_s *other_pseg); diff --git a/c7/stmgc.h b/c7/stmgc.h --- a/c7/stmgc.h +++ b/c7/stmgc.h @@ -408,6 +408,15 @@ void stm_become_globally_unique_transaction(stm_thread_local_t *tl, const char *msg); +/* Commit and start the next transaction as inevitable. Provided the + commit succeeds, the next transaction is immediately made + inevitable: no other transaction can commit (or turn inevitable) + between the two steps. The only reason to use this function + instead of stm_become_inevitable() is that the forced commit ends a + globally unique transaction. If there is already another inevitable + transaction, this function will come stm_become_inevitable() first. */ +void stm_commit_and_start_inevitable(stm_thread_local_t *tl, const char *msg); + /* Profiling events. In the comments: content of the markers, if any */ enum stm_event_e { diff --git a/c7/test/support.py b/c7/test/support.py --- a/c7/test/support.py +++ b/c7/test/support.py @@ -62,6 +62,7 @@ bool _check_abort_transaction(void); bool _check_become_inevitable(stm_thread_local_t *tl); bool _check_become_globally_unique_transaction(stm_thread_local_t *tl); +bool _check_commit_and_start_inevitable(stm_thread_local_t *tl); int stm_is_inevitable(void); long current_segment_num(void); @@ -254,13 +255,17 @@ } bool _check_become_inevitable(stm_thread_local_t *tl) { - CHECKED(stm_become_inevitable(tl, "TEST")); + CHECKED(stm_become_inevitable(tl, "TESTINEV")); } bool _check_become_globally_unique_transaction(stm_thread_local_t *tl) { CHECKED(stm_become_globally_unique_transaction(tl, "TESTGUT")); } +bool _check_commit_and_start_inevitable(stm_thread_local_t *tl) { + CHECKED(stm_commit_and_start_inevitable(tl, "TESTCSI")); +} + object_t *hashtable_read_result; bool _check_hashtable_read(object_t *hobj, stm_hashtable_t *h, uintptr_t key) @@ -781,3 +786,8 @@ tl = self.tls[self.current_thread] if lib._check_become_globally_unique_transaction(tl): raise Conflict() + + def commit_and_start_inevitable(self): + tl = self.tls[self.current_thread] + if lib._check_commit_and_start_inevitable(tl): + raise Conflict() diff --git a/c7/test/test_random.py b/c7/test/test_random.py --- a/c7/test/test_random.py +++ b/c7/test/test_random.py @@ -314,10 +314,14 @@ ################################################################### -def op_start_transaction(ex, global_state, thread_state): +def op_start_transaction(ex, global_state, thread_state, + commit_and_start_inevitable=False): thread_state.start_transaction(ex.thread_num) # - ex.do('self.start_transaction()') + if commit_and_start_inevitable: + ex.do('self.commit_and_start_inevitable()') + else: + ex.do('self.start_transaction()') thread_state.reload_roots(ex) # # assert that everything known is old: @@ -326,7 +330,8 @@ ex.do("assert not is_in_nursery(%s)" % o) -def op_commit_transaction(ex, global_state, thread_state): +def op_commit_transaction(ex, global_state, thread_state, + commit_and_start_inevitable=False): # # push all new roots ex.do("# push new objs before commit:") @@ -335,7 +340,8 @@ # if aborts: thread_state.abort_transaction() - ex.do(raising_call(aborts, "self.commit_transaction")) + if not commit_and_start_inevitable: + ex.do(raising_call(aborts, "self.commit_transaction")) def op_abort_transaction(ex, global_state, thread_state): trs = thread_state.transaction_state @@ -345,13 +351,17 @@ thread_state.abort_transaction() ex.do('self.abort_transaction()') -def op_become_inevitable(ex, global_state, thread_state): +def op_become_inevitable(ex, global_state, thread_state, + commit_and_start_inevitable=False): trs = thread_state.transaction_state global_state.check_if_can_become_inevitable(trs) thread_state.push_roots(ex) - ex.do(raising_call(trs.check_must_abort(), - "self.become_inevitable")) + if commit_and_start_inevitable: + assert not trs.check_must_abort() + else: + ex.do(raising_call(trs.check_must_abort(), + "self.become_inevitable")) if trs.check_must_abort(): thread_state.abort_transaction() else: @@ -359,6 +369,16 @@ thread_state.pop_roots(ex) thread_state.reload_roots(ex) +def op_commit_and_start_inevitable(ex, global_state, thread_state): + op_become_inevitable(ex, global_state, thread_state) + if thread_state.transaction_state is not None: + op_commit_transaction(ex, global_state, thread_state, + commit_and_start_inevitable=True) + op_start_transaction(ex, global_state, thread_state, + commit_and_start_inevitable=True) + op_become_inevitable(ex, global_state, thread_state, + commit_and_start_inevitable=True) + assert thread_state.transaction_state is not None def op_allocate(ex, global_state, thread_state): size = global_state.rnd.choice([ @@ -584,6 +604,7 @@ op_abort_transaction, op_forget_root, op_become_inevitable, + op_commit_and_start_inevitable, op_assert_size, op_assert_modified, op_minor_collect, From noreply at buildbot.pypy.org Fri Feb 13 18:44:35 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Feb 2015 18:44:35 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Add comment. (Unsure we'll use it, actually.) Message-ID: <20150213174435.957991C009F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r75860:ce4db3f0606e Date: 2015-02-13 17:20 +0100 http://bitbucket.org/pypy/pypy/changeset/ce4db3f0606e/ Log: Add comment. (Unsure we'll use it, actually.) diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py --- a/rpython/rlib/rstm.py +++ b/rpython/rlib/rstm.py @@ -164,6 +164,14 @@ @specialize.ll() def allocate_preexisting(p): + """Return a copy of p, which must be a Ptr(GcStruct), which + we pretend existed all along (including in other transactions). + Used in cases where other concurrent transactions have a non- + official way to get a pointer to that object even before we commit. + The copied content should be the "default initial" state which + the current transaction can then proceed to change normally. This + initial state must not contain GC pointers to any other uncommitted + object.""" TP = lltype.typeOf(p) size = llmemory.sizeof(TP.TO) return llop.stm_allocate_preexisting(TP, size, p) From noreply at buildbot.pypy.org Fri Feb 13 18:44:36 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Feb 2015 18:44:36 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Use stm_commit_and_start_inevitable() after starting a globally unique Message-ID: <20150213174436.BF6391C009F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r75861:4d74bdc4535a Date: 2015-02-13 18:43 +0100 http://bitbucket.org/pypy/pypy/changeset/4d74bdc4535a/ Log: Use stm_commit_and_start_inevitable() after starting a globally unique transaction. diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -248,16 +248,12 @@ for box in loop.inputargs: assert isinstance(box, Box) - if rgc.stm_is_enabled(): - rstm.stop_all_other_threads() target_token = loop.operations[-1].getdescr() resumekey.compile_and_attach(metainterp, loop) target_token = label.getdescr() assert isinstance(target_token, TargetToken) record_loop_or_bridge(metainterp_sd, loop) - if rgc.stm_is_enabled(): - rstm.partial_commit_and_resume_other_threads() return target_token @@ -824,13 +820,9 @@ if new_trace.operations[-1].getopnum() != rop.LABEL: # We managed to create a bridge. Dispatch to resumekey to # know exactly what we must do (ResumeGuardDescr/ResumeFromInterpDescr) - if rgc.stm_is_enabled(): - rstm.stop_all_other_threads() target_token = new_trace.operations[-1].getdescr() resumekey.compile_and_attach(metainterp, new_trace) record_loop_or_bridge(metainterp_sd, new_trace) - if rgc.stm_is_enabled(): - rstm.partial_commit_and_resume_other_threads() return target_token else: metainterp.retrace_needed(new_trace) diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py --- a/rpython/rlib/rstm.py +++ b/rpython/rlib/rstm.py @@ -73,8 +73,9 @@ def stop_all_other_threads(): llop.stm_become_globally_unique_transaction(lltype.Void) + at dont_look_inside def partial_commit_and_resume_other_threads(): - hint_commit_soon() # for now + llop.stm_commit_and_start_inevitable(lltype.Void) @specialize.arg(0) def should_break_transaction(keep): diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -429,6 +429,7 @@ 'stm_get_root_stack_top': LLOp(sideeffects=False), 'stm_become_inevitable': LLOp(canmallocgc=True), 'stm_become_globally_unique_transaction': LLOp(canmallocgc=True), + 'stm_commit_and_start_inevitable': LLOp(canmallocgc=True), 'stm_push_root': LLOp(), 'stm_pop_root_into': LLOp(), 'stm_commit_if_not_atomic': LLOp(canmallocgc=True), diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py --- a/rpython/translator/stm/funcgen.py +++ b/rpython/translator/stm/funcgen.py @@ -171,6 +171,9 @@ def stm_become_globally_unique_transaction(funcgen, op): return 'pypy_stm_become_globally_unique_transaction();' +def stm_commit_and_start_inevitable(funcgen, op): + return 'pypy_stm_commit_and_start_inevitable();' + def stm_push_root(funcgen, op): arg0 = funcgen.expr(op.args[0]) return 'STM_PUSH_ROOT(stm_thread_local, %s);' % (arg0,) diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -19bb1dde7aca +756a54dd920b diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c --- a/rpython/translator/stm/src_stm/stm/core.c +++ b/rpython/translator/stm/src_stm/stm/core.c @@ -331,10 +331,11 @@ static void _stm_start_transaction(stm_thread_local_t *tl) { - assert(!_stm_in_transaction(tl)); - - while (!acquire_thread_segment(tl)) - ; + if (!will_start_inevitable) { + assert(!_stm_in_transaction(tl)); + while (!acquire_thread_segment(tl)) + ; + } /* GS invalid before this point! */ assert(STM_PSEGMENT->safe_point == SP_NO_TRANSACTION); @@ -351,10 +352,12 @@ STM_PSEGMENT->shadowstack_at_start_of_transaction = tl->shadowstack; STM_PSEGMENT->threadlocal_at_start_of_transaction = tl->thread_local_obj; - enter_safe_point_if_requested(); - dprintf(("start_transaction\n")); + if (!will_start_inevitable) { + enter_safe_point_if_requested(); + dprintf(("start_transaction\n")); - s_mutex_unlock(); + s_mutex_unlock(); + } /* Now running the SP_RUNNING start. We can set our 'transaction_read_version' after releasing the mutex, @@ -828,7 +831,8 @@ stm_thread_local_t *tl = STM_SEGMENT->running_thread; timing_event(tl, event); - release_thread_segment(tl); + if (!will_start_inevitable) + release_thread_segment(tl); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ } @@ -848,6 +852,11 @@ s_mutex_lock(); + if (will_start_inevitable) { + assert(will_start_inevitable == 1); + assert(will_start_inevitable = 2); /* 1 -> 2, only if !NDEBUG */ + } + restart: /* force all other threads to be paused. They will unpause automatically when we are done here, i.e. at mutex_unlock(). @@ -906,6 +915,9 @@ _finish_transaction(STM_TRANSACTION_COMMIT); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ + if (will_start_inevitable) + return; /* hack: return with the mutex still held */ + s_mutex_unlock(); invoke_general_finalizers(tl); @@ -1161,3 +1173,29 @@ synchronize_all_threads(STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE); s_mutex_unlock(); } + +void stm_commit_and_start_inevitable(stm_thread_local_t *tl, const char *msg) +{ + stm_become_inevitable(tl, msg); /* may still abort */ + + /* cannot abort any more from here */ + will_start_inevitable = 1; + + /* as long as 'will_start_inevitable' is true, we cannot release the + mutex_lock or do a cond_wait. We must go through uninterrupted + with all the steps below. + */ + stm_commit_transaction(); + assert(will_start_inevitable == 2); + assert(_has_mutex()); + + _stm_start_transaction(tl); + assert(_has_mutex()); + + /* this line should be the only step from _stm_become_inevitable() we + must do in this case */ + STM_PSEGMENT->transaction_state = TS_INEVITABLE; + + will_start_inevitable = 0; + s_mutex_unlock(); +} diff --git a/rpython/translator/stm/src_stm/stm/sync.c b/rpython/translator/stm/src_stm/stm/sync.c --- a/rpython/translator/stm/src_stm/stm/sync.c +++ b/rpython/translator/stm/src_stm/stm/sync.c @@ -92,6 +92,7 @@ static inline void s_mutex_unlock(void) { assert(_has_mutex_here); + assert(!will_start_inevitable); if (UNLIKELY(pthread_mutex_unlock(&sync_ctl.global_mutex) != 0)) stm_fatalerror("pthread_mutex_unlock: %m"); assert((_has_mutex_here = false, 1)); @@ -104,6 +105,7 @@ #endif assert(_has_mutex_here); + assert(will_start_inevitable < 2); if (UNLIKELY(pthread_cond_wait(&sync_ctl.cond[ctype], &sync_ctl.global_mutex) != 0)) stm_fatalerror("pthread_cond_wait/%d: %m", (int)ctype); diff --git a/rpython/translator/stm/src_stm/stm/sync.h b/rpython/translator/stm/src_stm/stm/sync.h --- a/rpython/translator/stm/src_stm/stm/sync.h +++ b/rpython/translator/stm/src_stm/stm/sync.h @@ -39,5 +39,6 @@ static void committed_globally_unique_transaction(void); static bool pause_signalled, globally_unique_transaction; +static uint8_t will_start_inevitable; void signal_other_to_commit_soon(struct stm_priv_segment_info_s *other_pseg); diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h --- a/rpython/translator/stm/src_stm/stmgc.h +++ b/rpython/translator/stm/src_stm/stmgc.h @@ -409,6 +409,15 @@ void stm_become_globally_unique_transaction(stm_thread_local_t *tl, const char *msg); +/* Commit and start the next transaction as inevitable. Provided the + commit succeeds, the next transaction is immediately made + inevitable: no other transaction can commit (or turn inevitable) + between the two steps. The only reason to use this function + instead of stm_become_inevitable() is that the forced commit ends a + globally unique transaction. If there is already another inevitable + transaction, this function will come stm_become_inevitable() first. */ +void stm_commit_and_start_inevitable(stm_thread_local_t *tl, const char *msg); + /* Profiling events. In the comments: content of the markers, if any */ enum stm_event_e { diff --git a/rpython/translator/stm/src_stm/stmgcintf.c b/rpython/translator/stm/src_stm/stmgcintf.c --- a/rpython/translator/stm/src_stm/stmgcintf.c +++ b/rpython/translator/stm/src_stm/stmgcintf.c @@ -222,6 +222,11 @@ stm_become_globally_unique_transaction(&stm_thread_local, "for the JIT"); } +void pypy_stm_commit_and_start_inevitable(void) +{ + stm_commit_and_start_inevitable(&stm_thread_local, "for the JIT"); +} + long _pypy_stm_count(void) { static long value = 1; diff --git a/rpython/translator/stm/src_stm/stmgcintf.h b/rpython/translator/stm/src_stm/stmgcintf.h --- a/rpython/translator/stm/src_stm/stmgcintf.h +++ b/rpython/translator/stm/src_stm/stmgcintf.h @@ -30,6 +30,7 @@ void _pypy_stm_become_inevitable(const char *); void pypy_stm_become_globally_unique_transaction(void); +void pypy_stm_commit_and_start_inevitable(void); char *_pypy_stm_test_expand_marker(void); void pypy_stm_setup_expand_marker(long co_filename_ofs, From noreply at buildbot.pypy.org Fri Feb 13 19:06:45 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Feb 2015 19:06:45 +0100 (CET) Subject: [pypy-commit] stmgc default: Backed out changeset 756a54dd920b: does not work Message-ID: <20150213180645.635711C1204@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1621:e464ca5eca63 Date: 2015-02-13 19:07 +0100 http://bitbucket.org/pypy/stmgc/changeset/e464ca5eca63/ Log: Backed out changeset 756a54dd920b: does not work diff --git a/c7/stm/core.c b/c7/stm/core.c --- a/c7/stm/core.c +++ b/c7/stm/core.c @@ -330,11 +330,10 @@ static void _stm_start_transaction(stm_thread_local_t *tl) { - if (!will_start_inevitable) { - assert(!_stm_in_transaction(tl)); - while (!acquire_thread_segment(tl)) - ; - } + assert(!_stm_in_transaction(tl)); + + while (!acquire_thread_segment(tl)) + ; /* GS invalid before this point! */ assert(STM_PSEGMENT->safe_point == SP_NO_TRANSACTION); @@ -351,12 +350,10 @@ STM_PSEGMENT->shadowstack_at_start_of_transaction = tl->shadowstack; STM_PSEGMENT->threadlocal_at_start_of_transaction = tl->thread_local_obj; - if (!will_start_inevitable) { - enter_safe_point_if_requested(); - dprintf(("start_transaction\n")); + enter_safe_point_if_requested(); + dprintf(("start_transaction\n")); - s_mutex_unlock(); - } + s_mutex_unlock(); /* Now running the SP_RUNNING start. We can set our 'transaction_read_version' after releasing the mutex, @@ -830,8 +827,7 @@ stm_thread_local_t *tl = STM_SEGMENT->running_thread; timing_event(tl, event); - if (!will_start_inevitable) - release_thread_segment(tl); + release_thread_segment(tl); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ } @@ -851,11 +847,6 @@ s_mutex_lock(); - if (will_start_inevitable) { - assert(will_start_inevitable == 1); - assert(will_start_inevitable = 2); /* 1 -> 2, only if !NDEBUG */ - } - restart: /* force all other threads to be paused. They will unpause automatically when we are done here, i.e. at mutex_unlock(). @@ -914,9 +905,6 @@ _finish_transaction(STM_TRANSACTION_COMMIT); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ - if (will_start_inevitable) - return; /* hack: return with the mutex still held */ - s_mutex_unlock(); invoke_general_finalizers(tl); @@ -1172,29 +1160,3 @@ synchronize_all_threads(STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE); s_mutex_unlock(); } - -void stm_commit_and_start_inevitable(stm_thread_local_t *tl, const char *msg) -{ - stm_become_inevitable(tl, msg); /* may still abort */ - - /* cannot abort any more from here */ - will_start_inevitable = 1; - - /* as long as 'will_start_inevitable' is true, we cannot release the - mutex_lock or do a cond_wait. We must go through uninterrupted - with all the steps below. - */ - stm_commit_transaction(); - assert(will_start_inevitable == 2); - assert(_has_mutex()); - - _stm_start_transaction(tl); - assert(_has_mutex()); - - /* this line should be the only step from _stm_become_inevitable() we - must do in this case */ - STM_PSEGMENT->transaction_state = TS_INEVITABLE; - - will_start_inevitable = 0; - s_mutex_unlock(); -} diff --git a/c7/stm/sync.c b/c7/stm/sync.c --- a/c7/stm/sync.c +++ b/c7/stm/sync.c @@ -91,7 +91,6 @@ static inline void s_mutex_unlock(void) { assert(_has_mutex_here); - assert(!will_start_inevitable); if (UNLIKELY(pthread_mutex_unlock(&sync_ctl.global_mutex) != 0)) stm_fatalerror("pthread_mutex_unlock: %m"); assert((_has_mutex_here = false, 1)); @@ -104,7 +103,6 @@ #endif assert(_has_mutex_here); - assert(will_start_inevitable < 2); if (UNLIKELY(pthread_cond_wait(&sync_ctl.cond[ctype], &sync_ctl.global_mutex) != 0)) stm_fatalerror("pthread_cond_wait/%d: %m", (int)ctype); diff --git a/c7/stm/sync.h b/c7/stm/sync.h --- a/c7/stm/sync.h +++ b/c7/stm/sync.h @@ -38,6 +38,5 @@ static void committed_globally_unique_transaction(void); static bool pause_signalled, globally_unique_transaction; -static uint8_t will_start_inevitable; void signal_other_to_commit_soon(struct stm_priv_segment_info_s *other_pseg); diff --git a/c7/stmgc.h b/c7/stmgc.h --- a/c7/stmgc.h +++ b/c7/stmgc.h @@ -408,15 +408,6 @@ void stm_become_globally_unique_transaction(stm_thread_local_t *tl, const char *msg); -/* Commit and start the next transaction as inevitable. Provided the - commit succeeds, the next transaction is immediately made - inevitable: no other transaction can commit (or turn inevitable) - between the two steps. The only reason to use this function - instead of stm_become_inevitable() is that the forced commit ends a - globally unique transaction. If there is already another inevitable - transaction, this function will come stm_become_inevitable() first. */ -void stm_commit_and_start_inevitable(stm_thread_local_t *tl, const char *msg); - /* Profiling events. In the comments: content of the markers, if any */ enum stm_event_e { diff --git a/c7/test/support.py b/c7/test/support.py --- a/c7/test/support.py +++ b/c7/test/support.py @@ -62,7 +62,6 @@ bool _check_abort_transaction(void); bool _check_become_inevitable(stm_thread_local_t *tl); bool _check_become_globally_unique_transaction(stm_thread_local_t *tl); -bool _check_commit_and_start_inevitable(stm_thread_local_t *tl); int stm_is_inevitable(void); long current_segment_num(void); @@ -255,17 +254,13 @@ } bool _check_become_inevitable(stm_thread_local_t *tl) { - CHECKED(stm_become_inevitable(tl, "TESTINEV")); + CHECKED(stm_become_inevitable(tl, "TEST")); } bool _check_become_globally_unique_transaction(stm_thread_local_t *tl) { CHECKED(stm_become_globally_unique_transaction(tl, "TESTGUT")); } -bool _check_commit_and_start_inevitable(stm_thread_local_t *tl) { - CHECKED(stm_commit_and_start_inevitable(tl, "TESTCSI")); -} - object_t *hashtable_read_result; bool _check_hashtable_read(object_t *hobj, stm_hashtable_t *h, uintptr_t key) @@ -786,8 +781,3 @@ tl = self.tls[self.current_thread] if lib._check_become_globally_unique_transaction(tl): raise Conflict() - - def commit_and_start_inevitable(self): - tl = self.tls[self.current_thread] - if lib._check_commit_and_start_inevitable(tl): - raise Conflict() diff --git a/c7/test/test_random.py b/c7/test/test_random.py --- a/c7/test/test_random.py +++ b/c7/test/test_random.py @@ -314,14 +314,10 @@ ################################################################### -def op_start_transaction(ex, global_state, thread_state, - commit_and_start_inevitable=False): +def op_start_transaction(ex, global_state, thread_state): thread_state.start_transaction(ex.thread_num) # - if commit_and_start_inevitable: - ex.do('self.commit_and_start_inevitable()') - else: - ex.do('self.start_transaction()') + ex.do('self.start_transaction()') thread_state.reload_roots(ex) # # assert that everything known is old: @@ -330,8 +326,7 @@ ex.do("assert not is_in_nursery(%s)" % o) -def op_commit_transaction(ex, global_state, thread_state, - commit_and_start_inevitable=False): +def op_commit_transaction(ex, global_state, thread_state): # # push all new roots ex.do("# push new objs before commit:") @@ -340,8 +335,7 @@ # if aborts: thread_state.abort_transaction() - if not commit_and_start_inevitable: - ex.do(raising_call(aborts, "self.commit_transaction")) + ex.do(raising_call(aborts, "self.commit_transaction")) def op_abort_transaction(ex, global_state, thread_state): trs = thread_state.transaction_state @@ -351,17 +345,13 @@ thread_state.abort_transaction() ex.do('self.abort_transaction()') -def op_become_inevitable(ex, global_state, thread_state, - commit_and_start_inevitable=False): +def op_become_inevitable(ex, global_state, thread_state): trs = thread_state.transaction_state global_state.check_if_can_become_inevitable(trs) thread_state.push_roots(ex) - if commit_and_start_inevitable: - assert not trs.check_must_abort() - else: - ex.do(raising_call(trs.check_must_abort(), - "self.become_inevitable")) + ex.do(raising_call(trs.check_must_abort(), + "self.become_inevitable")) if trs.check_must_abort(): thread_state.abort_transaction() else: @@ -369,16 +359,6 @@ thread_state.pop_roots(ex) thread_state.reload_roots(ex) -def op_commit_and_start_inevitable(ex, global_state, thread_state): - op_become_inevitable(ex, global_state, thread_state) - if thread_state.transaction_state is not None: - op_commit_transaction(ex, global_state, thread_state, - commit_and_start_inevitable=True) - op_start_transaction(ex, global_state, thread_state, - commit_and_start_inevitable=True) - op_become_inevitable(ex, global_state, thread_state, - commit_and_start_inevitable=True) - assert thread_state.transaction_state is not None def op_allocate(ex, global_state, thread_state): size = global_state.rnd.choice([ @@ -604,7 +584,6 @@ op_abort_transaction, op_forget_root, op_become_inevitable, - op_commit_and_start_inevitable, op_assert_size, op_assert_modified, op_minor_collect, From noreply at buildbot.pypy.org Fri Feb 13 19:08:09 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 13 Feb 2015 19:08:09 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Backed out changeset 4d74bdc4535a Message-ID: <20150213180809.7A35B1C1213@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r75862:57c5136e7894 Date: 2015-02-13 19:07 +0100 http://bitbucket.org/pypy/pypy/changeset/57c5136e7894/ Log: Backed out changeset 4d74bdc4535a diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -248,12 +248,16 @@ for box in loop.inputargs: assert isinstance(box, Box) + if rgc.stm_is_enabled(): + rstm.stop_all_other_threads() target_token = loop.operations[-1].getdescr() resumekey.compile_and_attach(metainterp, loop) target_token = label.getdescr() assert isinstance(target_token, TargetToken) record_loop_or_bridge(metainterp_sd, loop) + if rgc.stm_is_enabled(): + rstm.partial_commit_and_resume_other_threads() return target_token @@ -820,9 +824,13 @@ if new_trace.operations[-1].getopnum() != rop.LABEL: # We managed to create a bridge. Dispatch to resumekey to # know exactly what we must do (ResumeGuardDescr/ResumeFromInterpDescr) + if rgc.stm_is_enabled(): + rstm.stop_all_other_threads() target_token = new_trace.operations[-1].getdescr() resumekey.compile_and_attach(metainterp, new_trace) record_loop_or_bridge(metainterp_sd, new_trace) + if rgc.stm_is_enabled(): + rstm.partial_commit_and_resume_other_threads() return target_token else: metainterp.retrace_needed(new_trace) diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py --- a/rpython/rlib/rstm.py +++ b/rpython/rlib/rstm.py @@ -73,9 +73,8 @@ def stop_all_other_threads(): llop.stm_become_globally_unique_transaction(lltype.Void) - at dont_look_inside def partial_commit_and_resume_other_threads(): - llop.stm_commit_and_start_inevitable(lltype.Void) + hint_commit_soon() # for now @specialize.arg(0) def should_break_transaction(keep): diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -429,7 +429,6 @@ 'stm_get_root_stack_top': LLOp(sideeffects=False), 'stm_become_inevitable': LLOp(canmallocgc=True), 'stm_become_globally_unique_transaction': LLOp(canmallocgc=True), - 'stm_commit_and_start_inevitable': LLOp(canmallocgc=True), 'stm_push_root': LLOp(), 'stm_pop_root_into': LLOp(), 'stm_commit_if_not_atomic': LLOp(canmallocgc=True), diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py --- a/rpython/translator/stm/funcgen.py +++ b/rpython/translator/stm/funcgen.py @@ -171,9 +171,6 @@ def stm_become_globally_unique_transaction(funcgen, op): return 'pypy_stm_become_globally_unique_transaction();' -def stm_commit_and_start_inevitable(funcgen, op): - return 'pypy_stm_commit_and_start_inevitable();' - def stm_push_root(funcgen, op): arg0 = funcgen.expr(op.args[0]) return 'STM_PUSH_ROOT(stm_thread_local, %s);' % (arg0,) diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision --- a/rpython/translator/stm/src_stm/revision +++ b/rpython/translator/stm/src_stm/revision @@ -1,1 +1,1 @@ -756a54dd920b +19bb1dde7aca diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c --- a/rpython/translator/stm/src_stm/stm/core.c +++ b/rpython/translator/stm/src_stm/stm/core.c @@ -331,11 +331,10 @@ static void _stm_start_transaction(stm_thread_local_t *tl) { - if (!will_start_inevitable) { - assert(!_stm_in_transaction(tl)); - while (!acquire_thread_segment(tl)) - ; - } + assert(!_stm_in_transaction(tl)); + + while (!acquire_thread_segment(tl)) + ; /* GS invalid before this point! */ assert(STM_PSEGMENT->safe_point == SP_NO_TRANSACTION); @@ -352,12 +351,10 @@ STM_PSEGMENT->shadowstack_at_start_of_transaction = tl->shadowstack; STM_PSEGMENT->threadlocal_at_start_of_transaction = tl->thread_local_obj; - if (!will_start_inevitable) { - enter_safe_point_if_requested(); - dprintf(("start_transaction\n")); + enter_safe_point_if_requested(); + dprintf(("start_transaction\n")); - s_mutex_unlock(); - } + s_mutex_unlock(); /* Now running the SP_RUNNING start. We can set our 'transaction_read_version' after releasing the mutex, @@ -831,8 +828,7 @@ stm_thread_local_t *tl = STM_SEGMENT->running_thread; timing_event(tl, event); - if (!will_start_inevitable) - release_thread_segment(tl); + release_thread_segment(tl); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ } @@ -852,11 +848,6 @@ s_mutex_lock(); - if (will_start_inevitable) { - assert(will_start_inevitable == 1); - assert(will_start_inevitable = 2); /* 1 -> 2, only if !NDEBUG */ - } - restart: /* force all other threads to be paused. They will unpause automatically when we are done here, i.e. at mutex_unlock(). @@ -915,9 +906,6 @@ _finish_transaction(STM_TRANSACTION_COMMIT); /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */ - if (will_start_inevitable) - return; /* hack: return with the mutex still held */ - s_mutex_unlock(); invoke_general_finalizers(tl); @@ -1173,29 +1161,3 @@ synchronize_all_threads(STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE); s_mutex_unlock(); } - -void stm_commit_and_start_inevitable(stm_thread_local_t *tl, const char *msg) -{ - stm_become_inevitable(tl, msg); /* may still abort */ - - /* cannot abort any more from here */ - will_start_inevitable = 1; - - /* as long as 'will_start_inevitable' is true, we cannot release the - mutex_lock or do a cond_wait. We must go through uninterrupted - with all the steps below. - */ - stm_commit_transaction(); - assert(will_start_inevitable == 2); - assert(_has_mutex()); - - _stm_start_transaction(tl); - assert(_has_mutex()); - - /* this line should be the only step from _stm_become_inevitable() we - must do in this case */ - STM_PSEGMENT->transaction_state = TS_INEVITABLE; - - will_start_inevitable = 0; - s_mutex_unlock(); -} diff --git a/rpython/translator/stm/src_stm/stm/sync.c b/rpython/translator/stm/src_stm/stm/sync.c --- a/rpython/translator/stm/src_stm/stm/sync.c +++ b/rpython/translator/stm/src_stm/stm/sync.c @@ -92,7 +92,6 @@ static inline void s_mutex_unlock(void) { assert(_has_mutex_here); - assert(!will_start_inevitable); if (UNLIKELY(pthread_mutex_unlock(&sync_ctl.global_mutex) != 0)) stm_fatalerror("pthread_mutex_unlock: %m"); assert((_has_mutex_here = false, 1)); @@ -105,7 +104,6 @@ #endif assert(_has_mutex_here); - assert(will_start_inevitable < 2); if (UNLIKELY(pthread_cond_wait(&sync_ctl.cond[ctype], &sync_ctl.global_mutex) != 0)) stm_fatalerror("pthread_cond_wait/%d: %m", (int)ctype); diff --git a/rpython/translator/stm/src_stm/stm/sync.h b/rpython/translator/stm/src_stm/stm/sync.h --- a/rpython/translator/stm/src_stm/stm/sync.h +++ b/rpython/translator/stm/src_stm/stm/sync.h @@ -39,6 +39,5 @@ static void committed_globally_unique_transaction(void); static bool pause_signalled, globally_unique_transaction; -static uint8_t will_start_inevitable; void signal_other_to_commit_soon(struct stm_priv_segment_info_s *other_pseg); diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h --- a/rpython/translator/stm/src_stm/stmgc.h +++ b/rpython/translator/stm/src_stm/stmgc.h @@ -409,15 +409,6 @@ void stm_become_globally_unique_transaction(stm_thread_local_t *tl, const char *msg); -/* Commit and start the next transaction as inevitable. Provided the - commit succeeds, the next transaction is immediately made - inevitable: no other transaction can commit (or turn inevitable) - between the two steps. The only reason to use this function - instead of stm_become_inevitable() is that the forced commit ends a - globally unique transaction. If there is already another inevitable - transaction, this function will come stm_become_inevitable() first. */ -void stm_commit_and_start_inevitable(stm_thread_local_t *tl, const char *msg); - /* Profiling events. In the comments: content of the markers, if any */ enum stm_event_e { diff --git a/rpython/translator/stm/src_stm/stmgcintf.c b/rpython/translator/stm/src_stm/stmgcintf.c --- a/rpython/translator/stm/src_stm/stmgcintf.c +++ b/rpython/translator/stm/src_stm/stmgcintf.c @@ -222,11 +222,6 @@ stm_become_globally_unique_transaction(&stm_thread_local, "for the JIT"); } -void pypy_stm_commit_and_start_inevitable(void) -{ - stm_commit_and_start_inevitable(&stm_thread_local, "for the JIT"); -} - long _pypy_stm_count(void) { static long value = 1; diff --git a/rpython/translator/stm/src_stm/stmgcintf.h b/rpython/translator/stm/src_stm/stmgcintf.h --- a/rpython/translator/stm/src_stm/stmgcintf.h +++ b/rpython/translator/stm/src_stm/stmgcintf.h @@ -30,7 +30,6 @@ void _pypy_stm_become_inevitable(const char *); void pypy_stm_become_globally_unique_transaction(void); -void pypy_stm_commit_and_start_inevitable(void); char *_pypy_stm_test_expand_marker(void); void pypy_stm_setup_expand_marker(long co_filename_ofs, From noreply at buildbot.pypy.org Sat Feb 14 11:18:07 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 14 Feb 2015 11:18:07 +0100 (CET) Subject: [pypy-commit] pypy default: issue #1983: test and fix Message-ID: <20150214101807.098D91C029E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75863:bc0fa15c04e1 Date: 2015-02-14 11:18 +0100 http://bitbucket.org/pypy/pypy/changeset/bc0fa15c04e1/ Log: issue #1983: test and fix diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -790,7 +790,8 @@ eqbox = self.implement_guard_value(eqbox, pc) isstandard = eqbox.getint() if isstandard: - self.metainterp.replace_box(box, standard_box) + if isinstance(box, history.BoxPtr): + self.metainterp.replace_box(box, standard_box) return False if not self.metainterp.heapcache.is_unescaped(box): self.emit_force_virtualizable(fielddescr, box) diff --git a/rpython/jit/metainterp/test/test_virtualizable.py b/rpython/jit/metainterp/test/test_virtualizable.py --- a/rpython/jit/metainterp/test/test_virtualizable.py +++ b/rpython/jit/metainterp/test/test_virtualizable.py @@ -1682,6 +1682,25 @@ res = self.meta_interp(f, [], listops=True) assert res == 0 + def test_constant_virtualizable(self): + class A: + _virtualizable_ = ['x'] + def __init__(self, x): + self.x = x + + driver = JitDriver(greens=['b'], reds=['a'], virtualizables=['a']) + + def f(): + a = A(10) + b = promote(a) + while a.x > 0: + driver.jit_merge_point(a=a, b=b) + a.x = b.x - 1 + return a.x + + res = self.meta_interp(f, [], listops=True) + assert res == 0 + class TestLLtype(ExplicitVirtualizableTests, ImplicitVirtualizableTests, From noreply at buildbot.pypy.org Sat Feb 14 12:39:41 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 14 Feb 2015 12:39:41 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.9: charmapdecode: pass all consecutive invalid chars to the error handler. Message-ID: <20150214113941.06E131C050C@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: stdlib-2.7.9 Changeset: r75864:3599c828e5f7 Date: 2015-02-14 11:39 +0100 http://bitbucket.org/pypy/pypy/changeset/3599c828e5f7/ Log: charmapdecode: pass all consecutive invalid chars to the error handler. This also ensure that on narrow builds, surrogate pairs are not split. Should fix test_codeccallbacks on win32. diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -1114,9 +1114,13 @@ c = mapping.get(ch, '') if len(c) == 0: + # collect all unencodable chars. Important for narrow builds. + collend = pos + 1 + while collend < size and mapping.get(s[collend], '') == '': + collend += 1 ru, rs, pos = errorhandler(errors, "charmap", "character maps to ", - s, pos, pos + 1) + s, pos, collend) if rs is not None: # py3k only result.append(rs) diff --git a/rpython/rlib/test/test_runicode.py b/rpython/rlib/test/test_runicode.py --- a/rpython/rlib/test/test_runicode.py +++ b/rpython/rlib/test/test_runicode.py @@ -118,6 +118,17 @@ if addstuff: assert result.endswith(u"some rest in ascii") + def test_charmap_encodeerror(self): + def errorhandler(errors, enc, msg, t, startingpos, + endingpos): + assert t[startingpos:endingpos] == u'\t\n \r' + return None, ' ', endingpos + s = u'aa\t\n \raa' + mapping = {u'a': 'a'} + r = runicode.unicode_encode_charmap(s, len(s), None, errorhandler, + mapping=mapping) + assert r == 'aa aa' + class TestDecoding(UnicodeTests): # XXX test bom recognition in utf-16 From noreply at buildbot.pypy.org Sat Feb 14 12:53:02 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 14 Feb 2015 12:53:02 +0100 (CET) Subject: [pypy-commit] pypy py3k: Fix a test, like in 2.7 branch Message-ID: <20150214115303.00C451C0F98@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r75865:dbd11cc004f0 Date: 2015-02-11 01:13 +0100 http://bitbucket.org/pypy/pypy/changeset/dbd11cc004f0/ Log: Fix a test, like in 2.7 branch diff --git a/lib-python/3/test/test_collections.py b/lib-python/3/test/test_collections.py --- a/lib-python/3/test/test_collections.py +++ b/lib-python/3/test/test_collections.py @@ -635,7 +635,12 @@ def __repr__(self): return "MySet(%s)" % repr(list(self)) s = MySet([5,43,2,1]) - self.assertEqual(s.pop(), 1) + # changed from CPython: it was "s.pop() == 1" but I see + # nothing that guarantees a particular order here. In the + # 'all_ordered_dicts' branch of PyPy (or with OrderedDict + # instead of sets), it consistently returns 5, but this test + # should not rely on this or any other order. + self.assert_(s.pop() in [5,43,2,1]) def test_issue8750(self): empty = WithSet() From noreply at buildbot.pypy.org Sat Feb 14 12:53:04 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 14 Feb 2015 12:53:04 +0100 (CET) Subject: [pypy-commit] pypy py3k: Workaround for a real libdb bug. Message-ID: <20150214115304.3E5661C0F98@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r75866:6226af3cba30 Date: 2015-02-14 12:52 +0100 http://bitbucket.org/pypy/pypy/changeset/6226af3cba30/ Log: Workaround for a real libdb bug. Uncovered by pypy because dictionaries are now ordered, and the first exercised key corresponds to an empty value. diff --git a/lib-python/3/test/test_dbm.py b/lib-python/3/test/test_dbm.py --- a/lib-python/3/test/test_dbm.py +++ b/lib-python/3/test/test_dbm.py @@ -106,6 +106,8 @@ f.close() def read_helper(self, f): + f['a'] # Work around a bug in BerkeleyDB: + # https://bugs.launchpad.net/ubuntu/+source/db5.3/+bug/1421223 keys = self.keys_helper(f) for key in self._dict: self.assertEqual(self._dict[key], f[key.encode("ascii")]) From noreply at buildbot.pypy.org Sat Feb 14 13:13:16 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 14 Feb 2015 13:13:16 +0100 (CET) Subject: [pypy-commit] pypy default: Add buffer._pypy_raw_address to memoryview as well. Message-ID: <20150214121316.48DC31C009F@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r75867:7063d1a5121f Date: 2015-02-14 13:11 +0100 http://bitbucket.org/pypy/pypy/changeset/7063d1a5121f/ Log: Add buffer._pypy_raw_address to memoryview as well. This will help python3... diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py --- a/pypy/module/_rawffi/test/test__rawffi.py +++ b/pypy/module/_rawffi/test/test__rawffi.py @@ -1153,6 +1153,11 @@ assert buffer(s)._pypy_raw_address() == addr assert buffer(s, 10)._pypy_raw_address() == addr + 10 + addr = memoryview(s)._pypy_raw_address() + assert type(addr) is int + assert memoryview(s)._pypy_raw_address() == addr + assert memoryview(s)[10:]._pypy_raw_address() == addr + 10 + def test_union(self): import _rawffi longsize = _rawffi.sizeof('l') diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py --- a/pypy/objspace/std/memoryobject.py +++ b/pypy/objspace/std/memoryobject.py @@ -121,6 +121,17 @@ # I've never seen anyone filling this field return space.w_None + def descr_pypy_raw_address(self, space): + from rpython.rtyper.lltypesystem import lltype, rffi + try: + ptr = self.buf.get_raw_address() + except ValueError: + # report the error using the RPython-level internal repr of self.buf + msg = ("cannot find the underlying address of buffer that " + "is internally %r" % (self.buf,)) + raise OperationError(space.w_ValueError, space.wrap(msg)) + return space.wrap(rffi.cast(lltype.Signed, ptr)) + W_MemoryView.typedef = TypeDef( "memoryview", __doc__ = """\ @@ -145,5 +156,6 @@ shape = GetSetProperty(W_MemoryView.w_get_shape), strides = GetSetProperty(W_MemoryView.w_get_strides), suboffsets = GetSetProperty(W_MemoryView.w_get_suboffsets), + _pypy_raw_address = interp2app(W_MemoryView.descr_pypy_raw_address), ) W_MemoryView.typedef.acceptable_as_base_class = False diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py --- a/pypy/objspace/std/test/test_memoryobject.py +++ b/pypy/objspace/std/test/test_memoryobject.py @@ -54,3 +54,8 @@ assert memoryview("abc") != u"abc" assert not u"abc" == memoryview("abc") assert u"abc" != memoryview("abc") + + def test_pypy_raw_address_base(self): + raises(ValueError, memoryview("foobar")._pypy_raw_address) + e = raises(ValueError, memoryview(bytearray("foobar"))._pypy_raw_address) + assert 'BytearrayBuffer' in str(e.value) From noreply at buildbot.pypy.org Sat Feb 14 14:14:45 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 14 Feb 2015 14:14:45 +0100 (CET) Subject: [pypy-commit] pypy py3k: Ignore difference in exception message Message-ID: <20150214131445.44FC41C009F@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r75868:6f256d6cb9c2 Date: 2015-02-14 13:01 +0100 http://bitbucket.org/pypy/pypy/changeset/6f256d6cb9c2/ Log: Ignore difference in exception message diff --git a/lib-python/3/test/test_xml_etree.py b/lib-python/3/test/test_xml_etree.py --- a/lib-python/3/test/test_xml_etree.py +++ b/lib-python/3/test/test_xml_etree.py @@ -241,7 +241,7 @@ >>> element.remove(subelement) >>> serialize(element) # 5 '' - >>> element.remove(subelement) + >>> element.remove(subelement) # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ValueError: list.remove(x): x not in list >>> serialize(element) # 6 From noreply at buildbot.pypy.org Sat Feb 14 14:14:47 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 14 Feb 2015 14:14:47 +0100 (CET) Subject: [pypy-commit] pypy py3k: hg merge default Message-ID: <20150214131447.C8C321C009F@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r75869:2f2ec61d2d55 Date: 2015-02-14 13:31 +0100 http://bitbucket.org/pypy/pypy/changeset/2f2ec61d2d55/ Log: hg merge default diff too long, truncating to 2000 out of 3186 lines diff --git a/lib-python/2.7/test/test_audioop.py b/lib-python/2.7/test/test_audioop.py --- a/lib-python/2.7/test/test_audioop.py +++ b/lib-python/2.7/test/test_audioop.py @@ -2,7 +2,7 @@ import sys import unittest import struct -from test.test_support import run_unittest, impl_detail +from test.test_support import run_unittest formats = { @@ -183,7 +183,6 @@ self.assertEqual(audioop.lin2lin(datas[4], 4, 2), packs[2](0, 0x1234, 0x4567, -0x4568, 0x7fff, -0x8000, -1)) - @impl_detail(pypy=False) def test_adpcm2lin(self): self.assertEqual(audioop.adpcm2lin(b'\x07\x7f\x7f', 1, None), (b'\x00\x00\x00\xff\x00\xff', (-179, 40))) @@ -198,7 +197,6 @@ self.assertEqual(audioop.adpcm2lin(b'\0' * 5, w, None), (b'\0' * w * 10, (0, 0))) - @impl_detail(pypy=False) def test_lin2adpcm(self): self.assertEqual(audioop.lin2adpcm(datas[1], 1, None), (b'\x07\x7f\x7f', (-221, 39))) @@ -212,7 +210,6 @@ self.assertEqual(audioop.lin2adpcm(b'\0' * w * 10, w, None), (b'\0' * 5, (0, 0))) - @impl_detail(pypy=False) def test_lin2alaw(self): self.assertEqual(audioop.lin2alaw(datas[1], 1), b'\xd5\x87\xa4\x24\xaa\x2a\x5a') @@ -221,7 +218,6 @@ self.assertEqual(audioop.lin2alaw(datas[4], 4), b'\xd5\x87\xa4\x24\xaa\x2a\x55') - @impl_detail(pypy=False) def test_alaw2lin(self): encoded = b'\x00\x03\x24\x2a\x51\x54\x55\x58\x6b\x71\x7f'\ b'\x80\x83\xa4\xaa\xd1\xd4\xd5\xd8\xeb\xf1\xff' @@ -236,7 +232,6 @@ decoded = audioop.alaw2lin(encoded, w) self.assertEqual(audioop.lin2alaw(decoded, w), encoded) - @impl_detail(pypy=False) def test_lin2ulaw(self): self.assertEqual(audioop.lin2ulaw(datas[1], 1), b'\xff\xad\x8e\x0e\x80\x00\x67') @@ -245,7 +240,6 @@ self.assertEqual(audioop.lin2ulaw(datas[4], 4), b'\xff\xad\x8e\x0e\x80\x00\x7e') - @impl_detail(pypy=False) def test_ulaw2lin(self): encoded = b'\x00\x0e\x28\x3f\x57\x6a\x76\x7c\x7e\x7f'\ b'\x80\x8e\xa8\xbf\xd7\xea\xf6\xfc\xfe\xff' @@ -360,7 +354,6 @@ self.assertRaises(audioop.error, audioop.findmax, ''.join( chr(x) for x in xrange(256)), -2392392) - @impl_detail(pypy=False) def test_issue7673(self): state = None for data, size in INVALID_DATA: @@ -385,7 +378,6 @@ self.assertRaises(audioop.error, audioop.lin2alaw, data, size) self.assertRaises(audioop.error, audioop.lin2adpcm, data, size, state) - @impl_detail(pypy=False) def test_wrongsize(self): data = b'abcdefgh' state = None diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py --- a/lib_pypy/_ctypes/array.py +++ b/lib_pypy/_ctypes/array.py @@ -11,7 +11,7 @@ if cls == (_CData,): # this is the Array class defined below return res - ffiarray = res._ffiarray = _rawffi.Array(res._type_._ffishape) + ffiarray = res._ffiarray = _rawffi.Array(res._type_._ffishape_) subletter = getattr(res._type_, '_type_', None) if subletter == 'c': def getvalue(self): @@ -58,8 +58,8 @@ target[len(val)] = '\x00' res.value = property(getvalue, setvalue) - res._ffishape = (ffiarray, res._length_) - res._fficompositesize = res._sizeofinstances() + res._ffishape_ = (ffiarray, res._length_) + res._fficompositesize_ = res._sizeofinstances() return res from_address = cdata_from_address @@ -154,7 +154,7 @@ return l class Array(_CData, metaclass=ArrayMeta): - _ffiargshape = 'P' + _ffiargshape_ = 'P' def __init__(self, *args): if not hasattr(self, '_buffer'): @@ -189,13 +189,13 @@ if ensure_objects(cobj) is not None: store_reference(self, index, cobj._objects) arg = cobj._get_buffer_value() - if self._type_._fficompositesize is None: + if self._type_._fficompositesize_ is None: self._buffer[index] = arg # something more sophisticated, cannot set field directly else: from ctypes import memmove dest = self._buffer.itemaddress(index) - memmove(dest, arg, self._type_._fficompositesize) + memmove(dest, arg, self._type_._fficompositesize_) def __getitem__(self, index): if isinstance(index, slice): diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py --- a/lib_pypy/_ctypes/basics.py +++ b/lib_pypy/_ctypes/basics.py @@ -52,7 +52,7 @@ def get_ffi_argtype(self): if self._ffiargtype: return self._ffiargtype - self._ffiargtype = _shape_to_ffi_type(self._ffiargshape) + self._ffiargtype = _shape_to_ffi_type(self._ffiargshape_) return self._ffiargtype def _CData_output(self, resbuffer, base=None, index=-1): diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py --- a/lib_pypy/_ctypes/function.py +++ b/lib_pypy/_ctypes/function.py @@ -64,9 +64,9 @@ _restype_ = None _errcheck_ = None _flags_ = 0 - _ffiargshape = 'P' - _ffishape = 'P' - _fficompositesize = None + _ffiargshape_ = 'P' + _ffishape_ = 'P' + _fficompositesize_ = None _ffiarray = _rawffi.Array('P') _needs_free = False callable = None @@ -97,7 +97,7 @@ argtypes = property(_getargtypes, _setargtypes) def _check_argtypes_for_fastpath(self): - if all([hasattr(argtype, '_ffiargshape') for argtype in self._argtypes_]): + if all([hasattr(argtype, '_ffiargshape_') for argtype in self._argtypes_]): fastpath_cls = make_fastpath_subclass(self.__class__) fastpath_cls.enable_fastpath_maybe(self) @@ -134,7 +134,7 @@ _flag = flag & PARAMFLAG_COMBINED if _flag == PARAMFLAG_FOUT: typ = self._argtypes_[idx] - if getattr(typ, '_ffiargshape', None) not in ('P', 'z', 'Z'): + if getattr(typ, '_ffiargshape_', None) not in ('P', 'z', 'Z'): raise TypeError( "'out' parameter %d must be a pointer type, not %s" % (idx+1, type(typ).__name__) @@ -181,11 +181,11 @@ def _ffishapes(self, args, restype): if args is None: args = [] - argtypes = [arg._ffiargshape for arg in args] + argtypes = [arg._ffiargshape_ for arg in args] if restype is not None: if not isinstance(restype, SimpleType): raise TypeError("invalid result type for callback function") - restype = restype._ffiargshape + restype = restype._ffiargshape_ else: restype = 'O' # void return argtypes, restype @@ -599,7 +599,7 @@ if self._is_primitive(restype) and not restype._is_pointer_like(): return result # - shape = restype._ffishape + shape = restype._ffishape_ if is_struct_shape(shape): buf = result else: diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py --- a/lib_pypy/_ctypes/pointer.py +++ b/lib_pypy/_ctypes/pointer.py @@ -21,9 +21,9 @@ size = _rawffi.sizeof('P'), align = _rawffi.alignment('P'), length = 1, - _ffiargshape = 'P', - _ffishape = 'P', - _fficompositesize = None, + _ffiargshape_ = 'P', + _ffishape_ = 'P', + _fficompositesize_ = None, ) # XXX check if typedict['_type_'] is any sane # XXX remember about paramfunc diff --git a/lib_pypy/_ctypes/primitive.py b/lib_pypy/_ctypes/primitive.py --- a/lib_pypy/_ctypes/primitive.py +++ b/lib_pypy/_ctypes/primitive.py @@ -117,9 +117,9 @@ default = TP_TO_DEFAULT[tp] ffiarray = _rawffi.Array(tp) result = type.__new__(self, name, bases, dct) - result._ffiargshape = tp - result._ffishape = tp - result._fficompositesize = None + result._ffiargshape_ = tp + result._ffishape_ = tp + result._fficompositesize_ = None result._ffiarray = ffiarray if tp == 'z': # c_char_p diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -36,9 +36,9 @@ rawfields = [] for f in all_fields: if len(f) > 2: - rawfields.append((f[0], f[1]._ffishape, f[2])) + rawfields.append((f[0], f[1]._ffishape_, f[2])) else: - rawfields.append((f[0], f[1]._ffishape)) + rawfields.append((f[0], f[1]._ffishape_)) _set_shape(self, rawfields, self._is_union) @@ -48,8 +48,8 @@ value = field[1] is_bitfield = (len(field) == 3) fields[name] = Field(name, - self._ffistruct.fieldoffset(name), - self._ffistruct.fieldsize(name), + self._ffistruct_.fieldoffset(name), + self._ffistruct_.fieldsize(name), value, i, is_bitfield) if anonymous_fields: @@ -58,9 +58,9 @@ name = field[0] value = field[1] is_bitfield = (len(field) == 3) - startpos = self._ffistruct.fieldoffset(name) + startpos = self._ffistruct_.fieldoffset(name) if name in anonymous_fields: - for subname in value._names: + for subname in value._names_: resnames.append(subname) subfield = getattr(value, subname) relpos = startpos + subfield.offset @@ -71,7 +71,7 @@ else: resnames.append(name) names = resnames - self._names = names + self._names_ = names for name, field in fields.items(): setattr(self, name, field) @@ -114,19 +114,19 @@ elif ensure_objects(cobj) is not None: store_reference(obj, key, cobj._objects) arg = cobj._get_buffer_value() - if fieldtype._fficompositesize is not None: + if fieldtype._fficompositesize_ is not None: from ctypes import memmove dest = obj._buffer.fieldaddress(self.name) - memmove(dest, arg, fieldtype._fficompositesize) + memmove(dest, arg, fieldtype._fficompositesize_) else: obj._buffer.__setattr__(self.name, arg) def _set_shape(tp, rawfields, is_union=False): - tp._ffistruct = _rawffi.Structure(rawfields, is_union, + tp._ffistruct_ = _rawffi.Structure(rawfields, is_union, getattr(tp, '_pack_', 0)) - tp._ffiargshape = tp._ffishape = (tp._ffistruct, 1) - tp._fficompositesize = tp._ffistruct.size + tp._ffiargshape_ = tp._ffishape_ = (tp._ffistruct_, 1) + tp._fficompositesize_ = tp._ffistruct_.size def struct_setattr(self, name, value): @@ -181,16 +181,16 @@ address = address.buffer # fix the address: turn it into as unsigned, in case it is negative address = address & (sys.maxsize * 2 + 1) - instance.__dict__['_buffer'] = self._ffistruct.fromaddress(address) + instance.__dict__['_buffer'] = self._ffistruct_.fromaddress(address) return instance def _sizeofinstances(self): - if not hasattr(self, '_ffistruct'): + if not hasattr(self, '_ffistruct_'): return 0 - return self._ffistruct.size + return self._ffistruct_.size def _alignmentofinstances(self): - return self._ffistruct.alignment + return self._ffistruct_.alignment def from_param(self, value): if isinstance(value, tuple): @@ -203,7 +203,7 @@ def _CData_output(self, resarray, base=None, index=-1): res = StructOrUnion.__new__(self) - ffistruct = self._ffistruct.fromaddress(resarray.buffer) + ffistruct = self._ffistruct_.fromaddress(resarray.buffer) res.__dict__['_buffer'] = ffistruct res.__dict__['_base'] = base res.__dict__['_index'] = index @@ -223,15 +223,15 @@ self = super(_CData, cls).__new__(cls, *args, **kwds) if '_abstract_' in cls.__dict__: raise TypeError("abstract class") - if hasattr(cls, '_ffistruct'): - self.__dict__['_buffer'] = self._ffistruct(autofree=True) + if hasattr(cls, '_ffistruct_'): + self.__dict__['_buffer'] = self._ffistruct_(autofree=True) return self def __init__(self, *args, **kwds): type(self)._make_final() - if len(args) > len(self._names): + if len(args) > len(self._names_): raise TypeError("too many initializers") - for name, arg in zip(self._names, args): + for name, arg in zip(self._names_, args): if name in kwds: raise TypeError("duplicate value for argument %r" % ( name,)) @@ -243,7 +243,7 @@ """Return a _rawffi array of length 1 whose address is the same as the address of the field 'name' of self.""" address = self._buffer.fieldaddress(name) - A = _rawffi.Array(fieldtype._ffishape) + A = _rawffi.Array(fieldtype._ffishape_) return A.fromaddress(address, 1) def _get_buffer_for_param(self): diff --git a/lib_pypy/_gdbm.py b/lib_pypy/_gdbm.py --- a/lib_pypy/_gdbm.py +++ b/lib_pypy/_gdbm.py @@ -1,5 +1,4 @@ -import cffi, os -import sys +import cffi, os, sys ffi = cffi.FFI() ffi.cdef(''' @@ -38,9 +37,19 @@ ''') try: - lib = ffi.verify(''' - #include "gdbm.h" - ''', libraries=['gdbm']) + if sys.platform.startswith('freebsd'): + import os.path + _localbase = os.environ.get('LOCALBASE', '/usr/local') + lib = ffi.verify(''' + #include "gdbm.h" + ''', libraries=['gdbm'], + include_dirs=[os.path.join(_localbase, 'include')], + library_dirs=[os.path.join(_localbase, 'lib')] + ) + else: + lib = ffi.verify(''' + #include "gdbm.h" + ''', libraries=['gdbm']) except cffi.VerificationError as e: # distutils does not preserve the actual message, # but the verification is simple enough that the diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -270,11 +270,14 @@ _ffi.cdef("int sqlite3_enable_load_extension(sqlite3 *db, int onoff);") if sys.platform.startswith('freebsd'): + import os + import os.path + _localbase = os.environ.get('LOCALBASE', '/usr/local') _lib = _ffi.verify(""" #include """, libraries=['sqlite3'], - include_dirs=['/usr/local/include'], - library_dirs=['/usr/local/lib'] + include_dirs=[os.path.join(_localbase, 'include')], + library_dirs=[os.path.join(_localbase, 'lib')] ) else: _lib = _ffi.verify(""" diff --git a/lib_pypy/audioop.py b/lib_pypy/audioop.py --- a/lib_pypy/audioop.py +++ b/lib_pypy/audioop.py @@ -2,7 +2,7 @@ import math import struct from fractions import gcd -from ctypes import create_string_buffer +from cffi import FFI _buffer = memoryview @@ -148,7 +148,7 @@ def _sum2(cp1, cp2, length): size = 2 return sum(getsample(cp1, size, i) * getsample(cp2, size, i) - for i in range(length)) + for i in range(length)) + 0.0 def findfit(cp1, cp2): @@ -327,13 +327,14 @@ _check_params(len(cp), size) clip = _get_clipfn(size) - result = create_string_buffer(len(cp)) + rv = ffi.new("unsigned char[]", len(cp)) + result = ffi.buffer(rv) for i, sample in enumerate(_get_samples(cp, size)): sample = clip(int(sample * factor)) _put_sample(result, size, i, sample) - return result.raw + return result[:] def tomono(cp, size, fac1, fac2): @@ -342,7 +343,8 @@ sample_count = _sample_count(cp, size) - result = create_string_buffer(len(cp) // 2) + rv = ffi.new("unsigned char[]", len(cp) // 2) + result = ffi.buffer(rv) for i in range(0, sample_count, 2): l_sample = getsample(cp, size, i) @@ -353,7 +355,7 @@ _put_sample(result, size, i // 2, sample) - return result.raw + return result[:] def tostereo(cp, size, fac1, fac2): @@ -361,19 +363,9 @@ sample_count = _sample_count(cp, size) - result = create_string_buffer(len(cp) * 2) - clip = _get_clipfn(size) - - for i in range(sample_count): - sample = _get_sample(cp, size, i) - - l_sample = clip(sample * fac1) - r_sample = clip(sample * fac2) - - _put_sample(result, size, i * 2, l_sample) - _put_sample(result, size, i * 2 + 1, r_sample) - - return result.raw + rv = ffi.new("unsigned char[]", len(cp) * 2) + lib.tostereo(rv, cp, len(cp), size, fac1, fac2) + return ffi.buffer(rv)[:] def add(cp1, cp2, size): @@ -382,42 +374,34 @@ if len(cp1) != len(cp2): raise error("Lengths should be the same") - clip = _get_clipfn(size) - sample_count = _sample_count(cp1, size) - result = create_string_buffer(len(cp1)) - - for i in range(sample_count): - sample1 = getsample(cp1, size, i) - sample2 = getsample(cp2, size, i) - - sample = clip(sample1 + sample2) - - _put_sample(result, size, i, sample) - - return result.raw + rv = ffi.new("unsigned char[]", len(cp1)) + lib.add(rv, cp1, cp2, len(cp1), size) + return ffi.buffer(rv)[:] def bias(cp, size, bias): _check_params(len(cp), size) - result = create_string_buffer(len(cp)) + rv = ffi.new("unsigned char[]", len(cp)) + result = ffi.buffer(rv) for i, sample in enumerate(_get_samples(cp, size)): sample = _overflow(sample + bias, size) _put_sample(result, size, i, sample) - return result.raw + return result[:] def reverse(cp, size): _check_params(len(cp), size) sample_count = _sample_count(cp, size) - result = create_string_buffer(len(cp)) + rv = ffi.new("unsigned char[]", len(cp)) + result = ffi.buffer(rv) for i, sample in enumerate(_get_samples(cp, size)): _put_sample(result, size, sample_count - i - 1, sample) - return result.raw + return result[:] def lin2lin(cp, size, size2): @@ -428,7 +412,8 @@ return cp new_len = (len(cp) // size) * size2 - result = create_string_buffer(new_len) + rv = ffi.new("unsigned char[]", new_len) + result = ffi.buffer(rv) for i in range(_sample_count(cp, size)): sample = _get_sample(cp, size, i) @@ -443,7 +428,7 @@ sample = _overflow(sample, size2) _put_sample(result, size2, i, sample) - return result.raw + return result[:] def ratecv(cp, size, nchannels, inrate, outrate, state, weightA=1, weightB=0): @@ -470,11 +455,10 @@ inrate //= d outrate //= d - prev_i = [0] * nchannels - cur_i = [0] * nchannels - if state is None: d = -outrate + prev_i = ffi.new('int[]', nchannels) + cur_i = ffi.new('int[]', nchannels) else: d, samps = state @@ -482,70 +466,709 @@ raise error("illegal state argument") prev_i, cur_i = zip(*samps) - prev_i, cur_i = list(prev_i), list(cur_i) + prev_i = ffi.new('int[]', prev_i) + cur_i = ffi.new('int[]', cur_i) + state_d = ffi.new('int[]', (d,)) q = frame_count // inrate ceiling = (q + 1) * outrate nbytes = ceiling * bytes_per_frame - result = create_string_buffer(nbytes) + rv = ffi.new("unsigned char[]", nbytes) + trim_index = lib.ratecv(rv, cp, frame_count, size, + nchannels, inrate, outrate, + state_d, prev_i, cur_i, + weightA, weightB) + result = ffi.buffer(rv)[:trim_index] + samps = zip(prev_i, cur_i) + return (result, (d, tuple(samps))) - samples = _get_samples(cp, size) - out_i = 0 - while True: - while d < 0: - if frame_count == 0: - samps = zip(prev_i, cur_i) - retval = result.raw - # slice off extra bytes - trim_index = (out_i * bytes_per_frame) - len(retval) - retval = retval[:trim_index] +ffi = FFI() +ffi.cdef(""" +typedef short PyInt16; - return (retval, (d, tuple(samps))) +int ratecv(char* rv, char* cp, size_t len, int size, + int nchannels, int inrate, int outrate, + int* state_d, int* prev_i, int* cur_i, + int weightA, int weightB); - for chan in range(nchannels): - prev_i[chan] = cur_i[chan] - cur_i[chan] = next(samples) +void tostereo(char* rv, char* cp, size_t len, int size, + double fac1, double fac2); +void add(char* rv, char* cp1, char* cp2, size_t len1, int size); - cur_i[chan] = ( - (weightA * cur_i[chan] + weightB * prev_i[chan]) - // (weightA + weightB) - ) +/* 2's complement (14-bit range) */ +unsigned char +st_14linear2ulaw(PyInt16 pcm_val); +PyInt16 st_ulaw2linear16(unsigned char); - frame_count -= 1 - d += outrate +/* 2's complement (13-bit range) */ +unsigned char +st_linear2alaw(PyInt16 pcm_val); +PyInt16 st_alaw2linear16(unsigned char); - while d >= 0: - for chan in range(nchannels): - cur_o = ( - (prev_i[chan] * d + cur_i[chan] * (outrate - d)) - // outrate - ) - _put_sample(result, size, out_i, _overflow(cur_o, size)) - out_i += 1 - d -= inrate +void lin2adcpm(unsigned char* rv, unsigned char* cp, size_t len, + size_t size, int* state); +void adcpm2lin(unsigned char* rv, unsigned char* cp, size_t len, + size_t size, int* state); +""") + +# This code is directly copied from CPython file: Modules/audioop.c +_AUDIOOP_C_MODULE = """ +typedef short PyInt16; +typedef int Py_Int32; + +/* Code shamelessly stolen from sox, 12.17.7, g711.c +** (c) Craig Reese, Joe Campbell and Jeff Poskanzer 1989 */ + +/* From g711.c: + * + * December 30, 1994: + * Functions linear2alaw, linear2ulaw have been updated to correctly + * convert unquantized 16 bit values. + * Tables for direct u- to A-law and A- to u-law conversions have been + * corrected. + * Borge Lindberg, Center for PersonKommunikation, Aalborg University. + * bli at cpk.auc.dk + * + */ +#define BIAS 0x84 /* define the add-in bias for 16 bit samples */ +#define CLIP 32635 +#define SIGN_BIT (0x80) /* Sign bit for a A-law byte. */ +#define QUANT_MASK (0xf) /* Quantization field mask. */ +#define SEG_SHIFT (4) /* Left shift for segment number. */ +#define SEG_MASK (0x70) /* Segment field mask. */ + +static PyInt16 seg_aend[8] = {0x1F, 0x3F, 0x7F, 0xFF, + 0x1FF, 0x3FF, 0x7FF, 0xFFF}; +static PyInt16 seg_uend[8] = {0x3F, 0x7F, 0xFF, 0x1FF, + 0x3FF, 0x7FF, 0xFFF, 0x1FFF}; + +static PyInt16 +search(PyInt16 val, PyInt16 *table, int size) +{ + int i; + + for (i = 0; i < size; i++) { + if (val <= *table++) + return (i); + } + return (size); +} +#define st_ulaw2linear16(uc) (_st_ulaw2linear16[uc]) +#define st_alaw2linear16(uc) (_st_alaw2linear16[uc]) + +static PyInt16 _st_ulaw2linear16[256] = { + -32124, -31100, -30076, -29052, -28028, -27004, -25980, + -24956, -23932, -22908, -21884, -20860, -19836, -18812, + -17788, -16764, -15996, -15484, -14972, -14460, -13948, + -13436, -12924, -12412, -11900, -11388, -10876, -10364, + -9852, -9340, -8828, -8316, -7932, -7676, -7420, + -7164, -6908, -6652, -6396, -6140, -5884, -5628, + -5372, -5116, -4860, -4604, -4348, -4092, -3900, + -3772, -3644, -3516, -3388, -3260, -3132, -3004, + -2876, -2748, -2620, -2492, -2364, -2236, -2108, + -1980, -1884, -1820, -1756, -1692, -1628, -1564, + -1500, -1436, -1372, -1308, -1244, -1180, -1116, + -1052, -988, -924, -876, -844, -812, -780, + -748, -716, -684, -652, -620, -588, -556, + -524, -492, -460, -428, -396, -372, -356, + -340, -324, -308, -292, -276, -260, -244, + -228, -212, -196, -180, -164, -148, -132, + -120, -112, -104, -96, -88, -80, -72, + -64, -56, -48, -40, -32, -24, -16, + -8, 0, 32124, 31100, 30076, 29052, 28028, + 27004, 25980, 24956, 23932, 22908, 21884, 20860, + 19836, 18812, 17788, 16764, 15996, 15484, 14972, + 14460, 13948, 13436, 12924, 12412, 11900, 11388, + 10876, 10364, 9852, 9340, 8828, 8316, 7932, + 7676, 7420, 7164, 6908, 6652, 6396, 6140, + 5884, 5628, 5372, 5116, 4860, 4604, 4348, + 4092, 3900, 3772, 3644, 3516, 3388, 3260, + 3132, 3004, 2876, 2748, 2620, 2492, 2364, + 2236, 2108, 1980, 1884, 1820, 1756, 1692, + 1628, 1564, 1500, 1436, 1372, 1308, 1244, + 1180, 1116, 1052, 988, 924, 876, 844, + 812, 780, 748, 716, 684, 652, 620, + 588, 556, 524, 492, 460, 428, 396, + 372, 356, 340, 324, 308, 292, 276, + 260, 244, 228, 212, 196, 180, 164, + 148, 132, 120, 112, 104, 96, 88, + 80, 72, 64, 56, 48, 40, 32, + 24, 16, 8, 0 +}; + +/* + * linear2ulaw() accepts a 14-bit signed integer and encodes it as u-law data + * stored in a unsigned char. This function should only be called with + * the data shifted such that it only contains information in the lower + * 14-bits. + * + * In order to simplify the encoding process, the original linear magnitude + * is biased by adding 33 which shifts the encoding range from (0 - 8158) to + * (33 - 8191). The result can be seen in the following encoding table: + * + * Biased Linear Input Code Compressed Code + * ------------------------ --------------- + * 00000001wxyza 000wxyz + * 0000001wxyzab 001wxyz + * 000001wxyzabc 010wxyz + * 00001wxyzabcd 011wxyz + * 0001wxyzabcde 100wxyz + * 001wxyzabcdef 101wxyz + * 01wxyzabcdefg 110wxyz + * 1wxyzabcdefgh 111wxyz + * + * Each biased linear code has a leading 1 which identifies the segment + * number. The value of the segment number is equal to 7 minus the number + * of leading 0's. The quantization interval is directly available as the + * four bits wxyz. * The trailing bits (a - h) are ignored. + * + * Ordinarily the complement of the resulting code word is used for + * transmission, and so the code word is complemented before it is returned. + * + * For further information see John C. Bellamy's Digital Telephony, 1982, + * John Wiley & Sons, pps 98-111 and 472-476. + */ +static unsigned char +st_14linear2ulaw(PyInt16 pcm_val) /* 2's complement (14-bit range) */ +{ + PyInt16 mask; + PyInt16 seg; + unsigned char uval; + + /* The original sox code does this in the calling function, not here */ + pcm_val = pcm_val >> 2; + + /* u-law inverts all bits */ + /* Get the sign and the magnitude of the value. */ + if (pcm_val < 0) { + pcm_val = -pcm_val; + mask = 0x7F; + } else { + mask = 0xFF; + } + if ( pcm_val > CLIP ) pcm_val = CLIP; /* clip the magnitude */ + pcm_val += (BIAS >> 2); + + /* Convert the scaled magnitude to segment number. */ + seg = search(pcm_val, seg_uend, 8); + + /* + * Combine the sign, segment, quantization bits; + * and complement the code word. + */ + if (seg >= 8) /* out of range, return maximum value. */ + return (unsigned char) (0x7F ^ mask); + else { + uval = (unsigned char) (seg << 4) | ((pcm_val >> (seg + 1)) & 0xF); + return (uval ^ mask); + } + +} + +static PyInt16 _st_alaw2linear16[256] = { + -5504, -5248, -6016, -5760, -4480, -4224, -4992, + -4736, -7552, -7296, -8064, -7808, -6528, -6272, + -7040, -6784, -2752, -2624, -3008, -2880, -2240, + -2112, -2496, -2368, -3776, -3648, -4032, -3904, + -3264, -3136, -3520, -3392, -22016, -20992, -24064, + -23040, -17920, -16896, -19968, -18944, -30208, -29184, + -32256, -31232, -26112, -25088, -28160, -27136, -11008, + -10496, -12032, -11520, -8960, -8448, -9984, -9472, + -15104, -14592, -16128, -15616, -13056, -12544, -14080, + -13568, -344, -328, -376, -360, -280, -264, + -312, -296, -472, -456, -504, -488, -408, + -392, -440, -424, -88, -72, -120, -104, + -24, -8, -56, -40, -216, -200, -248, + -232, -152, -136, -184, -168, -1376, -1312, + -1504, -1440, -1120, -1056, -1248, -1184, -1888, + -1824, -2016, -1952, -1632, -1568, -1760, -1696, + -688, -656, -752, -720, -560, -528, -624, + -592, -944, -912, -1008, -976, -816, -784, + -880, -848, 5504, 5248, 6016, 5760, 4480, + 4224, 4992, 4736, 7552, 7296, 8064, 7808, + 6528, 6272, 7040, 6784, 2752, 2624, 3008, + 2880, 2240, 2112, 2496, 2368, 3776, 3648, + 4032, 3904, 3264, 3136, 3520, 3392, 22016, + 20992, 24064, 23040, 17920, 16896, 19968, 18944, + 30208, 29184, 32256, 31232, 26112, 25088, 28160, + 27136, 11008, 10496, 12032, 11520, 8960, 8448, + 9984, 9472, 15104, 14592, 16128, 15616, 13056, + 12544, 14080, 13568, 344, 328, 376, 360, + 280, 264, 312, 296, 472, 456, 504, + 488, 408, 392, 440, 424, 88, 72, + 120, 104, 24, 8, 56, 40, 216, + 200, 248, 232, 152, 136, 184, 168, + 1376, 1312, 1504, 1440, 1120, 1056, 1248, + 1184, 1888, 1824, 2016, 1952, 1632, 1568, + 1760, 1696, 688, 656, 752, 720, 560, + 528, 624, 592, 944, 912, 1008, 976, + 816, 784, 880, 848 +}; + +/* + * linear2alaw() accepts an 13-bit signed integer and encodes it as A-law data + * stored in a unsigned char. This function should only be called with + * the data shifted such that it only contains information in the lower + * 13-bits. + * + * Linear Input Code Compressed Code + * ------------------------ --------------- + * 0000000wxyza 000wxyz + * 0000001wxyza 001wxyz + * 000001wxyzab 010wxyz + * 00001wxyzabc 011wxyz + * 0001wxyzabcd 100wxyz + * 001wxyzabcde 101wxyz + * 01wxyzabcdef 110wxyz + * 1wxyzabcdefg 111wxyz + * + * For further information see John C. Bellamy's Digital Telephony, 1982, + * John Wiley & Sons, pps 98-111 and 472-476. + */ +static unsigned char +st_linear2alaw(PyInt16 pcm_val) /* 2's complement (13-bit range) */ +{ + PyInt16 mask; + short seg; + unsigned char aval; + + /* The original sox code does this in the calling function, not here */ + pcm_val = pcm_val >> 3; + + /* A-law using even bit inversion */ + if (pcm_val >= 0) { + mask = 0xD5; /* sign (7th) bit = 1 */ + } else { + mask = 0x55; /* sign bit = 0 */ + pcm_val = -pcm_val - 1; + } + + /* Convert the scaled magnitude to segment number. */ + seg = search(pcm_val, seg_aend, 8); + + /* Combine the sign, segment, and quantization bits. */ + + if (seg >= 8) /* out of range, return maximum value. */ + return (unsigned char) (0x7F ^ mask); + else { + aval = (unsigned char) seg << SEG_SHIFT; + if (seg < 2) + aval |= (pcm_val >> 1) & QUANT_MASK; + else + aval |= (pcm_val >> seg) & QUANT_MASK; + return (aval ^ mask); + } +} +/* End of code taken from sox */ + +/* Intel ADPCM step variation table */ +static int indexTable[16] = { + -1, -1, -1, -1, 2, 4, 6, 8, + -1, -1, -1, -1, 2, 4, 6, 8, +}; + +static int stepsizeTable[89] = { + 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, + 19, 21, 23, 25, 28, 31, 34, 37, 41, 45, + 50, 55, 60, 66, 73, 80, 88, 97, 107, 118, + 130, 143, 157, 173, 190, 209, 230, 253, 279, 307, + 337, 371, 408, 449, 494, 544, 598, 658, 724, 796, + 876, 963, 1060, 1166, 1282, 1411, 1552, 1707, 1878, 2066, + 2272, 2499, 2749, 3024, 3327, 3660, 4026, 4428, 4871, 5358, + 5894, 6484, 7132, 7845, 8630, 9493, 10442, 11487, 12635, 13899, + 15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794, 32767 +}; + +#define CHARP(cp, i) ((signed char *)(cp+i)) +#define SHORTP(cp, i) ((short *)(cp+i)) +#define LONGP(cp, i) ((Py_Int32 *)(cp+i)) +""" + +lib = ffi.verify(_AUDIOOP_C_MODULE + r""" +#include + +static const int maxvals[] = {0, 0x7F, 0x7FFF, 0x7FFFFF, 0x7FFFFFFF}; +/* -1 trick is needed on Windows to support -0x80000000 without a warning */ +static const int minvals[] = {0, -0x80, -0x8000, -0x800000, -0x7FFFFFFF-1}; + +static int +fbound(double val, double minval, double maxval) +{ + if (val > maxval) + val = maxval; + else if (val < minval + 1) + val = minval; + return val; +} + +static int +gcd(int a, int b) +{ + while (b > 0) { + int tmp = a % b; + a = b; + b = tmp; + } + return a; +} + +int ratecv(char* rv, char* cp, size_t len, int size, + int nchannels, int inrate, int outrate, + int* state_d, int* prev_i, int* cur_i, + int weightA, int weightB) +{ + char *ncp = rv; + int d, chan; + + /* divide inrate and outrate by their greatest common divisor */ + d = gcd(inrate, outrate); + inrate /= d; + outrate /= d; + /* divide weightA and weightB by their greatest common divisor */ + d = gcd(weightA, weightB); + weightA /= d; + weightA /= d; + + d = *state_d; + + for (;;) { + while (d < 0) { + if (len == 0) { + *state_d = d; + return ncp - rv; + } + for (chan = 0; chan < nchannels; chan++) { + prev_i[chan] = cur_i[chan]; + if (size == 1) + cur_i[chan] = ((int)*CHARP(cp, 0)) << 24; + else if (size == 2) + cur_i[chan] = ((int)*SHORTP(cp, 0)) << 16; + else if (size == 4) + cur_i[chan] = (int)*LONGP(cp, 0); + cp += size; + /* implements a simple digital filter */ + cur_i[chan] = (int)( + ((double)weightA * (double)cur_i[chan] + + (double)weightB * (double)prev_i[chan]) / + ((double)weightA + (double)weightB)); + } + len--; + d += outrate; + } + while (d >= 0) { + for (chan = 0; chan < nchannels; chan++) { + int cur_o; + cur_o = (int)(((double)prev_i[chan] * (double)d + + (double)cur_i[chan] * (double)(outrate - d)) / + (double)outrate); + if (size == 1) + *CHARP(ncp, 0) = (signed char)(cur_o >> 24); + else if (size == 2) + *SHORTP(ncp, 0) = (short)(cur_o >> 16); + else if (size == 4) + *LONGP(ncp, 0) = (Py_Int32)(cur_o); + ncp += size; + } + d -= inrate; + } + } +} + +void tostereo(char* rv, char* cp, size_t len, int size, + double fac1, double fac2) +{ + int val1, val2, val = 0; + double fval, maxval, minval; + char *ncp = rv; + int i; + + maxval = (double) maxvals[size]; + minval = (double) minvals[size]; + + for ( i=0; i < len; i += size ) { + if ( size == 1 ) val = (int)*CHARP(cp, i); + else if ( size == 2 ) val = (int)*SHORTP(cp, i); + else if ( size == 4 ) val = (int)*LONGP(cp, i); + + fval = (double)val*fac1; + val1 = (int)floor(fbound(fval, minval, maxval)); + + fval = (double)val*fac2; + val2 = (int)floor(fbound(fval, minval, maxval)); + + if ( size == 1 ) *CHARP(ncp, i*2) = (signed char)val1; + else if ( size == 2 ) *SHORTP(ncp, i*2) = (short)val1; + else if ( size == 4 ) *LONGP(ncp, i*2) = (Py_Int32)val1; + + if ( size == 1 ) *CHARP(ncp, i*2+1) = (signed char)val2; + else if ( size == 2 ) *SHORTP(ncp, i*2+2) = (short)val2; + else if ( size == 4 ) *LONGP(ncp, i*2+4) = (Py_Int32)val2; + } +} + +void add(char* rv, char* cp1, char* cp2, size_t len1, int size) +{ + int i; + int val1 = 0, val2 = 0, minval, maxval, newval; + char* ncp = rv; + + maxval = maxvals[size]; + minval = minvals[size]; + + for ( i=0; i < len1; i += size ) { + if ( size == 1 ) val1 = (int)*CHARP(cp1, i); + else if ( size == 2 ) val1 = (int)*SHORTP(cp1, i); + else if ( size == 4 ) val1 = (int)*LONGP(cp1, i); + + if ( size == 1 ) val2 = (int)*CHARP(cp2, i); + else if ( size == 2 ) val2 = (int)*SHORTP(cp2, i); + else if ( size == 4 ) val2 = (int)*LONGP(cp2, i); + + if (size < 4) { + newval = val1 + val2; + /* truncate in case of overflow */ + if (newval > maxval) + newval = maxval; + else if (newval < minval) + newval = minval; + } + else { + double fval = (double)val1 + (double)val2; + /* truncate in case of overflow */ + newval = (int)floor(fbound(fval, minval, maxval)); + } + + if ( size == 1 ) *CHARP(ncp, i) = (signed char)newval; + else if ( size == 2 ) *SHORTP(ncp, i) = (short)newval; + else if ( size == 4 ) *LONGP(ncp, i) = (Py_Int32)newval; + } +} + +void lin2adcpm(unsigned char* ncp, unsigned char* cp, size_t len, + size_t size, int* state) +{ + int step, outputbuffer = 0, bufferstep; + int val = 0; + int diff, vpdiff, sign, delta; + size_t i; + int valpred = state[0]; + int index = state[1]; + + step = stepsizeTable[index]; + bufferstep = 1; + + for ( i=0; i < len; i += size ) { + if ( size == 1 ) val = ((int)*CHARP(cp, i)) << 8; + else if ( size == 2 ) val = (int)*SHORTP(cp, i); + else if ( size == 4 ) val = ((int)*LONGP(cp, i)) >> 16; + + /* Step 1 - compute difference with previous value */ + diff = val - valpred; + sign = (diff < 0) ? 8 : 0; + if ( sign ) diff = (-diff); + + /* Step 2 - Divide and clamp */ + /* Note: + ** This code *approximately* computes: + ** delta = diff*4/step; + ** vpdiff = (delta+0.5)*step/4; + ** but in shift step bits are dropped. The net result of this + ** is that even if you have fast mul/div hardware you cannot + ** put it to good use since the fixup would be too expensive. + */ + delta = 0; + vpdiff = (step >> 3); + + if ( diff >= step ) { + delta = 4; + diff -= step; + vpdiff += step; + } + step >>= 1; + if ( diff >= step ) { + delta |= 2; + diff -= step; + vpdiff += step; + } + step >>= 1; + if ( diff >= step ) { + delta |= 1; + vpdiff += step; + } + + /* Step 3 - Update previous value */ + if ( sign ) + valpred -= vpdiff; + else + valpred += vpdiff; + + /* Step 4 - Clamp previous value to 16 bits */ + if ( valpred > 32767 ) + valpred = 32767; + else if ( valpred < -32768 ) + valpred = -32768; + + /* Step 5 - Assemble value, update index and step values */ + delta |= sign; + + index += indexTable[delta]; + if ( index < 0 ) index = 0; + if ( index > 88 ) index = 88; + step = stepsizeTable[index]; + + /* Step 6 - Output value */ + if ( bufferstep ) { + outputbuffer = (delta << 4) & 0xf0; + } else { + *ncp++ = (delta & 0x0f) | outputbuffer; + } + bufferstep = !bufferstep; + } + state[0] = valpred; + state[1] = index; +} + + +void adcpm2lin(unsigned char* ncp, unsigned char* cp, size_t len, + size_t size, int* state) +{ + int step, inputbuffer = 0, bufferstep; + int val = 0; + int diff, vpdiff, sign, delta; + size_t i; + int valpred = state[0]; + int index = state[1]; + + step = stepsizeTable[index]; + bufferstep = 0; + + for ( i=0; i < len*size*2; i += size ) { + /* Step 1 - get the delta value and compute next index */ + if ( bufferstep ) { + delta = inputbuffer & 0xf; + } else { + inputbuffer = *cp++; + delta = (inputbuffer >> 4) & 0xf; + } + + bufferstep = !bufferstep; + + /* Step 2 - Find new index value (for later) */ + index += indexTable[delta]; + if ( index < 0 ) index = 0; + if ( index > 88 ) index = 88; + + /* Step 3 - Separate sign and magnitude */ + sign = delta & 8; + delta = delta & 7; + + /* Step 4 - Compute difference and new predicted value */ + /* + ** Computes 'vpdiff = (delta+0.5)*step/4', but see comment + ** in adpcm_coder. + */ + vpdiff = step >> 3; + if ( delta & 4 ) vpdiff += step; + if ( delta & 2 ) vpdiff += step>>1; + if ( delta & 1 ) vpdiff += step>>2; + + if ( sign ) + valpred -= vpdiff; + else + valpred += vpdiff; + + /* Step 5 - clamp output value */ + if ( valpred > 32767 ) + valpred = 32767; + else if ( valpred < -32768 ) + valpred = -32768; + + /* Step 6 - Update step value */ + step = stepsizeTable[index]; + + /* Step 6 - Output value */ + if ( size == 1 ) *CHARP(ncp, i) = (signed char)(valpred >> 8); + else if ( size == 2 ) *SHORTP(ncp, i) = (short)(valpred); + else if ( size == 4 ) *LONGP(ncp, i) = (Py_Int32)(valpred<<16); + } + state[0] = valpred; + state[1] = index; +} +""") + +def _get_lin_samples(cp, size): + for sample in _get_samples(cp, size): + if size == 1: + yield sample << 8 + elif size == 2: + yield sample + elif size == 4: + yield sample >> 16 + +def _put_lin_sample(result, size, i, sample): + if size == 1: + sample >>= 8 + elif size == 2: + pass + elif size == 4: + sample <<= 16 + _put_sample(result, size, i, sample) def lin2ulaw(cp, size): - raise NotImplementedError() + _check_params(len(cp), size) + rv = ffi.new("unsigned char[]", _sample_count(cp, size)) + for i, sample in enumerate(_get_lin_samples(cp, size)): + rv[i] = lib.st_14linear2ulaw(sample) + return ffi.buffer(rv)[:] def ulaw2lin(cp, size): - raise NotImplementedError() + _check_size(size) + rv = ffi.new("unsigned char[]", len(cp) * size) + result = ffi.buffer(rv) + for i, value in enumerate(cp): + sample = lib.st_ulaw2linear16(ord(value)) + _put_lin_sample(result, size, i, sample) + return result[:] def lin2alaw(cp, size): - raise NotImplementedError() + _check_params(len(cp), size) + rv = ffi.new("unsigned char[]", _sample_count(cp, size)) + for i, sample in enumerate(_get_lin_samples(cp, size)): + rv[i] = lib.st_linear2alaw(sample) + return ffi.buffer(rv)[:] def alaw2lin(cp, size): - raise NotImplementedError() + _check_size(size) + rv = ffi.new("unsigned char[]", len(cp) * size) + result = ffi.buffer(rv) + for i, value in enumerate(cp): + sample = lib.st_alaw2linear16(ord(value)) + _put_lin_sample(result, size, i, sample) + return result[:] def lin2adpcm(cp, size, state): - raise NotImplementedError() + _check_params(len(cp), size) + if state is None: + state = (0, 0) + rv = ffi.new("unsigned char[]", len(cp) // size // 2) + state_ptr = ffi.new("int[]", state) + lib.lin2adcpm(rv, cp, len(cp), size, state_ptr) + return ffi.buffer(rv)[:], tuple(state_ptr) def adpcm2lin(cp, size, state): - raise NotImplementedError() + _check_size(size) + if state is None: + state = (0, 0) + rv = ffi.new("unsigned char[]", len(cp) * size * 2) + state_ptr = ffi.new("int[]", state) + lib.adcpm2lin(rv, cp, len(cp), size, state_ptr) + return ffi.buffer(rv)[:], tuple(state_ptr) + diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.8.6" -__version_info__ = (0, 8, 6) +__version__ = "0.8.6+" +__version_info__ = (0, 8, 6, "plus") # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -55,7 +55,8 @@ # _cffi_backend.so compiled. import _cffi_backend as backend from . import __version__ - assert backend.__version__ == __version__ + assert backend.__version__ == __version__, \ + "version mismatch, %s != %s" % (backend.__version__, __version__) # (If you insist you can also try to pass the option # 'backend=backend_ctypes.CTypesBackend()', but don't # rely on it! It's probably not going to work well.) diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py --- a/lib_pypy/cffi/backend_ctypes.py +++ b/lib_pypy/cffi/backend_ctypes.py @@ -2,11 +2,10 @@ from . import model if sys.version_info < (3,): - integer_types = (int, long) bytechr = chr else: unicode = str - integer_types = int + long = int xrange = range bytechr = lambda num: bytes([num]) @@ -181,7 +180,7 @@ address = 0 elif isinstance(source, CTypesData): address = source._cast_to_integer() - elif isinstance(source, integer_types): + elif isinstance(source, (int, long)): address = source else: raise TypeError("bad type for cast to %r: %r" % @@ -358,7 +357,7 @@ is_signed = (ctype(-1).value == -1) # def _cast_source_to_int(source): - if isinstance(source, (integer_types, float)): + if isinstance(source, (int, long, float)): source = int(source) elif isinstance(source, CTypesData): source = source._cast_to_integer() @@ -399,7 +398,7 @@ if kind == 'bool': @classmethod def _cast_from(cls, source): - if not isinstance(source, (integer_types, float)): + if not isinstance(source, (int, long, float)): source = _cast_source_to_int(source) return cls(bool(source)) def __int__(self): @@ -438,7 +437,7 @@ if kind == 'int' or kind == 'byte' or kind == 'bool': @staticmethod def _to_ctypes(x): - if not isinstance(x, integer_types): + if not isinstance(x, (int, long)): if isinstance(x, CTypesData): x = int(x) else: @@ -465,7 +464,7 @@ if kind == 'float': @staticmethod def _to_ctypes(x): - if not isinstance(x, (integer_types, float, CTypesData)): + if not isinstance(x, (int, long, float, CTypesData)): raise TypeError("float expected, got %s" % type(x).__name__) return ctype(x).value @@ -529,14 +528,14 @@ self._own = True def __add__(self, other): - if isinstance(other, integer_types): + if isinstance(other, (int, long)): return self._new_pointer_at(self._address + other * self._bitem_size) else: return NotImplemented def __sub__(self, other): - if isinstance(other, integer_types): + if isinstance(other, (int, long)): return self._new_pointer_at(self._address - other * self._bitem_size) elif type(self) is type(other): @@ -611,7 +610,7 @@ def __init__(self, init): if length is None: - if isinstance(init, integer_types): + if isinstance(init, (int, long)): len1 = init init = None elif kind == 'char' and isinstance(init, bytes): @@ -686,7 +685,7 @@ return CTypesPtr._arg_to_ctypes(value) def __add__(self, other): - if isinstance(other, integer_types): + if isinstance(other, (int, long)): return CTypesPtr._new_pointer_at( ctypes.addressof(self._blob) + other * ctypes.sizeof(BItem._ctype)) diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py --- a/lib_pypy/cffi/verifier.py +++ b/lib_pypy/cffi/verifier.py @@ -1,4 +1,4 @@ -import sys, os, binascii, shutil +import sys, os, binascii, shutil, io from . import __version_verifier_modules__ from . import ffiplatform @@ -13,6 +13,16 @@ if type == imp.C_EXTENSION] +if sys.version_info >= (3,): + NativeIO = io.StringIO +else: + class NativeIO(io.BytesIO): + def write(self, s): + if isinstance(s, unicode): + s = s.encode('ascii') + super(NativeIO, self).write(s) + + class Verifier(object): def __init__(self, ffi, preamble, tmpdir=None, modulename=None, @@ -144,19 +154,36 @@ self._vengine.collect_types() self._has_module = True - def _write_source(self, file=None): - must_close = (file is None) - if must_close: - _ensure_dir(self.sourcefilename) - file = open(self.sourcefilename, 'w') + def _write_source_to(self, file): self._vengine._f = file try: self._vengine.write_source_to_f() finally: del self._vengine._f - if must_close: - file.close() - if must_close: + + def _write_source(self, file=None): + if file is not None: + self._write_source_to(file) + else: + # Write our source file to an in memory file. + f = NativeIO() + self._write_source_to(f) + source_data = f.getvalue() + + # Determine if this matches the current file + if os.path.exists(self.sourcefilename): + with open(self.sourcefilename, "r") as fp: + needs_written = not (fp.read() == source_data) + else: + needs_written = True + + # Actually write the file out if it doesn't match + if needs_written: + _ensure_dir(self.sourcefilename) + with open(self.sourcefilename, "w") as fp: + fp.write(source_data) + + # Set this flag self._has_source = True def _compile_module(self): diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py --- a/lib_pypy/pyrepl/readline.py +++ b/lib_pypy/pyrepl/readline.py @@ -77,6 +77,7 @@ assume_immutable_completions = False use_brackets = False sort_in_column = True + tab_insert_spaces_if_stem_is_empty = False def error(self, msg="none"): pass # don't show error messages by default @@ -90,6 +91,13 @@ return ''.join(b[p+1:self.pos]) def get_completions(self, stem): + if len(stem) == 0 and self.tab_insert_spaces_if_stem_is_empty: + b = self.buffer + p = self.pos + while p > 0 and b[p - 1] != '\n': + p -= 1 + num_spaces = 4 - ((self.pos - p) % 4) + return [' ' * num_spaces] result = [] function = self.config.readline_completer if function is not None: @@ -208,14 +216,15 @@ boolean value is true. """ reader = self.get_reader() - saved = reader.more_lines + saved = reader.more_lines, reader.tab_insert_spaces_if_stem_is_empty try: reader.more_lines = more_lines reader.ps1 = reader.ps2 = ps1 reader.ps3 = reader.ps4 = ps2 + reader.tab_insert_spaces_if_stem_is_empty = True return reader.readline(returns_unicode=returns_unicode) finally: - reader.more_lines = saved + reader.more_lines, reader.tab_insert_spaces_if_stem_is_empty = saved def parse_and_bind(self, string): pass # XXX we don't support parsing GNU-readline-style init files diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst --- a/pypy/doc/build.rst +++ b/pypy/doc/build.rst @@ -48,9 +48,13 @@ Install build-time dependencies ------------------------------- (**Note**: for some hints on how to translate the Python interpreter under -Windows, see the `windows document`_) +Windows, see the `windows document`_ . For hints on how to cross-compile in +a chroot using scratchbox2, see the `arm document`_ in the +`RPython documentation`_) .. _`windows document`: windows.html +.. _`arm document`: http://rpython.readthedocs.org/en/latest/arm.html +.. _`RPython documentation`: http://rpython.readthedocs.org To build PyPy on Unix using the C translation backend, you need at least a C diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -300,6 +300,18 @@ Notably missing from the list above are ``str`` and ``unicode``. If your code relies on comparing strings with ``is``, then it might break in PyPy. +Note that for floats there "``is``" only one object per "bit pattern" +of the float. So ``float('nan') is float('nan')`` is true on PyPy, +but not on CPython because they are two objects; but ``0.0 is -0.0`` +is always False, as the bit patterns are different. As usual, +``float('nan') == float('nan')`` is always False. When used in +containers (as list items or in sets for example), the exact rule of +equality used is "``if x is y or x == y``" (on both CPython and PyPy); +as a consequence, because all ``nans`` are identical in PyPy, you +cannot have several of them in a set, unlike in CPython. (Issue `#1974`__) + +.. __: https://bitbucket.org/pypy/pypy/issue/1974/different-behaviour-for-collections-of + Miscellaneous ------------- diff --git a/pypy/doc/interpreter-optimizations.rst b/pypy/doc/interpreter-optimizations.rst --- a/pypy/doc/interpreter-optimizations.rst +++ b/pypy/doc/interpreter-optimizations.rst @@ -146,7 +146,7 @@ We improved this by keeping method lookup separated from method call, unlike some other approaches, but using the value stack as a cache instead of building a temporary object. We extended the bytecode compiler to (optionally) generate -the following code for ``obj.meth(x)``:: +the following code for ``obj.meth(x, y)``:: LOAD_GLOBAL obj LOOKUP_METHOD meth @@ -162,7 +162,7 @@ the attribute actually refers to a function object from the class; when this is not the case, ``LOOKUP_METHOD`` still pushes two values, but one *(im_func)* is simply the regular result that ``LOAD_ATTR`` would have returned, and the other -*(im_self)* is a None placeholder. +*(im_self)* is an interpreter-level None placeholder. After pushing the arguments, the layout of the stack in the above example is as follows (the stack grows upwards): diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -701,7 +701,7 @@ return self.wrap(not self.is_true(w_obj)) def eq_w(self, w_obj1, w_obj2): - """shortcut for space.is_true(space.eq(w_obj1, w_obj2))""" + """Implements equality with the double check 'x is y or x == y'.""" return self.is_w(w_obj1, w_obj2) or self.is_true(self.eq(w_obj1, w_obj2)) def is_(self, w_one, w_two): diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -24,17 +24,20 @@ space.wrap(msg)])) return raise_unicode_exception_decode +class RUnicodeEncodeError(Exception): + def __init__(self, encoding, object, start, end, reason): + self.encoding = encoding + self.object = object + self.start = start + self.end = end + self.reason = reason + @specialize.memo() def encode_error_handler(space): # Fast version of the "strict" errors handler. def raise_unicode_exception_encode(errors, encoding, msg, u, startingpos, endingpos): - raise OperationError(space.w_UnicodeEncodeError, - space.newtuple([space.wrap(encoding), - space.wrap(u), - space.wrap(startingpos), - space.wrap(endingpos), - space.wrap(msg)])) + raise RUnicodeEncodeError(encoding, u, startingpos, endingpos, msg) return raise_unicode_exception_encode # ____________________________________________________________ diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -8,7 +8,7 @@ appleveldefs = { } interpleveldefs = { - '__version__': 'space.wrap("0.8.6")', + '__version__': 'space.wrap("0.8.6+")', 'load_library': 'libraryobj.load_library', diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -3241,4 +3241,4 @@ def test_version(): # this test is here mostly for PyPy - assert __version__ == "0.8.6" + assert __version__ == "0.8.6+" diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py --- a/pypy/module/_rawffi/test/test__rawffi.py +++ b/pypy/module/_rawffi/test/test__rawffi.py @@ -1141,10 +1141,10 @@ import _rawffi S = _rawffi.Structure((40, 1)) s = S(autofree=True) - addr = buffer(s)._pypy_raw_address() + addr = memoryview(s)._pypy_raw_address() assert type(addr) is int - assert buffer(s)._pypy_raw_address() == addr - assert buffer(s, 10)._pypy_raw_address() == addr + 10 + assert memoryview(s)._pypy_raw_address() == addr + assert memoryview(s)[10:]._pypy_raw_address() == addr + 10 def test_union(self): import _rawffi diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py --- a/pypy/module/micronumpy/ctors.py +++ b/pypy/module/micronumpy/ctors.py @@ -81,7 +81,10 @@ return w_object.descr_copy(space, w_order) elif not copy and (subok or type(w_object) is W_NDimArray): return w_object - # we have a ndarray, but need to copy or change dtype or create W_NDimArray + if subok: + raise oefmt(space.w_NotImplementedError, + "array(..., subok=True) only partially implemented") + # we have a ndarray, but need to copy or change dtype if dtype is None: dtype = w_object.get_dtype() if dtype != w_object.get_dtype(): @@ -89,13 +92,12 @@ copy = True if copy: shape = w_object.get_shape() - _elems_w = w_object.reshape(space, space.wrap(-1)) elems_w = [None] * w_object.get_size() - for i in range(len(elems_w)): - elems_w[i] = _elems_w.descr_getitem(space, space.wrap(i)) - elif subok: - raise oefmt(space.w_NotImplementedError, - "array(...copy=False, subok=True) not implemented yet") + elsize = w_object.get_dtype().elsize + # TODO - use w_object.implementation without copying to a list + # unfortunately that causes a union error in translation + for i in range(w_object.get_size()): + elems_w[i] = w_object.implementation.getitem(i * elsize) else: sz = support.product(w_object.get_shape()) * dtype.elsize return W_NDimArray.from_shape_and_storage(space, @@ -113,7 +115,7 @@ dtype = descriptor.variable_dtype(space, dtype.char + '1') w_arr = W_NDimArray.from_shape(space, shape, dtype, order=order) - if len(elems_w) == 1: + if support.product(shape) == 1: w_arr.set_scalar_value(dtype.coerce(space, elems_w[0])) else: loop.assign(space, w_arr, elems_w) diff --git a/pypy/module/micronumpy/flatiter.py b/pypy/module/micronumpy/flatiter.py --- a/pypy/module/micronumpy/flatiter.py +++ b/pypy/module/micronumpy/flatiter.py @@ -15,6 +15,7 @@ self._base = base self.dtype = base.get_dtype() self.shape = [base.get_size()] + self.storage = self._base.implementation.storage def base(self): return self._base diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -2843,6 +2843,13 @@ c.flat = ['defgh', 'ijklmnop'] assert (c.flatten() == ['def', 'ijk']*5).all() + def test_flatiter_subtype(self): + from numpy import array + x = array([[1, 2], [3, 4]]).T + y = array(x.flat) + assert (x == [[1, 3], [2, 4]]).all() + + def test_slice_copy(self): from numpy import zeros a = zeros((10, 10)) diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py --- a/pypy/module/micronumpy/test/test_subtype.py +++ b/pypy/module/micronumpy/test/test_subtype.py @@ -272,40 +272,103 @@ import numpy as N # numpy's matrix class caused an infinite loop class matrix(N.ndarray): - getcnt = 0 def __new__(subtype, data, dtype=None, copy=True): + print('matrix __new__') + if isinstance(data, matrix): + dtype2 = data.dtype + if (dtype is None): + dtype = dtype2 + if (dtype2 == dtype) and (not copy): + return data + return data.astype(dtype) + + if isinstance(data, N.ndarray): + if dtype is None: + intype = data.dtype + else: + intype = N.dtype(dtype) + new = data.view(subtype) + if intype != data.dtype: + return new.astype(intype) + if copy: return new.copy() + else: return new + + if isinstance(data, str): + data = _convert_from_string(data) + + # now convert data to an array arr = N.array(data, dtype=dtype, copy=copy) + ndim = arr.ndim shape = arr.shape + if (ndim > 2): + raise ValueError("matrix must be 2-dimensional") + elif ndim == 0: + shape = (1, 1) + elif ndim == 1: + shape = (1, shape[0]) + + order = False + if (ndim == 2) and arr.flags.fortran: + order = True + + if not (order or arr.flags.contiguous): + arr = arr.copy() ret = N.ndarray.__new__(subtype, shape, arr.dtype, buffer=arr, - order=True) + order=order) return ret + def __array_finalize__(self, obj): + print('matrix __array_finalize__') + self._getitem = False + if (isinstance(obj, matrix) and obj._getitem): return + ndim = self.ndim + if (ndim == 2): + return + if (ndim > 2): + newshape = tuple([x for x in self.shape if x > 1]) + ndim = len(newshape) + if ndim == 2: + self.shape = newshape + return + elif (ndim > 2): + raise ValueError("shape too large to be a matrix.") + else: + newshape = self.shape + if ndim == 0: + self.shape = (1, 1) + elif ndim == 1: + self.shape = (1, newshape[0]) + return + def __getitem__(self, index): - matrix.getcnt += 1 - if matrix.getcnt > 10: - # XXX strides.find_shape_and_elems is sensitive - # to shape modification - xxx - out = N.ndarray.__getitem__(self, index) + print('matrix __getitem__') + self._getitem = True + + try: + out = N.ndarray.__getitem__(self, index) + finally: + self._getitem = False if not isinstance(out, N.ndarray): return out + + if out.ndim == 0: + return out[()] + if out.ndim == 1: + sh = out.shape[0] # Determine when we should have a column array - old_shape = out.shape - if out.ndim < 2: - sh = out.shape[0] try: n = len(index) except: n = 0 - if n > 1: + if n > 1 and isscalar(index[1]): out.shape = (sh, 1) else: out.shape = (1, sh) - #print 'out, shape was',old_shape,'now',out.shape,'out',out return out + a = matrix([[1., 2.], [3., 4.]]) b = N.array([a]) assert (b == a).all() @@ -318,6 +381,17 @@ assert len(b.shape) == 2 assert (b == a).all() + b = N.array(a, copy=True, dtype=int) + assert len(b.shape) == 2 + assert (b == a).all() + + c = matrix(a, copy=False) + assert c.base is not None + c[0, 0] = 100 + assert a[0, 0] == 100 + b = N.array(c, copy=True) + assert (b == a).all() + def test_setstate_no_version(self): # Some subclasses of ndarray, like MaskedArray, do not use # version in __setstare__ diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py --- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py +++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py @@ -27,7 +27,7 @@ log = self.run(main, [libm_name]) pow_addr, res = log.result assert res == 8.0 * 300 - py.test.xfail() # XXX re-optimize _ffi for the JIT? + py.test.skip("XXX re-optimize _ffi for the JIT?") loop, = log.loops_by_filename(self.filepath) if 'ConstClass(pow)' in repr(loop): # e.g. OS/X pow_addr = 'ConstClass(pow)' @@ -134,7 +134,7 @@ ops = loop.allops() opnames = log.opnames(ops) assert opnames.count('new_with_vtable') == 1 # only the virtualref - py.test.xfail() # XXX re-optimize _ffi for the JIT? + py.test.skip("XXX re-optimize _ffi for the JIT?") assert opnames.count('call_release_gil') == 1 idx = opnames.index('call_release_gil') call = ops[idx] @@ -159,7 +159,7 @@ return struct.getfield('x') # log = self.run(main, []) - py.test.xfail() # XXX re-optimize _ffi for the JIT? + py.test.skip("XXX re-optimize _ffi for the JIT?") loop, = log.loops_by_filename(self.filepath) assert loop.match_by_id('getfield', """ guard_not_invalidated(descr=...) diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_ownlib.py b/pypy/module/test_lib_pypy/cffi_tests/test_ownlib.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_ownlib.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_ownlib.py @@ -8,34 +8,137 @@ SOURCE = """\ #include -int test_getting_errno(void) { +#ifdef _WIN32 +#define EXPORT __declspec(dllexport) +#else +#define EXPORT +#endif + +EXPORT int test_getting_errno(void) { errno = 123; return -1; } -int test_setting_errno(void) { +EXPORT int test_setting_errno(void) { return errno; +}; + +typedef struct { + long x; + long y; +} POINT; + +typedef struct { + long left; + long top; + long right; + long bottom; +} RECT; + + +EXPORT int PointInRect(RECT *prc, POINT pt) +{ + if (pt.x < prc->left) + return 0; + if (pt.x > prc->right) + return 0; + if (pt.y < prc->top) + return 0; + if (pt.y > prc->bottom) + return 0; + return 1; +}; + +EXPORT long left = 10; +EXPORT long top = 20; +EXPORT long right = 30; +EXPORT long bottom = 40; + +EXPORT RECT ReturnRect(int i, RECT ar, RECT* br, POINT cp, RECT dr, + RECT *er, POINT fp, RECT gr) +{ + /*Check input */ + if (ar.left + br->left + dr.left + er->left + gr.left != left * 5) + { + ar.left = 100; From noreply at buildbot.pypy.org Sat Feb 14 16:08:48 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Sat, 14 Feb 2015 16:08:48 +0100 (CET) Subject: [pypy-commit] pypy framestate: move RETURN_VALUE to bytecode.py and clean up Message-ID: <20150214150848.579271C034E@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: framestate Changeset: r75870:e578fa65e31c Date: 2015-02-14 14:48 +0000 http://bitbucket.org/pypy/pypy/changeset/e578fa65e31c/ Log: move RETURN_VALUE to bytecode.py and clean up diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py --- a/rpython/flowspace/bytecode.py +++ b/rpython/flowspace/bytecode.py @@ -233,22 +233,27 @@ instr.do_signals(self) def splice_finally_handler(self, block, context): + cell = [] def copy_block(handler): b = handler.copy() if handler is context.handler_end: instr = b.operations.pop() assert isinstance(instr, END_FINALLY) + cell.append(b) else: b.set_exits([copy_block(child) for child in handler._exits]) self.blocks.append(b) return b block.set_exits([copy_block(context.handler)]) + copy_of_handler_end, = cell + return copy_of_handler_end def check_graph(self): for b in self.blocks: if not b._exits: - assert any(instr.name in ('RETURN_VALUE', 'RAISE_VARARGS') - for instr in b.operations) + instr = b.operations[-1] + assert instr.name in ( + 'RETURN_VALUE', 'RAISE_VARARGS', 'EXEC_STMT') for x in b._exits: assert x in self.blocks @@ -614,6 +619,17 @@ "A continue statement should not escape from the function") @bc_reader.register_opcode +class RETURN_VALUE(BCInstruction): + def bc_flow(self, reader): + reader.curr_block.operations.append(self) + reader.end_block() + + def eval(self, ctx): + from rpython.flowspace.flowcontext import Return + w_returnvalue = ctx.popvalue() + raise Return(w_returnvalue) + + at bc_reader.register_opcode class END_FINALLY(BCInstruction): def bc_flow(self, reader): reader.curr_block.operations.append(self) diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py --- a/rpython/flowspace/flowcontext.py +++ b/rpython/flowspace/flowcontext.py @@ -606,9 +606,10 @@ w_module = self.peekvalue() self.pushvalue(self.import_from(w_module, w_name)) - def RETURN_VALUE(self, oparg): - w_returnvalue = self.popvalue() - raise Return(w_returnvalue) + def return_value(self, w_result): + link = Link([w_result], self.graph.returnblock) + self.recorder.crnt_block.closeblock(link) + raise StopFlowing def YIELD_VALUE(self, _): assert self.pycode.is_generator @@ -1014,10 +1015,7 @@ self.w_value = w_value def nomoreblocks(self, ctx): - w_result = self.w_value - link = Link([w_result], ctx.graph.returnblock) - ctx.recorder.crnt_block.closeblock(link) - raise StopFlowing + ctx.return_value(self.w_value) @property def args(self): diff --git a/rpython/flowspace/test/test_bytecode.py b/rpython/flowspace/test/test_bytecode.py --- a/rpython/flowspace/test/test_bytecode.py +++ b/rpython/flowspace/test/test_bytecode.py @@ -12,7 +12,7 @@ else: return 0 bc_graph = make_graph(f) - assert [lst[0].offset for lst in bc_graph.dump()] == [0, 6, 10] + assert [lst[0].offset for lst in bc_graph.dump()] == [0, 6, 10, 14] assert bc_graph.dump()[0][0] == bc_reader.new_instr('LOAD_FAST', 0) def test_blockstack(): From noreply at buildbot.pypy.org Sat Feb 14 19:28:50 2015 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 14 Feb 2015 19:28:50 +0100 (CET) Subject: [pypy-commit] pypy default: test, fix for array(..., dtype=xxx, subok=True) Message-ID: <20150214182850.202041C029E@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r75871:c745a42c6abe Date: 2015-02-14 20:28 +0200 http://bitbucket.org/pypy/pypy/changeset/c745a42c6abe/ Log: test, fix for array(..., dtype=xxx, subok=True) diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py --- a/pypy/module/micronumpy/ctors.py +++ b/pypy/module/micronumpy/ctors.py @@ -81,7 +81,7 @@ return w_object.descr_copy(space, w_order) elif not copy and (subok or type(w_object) is W_NDimArray): return w_object - if subok: + if subok and not type(w_object) is W_NDimArray: raise oefmt(space.w_NotImplementedError, "array(..., subok=True) only partially implemented") # we have a ndarray, but need to copy or change dtype diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py --- a/pypy/module/micronumpy/test/test_subtype.py +++ b/pypy/module/micronumpy/test/test_subtype.py @@ -26,6 +26,12 @@ self.called_finalize = True return SubType ''') + def test_subtype_ndarray(self): + from numpy import arange, array + a = arange(24, dtype='int32').reshape((6,4)) + b = array(a, dtype='float64', subok=True) + assert (a == b).all() + def test_subtype_base(self): from numpy import ndarray, dtype class C(ndarray): From noreply at buildbot.pypy.org Sat Feb 14 19:48:14 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Sat, 14 Feb 2015 19:48:14 +0100 (CET) Subject: [pypy-commit] pypy framestate: Don't print fake args in the repr of argless opcodes Message-ID: <20150214184814.14AAD1C034E@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: framestate Changeset: r75872:2c1cc95c3520 Date: 2015-02-14 16:10 +0000 http://bitbucket.org/pypy/pypy/changeset/2c1cc95c3520/ Log: Don't print fake args in the repr of argless opcodes diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py --- a/rpython/flowspace/bytecode.py +++ b/rpython/flowspace/bytecode.py @@ -429,6 +429,15 @@ return getattr(ctx, self.name)(self.arg) +class NullaryOpcode(BCInstruction): + def __init__(self, arg=NO_ARG, offset=-1): + self.arg = NO_ARG + self.offset = offset + + def __repr__(self): + return self.name + + def flow_opcode(func): name = func.__name__ class Op(BCInstruction): @@ -450,13 +459,13 @@ ctx.pushvalue(v_arg) @bc_reader.register_opcode -class DUP_TOP(BCInstruction): +class DUP_TOP(NullaryOpcode): def eval(self, ctx): w_1 = ctx.peekvalue() ctx.pushvalue(w_1) @bc_reader.register_opcode -class POP_TOP(BCInstruction): +class POP_TOP(NullaryOpcode): def eval(self, ctx): ctx.popvalue() @@ -511,7 +520,7 @@ return self.on_False -class SWITCH_BOOL(BCInstruction): +class SWITCH_BOOL(NullaryOpcode): name = 'SWITCH_BOOL' arg = NO_ARG def __init__(self, on_False, on_True, offset=-1): @@ -567,7 +576,7 @@ raise @bc_reader.register_opcode -class BREAK_LOOP(BCInstruction): +class BREAK_LOOP(NullaryOpcode): def bc_flow(self, reader): reader.curr_block.operations.append(self) reader.end_block() @@ -579,7 +588,7 @@ from rpython.flowspace.flowcontext import ExceptBlock, FinallyBlock while reader.blockstack: context = reader.blockstack.pop() - block.operations.append(POP_BLOCK(-1, self.offset)) + block.operations.append(POP_BLOCK(offset=self.offset)) if isinstance(context, ExceptBlock): pass elif isinstance(context, FinallyBlock): @@ -606,9 +615,9 @@ while reader.blockstack: context = reader.blockstack.pop() if isinstance(context, ExceptBlock): - block.operations.append(POP_BLOCK(-1, self.offset)) + block.operations.append(POP_BLOCK(offset=self.offset)) elif isinstance(context, FinallyBlock): - block.operations.append(POP_BLOCK(-1, self.offset)) + block.operations.append(POP_BLOCK(offset=self.offset)) reader.splice_finally_handler(block, context) block = context.handler_end else: # LoopBlock @@ -619,7 +628,7 @@ "A continue statement should not escape from the function") @bc_reader.register_opcode -class RETURN_VALUE(BCInstruction): +class RETURN_VALUE(NullaryOpcode): def bc_flow(self, reader): reader.curr_block.operations.append(self) reader.end_block() @@ -630,7 +639,7 @@ raise Return(w_returnvalue) @bc_reader.register_opcode -class END_FINALLY(BCInstruction): +class END_FINALLY(NullaryOpcode): def bc_flow(self, reader): reader.curr_block.operations.append(self) signal = reader.handlerstack.pop() @@ -737,7 +746,7 @@ ctx.pushvalue(w_result) @bc_reader.register_opcode -class POP_BLOCK(BCInstruction): +class POP_BLOCK(NullaryOpcode): def bc_flow(self, reader): reader.curr_block.operations.append(self) reader.end_block() @@ -758,7 +767,7 @@ ] def unaryoperation(OPCODE, oper): - class UNARY_OP(BCInstruction): + class UNARY_OP(NullaryOpcode): def eval(self, ctx): w_1 = ctx.popvalue() w_result = oper(w_1).eval(ctx) @@ -800,7 +809,7 @@ ] def binaryoperation(OPCODE, oper): - class BINARY_OP(BCInstruction): + class BINARY_OP(NullaryOpcode): def eval(self, ctx): w_2 = ctx.popvalue() w_1 = ctx.popvalue() From noreply at buildbot.pypy.org Sat Feb 14 19:48:15 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Sat, 14 Feb 2015 19:48:15 +0100 (CET) Subject: [pypy-commit] pypy framestate: Split RETURN_VALUE into 2 intructions and store the active return value on the flow context Message-ID: <20150214184815.51C2C1C034E@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: framestate Changeset: r75873:e4db2b64834f Date: 2015-02-14 18:41 +0000 http://bitbucket.org/pypy/pypy/changeset/e4db2b64834f/ Log: Split RETURN_VALUE into 2 intructions and store the active return value on the flow context diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py --- a/rpython/flowspace/bytecode.py +++ b/rpython/flowspace/bytecode.py @@ -253,7 +253,7 @@ if not b._exits: instr = b.operations[-1] assert instr.name in ( - 'RETURN_VALUE', 'RAISE_VARARGS', 'EXEC_STMT') + 'RETURN', 'RAISE_VARARGS', 'EXEC_STMT') for x in b._exits: assert x in self.blocks @@ -630,13 +630,24 @@ @bc_reader.register_opcode class RETURN_VALUE(NullaryOpcode): def bc_flow(self, reader): - reader.curr_block.operations.append(self) + block = reader.curr_block + block.operations.append(SET_RETURN_VALUE(offset=self.offset)) + block.operations.append(RETURN(offset=self.offset)) reader.end_block() +class SET_RETURN_VALUE(NullaryOpcode): + num = name = 'SET_RETURN_VALUE' + arg = NO_ARG + def eval(self, ctx): + w_value = ctx.popvalue() + ctx.w_return_value = w_value + +class RETURN(NullaryOpcode): + num = name = 'RETURN' + arg = NO_ARG def eval(self, ctx): from rpython.flowspace.flowcontext import Return - w_returnvalue = ctx.popvalue() - raise Return(w_returnvalue) + raise Return() @bc_reader.register_opcode class END_FINALLY(NullaryOpcode): diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py --- a/rpython/flowspace/flowcontext.py +++ b/rpython/flowspace/flowcontext.py @@ -291,13 +291,15 @@ def getstate(self, position): return FrameState(self.locals_w[:], self.stack[:], - self.last_exception, self.blockstack[:], position) + self.last_exception, self.w_return_value, + self.blockstack[:], position) def setstate(self, state): """ Reset the context to the given frame state. """ self.locals_w = state.locals_w[:] self.stack = state.stack[:] self.last_exception = state.last_exception + self.w_return_value = state.w_return_value self.blockstack = state.blocklist[:] self._normalize_raise_signals() @@ -606,7 +608,8 @@ w_module = self.peekvalue() self.pushvalue(self.import_from(w_module, w_name)) - def return_value(self, w_result): + def do_return(self): + w_result = self.w_return_value link = Link([w_result], self.graph.returnblock) self.recorder.crnt_block.closeblock(link) raise StopFlowing @@ -1008,22 +1011,18 @@ class Return(FlowSignal): - """Signals a 'return' statement. - Argument is the wrapped object to return. - """ - def __init__(self, w_value): - self.w_value = w_value - + """Signals a 'return' statement. """ def nomoreblocks(self, ctx): - ctx.return_value(self.w_value) + ctx.do_return() @property def args(self): - return [self.w_value] + return [] @staticmethod - def rebuild(w_value): - return Return(w_value) + def rebuild(): + return Return() + class Raise(FlowSignal): """Signals an application-level exception diff --git a/rpython/flowspace/framestate.py b/rpython/flowspace/framestate.py --- a/rpython/flowspace/framestate.py +++ b/rpython/flowspace/framestate.py @@ -16,10 +16,12 @@ class FrameState(object): - def __init__(self, locals_w, stack, last_exception, blocklist, position): + def __init__(self, locals_w, stack, last_exception, w_return_value, + blocklist, position): self.locals_w = locals_w self.stack = stack self.last_exception = last_exception + self.w_return_value = w_return_value self.blocklist = blocklist self.position = position self._mergeable = None @@ -35,6 +37,7 @@ else: data.append(self.last_exception.w_type) data.append(self.last_exception.w_value) + data.append(self.w_return_value) recursively_flatten(data) return data @@ -44,7 +47,7 @@ if exc is not None: exc = FSException(_copy(exc.w_type), _copy(exc.w_value)) return FrameState(map(_copy, self.locals_w), map(_copy, self.stack), - exc, self.blocklist, self.position) + exc, _copy(self.w_return_value), self.blocklist, self.position) def getvariables(self): return [w for w in self.mergeable if isinstance(w, Variable)] @@ -84,9 +87,10 @@ args2 = other._exc_args() exc = FSException(union(args1[0], args2[0]), union(args1[1], args2[1])) + w_rv = union(self.w_return_value, other.w_return_value) except UnionError: return None - return FrameState(locals, stack, exc, self.blocklist, self.position) + return FrameState(locals, stack, exc, w_rv, self.blocklist, self.position) def getoutputargs(self, targetstate): "Return the output arguments needed to link self to targetstate." diff --git a/rpython/flowspace/pygraph.py b/rpython/flowspace/pygraph.py --- a/rpython/flowspace/pygraph.py +++ b/rpython/flowspace/pygraph.py @@ -16,7 +16,7 @@ locals[i] = Variable(code.co_varnames[i]) bc_graph = code.graph start_pos = bc_graph.entry._exits[0], 0 - state = FrameState(locals, [], None, [], start_pos) + state = FrameState(locals, [], None, None, [], start_pos) initialblock = SpamBlock(state) super(PyGraph, self).__init__(self._sanitize_funcname(func), initialblock) self.func = func diff --git a/rpython/flowspace/test/test_flowcontext.py b/rpython/flowspace/test/test_flowcontext.py --- a/rpython/flowspace/test/test_flowcontext.py +++ b/rpython/flowspace/test/test_flowcontext.py @@ -5,7 +5,7 @@ Return, Raise, RaiseImplicit, Continue, Break) @pytest.mark.parametrize('signal', [ - Return(Variable()), + Return(), Raise(FSException(Variable(), Variable())), RaiseImplicit(FSException(Variable(), Variable())), Break(), From noreply at buildbot.pypy.org Sat Feb 14 20:26:49 2015 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 14 Feb 2015 20:26:49 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.9: merge default into branch Message-ID: <20150214192649.D57921C029E@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: stdlib-2.7.9 Changeset: r75874:efdb6e5046b6 Date: 2015-02-14 21:23 +0200 http://bitbucket.org/pypy/pypy/changeset/efdb6e5046b6/ Log: merge default into branch diff too long, truncating to 2000 out of 3194 lines diff --git a/lib-python/2.7/test/test_audioop.py b/lib-python/2.7/test/test_audioop.py --- a/lib-python/2.7/test/test_audioop.py +++ b/lib-python/2.7/test/test_audioop.py @@ -2,7 +2,7 @@ import sys import unittest import struct -from test.test_support import run_unittest, impl_detail +from test.test_support import run_unittest formats = { @@ -183,7 +183,6 @@ self.assertEqual(audioop.lin2lin(datas[4], 4, 2), packs[2](0, 0x1234, 0x4567, -0x4568, 0x7fff, -0x8000, -1)) - @impl_detail(pypy=False) def test_adpcm2lin(self): self.assertEqual(audioop.adpcm2lin(b'\x07\x7f\x7f', 1, None), (b'\x00\x00\x00\xff\x00\xff', (-179, 40))) @@ -198,7 +197,6 @@ self.assertEqual(audioop.adpcm2lin(b'\0' * 5, w, None), (b'\0' * w * 10, (0, 0))) - @impl_detail(pypy=False) def test_lin2adpcm(self): self.assertEqual(audioop.lin2adpcm(datas[1], 1, None), (b'\x07\x7f\x7f', (-221, 39))) @@ -212,7 +210,6 @@ self.assertEqual(audioop.lin2adpcm(b'\0' * w * 10, w, None), (b'\0' * 5, (0, 0))) - @impl_detail(pypy=False) def test_lin2alaw(self): self.assertEqual(audioop.lin2alaw(datas[1], 1), b'\xd5\x87\xa4\x24\xaa\x2a\x5a') @@ -221,7 +218,6 @@ self.assertEqual(audioop.lin2alaw(datas[4], 4), b'\xd5\x87\xa4\x24\xaa\x2a\x55') - @impl_detail(pypy=False) def test_alaw2lin(self): encoded = b'\x00\x03\x24\x2a\x51\x54\x55\x58\x6b\x71\x7f'\ b'\x80\x83\xa4\xaa\xd1\xd4\xd5\xd8\xeb\xf1\xff' @@ -236,7 +232,6 @@ decoded = audioop.alaw2lin(encoded, w) self.assertEqual(audioop.lin2alaw(decoded, w), encoded) - @impl_detail(pypy=False) def test_lin2ulaw(self): self.assertEqual(audioop.lin2ulaw(datas[1], 1), b'\xff\xad\x8e\x0e\x80\x00\x67') @@ -245,7 +240,6 @@ self.assertEqual(audioop.lin2ulaw(datas[4], 4), b'\xff\xad\x8e\x0e\x80\x00\x7e') - @impl_detail(pypy=False) def test_ulaw2lin(self): encoded = b'\x00\x0e\x28\x3f\x57\x6a\x76\x7c\x7e\x7f'\ b'\x80\x8e\xa8\xbf\xd7\xea\xf6\xfc\xfe\xff' @@ -360,7 +354,6 @@ self.assertRaises(audioop.error, audioop.findmax, ''.join( chr(x) for x in xrange(256)), -2392392) - @impl_detail(pypy=False) def test_issue7673(self): state = None for data, size in INVALID_DATA: @@ -385,7 +378,6 @@ self.assertRaises(audioop.error, audioop.lin2alaw, data, size) self.assertRaises(audioop.error, audioop.lin2adpcm, data, size, state) - @impl_detail(pypy=False) def test_wrongsize(self): data = b'abcdefgh' state = None diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py --- a/lib_pypy/_ctypes/array.py +++ b/lib_pypy/_ctypes/array.py @@ -9,7 +9,7 @@ def __new__(self, name, cls, typedict): res = type.__new__(self, name, cls, typedict) if '_type_' in typedict: - ffiarray = _rawffi.Array(typedict['_type_']._ffishape) + ffiarray = _rawffi.Array(typedict['_type_']._ffishape_) res._ffiarray = ffiarray subletter = getattr(typedict['_type_'], '_type_', None) if subletter == 'c': @@ -58,8 +58,8 @@ res.value = property(getvalue, setvalue) if '_length_' in typedict: - res._ffishape = (ffiarray, typedict['_length_']) - res._fficompositesize = res._sizeofinstances() + res._ffishape_ = (ffiarray, typedict['_length_']) + res._fficompositesize_ = res._sizeofinstances() else: res._ffiarray = None return res @@ -156,7 +156,7 @@ class Array(_CData): __metaclass__ = ArrayMeta - _ffiargshape = 'P' + _ffiargshape_ = 'P' def __init__(self, *args): if not hasattr(self, '_buffer'): @@ -191,13 +191,13 @@ if ensure_objects(cobj) is not None: store_reference(self, index, cobj._objects) arg = cobj._get_buffer_value() - if self._type_._fficompositesize is None: + if self._type_._fficompositesize_ is None: self._buffer[index] = arg # something more sophisticated, cannot set field directly else: from ctypes import memmove dest = self._buffer.itemaddress(index) - memmove(dest, arg, self._type_._fficompositesize) + memmove(dest, arg, self._type_._fficompositesize_) def __getitem__(self, index): if isinstance(index, slice): diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py --- a/lib_pypy/_ctypes/basics.py +++ b/lib_pypy/_ctypes/basics.py @@ -52,7 +52,7 @@ def get_ffi_argtype(self): if self._ffiargtype: return self._ffiargtype - self._ffiargtype = _shape_to_ffi_type(self._ffiargshape) + self._ffiargtype = _shape_to_ffi_type(self._ffiargshape_) return self._ffiargtype def _CData_output(self, resbuffer, base=None, index=-1): diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py --- a/lib_pypy/_ctypes/function.py +++ b/lib_pypy/_ctypes/function.py @@ -65,9 +65,9 @@ _restype_ = None _errcheck_ = None _flags_ = 0 - _ffiargshape = 'P' - _ffishape = 'P' - _fficompositesize = None + _ffiargshape_ = 'P' + _ffishape_ = 'P' + _fficompositesize_ = None _ffiarray = _rawffi.Array('P') _needs_free = False callable = None @@ -98,7 +98,7 @@ argtypes = property(_getargtypes, _setargtypes) def _check_argtypes_for_fastpath(self): - if all([hasattr(argtype, '_ffiargshape') for argtype in self._argtypes_]): + if all([hasattr(argtype, '_ffiargshape_') for argtype in self._argtypes_]): fastpath_cls = make_fastpath_subclass(self.__class__) fastpath_cls.enable_fastpath_maybe(self) @@ -135,7 +135,7 @@ _flag = flag & PARAMFLAG_COMBINED if _flag == PARAMFLAG_FOUT: typ = self._argtypes_[idx] - if getattr(typ, '_ffiargshape', None) not in ('P', 'z', 'Z'): + if getattr(typ, '_ffiargshape_', None) not in ('P', 'z', 'Z'): raise TypeError( "'out' parameter %d must be a pointer type, not %s" % (idx+1, type(typ).__name__) @@ -182,11 +182,11 @@ def _ffishapes(self, args, restype): if args is None: args = [] - argtypes = [arg._ffiargshape for arg in args] + argtypes = [arg._ffiargshape_ for arg in args] if restype is not None: if not isinstance(restype, SimpleType): raise TypeError("invalid result type for callback function") - restype = restype._ffiargshape + restype = restype._ffiargshape_ else: restype = 'O' # void return argtypes, restype @@ -599,7 +599,7 @@ if self._is_primitive(restype) and not restype._is_pointer_like(): return result # - shape = restype._ffishape + shape = restype._ffishape_ if is_struct_shape(shape): buf = result else: diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py --- a/lib_pypy/_ctypes/pointer.py +++ b/lib_pypy/_ctypes/pointer.py @@ -21,9 +21,9 @@ size = _rawffi.sizeof('P'), align = _rawffi.alignment('P'), length = 1, - _ffiargshape = 'P', - _ffishape = 'P', - _fficompositesize = None, + _ffiargshape_ = 'P', + _ffishape_ = 'P', + _fficompositesize_ = None, ) # XXX check if typedict['_type_'] is any sane # XXX remember about paramfunc diff --git a/lib_pypy/_ctypes/primitive.py b/lib_pypy/_ctypes/primitive.py --- a/lib_pypy/_ctypes/primitive.py +++ b/lib_pypy/_ctypes/primitive.py @@ -117,9 +117,9 @@ default = TP_TO_DEFAULT[tp] ffiarray = _rawffi.Array(tp) result = type.__new__(self, name, bases, dct) - result._ffiargshape = tp - result._ffishape = tp - result._fficompositesize = None + result._ffiargshape_ = tp + result._ffishape_ = tp + result._fficompositesize_ = None result._ffiarray = ffiarray if tp == 'z': # c_char_p diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -36,9 +36,9 @@ rawfields = [] for f in all_fields: if len(f) > 2: - rawfields.append((f[0], f[1]._ffishape, f[2])) + rawfields.append((f[0], f[1]._ffishape_, f[2])) else: - rawfields.append((f[0], f[1]._ffishape)) + rawfields.append((f[0], f[1]._ffishape_)) _set_shape(self, rawfields, self._is_union) @@ -48,8 +48,8 @@ value = field[1] is_bitfield = (len(field) == 3) fields[name] = Field(name, - self._ffistruct.fieldoffset(name), - self._ffistruct.fieldsize(name), + self._ffistruct_.fieldoffset(name), + self._ffistruct_.fieldsize(name), value, i, is_bitfield) if anonymous_fields: @@ -58,9 +58,9 @@ name = field[0] value = field[1] is_bitfield = (len(field) == 3) - startpos = self._ffistruct.fieldoffset(name) + startpos = self._ffistruct_.fieldoffset(name) if name in anonymous_fields: - for subname in value._names: + for subname in value._names_: resnames.append(subname) subfield = getattr(value, subname) relpos = startpos + subfield.offset @@ -71,7 +71,7 @@ else: resnames.append(name) names = resnames - self._names = names + self._names_ = names for name, field in fields.items(): setattr(self, name, field) @@ -114,19 +114,19 @@ elif ensure_objects(cobj) is not None: store_reference(obj, key, cobj._objects) arg = cobj._get_buffer_value() - if fieldtype._fficompositesize is not None: + if fieldtype._fficompositesize_ is not None: from ctypes import memmove dest = obj._buffer.fieldaddress(self.name) - memmove(dest, arg, fieldtype._fficompositesize) + memmove(dest, arg, fieldtype._fficompositesize_) else: obj._buffer.__setattr__(self.name, arg) def _set_shape(tp, rawfields, is_union=False): - tp._ffistruct = _rawffi.Structure(rawfields, is_union, + tp._ffistruct_ = _rawffi.Structure(rawfields, is_union, getattr(tp, '_pack_', 0)) - tp._ffiargshape = tp._ffishape = (tp._ffistruct, 1) - tp._fficompositesize = tp._ffistruct.size + tp._ffiargshape_ = tp._ffishape_ = (tp._ffistruct_, 1) + tp._fficompositesize_ = tp._ffistruct_.size def struct_setattr(self, name, value): @@ -181,16 +181,16 @@ address = address.buffer # fix the address: turn it into as unsigned, in case it is negative address = address & (sys.maxint * 2 + 1) - instance.__dict__['_buffer'] = self._ffistruct.fromaddress(address) + instance.__dict__['_buffer'] = self._ffistruct_.fromaddress(address) return instance def _sizeofinstances(self): - if not hasattr(self, '_ffistruct'): + if not hasattr(self, '_ffistruct_'): return 0 - return self._ffistruct.size + return self._ffistruct_.size def _alignmentofinstances(self): - return self._ffistruct.alignment + return self._ffistruct_.alignment def from_param(self, value): if isinstance(value, tuple): @@ -203,7 +203,7 @@ def _CData_output(self, resarray, base=None, index=-1): res = StructOrUnion.__new__(self) - ffistruct = self._ffistruct.fromaddress(resarray.buffer) + ffistruct = self._ffistruct_.fromaddress(resarray.buffer) res.__dict__['_buffer'] = ffistruct res.__dict__['_base'] = base res.__dict__['_index'] = index @@ -224,15 +224,15 @@ self = super(_CData, cls).__new__(cls, *args, **kwds) if '_abstract_' in cls.__dict__: raise TypeError("abstract class") - if hasattr(cls, '_ffistruct'): - self.__dict__['_buffer'] = self._ffistruct(autofree=True) + if hasattr(cls, '_ffistruct_'): + self.__dict__['_buffer'] = self._ffistruct_(autofree=True) return self def __init__(self, *args, **kwds): type(self)._make_final() - if len(args) > len(self._names): + if len(args) > len(self._names_): raise TypeError("too many initializers") - for name, arg in zip(self._names, args): + for name, arg in zip(self._names_, args): if name in kwds: raise TypeError("duplicate value for argument %r" % ( name,)) @@ -244,7 +244,7 @@ """Return a _rawffi array of length 1 whose address is the same as the address of the field 'name' of self.""" address = self._buffer.fieldaddress(name) - A = _rawffi.Array(fieldtype._ffishape) + A = _rawffi.Array(fieldtype._ffishape_) return A.fromaddress(address, 1) def _get_buffer_for_param(self): diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -270,11 +270,14 @@ _ffi.cdef("int sqlite3_enable_load_extension(sqlite3 *db, int onoff);") if sys.platform.startswith('freebsd'): + import os + import os.path + _localbase = os.environ.get('LOCALBASE', '/usr/local') _lib = _ffi.verify(""" #include """, libraries=['sqlite3'], - include_dirs=['/usr/local/include'], - library_dirs=['/usr/local/lib'] + include_dirs=[os.path.join(_localbase, 'include')], + library_dirs=[os.path.join(_localbase, 'lib')] ) else: _lib = _ffi.verify(""" diff --git a/lib_pypy/audioop.py b/lib_pypy/audioop.py --- a/lib_pypy/audioop.py +++ b/lib_pypy/audioop.py @@ -1,12 +1,11 @@ -from __future__ import division import __builtin__ as builtins import math import struct from fractions import gcd -from ctypes import create_string_buffer +from cffi import FFI -_buffer = buffer +_buffer = memoryview class error(Exception): @@ -149,7 +148,7 @@ def _sum2(cp1, cp2, length): size = 2 return sum(getsample(cp1, size, i) * getsample(cp2, size, i) - for i in range(length)) + for i in range(length)) + 0.0 def findfit(cp1, cp2): @@ -328,13 +327,14 @@ _check_params(len(cp), size) clip = _get_clipfn(size) - result = create_string_buffer(len(cp)) + rv = ffi.new("unsigned char[]", len(cp)) + result = ffi.buffer(rv) for i, sample in enumerate(_get_samples(cp, size)): sample = clip(int(sample * factor)) _put_sample(result, size, i, sample) - return result.raw + return result[:] def tomono(cp, size, fac1, fac2): @@ -343,7 +343,8 @@ sample_count = _sample_count(cp, size) - result = create_string_buffer(len(cp) // 2) + rv = ffi.new("unsigned char[]", len(cp) // 2) + result = ffi.buffer(rv) for i in range(0, sample_count, 2): l_sample = getsample(cp, size, i) @@ -354,7 +355,7 @@ _put_sample(result, size, i // 2, sample) - return result.raw + return result[:] def tostereo(cp, size, fac1, fac2): @@ -362,19 +363,9 @@ sample_count = _sample_count(cp, size) - result = create_string_buffer(len(cp) * 2) - clip = _get_clipfn(size) - - for i in range(sample_count): - sample = _get_sample(cp, size, i) - - l_sample = clip(sample * fac1) - r_sample = clip(sample * fac2) - - _put_sample(result, size, i * 2, l_sample) - _put_sample(result, size, i * 2 + 1, r_sample) - - return result.raw + rv = ffi.new("unsigned char[]", len(cp) * 2) + lib.tostereo(rv, cp, len(cp), size, fac1, fac2) + return ffi.buffer(rv)[:] def add(cp1, cp2, size): @@ -383,42 +374,34 @@ if len(cp1) != len(cp2): raise error("Lengths should be the same") - clip = _get_clipfn(size) - sample_count = _sample_count(cp1, size) - result = create_string_buffer(len(cp1)) - - for i in range(sample_count): - sample1 = getsample(cp1, size, i) - sample2 = getsample(cp2, size, i) - - sample = clip(sample1 + sample2) - - _put_sample(result, size, i, sample) - - return result.raw + rv = ffi.new("unsigned char[]", len(cp1)) + lib.add(rv, cp1, cp2, len(cp1), size) + return ffi.buffer(rv)[:] def bias(cp, size, bias): _check_params(len(cp), size) - result = create_string_buffer(len(cp)) + rv = ffi.new("unsigned char[]", len(cp)) + result = ffi.buffer(rv) for i, sample in enumerate(_get_samples(cp, size)): sample = _overflow(sample + bias, size) _put_sample(result, size, i, sample) - return result.raw + return result[:] def reverse(cp, size): _check_params(len(cp), size) sample_count = _sample_count(cp, size) - result = create_string_buffer(len(cp)) + rv = ffi.new("unsigned char[]", len(cp)) + result = ffi.buffer(rv) for i, sample in enumerate(_get_samples(cp, size)): _put_sample(result, size, sample_count - i - 1, sample) - return result.raw + return result[:] def lin2lin(cp, size, size2): @@ -429,7 +412,8 @@ return cp new_len = (len(cp) // size) * size2 - result = create_string_buffer(new_len) + rv = ffi.new("unsigned char[]", new_len) + result = ffi.buffer(rv) for i in range(_sample_count(cp, size)): sample = _get_sample(cp, size, i) @@ -444,7 +428,7 @@ sample = _overflow(sample, size2) _put_sample(result, size2, i, sample) - return result.raw + return result[:] def ratecv(cp, size, nchannels, inrate, outrate, state, weightA=1, weightB=0): @@ -471,11 +455,10 @@ inrate //= d outrate //= d - prev_i = [0] * nchannels - cur_i = [0] * nchannels - if state is None: d = -outrate + prev_i = ffi.new('int[]', nchannels) + cur_i = ffi.new('int[]', nchannels) else: d, samps = state @@ -483,70 +466,709 @@ raise error("illegal state argument") prev_i, cur_i = zip(*samps) - prev_i, cur_i = list(prev_i), list(cur_i) + prev_i = ffi.new('int[]', prev_i) + cur_i = ffi.new('int[]', cur_i) + state_d = ffi.new('int[]', (d,)) q = frame_count // inrate ceiling = (q + 1) * outrate nbytes = ceiling * bytes_per_frame - result = create_string_buffer(nbytes) + rv = ffi.new("unsigned char[]", nbytes) + trim_index = lib.ratecv(rv, cp, frame_count, size, + nchannels, inrate, outrate, + state_d, prev_i, cur_i, + weightA, weightB) + result = ffi.buffer(rv)[:trim_index] + samps = zip(prev_i, cur_i) + return (result, (d, tuple(samps))) - samples = _get_samples(cp, size) - out_i = 0 - while True: - while d < 0: - if frame_count == 0: - samps = zip(prev_i, cur_i) - retval = result.raw - # slice off extra bytes - trim_index = (out_i * bytes_per_frame) - len(retval) - retval = retval[:trim_index] +ffi = FFI() +ffi.cdef(""" +typedef short PyInt16; - return (retval, (d, tuple(samps))) +int ratecv(char* rv, char* cp, size_t len, int size, + int nchannels, int inrate, int outrate, + int* state_d, int* prev_i, int* cur_i, + int weightA, int weightB); - for chan in range(nchannels): - prev_i[chan] = cur_i[chan] - cur_i[chan] = next(samples) +void tostereo(char* rv, char* cp, size_t len, int size, + double fac1, double fac2); +void add(char* rv, char* cp1, char* cp2, size_t len1, int size); - cur_i[chan] = ( - (weightA * cur_i[chan] + weightB * prev_i[chan]) - // (weightA + weightB) - ) +/* 2's complement (14-bit range) */ +unsigned char +st_14linear2ulaw(PyInt16 pcm_val); +PyInt16 st_ulaw2linear16(unsigned char); - frame_count -= 1 - d += outrate +/* 2's complement (13-bit range) */ +unsigned char +st_linear2alaw(PyInt16 pcm_val); +PyInt16 st_alaw2linear16(unsigned char); - while d >= 0: - for chan in range(nchannels): - cur_o = ( - (prev_i[chan] * d + cur_i[chan] * (outrate - d)) - // outrate - ) - _put_sample(result, size, out_i, _overflow(cur_o, size)) - out_i += 1 - d -= inrate +void lin2adcpm(unsigned char* rv, unsigned char* cp, size_t len, + size_t size, int* state); +void adcpm2lin(unsigned char* rv, unsigned char* cp, size_t len, + size_t size, int* state); +""") + +# This code is directly copied from CPython file: Modules/audioop.c +_AUDIOOP_C_MODULE = """ +typedef short PyInt16; +typedef int Py_Int32; + +/* Code shamelessly stolen from sox, 12.17.7, g711.c +** (c) Craig Reese, Joe Campbell and Jeff Poskanzer 1989 */ + +/* From g711.c: + * + * December 30, 1994: + * Functions linear2alaw, linear2ulaw have been updated to correctly + * convert unquantized 16 bit values. + * Tables for direct u- to A-law and A- to u-law conversions have been + * corrected. + * Borge Lindberg, Center for PersonKommunikation, Aalborg University. + * bli at cpk.auc.dk + * + */ +#define BIAS 0x84 /* define the add-in bias for 16 bit samples */ +#define CLIP 32635 +#define SIGN_BIT (0x80) /* Sign bit for a A-law byte. */ +#define QUANT_MASK (0xf) /* Quantization field mask. */ +#define SEG_SHIFT (4) /* Left shift for segment number. */ +#define SEG_MASK (0x70) /* Segment field mask. */ + +static PyInt16 seg_aend[8] = {0x1F, 0x3F, 0x7F, 0xFF, + 0x1FF, 0x3FF, 0x7FF, 0xFFF}; +static PyInt16 seg_uend[8] = {0x3F, 0x7F, 0xFF, 0x1FF, + 0x3FF, 0x7FF, 0xFFF, 0x1FFF}; + +static PyInt16 +search(PyInt16 val, PyInt16 *table, int size) +{ + int i; + + for (i = 0; i < size; i++) { + if (val <= *table++) + return (i); + } + return (size); +} +#define st_ulaw2linear16(uc) (_st_ulaw2linear16[uc]) +#define st_alaw2linear16(uc) (_st_alaw2linear16[uc]) + +static PyInt16 _st_ulaw2linear16[256] = { + -32124, -31100, -30076, -29052, -28028, -27004, -25980, + -24956, -23932, -22908, -21884, -20860, -19836, -18812, + -17788, -16764, -15996, -15484, -14972, -14460, -13948, + -13436, -12924, -12412, -11900, -11388, -10876, -10364, + -9852, -9340, -8828, -8316, -7932, -7676, -7420, + -7164, -6908, -6652, -6396, -6140, -5884, -5628, + -5372, -5116, -4860, -4604, -4348, -4092, -3900, + -3772, -3644, -3516, -3388, -3260, -3132, -3004, + -2876, -2748, -2620, -2492, -2364, -2236, -2108, + -1980, -1884, -1820, -1756, -1692, -1628, -1564, + -1500, -1436, -1372, -1308, -1244, -1180, -1116, + -1052, -988, -924, -876, -844, -812, -780, + -748, -716, -684, -652, -620, -588, -556, + -524, -492, -460, -428, -396, -372, -356, + -340, -324, -308, -292, -276, -260, -244, + -228, -212, -196, -180, -164, -148, -132, + -120, -112, -104, -96, -88, -80, -72, + -64, -56, -48, -40, -32, -24, -16, + -8, 0, 32124, 31100, 30076, 29052, 28028, + 27004, 25980, 24956, 23932, 22908, 21884, 20860, + 19836, 18812, 17788, 16764, 15996, 15484, 14972, + 14460, 13948, 13436, 12924, 12412, 11900, 11388, + 10876, 10364, 9852, 9340, 8828, 8316, 7932, + 7676, 7420, 7164, 6908, 6652, 6396, 6140, + 5884, 5628, 5372, 5116, 4860, 4604, 4348, + 4092, 3900, 3772, 3644, 3516, 3388, 3260, + 3132, 3004, 2876, 2748, 2620, 2492, 2364, + 2236, 2108, 1980, 1884, 1820, 1756, 1692, + 1628, 1564, 1500, 1436, 1372, 1308, 1244, + 1180, 1116, 1052, 988, 924, 876, 844, + 812, 780, 748, 716, 684, 652, 620, + 588, 556, 524, 492, 460, 428, 396, + 372, 356, 340, 324, 308, 292, 276, + 260, 244, 228, 212, 196, 180, 164, + 148, 132, 120, 112, 104, 96, 88, + 80, 72, 64, 56, 48, 40, 32, + 24, 16, 8, 0 +}; + +/* + * linear2ulaw() accepts a 14-bit signed integer and encodes it as u-law data + * stored in a unsigned char. This function should only be called with + * the data shifted such that it only contains information in the lower + * 14-bits. + * + * In order to simplify the encoding process, the original linear magnitude + * is biased by adding 33 which shifts the encoding range from (0 - 8158) to + * (33 - 8191). The result can be seen in the following encoding table: + * + * Biased Linear Input Code Compressed Code + * ------------------------ --------------- + * 00000001wxyza 000wxyz + * 0000001wxyzab 001wxyz + * 000001wxyzabc 010wxyz + * 00001wxyzabcd 011wxyz + * 0001wxyzabcde 100wxyz + * 001wxyzabcdef 101wxyz + * 01wxyzabcdefg 110wxyz + * 1wxyzabcdefgh 111wxyz + * + * Each biased linear code has a leading 1 which identifies the segment + * number. The value of the segment number is equal to 7 minus the number + * of leading 0's. The quantization interval is directly available as the + * four bits wxyz. * The trailing bits (a - h) are ignored. + * + * Ordinarily the complement of the resulting code word is used for + * transmission, and so the code word is complemented before it is returned. + * + * For further information see John C. Bellamy's Digital Telephony, 1982, + * John Wiley & Sons, pps 98-111 and 472-476. + */ +static unsigned char +st_14linear2ulaw(PyInt16 pcm_val) /* 2's complement (14-bit range) */ +{ + PyInt16 mask; + PyInt16 seg; + unsigned char uval; + + /* The original sox code does this in the calling function, not here */ + pcm_val = pcm_val >> 2; + + /* u-law inverts all bits */ + /* Get the sign and the magnitude of the value. */ + if (pcm_val < 0) { + pcm_val = -pcm_val; + mask = 0x7F; + } else { + mask = 0xFF; + } + if ( pcm_val > CLIP ) pcm_val = CLIP; /* clip the magnitude */ + pcm_val += (BIAS >> 2); + + /* Convert the scaled magnitude to segment number. */ + seg = search(pcm_val, seg_uend, 8); + + /* + * Combine the sign, segment, quantization bits; + * and complement the code word. + */ + if (seg >= 8) /* out of range, return maximum value. */ + return (unsigned char) (0x7F ^ mask); + else { + uval = (unsigned char) (seg << 4) | ((pcm_val >> (seg + 1)) & 0xF); + return (uval ^ mask); + } + +} + +static PyInt16 _st_alaw2linear16[256] = { + -5504, -5248, -6016, -5760, -4480, -4224, -4992, + -4736, -7552, -7296, -8064, -7808, -6528, -6272, + -7040, -6784, -2752, -2624, -3008, -2880, -2240, + -2112, -2496, -2368, -3776, -3648, -4032, -3904, + -3264, -3136, -3520, -3392, -22016, -20992, -24064, + -23040, -17920, -16896, -19968, -18944, -30208, -29184, + -32256, -31232, -26112, -25088, -28160, -27136, -11008, + -10496, -12032, -11520, -8960, -8448, -9984, -9472, + -15104, -14592, -16128, -15616, -13056, -12544, -14080, + -13568, -344, -328, -376, -360, -280, -264, + -312, -296, -472, -456, -504, -488, -408, + -392, -440, -424, -88, -72, -120, -104, + -24, -8, -56, -40, -216, -200, -248, + -232, -152, -136, -184, -168, -1376, -1312, + -1504, -1440, -1120, -1056, -1248, -1184, -1888, + -1824, -2016, -1952, -1632, -1568, -1760, -1696, + -688, -656, -752, -720, -560, -528, -624, + -592, -944, -912, -1008, -976, -816, -784, + -880, -848, 5504, 5248, 6016, 5760, 4480, + 4224, 4992, 4736, 7552, 7296, 8064, 7808, + 6528, 6272, 7040, 6784, 2752, 2624, 3008, + 2880, 2240, 2112, 2496, 2368, 3776, 3648, + 4032, 3904, 3264, 3136, 3520, 3392, 22016, + 20992, 24064, 23040, 17920, 16896, 19968, 18944, + 30208, 29184, 32256, 31232, 26112, 25088, 28160, + 27136, 11008, 10496, 12032, 11520, 8960, 8448, + 9984, 9472, 15104, 14592, 16128, 15616, 13056, + 12544, 14080, 13568, 344, 328, 376, 360, + 280, 264, 312, 296, 472, 456, 504, + 488, 408, 392, 440, 424, 88, 72, + 120, 104, 24, 8, 56, 40, 216, + 200, 248, 232, 152, 136, 184, 168, + 1376, 1312, 1504, 1440, 1120, 1056, 1248, + 1184, 1888, 1824, 2016, 1952, 1632, 1568, + 1760, 1696, 688, 656, 752, 720, 560, + 528, 624, 592, 944, 912, 1008, 976, + 816, 784, 880, 848 +}; + +/* + * linear2alaw() accepts an 13-bit signed integer and encodes it as A-law data + * stored in a unsigned char. This function should only be called with + * the data shifted such that it only contains information in the lower + * 13-bits. + * + * Linear Input Code Compressed Code + * ------------------------ --------------- + * 0000000wxyza 000wxyz + * 0000001wxyza 001wxyz + * 000001wxyzab 010wxyz + * 00001wxyzabc 011wxyz + * 0001wxyzabcd 100wxyz + * 001wxyzabcde 101wxyz + * 01wxyzabcdef 110wxyz + * 1wxyzabcdefg 111wxyz + * + * For further information see John C. Bellamy's Digital Telephony, 1982, + * John Wiley & Sons, pps 98-111 and 472-476. + */ +static unsigned char +st_linear2alaw(PyInt16 pcm_val) /* 2's complement (13-bit range) */ +{ + PyInt16 mask; + short seg; + unsigned char aval; + + /* The original sox code does this in the calling function, not here */ + pcm_val = pcm_val >> 3; + + /* A-law using even bit inversion */ + if (pcm_val >= 0) { + mask = 0xD5; /* sign (7th) bit = 1 */ + } else { + mask = 0x55; /* sign bit = 0 */ + pcm_val = -pcm_val - 1; + } + + /* Convert the scaled magnitude to segment number. */ + seg = search(pcm_val, seg_aend, 8); + + /* Combine the sign, segment, and quantization bits. */ + + if (seg >= 8) /* out of range, return maximum value. */ + return (unsigned char) (0x7F ^ mask); + else { + aval = (unsigned char) seg << SEG_SHIFT; + if (seg < 2) + aval |= (pcm_val >> 1) & QUANT_MASK; + else + aval |= (pcm_val >> seg) & QUANT_MASK; + return (aval ^ mask); + } +} +/* End of code taken from sox */ + +/* Intel ADPCM step variation table */ +static int indexTable[16] = { + -1, -1, -1, -1, 2, 4, 6, 8, + -1, -1, -1, -1, 2, 4, 6, 8, +}; + +static int stepsizeTable[89] = { + 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, + 19, 21, 23, 25, 28, 31, 34, 37, 41, 45, + 50, 55, 60, 66, 73, 80, 88, 97, 107, 118, + 130, 143, 157, 173, 190, 209, 230, 253, 279, 307, + 337, 371, 408, 449, 494, 544, 598, 658, 724, 796, + 876, 963, 1060, 1166, 1282, 1411, 1552, 1707, 1878, 2066, + 2272, 2499, 2749, 3024, 3327, 3660, 4026, 4428, 4871, 5358, + 5894, 6484, 7132, 7845, 8630, 9493, 10442, 11487, 12635, 13899, + 15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794, 32767 +}; + +#define CHARP(cp, i) ((signed char *)(cp+i)) +#define SHORTP(cp, i) ((short *)(cp+i)) +#define LONGP(cp, i) ((Py_Int32 *)(cp+i)) +""" + +lib = ffi.verify(_AUDIOOP_C_MODULE + r""" +#include + +static const int maxvals[] = {0, 0x7F, 0x7FFF, 0x7FFFFF, 0x7FFFFFFF}; +/* -1 trick is needed on Windows to support -0x80000000 without a warning */ +static const int minvals[] = {0, -0x80, -0x8000, -0x800000, -0x7FFFFFFF-1}; + +static int +fbound(double val, double minval, double maxval) +{ + if (val > maxval) + val = maxval; + else if (val < minval + 1) + val = minval; + return val; +} + +static int +gcd(int a, int b) +{ + while (b > 0) { + int tmp = a % b; + a = b; + b = tmp; + } + return a; +} + +int ratecv(char* rv, char* cp, size_t len, int size, + int nchannels, int inrate, int outrate, + int* state_d, int* prev_i, int* cur_i, + int weightA, int weightB) +{ + char *ncp = rv; + int d, chan; + + /* divide inrate and outrate by their greatest common divisor */ + d = gcd(inrate, outrate); + inrate /= d; + outrate /= d; + /* divide weightA and weightB by their greatest common divisor */ + d = gcd(weightA, weightB); + weightA /= d; + weightA /= d; + + d = *state_d; + + for (;;) { + while (d < 0) { + if (len == 0) { + *state_d = d; + return ncp - rv; + } + for (chan = 0; chan < nchannels; chan++) { + prev_i[chan] = cur_i[chan]; + if (size == 1) + cur_i[chan] = ((int)*CHARP(cp, 0)) << 24; + else if (size == 2) + cur_i[chan] = ((int)*SHORTP(cp, 0)) << 16; + else if (size == 4) + cur_i[chan] = (int)*LONGP(cp, 0); + cp += size; + /* implements a simple digital filter */ + cur_i[chan] = (int)( + ((double)weightA * (double)cur_i[chan] + + (double)weightB * (double)prev_i[chan]) / + ((double)weightA + (double)weightB)); + } + len--; + d += outrate; + } + while (d >= 0) { + for (chan = 0; chan < nchannels; chan++) { + int cur_o; + cur_o = (int)(((double)prev_i[chan] * (double)d + + (double)cur_i[chan] * (double)(outrate - d)) / + (double)outrate); + if (size == 1) + *CHARP(ncp, 0) = (signed char)(cur_o >> 24); + else if (size == 2) + *SHORTP(ncp, 0) = (short)(cur_o >> 16); + else if (size == 4) + *LONGP(ncp, 0) = (Py_Int32)(cur_o); + ncp += size; + } + d -= inrate; + } + } +} + +void tostereo(char* rv, char* cp, size_t len, int size, + double fac1, double fac2) +{ + int val1, val2, val = 0; + double fval, maxval, minval; + char *ncp = rv; + int i; + + maxval = (double) maxvals[size]; + minval = (double) minvals[size]; + + for ( i=0; i < len; i += size ) { + if ( size == 1 ) val = (int)*CHARP(cp, i); + else if ( size == 2 ) val = (int)*SHORTP(cp, i); + else if ( size == 4 ) val = (int)*LONGP(cp, i); + + fval = (double)val*fac1; + val1 = (int)floor(fbound(fval, minval, maxval)); + + fval = (double)val*fac2; + val2 = (int)floor(fbound(fval, minval, maxval)); + + if ( size == 1 ) *CHARP(ncp, i*2) = (signed char)val1; + else if ( size == 2 ) *SHORTP(ncp, i*2) = (short)val1; + else if ( size == 4 ) *LONGP(ncp, i*2) = (Py_Int32)val1; + + if ( size == 1 ) *CHARP(ncp, i*2+1) = (signed char)val2; + else if ( size == 2 ) *SHORTP(ncp, i*2+2) = (short)val2; + else if ( size == 4 ) *LONGP(ncp, i*2+4) = (Py_Int32)val2; + } +} + +void add(char* rv, char* cp1, char* cp2, size_t len1, int size) +{ + int i; + int val1 = 0, val2 = 0, minval, maxval, newval; + char* ncp = rv; + + maxval = maxvals[size]; + minval = minvals[size]; + + for ( i=0; i < len1; i += size ) { + if ( size == 1 ) val1 = (int)*CHARP(cp1, i); + else if ( size == 2 ) val1 = (int)*SHORTP(cp1, i); + else if ( size == 4 ) val1 = (int)*LONGP(cp1, i); + + if ( size == 1 ) val2 = (int)*CHARP(cp2, i); + else if ( size == 2 ) val2 = (int)*SHORTP(cp2, i); + else if ( size == 4 ) val2 = (int)*LONGP(cp2, i); + + if (size < 4) { + newval = val1 + val2; + /* truncate in case of overflow */ + if (newval > maxval) + newval = maxval; + else if (newval < minval) + newval = minval; + } + else { + double fval = (double)val1 + (double)val2; + /* truncate in case of overflow */ + newval = (int)floor(fbound(fval, minval, maxval)); + } + + if ( size == 1 ) *CHARP(ncp, i) = (signed char)newval; + else if ( size == 2 ) *SHORTP(ncp, i) = (short)newval; + else if ( size == 4 ) *LONGP(ncp, i) = (Py_Int32)newval; + } +} + +void lin2adcpm(unsigned char* ncp, unsigned char* cp, size_t len, + size_t size, int* state) +{ + int step, outputbuffer = 0, bufferstep; + int val = 0; + int diff, vpdiff, sign, delta; + size_t i; + int valpred = state[0]; + int index = state[1]; + + step = stepsizeTable[index]; + bufferstep = 1; + + for ( i=0; i < len; i += size ) { + if ( size == 1 ) val = ((int)*CHARP(cp, i)) << 8; + else if ( size == 2 ) val = (int)*SHORTP(cp, i); + else if ( size == 4 ) val = ((int)*LONGP(cp, i)) >> 16; + + /* Step 1 - compute difference with previous value */ + diff = val - valpred; + sign = (diff < 0) ? 8 : 0; + if ( sign ) diff = (-diff); + + /* Step 2 - Divide and clamp */ + /* Note: + ** This code *approximately* computes: + ** delta = diff*4/step; + ** vpdiff = (delta+0.5)*step/4; + ** but in shift step bits are dropped. The net result of this + ** is that even if you have fast mul/div hardware you cannot + ** put it to good use since the fixup would be too expensive. + */ + delta = 0; + vpdiff = (step >> 3); + + if ( diff >= step ) { + delta = 4; + diff -= step; + vpdiff += step; + } + step >>= 1; + if ( diff >= step ) { + delta |= 2; + diff -= step; + vpdiff += step; + } + step >>= 1; + if ( diff >= step ) { + delta |= 1; + vpdiff += step; + } + + /* Step 3 - Update previous value */ + if ( sign ) + valpred -= vpdiff; + else + valpred += vpdiff; + + /* Step 4 - Clamp previous value to 16 bits */ + if ( valpred > 32767 ) + valpred = 32767; + else if ( valpred < -32768 ) + valpred = -32768; + + /* Step 5 - Assemble value, update index and step values */ + delta |= sign; + + index += indexTable[delta]; + if ( index < 0 ) index = 0; + if ( index > 88 ) index = 88; + step = stepsizeTable[index]; + + /* Step 6 - Output value */ + if ( bufferstep ) { + outputbuffer = (delta << 4) & 0xf0; + } else { + *ncp++ = (delta & 0x0f) | outputbuffer; + } + bufferstep = !bufferstep; + } + state[0] = valpred; + state[1] = index; +} + + +void adcpm2lin(unsigned char* ncp, unsigned char* cp, size_t len, + size_t size, int* state) +{ + int step, inputbuffer = 0, bufferstep; + int val = 0; + int diff, vpdiff, sign, delta; + size_t i; + int valpred = state[0]; + int index = state[1]; + + step = stepsizeTable[index]; + bufferstep = 0; + + for ( i=0; i < len*size*2; i += size ) { + /* Step 1 - get the delta value and compute next index */ + if ( bufferstep ) { + delta = inputbuffer & 0xf; + } else { + inputbuffer = *cp++; + delta = (inputbuffer >> 4) & 0xf; + } + + bufferstep = !bufferstep; + + /* Step 2 - Find new index value (for later) */ + index += indexTable[delta]; + if ( index < 0 ) index = 0; + if ( index > 88 ) index = 88; + + /* Step 3 - Separate sign and magnitude */ + sign = delta & 8; + delta = delta & 7; + + /* Step 4 - Compute difference and new predicted value */ + /* + ** Computes 'vpdiff = (delta+0.5)*step/4', but see comment + ** in adpcm_coder. + */ + vpdiff = step >> 3; + if ( delta & 4 ) vpdiff += step; + if ( delta & 2 ) vpdiff += step>>1; + if ( delta & 1 ) vpdiff += step>>2; + + if ( sign ) + valpred -= vpdiff; + else + valpred += vpdiff; + + /* Step 5 - clamp output value */ + if ( valpred > 32767 ) + valpred = 32767; + else if ( valpred < -32768 ) + valpred = -32768; + + /* Step 6 - Update step value */ + step = stepsizeTable[index]; + + /* Step 6 - Output value */ + if ( size == 1 ) *CHARP(ncp, i) = (signed char)(valpred >> 8); + else if ( size == 2 ) *SHORTP(ncp, i) = (short)(valpred); + else if ( size == 4 ) *LONGP(ncp, i) = (Py_Int32)(valpred<<16); + } + state[0] = valpred; + state[1] = index; +} +""") + +def _get_lin_samples(cp, size): + for sample in _get_samples(cp, size): + if size == 1: + yield sample << 8 + elif size == 2: + yield sample + elif size == 4: + yield sample >> 16 + +def _put_lin_sample(result, size, i, sample): + if size == 1: + sample >>= 8 + elif size == 2: + pass + elif size == 4: + sample <<= 16 + _put_sample(result, size, i, sample) def lin2ulaw(cp, size): - raise NotImplementedError() + _check_params(len(cp), size) + rv = ffi.new("unsigned char[]", _sample_count(cp, size)) + for i, sample in enumerate(_get_lin_samples(cp, size)): + rv[i] = lib.st_14linear2ulaw(sample) + return ffi.buffer(rv)[:] def ulaw2lin(cp, size): - raise NotImplementedError() + _check_size(size) + rv = ffi.new("unsigned char[]", len(cp) * size) + result = ffi.buffer(rv) + for i, value in enumerate(cp): + sample = lib.st_ulaw2linear16(ord(value)) + _put_lin_sample(result, size, i, sample) + return result[:] def lin2alaw(cp, size): - raise NotImplementedError() + _check_params(len(cp), size) + rv = ffi.new("unsigned char[]", _sample_count(cp, size)) + for i, sample in enumerate(_get_lin_samples(cp, size)): + rv[i] = lib.st_linear2alaw(sample) + return ffi.buffer(rv)[:] def alaw2lin(cp, size): - raise NotImplementedError() + _check_size(size) + rv = ffi.new("unsigned char[]", len(cp) * size) + result = ffi.buffer(rv) + for i, value in enumerate(cp): + sample = lib.st_alaw2linear16(ord(value)) + _put_lin_sample(result, size, i, sample) + return result[:] def lin2adpcm(cp, size, state): - raise NotImplementedError() + _check_params(len(cp), size) + if state is None: + state = (0, 0) + rv = ffi.new("unsigned char[]", len(cp) // size // 2) + state_ptr = ffi.new("int[]", state) + lib.lin2adcpm(rv, cp, len(cp), size, state_ptr) + return ffi.buffer(rv)[:], tuple(state_ptr) def adpcm2lin(cp, size, state): - raise NotImplementedError() + _check_size(size) + if state is None: + state = (0, 0) + rv = ffi.new("unsigned char[]", len(cp) * size * 2) + state_ptr = ffi.new("int[]", state) + lib.adcpm2lin(rv, cp, len(cp), size, state_ptr) + return ffi.buffer(rv)[:], tuple(state_ptr) + diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.8.6" -__version_info__ = (0, 8, 6) +__version__ = "0.8.6+" +__version_info__ = (0, 8, 6, "plus") # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -55,7 +55,8 @@ # _cffi_backend.so compiled. import _cffi_backend as backend from . import __version__ - assert backend.__version__ == __version__ + assert backend.__version__ == __version__, \ + "version mismatch, %s != %s" % (backend.__version__, __version__) # (If you insist you can also try to pass the option # 'backend=backend_ctypes.CTypesBackend()', but don't # rely on it! It's probably not going to work well.) diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py --- a/lib_pypy/cffi/backend_ctypes.py +++ b/lib_pypy/cffi/backend_ctypes.py @@ -2,11 +2,10 @@ from . import model if sys.version_info < (3,): - integer_types = (int, long) bytechr = chr else: unicode = str - integer_types = int + long = int xrange = range bytechr = lambda num: bytes([num]) @@ -181,7 +180,7 @@ address = 0 elif isinstance(source, CTypesData): address = source._cast_to_integer() - elif isinstance(source, integer_types): + elif isinstance(source, (int, long)): address = source else: raise TypeError("bad type for cast to %r: %r" % @@ -358,7 +357,7 @@ is_signed = (ctype(-1).value == -1) # def _cast_source_to_int(source): - if isinstance(source, (integer_types, float)): + if isinstance(source, (int, long, float)): source = int(source) elif isinstance(source, CTypesData): source = source._cast_to_integer() @@ -399,7 +398,7 @@ if kind == 'bool': @classmethod def _cast_from(cls, source): - if not isinstance(source, (integer_types, float)): + if not isinstance(source, (int, long, float)): source = _cast_source_to_int(source) return cls(bool(source)) def __int__(self): @@ -438,7 +437,7 @@ if kind == 'int' or kind == 'byte' or kind == 'bool': @staticmethod def _to_ctypes(x): - if not isinstance(x, integer_types): + if not isinstance(x, (int, long)): if isinstance(x, CTypesData): x = int(x) else: @@ -465,7 +464,7 @@ if kind == 'float': @staticmethod def _to_ctypes(x): - if not isinstance(x, (integer_types, float, CTypesData)): + if not isinstance(x, (int, long, float, CTypesData)): raise TypeError("float expected, got %s" % type(x).__name__) return ctype(x).value @@ -529,14 +528,14 @@ self._own = True def __add__(self, other): - if isinstance(other, integer_types): + if isinstance(other, (int, long)): return self._new_pointer_at(self._address + other * self._bitem_size) else: return NotImplemented def __sub__(self, other): - if isinstance(other, integer_types): + if isinstance(other, (int, long)): return self._new_pointer_at(self._address - other * self._bitem_size) elif type(self) is type(other): @@ -611,7 +610,7 @@ def __init__(self, init): if length is None: - if isinstance(init, integer_types): + if isinstance(init, (int, long)): len1 = init init = None elif kind == 'char' and isinstance(init, bytes): @@ -686,7 +685,7 @@ return CTypesPtr._arg_to_ctypes(value) def __add__(self, other): - if isinstance(other, integer_types): + if isinstance(other, (int, long)): return CTypesPtr._new_pointer_at( ctypes.addressof(self._blob) + other * ctypes.sizeof(BItem._ctype)) diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py --- a/lib_pypy/cffi/verifier.py +++ b/lib_pypy/cffi/verifier.py @@ -1,4 +1,4 @@ -import sys, os, binascii, shutil +import sys, os, binascii, shutil, io from . import __version_verifier_modules__ from . import ffiplatform @@ -13,6 +13,16 @@ if type == imp.C_EXTENSION] +if sys.version_info >= (3,): + NativeIO = io.StringIO +else: + class NativeIO(io.BytesIO): + def write(self, s): + if isinstance(s, unicode): + s = s.encode('ascii') + super(NativeIO, self).write(s) + + class Verifier(object): def __init__(self, ffi, preamble, tmpdir=None, modulename=None, @@ -144,19 +154,36 @@ self._vengine.collect_types() self._has_module = True - def _write_source(self, file=None): - must_close = (file is None) - if must_close: - _ensure_dir(self.sourcefilename) - file = open(self.sourcefilename, 'w') + def _write_source_to(self, file): self._vengine._f = file try: self._vengine.write_source_to_f() finally: del self._vengine._f - if must_close: - file.close() - if must_close: + + def _write_source(self, file=None): + if file is not None: + self._write_source_to(file) + else: + # Write our source file to an in memory file. + f = NativeIO() + self._write_source_to(f) + source_data = f.getvalue() + + # Determine if this matches the current file + if os.path.exists(self.sourcefilename): + with open(self.sourcefilename, "r") as fp: + needs_written = not (fp.read() == source_data) + else: + needs_written = True + + # Actually write the file out if it doesn't match + if needs_written: + _ensure_dir(self.sourcefilename) + with open(self.sourcefilename, "w") as fp: + fp.write(source_data) + + # Set this flag self._has_source = True def _compile_module(self): diff --git a/lib_pypy/gdbm.py b/lib_pypy/gdbm.py --- a/lib_pypy/gdbm.py +++ b/lib_pypy/gdbm.py @@ -1,4 +1,4 @@ -import cffi, os +import cffi, os, sys ffi = cffi.FFI() ffi.cdef(''' @@ -37,9 +37,19 @@ ''') try: - lib = ffi.verify(''' - #include "gdbm.h" - ''', libraries=['gdbm']) + if sys.platform.startswith('freebsd'): + import os.path + _localbase = os.environ.get('LOCALBASE', '/usr/local') + lib = ffi.verify(''' + #include "gdbm.h" + ''', libraries=['gdbm'], + include_dirs=[os.path.join(_localbase, 'include')], + library_dirs=[os.path.join(_localbase, 'lib')] + ) + else: + lib = ffi.verify(''' + #include "gdbm.h" + ''', libraries=['gdbm']) except cffi.VerificationError as e: # distutils does not preserve the actual message, # but the verification is simple enough that the diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py --- a/lib_pypy/pyrepl/readline.py +++ b/lib_pypy/pyrepl/readline.py @@ -73,6 +73,7 @@ assume_immutable_completions = False use_brackets = False sort_in_column = True + tab_insert_spaces_if_stem_is_empty = False def error(self, msg="none"): pass # don't show error messages by default @@ -86,6 +87,13 @@ return ''.join(b[p+1:self.pos]) def get_completions(self, stem): + if len(stem) == 0 and self.tab_insert_spaces_if_stem_is_empty: + b = self.buffer + p = self.pos + while p > 0 and b[p - 1] != '\n': + p -= 1 + num_spaces = 4 - ((self.pos - p) % 4) + return [' ' * num_spaces] result = [] function = self.config.readline_completer if function is not None: @@ -204,14 +212,15 @@ boolean value is true. """ reader = self.get_reader() - saved = reader.more_lines + saved = reader.more_lines, reader.tab_insert_spaces_if_stem_is_empty try: reader.more_lines = more_lines reader.ps1 = reader.ps2 = ps1 reader.ps3 = reader.ps4 = ps2 + reader.tab_insert_spaces_if_stem_is_empty = True return reader.readline(returns_unicode=returns_unicode) finally: - reader.more_lines = saved + reader.more_lines, reader.tab_insert_spaces_if_stem_is_empty = saved def parse_and_bind(self, string): pass # XXX we don't support parsing GNU-readline-style init files diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst --- a/pypy/doc/build.rst +++ b/pypy/doc/build.rst @@ -48,9 +48,13 @@ Install build-time dependencies ------------------------------- (**Note**: for some hints on how to translate the Python interpreter under -Windows, see the `windows document`_) +Windows, see the `windows document`_ . For hints on how to cross-compile in +a chroot using scratchbox2, see the `arm document`_ in the +`RPython documentation`_) .. _`windows document`: windows.html +.. _`arm document`: http://rpython.readthedocs.org/en/latest/arm.html +.. _`RPython documentation`: http://rpython.readthedocs.org To build PyPy on Unix using the C translation backend, you need at least a C diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -300,6 +300,18 @@ Notably missing from the list above are ``str`` and ``unicode``. If your code relies on comparing strings with ``is``, then it might break in PyPy. +Note that for floats there "``is``" only one object per "bit pattern" +of the float. So ``float('nan') is float('nan')`` is true on PyPy, +but not on CPython because they are two objects; but ``0.0 is -0.0`` +is always False, as the bit patterns are different. As usual, +``float('nan') == float('nan')`` is always False. When used in +containers (as list items or in sets for example), the exact rule of +equality used is "``if x is y or x == y``" (on both CPython and PyPy); +as a consequence, because all ``nans`` are identical in PyPy, you +cannot have several of them in a set, unlike in CPython. (Issue `#1974`__) + +.. __: https://bitbucket.org/pypy/pypy/issue/1974/different-behaviour-for-collections-of + Miscellaneous ------------- diff --git a/pypy/doc/interpreter-optimizations.rst b/pypy/doc/interpreter-optimizations.rst --- a/pypy/doc/interpreter-optimizations.rst +++ b/pypy/doc/interpreter-optimizations.rst @@ -165,7 +165,7 @@ We improved this by keeping method lookup separated from method call, unlike some other approaches, but using the value stack as a cache instead of building a temporary object. We extended the bytecode compiler to (optionally) generate -the following code for ``obj.meth(x)``:: +the following code for ``obj.meth(x, y)``:: LOAD_GLOBAL obj LOOKUP_METHOD meth @@ -181,7 +181,7 @@ the attribute actually refers to a function object from the class; when this is not the case, ``LOOKUP_METHOD`` still pushes two values, but one *(im_func)* is simply the regular result that ``LOAD_ATTR`` would have returned, and the other -*(im_self)* is a None placeholder. +*(im_self)* is an interpreter-level None placeholder. After pushing the arguments, the layout of the stack in the above example is as follows (the stack grows upwards): diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -712,7 +712,7 @@ return self.wrap(not self.is_true(w_obj)) def eq_w(self, w_obj1, w_obj2): - """shortcut for space.is_true(space.eq(w_obj1, w_obj2))""" + """Implements equality with the double check 'x is y or x == y'.""" return self.is_w(w_obj1, w_obj2) or self.is_true(self.eq(w_obj1, w_obj2)) def is_(self, w_one, w_two): diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -16,17 +16,20 @@ space.wrap(msg)])) return raise_unicode_exception_decode +class RUnicodeEncodeError(Exception): + def __init__(self, encoding, object, start, end, reason): + self.encoding = encoding + self.object = object + self.start = start + self.end = end + self.reason = reason + @specialize.memo() def encode_error_handler(space): # Fast version of the "strict" errors handler. def raise_unicode_exception_encode(errors, encoding, msg, u, startingpos, endingpos): - raise OperationError(space.w_UnicodeEncodeError, - space.newtuple([space.wrap(encoding), - space.wrap(u), - space.wrap(startingpos), - space.wrap(endingpos), - space.wrap(msg)])) + raise RUnicodeEncodeError(encoding, u, startingpos, endingpos, msg) return raise_unicode_exception_encode # ____________________________________________________________ diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -8,7 +8,7 @@ appleveldefs = { } interpleveldefs = { - '__version__': 'space.wrap("0.8.6")', + '__version__': 'space.wrap("0.8.6+")', 'load_library': 'libraryobj.load_library', diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -3241,4 +3241,4 @@ def test_version(): # this test is here mostly for PyPy - assert __version__ == "0.8.6" + assert __version__ == "0.8.6+" diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py --- a/pypy/module/_rawffi/test/test__rawffi.py +++ b/pypy/module/_rawffi/test/test__rawffi.py @@ -1153,6 +1153,11 @@ assert buffer(s)._pypy_raw_address() == addr assert buffer(s, 10)._pypy_raw_address() == addr + 10 + addr = memoryview(s)._pypy_raw_address() + assert type(addr) is int + assert memoryview(s)._pypy_raw_address() == addr + assert memoryview(s)[10:]._pypy_raw_address() == addr + 10 + def test_union(self): import _rawffi longsize = _rawffi.sizeof('l') diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py --- a/pypy/module/micronumpy/ctors.py +++ b/pypy/module/micronumpy/ctors.py @@ -81,7 +81,10 @@ return w_object.descr_copy(space, w_order) elif not copy and (subok or type(w_object) is W_NDimArray): return w_object - # we have a ndarray, but need to copy or change dtype or create W_NDimArray + if subok and not type(w_object) is W_NDimArray: + raise oefmt(space.w_NotImplementedError, + "array(..., subok=True) only partially implemented") + # we have a ndarray, but need to copy or change dtype if dtype is None: dtype = w_object.get_dtype() if dtype != w_object.get_dtype(): @@ -89,13 +92,12 @@ copy = True if copy: shape = w_object.get_shape() - _elems_w = w_object.reshape(space, space.wrap(-1)) elems_w = [None] * w_object.get_size() - for i in range(len(elems_w)): - elems_w[i] = _elems_w.descr_getitem(space, space.wrap(i)) - elif subok: - raise oefmt(space.w_NotImplementedError, - "array(...copy=False, subok=True) not implemented yet") + elsize = w_object.get_dtype().elsize + # TODO - use w_object.implementation without copying to a list + # unfortunately that causes a union error in translation + for i in range(w_object.get_size()): + elems_w[i] = w_object.implementation.getitem(i * elsize) else: sz = support.product(w_object.get_shape()) * dtype.elsize return W_NDimArray.from_shape_and_storage(space, @@ -113,7 +115,7 @@ dtype = descriptor.variable_dtype(space, dtype.char + '1') w_arr = W_NDimArray.from_shape(space, shape, dtype, order=order) - if len(elems_w) == 1: + if support.product(shape) == 1: w_arr.set_scalar_value(dtype.coerce(space, elems_w[0])) else: loop.assign(space, w_arr, elems_w) diff --git a/pypy/module/micronumpy/flatiter.py b/pypy/module/micronumpy/flatiter.py --- a/pypy/module/micronumpy/flatiter.py +++ b/pypy/module/micronumpy/flatiter.py @@ -15,6 +15,7 @@ self._base = base self.dtype = base.get_dtype() self.shape = [base.get_size()] + self.storage = self._base.implementation.storage def base(self): return self._base diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -2852,6 +2852,13 @@ c.flat = ['defgh', 'ijklmnop'] assert (c.flatten() == ['def', 'ijk']*5).all() + def test_flatiter_subtype(self): + from numpy import array + x = array([[1, 2], [3, 4]]).T + y = array(x.flat) + assert (x == [[1, 3], [2, 4]]).all() + + def test_slice_copy(self): from numpy import zeros a = zeros((10, 10)) diff --git a/pypy/module/micronumpy/test/test_subtype.py b/pypy/module/micronumpy/test/test_subtype.py --- a/pypy/module/micronumpy/test/test_subtype.py +++ b/pypy/module/micronumpy/test/test_subtype.py @@ -26,6 +26,12 @@ self.called_finalize = True return SubType ''') + def test_subtype_ndarray(self): + from numpy import arange, array + a = arange(24, dtype='int32').reshape((6,4)) + b = array(a, dtype='float64', subok=True) + assert (a == b).all() + def test_subtype_base(self): from numpy import ndarray, dtype class C(ndarray): @@ -272,40 +278,103 @@ import numpy as N # numpy's matrix class caused an infinite loop class matrix(N.ndarray): - getcnt = 0 def __new__(subtype, data, dtype=None, copy=True): + print('matrix __new__') + if isinstance(data, matrix): + dtype2 = data.dtype + if (dtype is None): + dtype = dtype2 + if (dtype2 == dtype) and (not copy): + return data + return data.astype(dtype) + + if isinstance(data, N.ndarray): + if dtype is None: + intype = data.dtype + else: + intype = N.dtype(dtype) + new = data.view(subtype) + if intype != data.dtype: + return new.astype(intype) + if copy: return new.copy() + else: return new + + if isinstance(data, str): + data = _convert_from_string(data) + + # now convert data to an array arr = N.array(data, dtype=dtype, copy=copy) + ndim = arr.ndim shape = arr.shape + if (ndim > 2): + raise ValueError("matrix must be 2-dimensional") + elif ndim == 0: + shape = (1, 1) + elif ndim == 1: + shape = (1, shape[0]) + + order = False + if (ndim == 2) and arr.flags.fortran: + order = True + + if not (order or arr.flags.contiguous): + arr = arr.copy() ret = N.ndarray.__new__(subtype, shape, arr.dtype, buffer=arr, - order=True) + order=order) return ret + def __array_finalize__(self, obj): + print('matrix __array_finalize__') + self._getitem = False + if (isinstance(obj, matrix) and obj._getitem): return + ndim = self.ndim + if (ndim == 2): + return + if (ndim > 2): + newshape = tuple([x for x in self.shape if x > 1]) + ndim = len(newshape) + if ndim == 2: + self.shape = newshape + return + elif (ndim > 2): + raise ValueError("shape too large to be a matrix.") + else: + newshape = self.shape + if ndim == 0: + self.shape = (1, 1) + elif ndim == 1: + self.shape = (1, newshape[0]) + return + def __getitem__(self, index): - matrix.getcnt += 1 - if matrix.getcnt > 10: - # XXX strides.find_shape_and_elems is sensitive - # to shape modification - xxx - out = N.ndarray.__getitem__(self, index) + print('matrix __getitem__') + self._getitem = True + + try: + out = N.ndarray.__getitem__(self, index) + finally: + self._getitem = False if not isinstance(out, N.ndarray): return out + + if out.ndim == 0: + return out[()] + if out.ndim == 1: + sh = out.shape[0] # Determine when we should have a column array - old_shape = out.shape - if out.ndim < 2: - sh = out.shape[0] try: n = len(index) except: n = 0 - if n > 1: + if n > 1 and isscalar(index[1]): out.shape = (sh, 1) else: out.shape = (1, sh) - #print 'out, shape was',old_shape,'now',out.shape,'out',out return out + a = matrix([[1., 2.], [3., 4.]]) b = N.array([a]) assert (b == a).all() @@ -318,6 +387,17 @@ assert len(b.shape) == 2 assert (b == a).all() + b = N.array(a, copy=True, dtype=int) + assert len(b.shape) == 2 + assert (b == a).all() + + c = matrix(a, copy=False) + assert c.base is not None + c[0, 0] = 100 + assert a[0, 0] == 100 + b = N.array(c, copy=True) + assert (b == a).all() + def test_setstate_no_version(self): # Some subclasses of ndarray, like MaskedArray, do not use # version in __setstare__ diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py --- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py +++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py @@ -27,7 +27,7 @@ log = self.run(main, [libm_name]) pow_addr, res = log.result assert res == 8.0 * 300 - py.test.xfail() # XXX re-optimize _ffi for the JIT? + py.test.skip("XXX re-optimize _ffi for the JIT?") loop, = log.loops_by_filename(self.filepath) if 'ConstClass(pow)' in repr(loop): # e.g. OS/X pow_addr = 'ConstClass(pow)' @@ -134,7 +134,7 @@ ops = loop.allops() opnames = log.opnames(ops) assert opnames.count('new_with_vtable') == 1 # only the virtualref - py.test.xfail() # XXX re-optimize _ffi for the JIT? + py.test.skip("XXX re-optimize _ffi for the JIT?") assert opnames.count('call_release_gil') == 1 idx = opnames.index('call_release_gil') call = ops[idx] @@ -159,7 +159,7 @@ return struct.getfield('x') # log = self.run(main, []) - py.test.xfail() # XXX re-optimize _ffi for the JIT? + py.test.skip("XXX re-optimize _ffi for the JIT?") loop, = log.loops_by_filename(self.filepath) assert loop.match_by_id('getfield', """ guard_not_invalidated(descr=...) diff --git a/pypy/module/test_lib_pypy/cffi_tests/test_ownlib.py b/pypy/module/test_lib_pypy/cffi_tests/test_ownlib.py --- a/pypy/module/test_lib_pypy/cffi_tests/test_ownlib.py +++ b/pypy/module/test_lib_pypy/cffi_tests/test_ownlib.py @@ -8,34 +8,137 @@ SOURCE = """\ #include -int test_getting_errno(void) { +#ifdef _WIN32 +#define EXPORT __declspec(dllexport) +#else +#define EXPORT +#endif + +EXPORT int test_getting_errno(void) { errno = 123; return -1; } -int test_setting_errno(void) { +EXPORT int test_setting_errno(void) { return errno; +}; + +typedef struct { + long x; + long y; +} POINT; + +typedef struct { + long left; + long top; + long right; + long bottom; +} RECT; + + +EXPORT int PointInRect(RECT *prc, POINT pt) +{ + if (pt.x < prc->left) + return 0; + if (pt.x > prc->right) + return 0; + if (pt.y < prc->top) + return 0; + if (pt.y > prc->bottom) From noreply at buildbot.pypy.org Sat Feb 14 21:09:37 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 14 Feb 2015 21:09:37 +0100 (CET) Subject: [pypy-commit] pypy default: fix test_egg_version Message-ID: <20150214200937.1A7E11C029E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75875:c737b98a841c Date: 2015-02-14 21:09 +0100 http://bitbucket.org/pypy/pypy/changeset/c737b98a841c/ Log: fix test_egg_version diff --git a/lib_pypy/cffi.egg-info b/lib_pypy/cffi.egg-info --- a/lib_pypy/cffi.egg-info +++ b/lib_pypy/cffi.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: cffi -Version: 0.8.6 +Version: 0.8.6+ Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski From noreply at buildbot.pypy.org Sat Feb 14 21:11:40 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 14 Feb 2015 21:11:40 +0100 (CET) Subject: [pypy-commit] pypy default: fix test_whatsnew Message-ID: <20150214201140.6130E1C029E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75876:1dcdfe298330 Date: 2015-02-14 21:11 +0100 http://bitbucket.org/pypy/pypy/changeset/1dcdfe298330/ Log: fix test_whatsnew diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,3 +5,6 @@ .. this is a revision shortly after release-2.5.x .. startrev: 397b96217b85 +.. branch: vmprof + +.. branch: stackroot-speedup-2 From noreply at buildbot.pypy.org Sat Feb 14 21:26:31 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 14 Feb 2015 21:26:31 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: update the values Message-ID: <20150214202631.C30441C029E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r571:fb7fcfce420f Date: 2015-02-14 21:27 +0100 http://bitbucket.org/pypy/pypy.org/changeset/fb7fcfce420f/ Log: update the values diff --git a/don1.html b/don1.html --- a/don1.html +++ b/don1.html @@ -9,13 +9,13 @@ - $58667 of $105000 (55.9%) + $58764 of $105000 (56.0%)
    diff --git a/don3.html b/don3.html --- a/don3.html +++ b/don3.html @@ -15,7 +15,7 @@ - $51288 of $60000 (85.5%) + $51316 of $60000 (85.5%)
    diff --git a/don4.html b/don4.html --- a/don4.html +++ b/don4.html @@ -9,7 +9,7 @@ @@ -17,7 +17,7 @@ 2nd call: - $22041 of $80000 (27.6%) + $22357 of $80000 (27.9%)
    From noreply at buildbot.pypy.org Sat Feb 14 22:35:43 2015 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 14 Feb 2015 22:35:43 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.9: merge stdlib-2.7.9 which updates stdlib Message-ID: <20150214213543.2801B1C050C@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: stdlib-2.7.9 Changeset: r75878:cca109a5138b Date: 2015-02-14 23:28 +0200 http://bitbucket.org/pypy/pypy/changeset/cca109a5138b/ Log: merge stdlib-2.7.9 which updates stdlib diff --git a/lib_pypy/cffi.egg-info b/lib_pypy/cffi.egg-info --- a/lib_pypy/cffi.egg-info +++ b/lib_pypy/cffi.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: cffi -Version: 0.8.6 +Version: 0.8.6+ Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,3 +5,6 @@ .. this is a revision shortly after release-2.5.x .. startrev: 397b96217b85 +.. branch: vmprof + +.. branch: stackroot-speedup-2 From noreply at buildbot.pypy.org Sat Feb 14 22:35:41 2015 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 14 Feb 2015 22:35:41 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.9: close branch to be merged Message-ID: <20150214213541.DA2571C050C@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: stdlib-2.7.9 Changeset: r75877:784c6261416a Date: 2015-02-14 23:27 +0200 http://bitbucket.org/pypy/pypy/changeset/784c6261416a/ Log: close branch to be merged From noreply at buildbot.pypy.org Sat Feb 14 22:35:44 2015 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 14 Feb 2015 22:35:44 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.9: update what's new Message-ID: <20150214213544.545D51C050C@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: stdlib-2.7.9 Changeset: r75879:05f4ad78e358 Date: 2015-02-14 23:35 +0200 http://bitbucket.org/pypy/pypy/changeset/05f4ad78e358/ Log: update what's new diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -8,3 +8,8 @@ .. branch: vmprof .. branch: stackroot-speedup-2 +Avoid tracing all stack roots during repeated minor collections, +by ignoring the part of the stack that didn't change + +.. branch: stdlib-2.7.9 +Update stdlib to version 2.7.9 From noreply at buildbot.pypy.org Sat Feb 14 22:35:45 2015 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 14 Feb 2015 22:35:45 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.9: update trunk for post-2.5.0 Message-ID: <20150214213545.7B8881C050C@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: stdlib-2.7.9 Changeset: r75880:a72780de7739 Date: 2015-02-14 23:36 +0200 http://bitbucket.org/pypy/pypy/changeset/a72780de7739/ Log: update trunk for post-2.5.0 diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,7 +29,7 @@ #define PY_VERSION "2.7.9" /* PyPy version as a string */ -#define PYPY_VERSION "2.5.0" +#define PYPY_VERSION "2.6.0" /* Subversion Revision number of this file (not of the repository). * Empty since Mercurial migration. */ diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -10,7 +10,7 @@ #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h -PYPY_VERSION = (2, 5, 0, "final", 0) #XXX # sync patchlevel.h +PYPY_VERSION = (2, 6, 0, "alpha", 0) #XXX # sync patchlevel.h if platform.name == 'msvc': COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600) From noreply at buildbot.pypy.org Sat Feb 14 22:59:00 2015 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 14 Feb 2015 22:59:00 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.9: reclose branch to be merged Message-ID: <20150214215900.6DC611C0F98@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: stdlib-2.7.9 Changeset: r75881:5a697895c2eb Date: 2015-02-14 23:59 +0200 http://bitbucket.org/pypy/pypy/changeset/5a697895c2eb/ Log: reclose branch to be merged From noreply at buildbot.pypy.org Sat Feb 14 22:59:05 2015 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 14 Feb 2015 22:59:05 +0100 (CET) Subject: [pypy-commit] pypy default: really merge stdlib-2.7.9 into default, which updates stdlib to 2.7.9 Message-ID: <20150214215905.B655B1C0F98@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r75882:7db3e6dd2efe Date: 2015-02-14 23:59 +0200 http://bitbucket.org/pypy/pypy/changeset/7db3e6dd2efe/ Log: really merge stdlib-2.7.9 into default, which updates stdlib to 2.7.9 diff too long, truncating to 2000 out of 28484 lines diff --git a/lib-python/2.7/CGIHTTPServer.py b/lib-python/2.7/CGIHTTPServer.py --- a/lib-python/2.7/CGIHTTPServer.py +++ b/lib-python/2.7/CGIHTTPServer.py @@ -106,16 +106,16 @@ def run_cgi(self): """Execute a CGI script.""" dir, rest = self.cgi_info - - i = rest.find('/') + path = dir + '/' + rest + i = path.find('/', len(dir)+1) while i >= 0: - nextdir = rest[:i] - nextrest = rest[i+1:] + nextdir = path[:i] + nextrest = path[i+1:] scriptdir = self.translate_path(nextdir) if os.path.isdir(scriptdir): dir, rest = nextdir, nextrest - i = rest.find('/') + i = path.find('/', len(dir)+1) else: break diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py --- a/lib-python/2.7/Cookie.py +++ b/lib-python/2.7/Cookie.py @@ -56,7 +56,7 @@ >>> C = Cookie.SmartCookie() [Note: Long-time users of Cookie.py will remember using -Cookie.Cookie() to create an Cookie object. Although deprecated, it +Cookie.Cookie() to create a Cookie object. Although deprecated, it is still supported by the code. See the Backward Compatibility notes for more information.] @@ -426,6 +426,8 @@ "version" : "Version", } + _flags = {'secure', 'httponly'} + def __init__(self): # Set defaults self.key = self.value = self.coded_value = None @@ -529,9 +531,11 @@ _LegalCharsPatt = r"[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=]" _CookiePattern = re.compile( r"(?x)" # This is a Verbose pattern + r"\s*" # Optional whitespace at start of cookie r"(?P" # Start of group 'key' ""+ _LegalCharsPatt +"+?" # Any word of at least one letter, nongreedy r")" # End of group 'key' + r"(" # Optional group: there may not be a value. r"\s*=\s*" # Equal Sign r"(?P" # Start of group 'val' r'"(?:[^\\"]|\\.)*"' # Any doublequoted string @@ -540,7 +544,9 @@ r"|" # or ""+ _LegalCharsPatt +"*" # Any word or empty string r")" # End of group 'val' - r"\s*;?" # Probably ending in a semi-colon + r")?" # End of optional value group + r"\s*" # Any number of spaces. + r"(\s+|;|$)" # Ending either at space, semicolon, or EOS. ) @@ -585,8 +591,12 @@ def __setitem__(self, key, value): """Dictionary style assignment.""" - rval, cval = self.value_encode(value) - self.__set(key, rval, cval) + if isinstance(value, Morsel): + # allow assignment of constructed Morsels (e.g. for pickling) + dict.__setitem__(self, key, value) + else: + rval, cval = self.value_encode(value) + self.__set(key, rval, cval) # end __setitem__ def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"): @@ -641,7 +651,7 @@ while 0 <= i < n: # Start looking for a cookie - match = patt.search(str, i) + match = patt.match(str, i) if not match: break # No more cookies K,V = match.group("key"), match.group("val") @@ -656,8 +666,12 @@ M[ K[1:] ] = V elif K.lower() in Morsel._reserved: if M: - M[ K ] = _unquote(V) - else: + if V is None: + if K.lower() in Morsel._flags: + M[K] = True + else: + M[K] = _unquote(V) + elif V is not None: rval, cval = self.value_decode(V) self.__set(K, rval, cval) M = self[K] diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py --- a/lib-python/2.7/SocketServer.py +++ b/lib-python/2.7/SocketServer.py @@ -416,8 +416,12 @@ self.socket = socket.socket(self.address_family, self.socket_type) if bind_and_activate: - self.server_bind() - self.server_activate() + try: + self.server_bind() + self.server_activate() + except: + self.server_close() + raise def server_bind(self): """Called by constructor to bind the socket. diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py --- a/lib-python/2.7/_abcoll.py +++ b/lib-python/2.7/_abcoll.py @@ -143,7 +143,7 @@ methods except for __contains__, __iter__ and __len__. To override the comparisons (presumably for speed, as the - semantics are fixed), all you have to do is redefine __le__ and + semantics are fixed), redefine __le__ and __ge__, then the other operations will automatically follow suit. """ diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py --- a/lib-python/2.7/argparse.py +++ b/lib-python/2.7/argparse.py @@ -1089,7 +1089,14 @@ # parse all the remaining options into the namespace # store any unrecognized options on the object, so that the top # level parser can decide what to do with them - namespace, arg_strings = parser.parse_known_args(arg_strings, namespace) + + # In case this subparser defines new defaults, we parse them + # in a new namespace object and then update the original + # namespace for the relevant parts. + subnamespace, arg_strings = parser.parse_known_args(arg_strings, None) + for key, value in vars(subnamespace).items(): + setattr(namespace, key, value) + if arg_strings: vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, []) getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings) diff --git a/lib-python/2.7/asynchat.py b/lib-python/2.7/asynchat.py --- a/lib-python/2.7/asynchat.py +++ b/lib-python/2.7/asynchat.py @@ -46,12 +46,17 @@ you - by calling your self.found_terminator() method. """ +import asyncore +import errno import socket -import asyncore from collections import deque from sys import py3kwarning from warnings import filterwarnings, catch_warnings +_BLOCKING_IO_ERRORS = (errno.EAGAIN, errno.EALREADY, errno.EINPROGRESS, + errno.EWOULDBLOCK) + + class async_chat (asyncore.dispatcher): """This is an abstract class. You must derive from this class, and add the two methods collect_incoming_data() and found_terminator()""" @@ -109,6 +114,8 @@ try: data = self.recv (self.ac_in_buffer_size) except socket.error, why: + if why.args[0] in _BLOCKING_IO_ERRORS: + return self.handle_error() return diff --git a/lib-python/2.7/bsddb/test/test_queue.py b/lib-python/2.7/bsddb/test/test_queue.py --- a/lib-python/2.7/bsddb/test/test_queue.py +++ b/lib-python/2.7/bsddb/test/test_queue.py @@ -10,6 +10,7 @@ #---------------------------------------------------------------------- + at unittest.skip("fails on Windows; see issue 22943") class SimpleQueueTestCase(unittest.TestCase): def setUp(self): self.filename = get_new_database_path() diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py --- a/lib-python/2.7/cookielib.py +++ b/lib-python/2.7/cookielib.py @@ -1719,12 +1719,12 @@ def __repr__(self): r = [] for cookie in self: r.append(repr(cookie)) - return "<%s[%s]>" % (self.__class__, ", ".join(r)) + return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r)) def __str__(self): r = [] for cookie in self: r.append(str(cookie)) - return "<%s[%s]>" % (self.__class__, ", ".join(r)) + return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r)) # derives from IOError for backwards-compatibility with Python 2.4.0 diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py --- a/lib-python/2.7/ctypes/test/test_pointers.py +++ b/lib-python/2.7/ctypes/test/test_pointers.py @@ -7,6 +7,8 @@ c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float] python_types = [int, int, int, int, int, long, int, long, long, long, float, float] +LargeNamedType = type('T' * 2 ** 25, (Structure,), {}) +large_string = 'T' * 2 ** 25 class PointersTestCase(unittest.TestCase): @@ -188,5 +190,11 @@ mth = WINFUNCTYPE(None)(42, "name", (), None) self.assertEqual(bool(mth), True) + def test_pointer_type_name(self): + self.assertTrue(POINTER(LargeNamedType)) + + def test_pointer_type_str_name(self): + self.assertTrue(POINTER(large_string)) + if __name__ == '__main__': unittest.main() diff --git a/lib-python/2.7/ctypes/test/test_python_api.py b/lib-python/2.7/ctypes/test/test_python_api.py --- a/lib-python/2.7/ctypes/test/test_python_api.py +++ b/lib-python/2.7/ctypes/test/test_python_api.py @@ -46,8 +46,8 @@ # This test is unreliable, because it is possible that code in # unittest changes the refcount of the '42' integer. So, it # is disabled by default. - @requires("refcount") def test_PyInt_Long(self): + requires("refcount") ref42 = grc(42) pythonapi.PyInt_FromLong.restype = py_object self.assertEqual(pythonapi.PyInt_FromLong(42), 42) diff --git a/lib-python/2.7/ctypes/test/test_win32.py b/lib-python/2.7/ctypes/test/test_win32.py --- a/lib-python/2.7/ctypes/test/test_win32.py +++ b/lib-python/2.7/ctypes/test/test_win32.py @@ -38,8 +38,11 @@ @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') class FunctionCallTestCase(unittest.TestCase): - @requires("SEH") + @unittest.skipUnless('MSC' in sys.version, "SEH only supported by MSC") + @unittest.skipIf(sys.executable.endswith('_d.exe'), + "SEH not enabled in debug builds") def test_SEH(self): + requires("SEH") # Call functions with invalid arguments, and make sure # that access violations are trapped and raise an # exception. @@ -87,9 +90,29 @@ dll = CDLL(_ctypes_test.__file__) - pt = POINT(10, 10) - rect = RECT(0, 0, 20, 20) - self.assertEqual(1, dll.PointInRect(byref(rect), pt)) + pt = POINT(15, 25) + left = c_long.in_dll(dll, 'left') + top = c_long.in_dll(dll, 'top') + right = c_long.in_dll(dll, 'right') + bottom = c_long.in_dll(dll, 'bottom') + rect = RECT(left, top, right, bottom) + PointInRect = dll.PointInRect + PointInRect.argtypes = [POINTER(RECT), POINT] + self.assertEqual(1, PointInRect(byref(rect), pt)) + + ReturnRect = dll.ReturnRect + ReturnRect.argtypes = [c_int, RECT, POINTER(RECT), POINT, RECT, + POINTER(RECT), POINT, RECT] + ReturnRect.restype = RECT + for i in range(4): + ret = ReturnRect(i, rect, pointer(rect), pt, rect, + byref(rect), pt, rect) + # the c function will check and modify ret if something is + # passed in improperly + self.assertEqual(ret.left, left.value) + self.assertEqual(ret.right, right.value) + self.assertEqual(ret.top, top.value) + self.assertEqual(ret.bottom, bottom.value) if __name__ == '__main__': unittest.main() diff --git a/lib-python/2.7/decimal.py b/lib-python/2.7/decimal.py --- a/lib-python/2.7/decimal.py +++ b/lib-python/2.7/decimal.py @@ -136,7 +136,6 @@ __version__ = '1.70' # Highest version of the spec this complies with -import copy as _copy import math as _math import numbers as _numbers @@ -3665,6 +3664,8 @@ if self._is_special: sign = _format_sign(self._sign, spec) body = str(self.copy_abs()) + if spec['type'] == '%': + body += '%' return _format_align(sign, body, spec) # a type of None defaults to 'g' or 'G', depending on context @@ -6033,7 +6034,10 @@ format_dict['decimal_point'] = '.' # record whether return type should be str or unicode - format_dict['unicode'] = isinstance(format_spec, unicode) + try: + format_dict['unicode'] = isinstance(format_spec, unicode) + except NameError: + format_dict['unicode'] = False return format_dict diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py --- a/lib-python/2.7/distutils/__init__.py +++ b/lib-python/2.7/distutils/__init__.py @@ -15,5 +15,5 @@ # Updated automatically by the Python release process. # #--start constants-- -__version__ = "2.7.8" +__version__ = "2.7.9" #--end constants-- diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py --- a/lib-python/2.7/distutils/command/build_ext.py +++ b/lib-python/2.7/distutils/command/build_ext.py @@ -245,7 +245,7 @@ # Python's library directory must be appended to library_dirs # See Issues: #1600860, #4366 if (sysconfig.get_config_var('Py_ENABLE_SHARED')): - if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): + if not sysconfig.python_build: # building third party extensions self.library_dirs.append(sysconfig.get_config_var('LIBDIR')) else: diff --git a/lib-python/2.7/distutils/command/upload.py b/lib-python/2.7/distutils/command/upload.py --- a/lib-python/2.7/distutils/command/upload.py +++ b/lib-python/2.7/distutils/command/upload.py @@ -136,8 +136,8 @@ # Build up the MIME payload for the POST data boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254' - sep_boundary = '\n--' + boundary - end_boundary = sep_boundary + '--' + sep_boundary = '\r\n--' + boundary + end_boundary = sep_boundary + '--\r\n' body = StringIO.StringIO() for key, value in data.items(): # handle multiple entries for the same name @@ -151,14 +151,13 @@ fn = "" body.write(sep_boundary) - body.write('\nContent-Disposition: form-data; name="%s"'%key) + body.write('\r\nContent-Disposition: form-data; name="%s"' % key) body.write(fn) - body.write("\n\n") + body.write("\r\n\r\n") body.write(value) if value and value[-1] == '\r': body.write('\n') # write an extra newline (lurve Macs) body.write(end_boundary) - body.write("\n") body = body.getvalue() self.announce("Submitting %s to %s" % (filename, self.repository), log.INFO) diff --git a/lib-python/2.7/distutils/file_util.py b/lib-python/2.7/distutils/file_util.py --- a/lib-python/2.7/distutils/file_util.py +++ b/lib-python/2.7/distutils/file_util.py @@ -85,7 +85,8 @@ (os.symlink) instead of copying: set it to "hard" or "sym"; if it is None (the default), files are copied. Don't set 'link' on systems that don't support it: 'copy_file()' doesn't check if hard or symbolic - linking is available. + linking is available. If hardlink fails, falls back to + _copy_file_contents(). Under Mac OS, uses the native file copy function in macostools; on other systems, uses '_copy_file_contents()' to copy file contents. @@ -137,24 +138,31 @@ # (Unix only, of course, but that's the caller's responsibility) if link == 'hard': if not (os.path.exists(dst) and os.path.samefile(src, dst)): - os.link(src, dst) + try: + os.link(src, dst) + return (dst, 1) + except OSError: + # If hard linking fails, fall back on copying file + # (some special filesystems don't support hard linking + # even under Unix, see issue #8876). + pass elif link == 'sym': if not (os.path.exists(dst) and os.path.samefile(src, dst)): os.symlink(src, dst) + return (dst, 1) # Otherwise (non-Mac, not linking), copy the file contents and # (optionally) copy the times and mode. - else: - _copy_file_contents(src, dst) - if preserve_mode or preserve_times: - st = os.stat(src) + _copy_file_contents(src, dst) + if preserve_mode or preserve_times: + st = os.stat(src) - # According to David Ascher , utime() should be done - # before chmod() (at least under NT). - if preserve_times: - os.utime(dst, (st[ST_ATIME], st[ST_MTIME])) - if preserve_mode: - os.chmod(dst, S_IMODE(st[ST_MODE])) + # According to David Ascher , utime() should be done + # before chmod() (at least under NT). + if preserve_times: + os.utime(dst, (st[ST_ATIME], st[ST_MTIME])) + if preserve_mode: + os.chmod(dst, S_IMODE(st[ST_MODE])) return (dst, 1) diff --git a/lib-python/2.7/distutils/sysconfig_cpython.py b/lib-python/2.7/distutils/sysconfig_cpython.py --- a/lib-python/2.7/distutils/sysconfig_cpython.py +++ b/lib-python/2.7/distutils/sysconfig_cpython.py @@ -165,7 +165,8 @@ # version and build tools may not support the same set # of CPU architectures for universal builds. global _config_vars - if not _config_vars.get('CUSTOMIZED_OSX_COMPILER', ''): + # Use get_config_var() to ensure _config_vars is initialized. + if not get_config_var('CUSTOMIZED_OSX_COMPILER'): import _osx_support _osx_support.customize_compiler(_config_vars) _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True' diff --git a/lib-python/2.7/distutils/tests/test_bdist_rpm.py b/lib-python/2.7/distutils/tests/test_bdist_rpm.py --- a/lib-python/2.7/distutils/tests/test_bdist_rpm.py +++ b/lib-python/2.7/distutils/tests/test_bdist_rpm.py @@ -25,6 +25,7 @@ """ class BuildRpmTestCase(support.TempdirManager, + support.EnvironGuard, support.LoggingSilencer, unittest.TestCase): @@ -50,6 +51,7 @@ def test_quiet(self): # let's create a package tmp_dir = self.mkdtemp() + os.environ['HOME'] = tmp_dir # to confine dir '.rpmdb' creation pkg_dir = os.path.join(tmp_dir, 'foo') os.mkdir(pkg_dir) self.write_file((pkg_dir, 'setup.py'), SETUP_PY) @@ -92,6 +94,7 @@ def test_no_optimize_flag(self): # let's create a package that brakes bdist_rpm tmp_dir = self.mkdtemp() + os.environ['HOME'] = tmp_dir # to confine dir '.rpmdb' creation pkg_dir = os.path.join(tmp_dir, 'foo') os.mkdir(pkg_dir) self.write_file((pkg_dir, 'setup.py'), SETUP_PY) diff --git a/lib-python/2.7/distutils/tests/test_dist.py b/lib-python/2.7/distutils/tests/test_dist.py --- a/lib-python/2.7/distutils/tests/test_dist.py +++ b/lib-python/2.7/distutils/tests/test_dist.py @@ -11,7 +11,7 @@ from distutils.dist import Distribution, fix_help_options from distutils.cmd import Command import distutils.dist -from test.test_support import TESTFN, captured_stdout, run_unittest +from test.test_support import TESTFN, captured_stdout, run_unittest, unlink from distutils.tests import support @@ -64,6 +64,7 @@ with open(TESTFN, "w") as f: f.write("[global]\n") f.write("command_packages = foo.bar, splat") + self.addCleanup(unlink, TESTFN) files = [TESTFN] sys.argv.append("build") diff --git a/lib-python/2.7/distutils/tests/test_file_util.py b/lib-python/2.7/distutils/tests/test_file_util.py --- a/lib-python/2.7/distutils/tests/test_file_util.py +++ b/lib-python/2.7/distutils/tests/test_file_util.py @@ -8,6 +8,11 @@ from distutils.tests import support from test.test_support import run_unittest + +requires_os_link = unittest.skipUnless(hasattr(os, "link"), + "test requires os.link()") + + class FileUtilTestCase(support.TempdirManager, unittest.TestCase): def _log(self, msg, *args): @@ -74,6 +79,44 @@ copy_file(foo, dst_dir) self.assertTrue(os.path.exists(os.path.join(dst_dir, 'foo'))) + @requires_os_link + def test_copy_file_hard_link(self): + with open(self.source, 'w') as f: + f.write('some content') + st = os.stat(self.source) + copy_file(self.source, self.target, link='hard') + st2 = os.stat(self.source) + st3 = os.stat(self.target) + self.assertTrue(os.path.samestat(st, st2), (st, st2)) + self.assertTrue(os.path.samestat(st2, st3), (st2, st3)) + with open(self.source, 'r') as f: + self.assertEqual(f.read(), 'some content') + + @requires_os_link + def test_copy_file_hard_link_failure(self): + # If hard linking fails, copy_file() falls back on copying file + # (some special filesystems don't support hard linking even under + # Unix, see issue #8876). + with open(self.source, 'w') as f: + f.write('some content') + st = os.stat(self.source) + def _os_link(*args): + raise OSError(0, "linking unsupported") + old_link = os.link + os.link = _os_link + try: + copy_file(self.source, self.target, link='hard') + finally: + os.link = old_link + st2 = os.stat(self.source) + st3 = os.stat(self.target) + self.assertTrue(os.path.samestat(st, st2), (st, st2)) + self.assertFalse(os.path.samestat(st2, st3), (st2, st3)) + for fn in (self.source, self.target): + with open(fn, 'r') as f: + self.assertEqual(f.read(), 'some content') + + def test_suite(): return unittest.makeSuite(FileUtilTestCase) diff --git a/lib-python/2.7/distutils/tests/test_sysconfig.py b/lib-python/2.7/distutils/tests/test_sysconfig.py --- a/lib-python/2.7/distutils/tests/test_sysconfig.py +++ b/lib-python/2.7/distutils/tests/test_sysconfig.py @@ -3,6 +3,9 @@ import test import unittest import shutil +import subprocess +import sys +import textwrap from distutils import sysconfig from distutils.tests import support @@ -99,6 +102,24 @@ self.assertEqual(global_sysconfig.get_config_var('LDSHARED'), sysconfig.get_config_var('LDSHARED')) self.assertEqual(global_sysconfig.get_config_var('CC'), sysconfig.get_config_var('CC')) + def test_customize_compiler_before_get_config_vars(self): + # Issue #21923: test that a Distribution compiler + # instance can be called without an explicit call to + # get_config_vars(). + with open(TESTFN, 'w') as f: + f.writelines(textwrap.dedent('''\ + from distutils.core import Distribution + config = Distribution().get_command_obj('config') + # try_compile may pass or it may fail if no compiler + # is found but it should not raise an exception. + rc = config.try_compile('int x;') + ''')) + p = subprocess.Popen([str(sys.executable), TESTFN], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True) + outs, errs = p.communicate() + self.assertEqual(0, p.returncode, "Subprocess failed: " + outs) def test_suite(): diff --git a/lib-python/2.7/distutils/tests/test_upload.py b/lib-python/2.7/distutils/tests/test_upload.py --- a/lib-python/2.7/distutils/tests/test_upload.py +++ b/lib-python/2.7/distutils/tests/test_upload.py @@ -119,7 +119,7 @@ # what did we send ? self.assertIn('dédé', self.last_open.req.data) headers = dict(self.last_open.req.headers) - self.assertEqual(headers['Content-length'], '2085') + self.assertEqual(headers['Content-length'], '2159') self.assertTrue(headers['Content-type'].startswith('multipart/form-data')) self.assertEqual(self.last_open.req.get_method(), 'POST') self.assertEqual(self.last_open.req.get_full_url(), diff --git a/lib-python/2.7/doctest.py b/lib-python/2.7/doctest.py --- a/lib-python/2.7/doctest.py +++ b/lib-python/2.7/doctest.py @@ -216,7 +216,7 @@ # get_data() opens files as 'rb', so one must do the equivalent # conversion as universal newlines would do. return file_contents.replace(os.linesep, '\n'), filename - with open(filename) as f: + with open(filename, 'U') as f: return f.read(), filename # Use sys.stdout encoding for ouput. diff --git a/lib-python/2.7/email/feedparser.py b/lib-python/2.7/email/feedparser.py --- a/lib-python/2.7/email/feedparser.py +++ b/lib-python/2.7/email/feedparser.py @@ -49,8 +49,8 @@ simple abstraction -- it parses until EOF closes the current message. """ def __init__(self): - # The last partial line pushed into this object. - self._partial = '' + # Chunks of the last partial line pushed into this object. + self._partial = [] # The list of full, pushed lines, in reverse order self._lines = [] # The stack of false-EOF checking predicates. @@ -66,8 +66,8 @@ def close(self): # Don't forget any trailing partial line. - self._lines.append(self._partial) - self._partial = '' + self.pushlines(''.join(self._partial).splitlines(True)) + self._partial = [] self._closed = True def readline(self): @@ -95,8 +95,29 @@ def push(self, data): """Push some new data into this object.""" - # Handle any previous leftovers - data, self._partial = self._partial + data, '' + # Crack into lines, but preserve the linesep characters on the end of each + parts = data.splitlines(True) + + if not parts or not parts[0].endswith(('\n', '\r')): + # No new complete lines, so just accumulate partials + self._partial += parts + return + + if self._partial: + # If there are previous leftovers, complete them now + self._partial.append(parts[0]) + parts[0:1] = ''.join(self._partial).splitlines(True) + del self._partial[:] + + # If the last element of the list does not end in a newline, then treat + # it as a partial line. We only check for '\n' here because a line + # ending with '\r' might be a line that was split in the middle of a + # '\r\n' sequence (see bugs 1555570 and 1721862). + if not parts[-1].endswith('\n'): + self._partial = [parts.pop()] + self.pushlines(parts) + + def pushlines(self, lines): # Crack into lines, but preserve the newlines on the end of each parts = NLCRE_crack.split(data) # The *ahem* interesting behaviour of re.split when supplied grouping diff --git a/lib-python/2.7/email/mime/nonmultipart.py b/lib-python/2.7/email/mime/nonmultipart.py --- a/lib-python/2.7/email/mime/nonmultipart.py +++ b/lib-python/2.7/email/mime/nonmultipart.py @@ -12,7 +12,7 @@ class MIMENonMultipart(MIMEBase): - """Base class for MIME multipart/* type messages.""" + """Base class for MIME non-multipart type messages.""" def attach(self, payload): # The public API prohibits attaching multiple subparts to MIMEBase diff --git a/lib-python/2.7/email/test/test_email.py b/lib-python/2.7/email/test/test_email.py --- a/lib-python/2.7/email/test/test_email.py +++ b/lib-python/2.7/email/test/test_email.py @@ -11,6 +11,7 @@ import warnings import textwrap from cStringIO import StringIO +from random import choice import email @@ -2578,16 +2579,64 @@ bsf.push(il) nt += n n1 = 0 - while True: - ol = bsf.readline() - if ol == NeedMoreData: - break + for ol in iter(bsf.readline, NeedMoreData): om.append(ol) n1 += 1 self.assertEqual(n, n1) self.assertEqual(len(om), nt) self.assertEqual(''.join([il for il, n in imt]), ''.join(om)) + def test_push_random(self): + from email.feedparser import BufferedSubFile, NeedMoreData + + n = 10000 + chunksize = 5 + chars = 'abcd \t\r\n' + + s = ''.join(choice(chars) for i in range(n)) + '\n' + target = s.splitlines(True) + + bsf = BufferedSubFile() + lines = [] + for i in range(0, len(s), chunksize): + chunk = s[i:i+chunksize] + bsf.push(chunk) + lines.extend(iter(bsf.readline, NeedMoreData)) + self.assertEqual(lines, target) + + +class TestFeedParsers(TestEmailBase): + + def parse(self, chunks): + from email.feedparser import FeedParser + feedparser = FeedParser() + for chunk in chunks: + feedparser.feed(chunk) + return feedparser.close() + + def test_newlines(self): + m = self.parse(['a:\nb:\rc:\r\nd:\n']) + self.assertEqual(m.keys(), ['a', 'b', 'c', 'd']) + m = self.parse(['a:\nb:\rc:\r\nd:']) + self.assertEqual(m.keys(), ['a', 'b', 'c', 'd']) + m = self.parse(['a:\rb', 'c:\n']) + self.assertEqual(m.keys(), ['a', 'bc']) + m = self.parse(['a:\r', 'b:\n']) + self.assertEqual(m.keys(), ['a', 'b']) + m = self.parse(['a:\r', '\nb:\n']) + self.assertEqual(m.keys(), ['a', 'b']) + + def test_long_lines(self): + # Expected peak memory use on 32-bit platform: 4*N*M bytes. + M, N = 1000, 20000 + m = self.parse(['a:b\n\n'] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', 'b')]) + self.assertEqual(m.get_payload(), 'x'*M*N) + m = self.parse(['a:b\r\r'] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', 'b')]) + self.assertEqual(m.get_payload(), 'x'*M*N) + m = self.parse(['a:\r', 'b: '] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', ''), ('b', 'x'*M*N)]) class TestParsers(TestEmailBase): @@ -3180,7 +3229,6 @@ self.assertEqual(res, '=?iso-8859-2?q?abc?=') self.assertIsInstance(res, str) - # Test RFC 2231 header parameters (en/de)coding class TestRFC2231(TestEmailBase): def test_get_param(self): diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/__init__.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python2 +from __future__ import print_function + +import os +import os.path +import pkgutil +import shutil +import sys +import tempfile + + +__all__ = ["version", "bootstrap"] + + +_SETUPTOOLS_VERSION = "7.0" + +_PIP_VERSION = "1.5.6" + +# pip currently requires ssl support, so we try to provide a nicer +# error message when that is missing (http://bugs.python.org/issue19744) +_MISSING_SSL_MESSAGE = ("pip {} requires SSL/TLS".format(_PIP_VERSION)) +try: + import ssl +except ImportError: + ssl = None + + def _require_ssl_for_pip(): + raise RuntimeError(_MISSING_SSL_MESSAGE) +else: + def _require_ssl_for_pip(): + pass + +_PROJECTS = [ + ("setuptools", _SETUPTOOLS_VERSION), + ("pip", _PIP_VERSION), +] + + +def _run_pip(args, additional_paths=None): + # Add our bundled software to the sys.path so we can import it + if additional_paths is not None: + sys.path = additional_paths + sys.path + + # Install the bundled software + import pip + pip.main(args) + + +def version(): + """ + Returns a string specifying the bundled version of pip. + """ + return _PIP_VERSION + + +def _disable_pip_configuration_settings(): + # We deliberately ignore all pip environment variables + # when invoking pip + # See http://bugs.python.org/issue19734 for details + keys_to_remove = [k for k in os.environ if k.startswith("PIP_")] + for k in keys_to_remove: + del os.environ[k] + # We also ignore the settings in the default pip configuration file + # See http://bugs.python.org/issue20053 for details + os.environ['PIP_CONFIG_FILE'] = os.devnull + + +def bootstrap(root=None, upgrade=False, user=False, + altinstall=False, default_pip=True, + verbosity=0): + """ + Bootstrap pip into the current Python installation (or the given root + directory). + + Note that calling this function will alter both sys.path and os.environ. + """ + if altinstall and default_pip: + raise ValueError("Cannot use altinstall and default_pip together") + + _require_ssl_for_pip() + _disable_pip_configuration_settings() + + # By default, installing pip and setuptools installs all of the + # following scripts (X.Y == running Python version): + # + # pip, pipX, pipX.Y, easy_install, easy_install-X.Y + # + # pip 1.5+ allows ensurepip to request that some of those be left out + if altinstall: + # omit pip, pipX and easy_install + os.environ["ENSUREPIP_OPTIONS"] = "altinstall" + elif not default_pip: + # omit pip and easy_install + os.environ["ENSUREPIP_OPTIONS"] = "install" + + tmpdir = tempfile.mkdtemp() + try: + # Put our bundled wheels into a temporary directory and construct the + # additional paths that need added to sys.path + additional_paths = [] + for project, version in _PROJECTS: + wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version) + whl = pkgutil.get_data( + "ensurepip", + "_bundled/{}".format(wheel_name), + ) + with open(os.path.join(tmpdir, wheel_name), "wb") as fp: + fp.write(whl) + + additional_paths.append(os.path.join(tmpdir, wheel_name)) + + # Construct the arguments to be passed to the pip command + args = ["install", "--no-index", "--find-links", tmpdir] + if root: + args += ["--root", root] + if upgrade: + args += ["--upgrade"] + if user: + args += ["--user"] + if verbosity: + args += ["-" + "v" * verbosity] + + _run_pip(args + [p[0] for p in _PROJECTS], additional_paths) + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def _uninstall_helper(verbosity=0): + """Helper to support a clean default uninstall process on Windows + + Note that calling this function may alter os.environ. + """ + # Nothing to do if pip was never installed, or has been removed + try: + import pip + except ImportError: + return + + # If the pip version doesn't match the bundled one, leave it alone + if pip.__version__ != _PIP_VERSION: + msg = ("ensurepip will only uninstall a matching version " + "({!r} installed, {!r} bundled)") + print(msg.format(pip.__version__, _PIP_VERSION), file=sys.stderr) + return + + _require_ssl_for_pip() + _disable_pip_configuration_settings() + + # Construct the arguments to be passed to the pip command + args = ["uninstall", "-y"] + if verbosity: + args += ["-" + "v" * verbosity] + + _run_pip(args + [p[0] for p in reversed(_PROJECTS)]) + + +def _main(argv=None): + if ssl is None: + print("Ignoring ensurepip failure: {}".format(_MISSING_SSL_MESSAGE), + file=sys.stderr) + return + + import argparse + parser = argparse.ArgumentParser(prog="python -m ensurepip") + parser.add_argument( + "--version", + action="version", + version="pip {}".format(version()), + help="Show the version of pip that is bundled with this Python.", + ) + parser.add_argument( + "-v", "--verbose", + action="count", + default=0, + dest="verbosity", + help=("Give more output. Option is additive, and can be used up to 3 " + "times."), + ) + parser.add_argument( + "-U", "--upgrade", + action="store_true", + default=False, + help="Upgrade pip and dependencies, even if already installed.", + ) + parser.add_argument( + "--user", + action="store_true", + default=False, + help="Install using the user scheme.", + ) + parser.add_argument( + "--root", + default=None, + help="Install everything relative to this alternate root directory.", + ) + parser.add_argument( + "--altinstall", + action="store_true", + default=False, + help=("Make an alternate install, installing only the X.Y versioned" + "scripts (Default: pipX, pipX.Y, easy_install-X.Y)"), + ) + parser.add_argument( + "--default-pip", + action="store_true", + default=True, + dest="default_pip", + help=argparse.SUPPRESS, + ) + parser.add_argument( + "--no-default-pip", + action="store_false", + dest="default_pip", + help=("Make a non default install, installing only the X and X.Y " + "versioned scripts."), + ) + + args = parser.parse_args(argv) + + bootstrap( + root=args.root, + upgrade=args.upgrade, + user=args.user, + verbosity=args.verbosity, + altinstall=args.altinstall, + default_pip=args.default_pip, + ) diff --git a/lib-python/2.7/ensurepip/__main__.py b/lib-python/2.7/ensurepip/__main__.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/__main__.py @@ -0,0 +1,4 @@ +import ensurepip + +if __name__ == "__main__": + ensurepip._main() diff --git a/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..097ab43430d4c1302b0be353a8c16407c370693b GIT binary patch [cut] diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..fa1d1054da1dab98f8906555d31a9fda271b3a85 GIT binary patch [cut] diff --git a/lib-python/2.7/ensurepip/_uninstall.py b/lib-python/2.7/ensurepip/_uninstall.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/_uninstall.py @@ -0,0 +1,30 @@ +"""Basic pip uninstallation support, helper for the Windows uninstaller""" + +import argparse +import ensurepip + + +def _main(argv=None): + parser = argparse.ArgumentParser(prog="python -m ensurepip._uninstall") + parser.add_argument( + "--version", + action="version", + version="pip {}".format(ensurepip.version()), + help="Show the version of pip this will attempt to uninstall.", + ) + parser.add_argument( + "-v", "--verbose", + action="count", + default=0, + dest="verbosity", + help=("Give more output. Option is additive, and can be used up to 3 " + "times."), + ) + + args = parser.parse_args(argv) + + ensurepip._uninstall_helper(verbosity=args.verbosity) + + +if __name__ == "__main__": + _main() diff --git a/lib-python/2.7/glob.py b/lib-python/2.7/glob.py --- a/lib-python/2.7/glob.py +++ b/lib-python/2.7/glob.py @@ -35,11 +35,16 @@ patterns. """ + dirname, basename = os.path.split(pathname) if not has_magic(pathname): - if os.path.lexists(pathname): - yield pathname + if basename: + if os.path.lexists(pathname): + yield pathname + else: + # Patterns ending with a slash should match only directories + if os.path.isdir(dirname): + yield pathname return - dirname, basename = os.path.split(pathname) if not dirname: for name in glob1(os.curdir, basename): yield name diff --git a/lib-python/2.7/gzip.py b/lib-python/2.7/gzip.py --- a/lib-python/2.7/gzip.py +++ b/lib-python/2.7/gzip.py @@ -164,9 +164,16 @@ def _write_gzip_header(self): self.fileobj.write('\037\213') # magic header self.fileobj.write('\010') # compression method - fname = os.path.basename(self.name) - if fname.endswith(".gz"): - fname = fname[:-3] + try: + # RFC 1952 requires the FNAME field to be Latin-1. Do not + # include filenames that cannot be represented that way. + fname = os.path.basename(self.name) + if not isinstance(fname, str): + fname = fname.encode('latin-1') + if fname.endswith('.gz'): + fname = fname[:-3] + except UnicodeEncodeError: + fname = '' flags = 0 if fname: flags = FNAME diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py --- a/lib-python/2.7/hashlib.py +++ b/lib-python/2.7/hashlib.py @@ -15,8 +15,9 @@ md5(), sha1(), sha224(), sha256(), sha384(), and sha512() -More algorithms may be available on your platform but the above are -guaranteed to exist. +More algorithms may be available on your platform but the above are guaranteed +to exist. See the algorithms_guaranteed and algorithms_available attributes +to find out what algorithm names can be passed to new(). NOTE: If you want the adler32 or crc32 hash functions they are available in the zlib module. @@ -58,9 +59,14 @@ # always available algorithm is added. __always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512') +algorithms_guaranteed = set(__always_supported) +algorithms_available = set(__always_supported) + algorithms = __always_supported -__all__ = __always_supported + ('new', 'algorithms', 'pbkdf2_hmac') +__all__ = __always_supported + ('new', 'algorithms_guaranteed', + 'algorithms_available', 'algorithms', + 'pbkdf2_hmac') def __get_builtin_constructor(name): @@ -128,6 +134,8 @@ import _hashlib new = __hash_new __get_hash = __get_openssl_constructor + algorithms_available = algorithms_available.union( + _hashlib.openssl_md_meth_names) except ImportError: new = __py_new __get_hash = __get_builtin_constructor diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py --- a/lib-python/2.7/httplib.py +++ b/lib-python/2.7/httplib.py @@ -215,6 +215,10 @@ # maximal line length when calling readline(). _MAXLINE = 65536 +# maximum amount of headers accepted +_MAXHEADERS = 100 + + class HTTPMessage(mimetools.Message): def addheader(self, key, value): @@ -271,6 +275,8 @@ elif self.seekable: tell = self.fp.tell while True: + if len(hlist) > _MAXHEADERS: + raise HTTPException("got more than %d headers" % _MAXHEADERS) if tell: try: startofline = tell() @@ -1185,21 +1191,29 @@ def __init__(self, host, port=None, key_file=None, cert_file=None, strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - source_address=None): + source_address=None, context=None): HTTPConnection.__init__(self, host, port, strict, timeout, source_address) self.key_file = key_file self.cert_file = cert_file + if context is None: + context = ssl._create_default_https_context() + if key_file or cert_file: + context.load_cert_chain(cert_file, key_file) + self._context = context def connect(self): "Connect to a host on a given (SSL) port." - sock = self._create_connection((self.host, self.port), - self.timeout, self.source_address) + HTTPConnection.connect(self) + if self._tunnel_host: - self.sock = sock - self._tunnel() - self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file) + server_hostname = self._tunnel_host + else: + server_hostname = self.host + + self.sock = self._context.wrap_socket(self.sock, + server_hostname=server_hostname) __all__.append("HTTPSConnection") @@ -1214,14 +1228,15 @@ _connection_class = HTTPSConnection def __init__(self, host='', port=None, key_file=None, cert_file=None, - strict=None): + strict=None, context=None): # provide a default host, pass the X509 cert info # urf. compensate for bad input. if port == 0: port = None self._setup(self._connection_class(host, port, key_file, - cert_file, strict)) + cert_file, strict, + context=context)) # we never actually use these for anything, but we keep them # here for compatibility with post-1.5.2 CVS. diff --git a/lib-python/2.7/idlelib/Bindings.py b/lib-python/2.7/idlelib/Bindings.py --- a/lib-python/2.7/idlelib/Bindings.py +++ b/lib-python/2.7/idlelib/Bindings.py @@ -75,7 +75,8 @@ ('!_Auto-open Stack Viewer', '<>'), ]), ('options', [ - ('_Configure IDLE...', '<>'), + ('Configure _IDLE', '<>'), + ('Configure _Extensions', '<>'), None, ]), ('help', [ diff --git a/lib-python/2.7/idlelib/CallTipWindow.py b/lib-python/2.7/idlelib/CallTipWindow.py --- a/lib-python/2.7/idlelib/CallTipWindow.py +++ b/lib-python/2.7/idlelib/CallTipWindow.py @@ -2,9 +2,8 @@ After ToolTip.py, which uses ideas gleaned from PySol Used by the CallTips IDLE extension. - """ -from Tkinter import * +from Tkinter import Toplevel, Label, LEFT, SOLID, TclError HIDE_VIRTUAL_EVENT_NAME = "<>" HIDE_SEQUENCES = ("", "") @@ -133,35 +132,28 @@ return bool(self.tipwindow) -def _calltip_window(parent): - root = Tk() - root.title("Test calltips") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) +def _calltip_window(parent): # htest # + from Tkinter import Toplevel, Text, LEFT, BOTH - class MyEditWin: # comparenceptually an editor_window - def __init__(self): - text = self.text = Text(root) - text.pack(side=LEFT, fill=BOTH, expand=1) - text.insert("insert", "string.split") - root.update() - self.calltip = CallTip(text) + top = Toplevel(parent) + top.title("Test calltips") + top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, + parent.winfo_rooty() + 150)) + text = Text(top) + text.pack(side=LEFT, fill=BOTH, expand=1) + text.insert("insert", "string.split") + top.update() + calltip = CallTip(text) - text.event_add("<>", "(") - text.event_add("<>", ")") - text.bind("<>", self.calltip_show) - text.bind("<>", self.calltip_hide) - - text.focus_set() - root.mainloop() - - def calltip_show(self, event): - self.calltip.showtip("Hello world", "insert", "end") - - def calltip_hide(self, event): - self.calltip.hidetip() - - editwin = MyEditWin() + def calltip_show(event): + calltip.showtip("(s=Hello world)", "insert", "end") + def calltip_hide(event): + calltip.hidetip() + text.event_add("<>", "(") + text.event_add("<>", ")") + text.bind("<>", calltip_show) + text.bind("<>", calltip_hide) + text.focus_set() if __name__=='__main__': from idlelib.idle_test.htest import run diff --git a/lib-python/2.7/idlelib/ClassBrowser.py b/lib-python/2.7/idlelib/ClassBrowser.py --- a/lib-python/2.7/idlelib/ClassBrowser.py +++ b/lib-python/2.7/idlelib/ClassBrowser.py @@ -19,6 +19,9 @@ from idlelib.TreeWidget import TreeNode, TreeItem, ScrolledCanvas from idlelib.configHandler import idleConf +file_open = None # Method...Item and Class...Item use this. +# Normally PyShell.flist.open, but there is no PyShell.flist for htest. + class ClassBrowser: def __init__(self, flist, name, path, _htest=False): @@ -27,6 +30,9 @@ """ _htest - bool, change box when location running htest. """ + global file_open + if not _htest: + file_open = PyShell.flist.open self.name = name self.file = os.path.join(path[0], self.name + ".py") self._htest = _htest @@ -101,7 +107,7 @@ return [] try: dict = pyclbr.readmodule_ex(name, [dir] + sys.path) - except ImportError, msg: + except ImportError: return [] items = [] self.classes = {} @@ -170,7 +176,7 @@ def OnDoubleClick(self): if not os.path.exists(self.file): return - edit = PyShell.flist.open(self.file) + edit = file_open(self.file) if hasattr(self.cl, 'lineno'): lineno = self.cl.lineno edit.gotoline(lineno) @@ -206,7 +212,7 @@ def OnDoubleClick(self): if not os.path.exists(self.file): return - edit = PyShell.flist.open(self.file) + edit = file_open(self.file) edit.gotoline(self.cl.methods[self.name]) def _class_browser(parent): #Wrapper for htest @@ -221,8 +227,9 @@ dir, file = os.path.split(file) name = os.path.splitext(file)[0] flist = PyShell.PyShellFileList(parent) + global file_open + file_open = flist.open ClassBrowser(flist, name, [dir], _htest=True) - parent.mainloop() if __name__ == "__main__": from idlelib.idle_test.htest import run diff --git a/lib-python/2.7/idlelib/ColorDelegator.py b/lib-python/2.7/idlelib/ColorDelegator.py --- a/lib-python/2.7/idlelib/ColorDelegator.py +++ b/lib-python/2.7/idlelib/ColorDelegator.py @@ -2,7 +2,6 @@ import re import keyword import __builtin__ -from Tkinter import * from idlelib.Delegator import Delegator from idlelib.configHandler import idleConf @@ -34,7 +33,6 @@ prog = re.compile(make_pat(), re.S) idprog = re.compile(r"\s+(\w+)", re.S) -asprog = re.compile(r".*?\b(as)\b") class ColorDelegator(Delegator): @@ -42,7 +40,6 @@ Delegator.__init__(self) self.prog = prog self.idprog = idprog - self.asprog = asprog self.LoadTagDefs() def setdelegate(self, delegate): @@ -74,7 +71,6 @@ "DEFINITION": idleConf.GetHighlight(theme, "definition"), "SYNC": {'background':None,'foreground':None}, "TODO": {'background':None,'foreground':None}, - "BREAK": idleConf.GetHighlight(theme, "break"), "ERROR": idleConf.GetHighlight(theme, "error"), # The following is used by ReplaceDialog: "hit": idleConf.GetHighlight(theme, "hit"), @@ -216,22 +212,6 @@ self.tag_add("DEFINITION", head + "+%dc" % a, head + "+%dc" % b) - elif value == "import": - # color all the "as" words on same line, except - # if in a comment; cheap approximation to the - # truth - if '#' in chars: - endpos = chars.index('#') - else: - endpos = len(chars) - while True: - m1 = self.asprog.match(chars, b, endpos) - if not m1: - break - a, b = m1.span(1) - self.tag_add("KEYWORD", - head + "+%dc" % a, - head + "+%dc" % b) m = self.prog.search(chars, m.end()) if "SYNC" in self.tag_names(next + "-1c"): head = next @@ -255,20 +235,23 @@ for tag in self.tagdefs.keys(): self.tag_remove(tag, "1.0", "end") -def _color_delegator(parent): +def _color_delegator(parent): # htest # + from Tkinter import Toplevel, Text from idlelib.Percolator import Percolator - root = Tk() - root.title("Test ColorDelegator") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) - source = "if somename: x = 'abc' # comment\nprint" - text = Text(root, background="white") + + top = Toplevel(parent) + top.title("Test ColorDelegator") + top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, + parent.winfo_rooty() + 150)) + source = "if somename: x = 'abc' # comment\nprint\n" + text = Text(top, background="white") + text.pack(expand=1, fill="both") text.insert("insert", source) - text.pack(expand=1, fill="both") + text.focus_set() + p = Percolator(text) d = ColorDelegator() p.insertfilter(d) - root.mainloop() if __name__ == "__main__": from idlelib.idle_test.htest import run diff --git a/lib-python/2.7/idlelib/Debugger.py b/lib-python/2.7/idlelib/Debugger.py --- a/lib-python/2.7/idlelib/Debugger.py +++ b/lib-python/2.7/idlelib/Debugger.py @@ -1,6 +1,5 @@ import os import bdb -import types from Tkinter import * from idlelib.WindowList import ListedToplevel from idlelib.ScrolledList import ScrolledList diff --git a/lib-python/2.7/idlelib/EditorWindow.py b/lib-python/2.7/idlelib/EditorWindow.py --- a/lib-python/2.7/idlelib/EditorWindow.py +++ b/lib-python/2.7/idlelib/EditorWindow.py @@ -1,6 +1,6 @@ import sys import os -from platform import python_version +import platform import re import imp from Tkinter import * @@ -22,6 +22,8 @@ # The default tab setting for a Text widget, in average-width characters. TK_TABWIDTH_DEFAULT = 8 +_py_version = ' (%s)' % platform.python_version() + def _sphinx_version(): "Format sys.version_info to produce the Sphinx version string used to install the chm docs" major, minor, micro, level, serial = sys.version_info @@ -151,7 +153,7 @@ # Safari requires real file:-URLs EditorWindow.help_url = 'file://' + EditorWindow.help_url else: - EditorWindow.help_url = "http://docs.python.org/%d.%d" % sys.version_info[:2] + EditorWindow.help_url = "https://docs.python.org/%d.%d/" % sys.version_info[:2] currentTheme=idleConf.CurrentTheme() self.flist = flist root = root or flist.root @@ -214,6 +216,8 @@ text.bind("<>", self.python_docs) text.bind("<>", self.about_dialog) text.bind("<>", self.config_dialog) + text.bind("<>", + self.config_extensions_dialog) text.bind("<>", self.open_module) text.bind("<>", lambda event: "break") text.bind("<>", self.select_all) @@ -568,6 +572,8 @@ def config_dialog(self, event=None): configDialog.ConfigDialog(self.top,'Settings') + def config_extensions_dialog(self, event=None): + configDialog.ConfigExtensionsDialog(self.top) def help_dialog(self, event=None): if self.root: @@ -691,30 +697,29 @@ return # XXX Ought to insert current file's directory in front of path try: - (f, file, (suffix, mode, type)) = _find_module(name) + (f, file_path, (suffix, mode, mtype)) = _find_module(name) except (NameError, ImportError) as msg: tkMessageBox.showerror("Import error", str(msg), parent=self.text) return - if type != imp.PY_SOURCE: + if mtype != imp.PY_SOURCE: tkMessageBox.showerror("Unsupported type", "%s is not a source module" % name, parent=self.text) return if f: f.close() if self.flist: - self.flist.open(file) + self.flist.open(file_path) else: - self.io.loadfile(file) + self.io.loadfile(file_path) + return file_path def open_class_browser(self, event=None): filename = self.io.filename - if not filename: - tkMessageBox.showerror( - "No filename", - "This buffer has no associated filename", - master=self.text) - self.text.focus_set() - return None + if not (self.__class__.__name__ == 'PyShellEditorWindow' + and filename): + filename = self.open_module() + if filename is None: + return head, tail = os.path.split(filename) base, ext = os.path.splitext(tail) from idlelib import ClassBrowser @@ -779,7 +784,7 @@ self.color = None def ResetColorizer(self): - "Update the colour theme" + "Update the color theme" # Called from self.filename_change_hook and from configDialog.py self._rmcolorizer() self._addcolorizer() @@ -944,7 +949,7 @@ short = self.short_title() long = self.long_title() if short and long: - title = short + " - " + long + title = short + " - " + long + _py_version elif short: title = short elif long: @@ -968,14 +973,13 @@ self.undo.reset_undo() def short_title(self): - pyversion = "Python " + python_version() + ": " filename = self.io.filename if filename: filename = os.path.basename(filename) else: filename = "Untitled" # return unicode string to display non-ASCII chars correctly - return pyversion + self._filename_to_unicode(filename) + return self._filename_to_unicode(filename) def long_title(self): # return unicode string to display non-ASCII chars correctly @@ -1711,7 +1715,8 @@ tk.call('set', 'tcl_nonwordchars', '[^a-zA-Z0-9_]') -def _editor_window(parent): +def _editor_window(parent): # htest # + # error if close master window first - timer event, after script root = parent fixwordbreaks(root) if sys.argv[1:]: @@ -1721,7 +1726,8 @@ macosxSupport.setupApp(root, None) edit = EditorWindow(root=root, filename=filename) edit.text.bind("<>", edit.close_event) - parent.mainloop() + # Does not stop error, neither does following + # edit.text.bind("<>", edit.close_event) if __name__ == '__main__': diff --git a/lib-python/2.7/idlelib/GrepDialog.py b/lib-python/2.7/idlelib/GrepDialog.py --- a/lib-python/2.7/idlelib/GrepDialog.py +++ b/lib-python/2.7/idlelib/GrepDialog.py @@ -45,10 +45,10 @@ def create_entries(self): SearchDialogBase.create_entries(self) - self.globent = self.make_entry("In files:", self.globvar) + self.globent = self.make_entry("In files:", self.globvar)[0] def create_other_buttons(self): - f = self.make_frame() + f = self.make_frame()[0] btn = Checkbutton(f, anchor="w", variable=self.recvar, @@ -131,7 +131,7 @@ self.top.withdraw() -def _grep_dialog(parent): # for htest +def _grep_dialog(parent): # htest # from idlelib.PyShell import PyShellFileList root = Tk() root.title("Test GrepDialog") diff --git a/lib-python/2.7/idlelib/IOBinding.py b/lib-python/2.7/idlelib/IOBinding.py --- a/lib-python/2.7/idlelib/IOBinding.py +++ b/lib-python/2.7/idlelib/IOBinding.py @@ -19,11 +19,7 @@ from idlelib.configHandler import idleConf -try: - from codecs import BOM_UTF8 -except ImportError: - # only available since Python 2.3 - BOM_UTF8 = '\xef\xbb\xbf' +from codecs import BOM_UTF8 # Try setting the locale, so that we can find out # what encoding to use @@ -72,6 +68,7 @@ encoding = encoding.lower() coding_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)') +blank_re = re.compile(r'^[ \t\f]*(?:[#\r\n]|$)') class EncodingMessage(SimpleDialog): "Inform user that an encoding declaration is needed." @@ -130,6 +127,8 @@ match = coding_re.match(line) if match is not None: break + if not blank_re.match(line): + return None else: return None name = match.group(1) @@ -529,6 +528,8 @@ ("All files", "*"), ] + defaultextension = '.py' if sys.platform == 'darwin' else '' + def askopenfile(self): dir, base = self.defaultfilename("open") if not self.opendialog: @@ -554,8 +555,10 @@ def asksavefile(self): dir, base = self.defaultfilename("save") if not self.savedialog: - self.savedialog = tkFileDialog.SaveAs(master=self.text, - filetypes=self.filetypes) + self.savedialog = tkFileDialog.SaveAs( + master=self.text, + filetypes=self.filetypes, + defaultextension=self.defaultextension) filename = self.savedialog.show(initialdir=dir, initialfile=base) if isinstance(filename, unicode): filename = filename.encode(filesystemencoding) diff --git a/lib-python/2.7/idlelib/NEWS.txt b/lib-python/2.7/idlelib/NEWS.txt --- a/lib-python/2.7/idlelib/NEWS.txt +++ b/lib-python/2.7/idlelib/NEWS.txt @@ -1,6 +1,183 @@ +What's New in IDLE 2.7.9? +========================= + +*Release data: 2014-12-07* (projected) + +- Issue #16893: Update Idle doc chapter to match current Idle and add new + information. + +- Issue #3068: Add Idle extension configuration dialog to Options menu. + Changes are written to HOME/.idlerc/config-extensions.cfg. + Original patch by Tal Einat. + +- Issue #16233: A module browser (File : Class Browser, Alt+C) requires a + editor window with a filename. When Class Browser is requested otherwise, + from a shell, output window, or 'Untitled' editor, Idle no longer displays + an error box. It now pops up an Open Module box (Alt+M). If a valid name + is entered and a module is opened, a corresponding browser is also opened. + +- Issue #4832: Save As to type Python files automatically adds .py to the + name you enter (even if your system does not display it). Some systems + automatically add .txt when type is Text files. + +- Issue #21986: Code objects are not normally pickled by the pickle module. + To match this, they are no longer pickled when running under Idle. + +- Issue #22221: IDLE now ignores the source encoding declaration on the second + line if the first line contains anything except a comment. + +- Issue #17390: Adjust Editor window title; remove 'Python', + move version to end. + +- Issue #14105: Idle debugger breakpoints no longer disappear + when inseting or deleting lines. + + +What's New in IDLE 2.7.8? +========================= + +*Release date: 2014-06-29* + +- Issue #21940: Add unittest for WidgetRedirector. Initial patch by Saimadhav + Heblikar. + +- Issue #18592: Add unittest for SearchDialogBase. Patch by Phil Webster. + +- Issue #21694: Add unittest for ParenMatch. Patch by Saimadhav Heblikar. + +- Issue #21686: add unittest for HyperParser. Original patch by Saimadhav + Heblikar. + +- Issue #12387: Add missing upper(lower)case versions of default Windows key + bindings for Idle so Caps Lock does not disable them. Patch by Roger Serwy. + +- Issue #21695: Closing a Find-in-files output window while the search is + still in progress no longer closes Idle. + +- Issue #18910: Add unittest for textView. Patch by Phil Webster. + +- Issue #18292: Add unittest for AutoExpand. Patch by Saihadhav Heblikar. + +- Issue #18409: Add unittest for AutoComplete. Patch by Phil Webster. + + +What's New in IDLE 2.7.7? +========================= + +*Release date: 2014-05-31* + +- Issue #18104: Add idlelib/idle_test/htest.py with a few sample tests to begin + consolidating and improving human-validated tests of Idle. Change other files + as needed to work with htest. Running the module as __main__ runs all tests. + +- Issue #21139: Change default paragraph width to 72, the PEP 8 recommendation. + +- Issue #21284: Paragraph reformat test passes after user changes reformat width. + +- Issue #20406: Use Python application icons for Idle window title bars. + Patch mostly by Serhiy Storchaka. + +- Issue #21029: Occurrences of "print" are now consistently colored as + being a keyword (the colorizer doesn't know if print functions are + enabled in the source). + +- Issue #17721: Remove non-functional configuration dialog help button until we + make it actually gives some help when clicked. Patch by Guilherme Sim�es. + +- Issue #17390: Add Python version to Idle editor window title bar. + Original patches by Edmond Burnett and Kent Johnson. + +- Issue #20058: sys.stdin.readline() in IDLE now always returns only one line. + +- Issue #19481: print() of unicode, str or bytearray subclass instance in IDLE + no more hangs. + +- Issue #18270: Prevent possible IDLE AttributeError on OS X when no initial + shell window is present. + +- Issue #17654: Ensure IDLE menus are customized properly on OS X for + non-framework builds and for all variants of Tk. + + +What's New in IDLE 2.7.6? +========================= + +*Release date: 2013-11-10* + +- Issue #19426: Fixed the opening of Python source file with specified encoding. + +- Issue #18873: IDLE now detects Python source code encoding only in comment + lines. + +- Issue #18988: The "Tab" key now works when a word is already autocompleted. + +- Issue #18489: Add tests for SearchEngine. Original patch by Phil Webster. + +- Issue #18429: Format / Format Paragraph, now works when comment blocks + are selected. As with text blocks, this works best when the selection + only includes complete lines. + +- Issue #18226: Add docstrings and unittests for FormatParagraph.py. + Original patches by Todd Rovito and Phil Webster. + +- Issue #18279: Format - Strip trailing whitespace no longer marks a file as + changed when it has not been changed. This fix followed the addition of a + test file originally written by Phil Webster (the issue's main goal). + +- Issue #18539: Calltips now work for float default arguments. + +- Issue #7136: In the Idle File menu, "New Window" is renamed "New File". + Patch by Tal Einat, Roget Serwy, and Todd Rovito. + +- Issue #8515: Set __file__ when run file in IDLE. + Initial patch by Bruce Frederiksen. + +- Issue #5492: Avoid traceback when exiting IDLE caused by a race condition. + +- Issue #17511: Keep IDLE find dialog open after clicking "Find Next". + Original patch by Sarah K. + +- Issue #15392: Create a unittest framework for IDLE. + Preliminary patch by Rajagopalasarma Jayakrishnan + See Lib/idlelib/idle_test/README.txt for how to run Idle tests. + +- Issue #14146: Highlight source line while debugging on Windows. + +- Issue #17532: Always include Options menu for IDLE on OS X. + Patch by Guilherme Sim�es. + + What's New in IDLE 2.7.5? ========================= +*Release date: 2013-05-12* + +- Issue #17838: Allow sys.stdin to be reassigned. + +- Issue #14735: Update IDLE docs to omit "Control-z on Windows". + +- Issue #17585: Fixed IDLE regression. Now closes when using exit() or quit(). + +- Issue #17657: Show full Tk version in IDLE's about dialog. + Patch by Todd Rovito. + +- Issue #17613: Prevent traceback when removing syntax colorizer in IDLE. + +- Issue #1207589: Backwards-compatibility patch for right-click menu in IDLE. + +- Issue #16887: IDLE now accepts Cancel in tabify/untabify dialog box. + +- Issue #14254: IDLE now handles readline correctly across shell restarts. + +- Issue #17614: IDLE no longer raises exception when quickly closing a file. + +- Issue #6698: IDLE now opens just an editor window when configured to do so. + +- Issue #8900: Using keyboard shortcuts in IDLE to open a file no longer + raises an exception. + +- Issue #6649: Fixed missing exit status in IDLE. Patch by Guilherme Polo. + - Issue #17390: Display Python version on Idle title bar. Initial patch by Edmond Burnett. @@ -8,17 +185,67 @@ What's New in IDLE 2.7.4? ========================= +*Release date: 2013-04-06* + +- Issue #17625: In IDLE, close the replace dialog after it is used. + +- IDLE was displaying spurious SystemExit tracebacks when running scripts + that terminated by raising SystemExit (i.e. unittest and turtledemo). + +- Issue #9290: In IDLE the sys.std* streams now implement io.TextIOBase + interface and support all mandatory methods and properties. + +- Issue #16829: IDLE printing no longer fails if there are spaces or other + special characters in the file path. + +- Issue #16819: IDLE method completion now correctly works for unicode literals. + +- Issue #16504: IDLE now catches SyntaxErrors raised by tokenizer. Patch by + Roger Serwy. + +- Issue #1207589: Add Cut/Copy/Paste items to IDLE right click Context Menu + Patch by Todd Rovito. + +- Issue #13052: Fix IDLE crashing when replace string in Search/Replace dialog + ended with '\'. Patch by Roger Serwy. + +- Issue #9803: Don't close IDLE on saving if breakpoint is open. + Patch by Roger Serwy. + +- Issue #14958: Change IDLE systax highlighting to recognize all string and byte + literals currently supported in Python 2.7. + +- Issue #14962: Update text coloring in IDLE shell window after changing + options. Patch by Roger Serwy. + +- Issue #10997: Prevent a duplicate entry in IDLE's "Recent Files" menu. + +- Issue #12510: Attempting to get invalid tooltip no longer closes IDLE. + Original patch by Roger Serwy. + +- Issue #10365: File open dialog now works instead of crashing + even when parent window is closed. Patch by Roger Serwy. + +- Issue #14876: Use user-selected font for highlight configuration. + Patch by Roger Serwy. + +- Issue #14409: IDLE now properly executes commands in the Shell window + when it cannot read the normal config files on startup and + has to use the built-in default key bindings. + There was previously a bug in one of the defaults. + +- Issue #3573: IDLE hangs when passing invalid command line args + (directory(ies) instead of file(s)) (Patch by Guilherme Polo) + +- Issue #5219: Prevent event handler cascade in IDLE. + - Issue #15318: Prevent writing to sys.stdin. - Issue #13532, #15319: Check that arguments to sys.stdout.write are strings. -- Issue # 12510: Attempt to get certain tool tips no longer crashes IDLE. - -- Issue10365: File open dialog now works instead of crashing even when +- Issue #10365: File open dialog now works instead of crashing even when parent window is closed while dialog is open. -- Issue 14876: use user-selected font for highlight configuration. - - Issue #14018: Update checks for unstable system Tcl/Tk versions on OS X to include versions shipped with OS X 10.7 and 10.8 in addition to 10.6. @@ -29,6 +256,27 @@ What's New in IDLE 2.7.3? ========================= +*Release date: 2012-04-09* + +- Issue #964437 Make IDLE help window non-modal. + Patch by Guilherme Polo and Roger Serwy. + +- Issue #13933: IDLE auto-complete did not work with some imported + module, like hashlib. (Patch by Roger Serwy) + +- Issue #13506: Add '' to path for IDLE Shell when started and restarted with Restart Shell. + Original patches by Marco Scataglini and Roger Serwy. + +- Issue #4625: If IDLE cannot write to its recent file or breakpoint + files, display a message popup and continue rather than crash. + (original patch by Roger Serwy) + +- Issue #8793: Prevent IDLE crash when given strings with invalid hex escape + sequences. + +- Issue #13296: Fix IDLE to clear compile __future__ flags on shell restart. + (Patch by Roger Serwy) + - Issue #14409: IDLE now properly executes commands in the Shell window when it cannot read the normal config files on startup and has to use the built-in default key bindings. @@ -41,41 +289,85 @@ What's New in IDLE 2.7.2? ========================= -*Release date: 29-May-2011* +*Release date: 2011-06-11* + +- Issue #11718: IDLE's open module dialog couldn't find the __init__.py + file in a package. + From noreply at buildbot.pypy.org Sun Feb 15 09:46:37 2015 From: noreply at buildbot.pypy.org (jstasiak) Date: Sun, 15 Feb 2015 09:46:37 +0100 (CET) Subject: [pypy-commit] pypy default: Mention --withmod-... options in the documentation Message-ID: <20150215084637.870031C009F@cobra.cs.uni-duesseldorf.de> Author: Jakub Stasiak Branch: Changeset: r75883:4fac268cd818 Date: 2015-02-15 01:12 +0100 http://bitbucket.org/pypy/pypy/changeset/4fac268cd818/ Log: Mention --withmod-... options in the documentation diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst --- a/pypy/doc/getting-started-dev.rst +++ b/pypy/doc/getting-started-dev.rst @@ -188,6 +188,10 @@ it runs on CPython level. If you want to return to PyPy, press (under Linux) or , (under Windows). +Also note that not all modules are available by default in this mode (for +example: ``_continuation`` needed by ``greenlet``) , you may need to use one of +``--withmod-...`` command line options. + You may be interested in reading more about the distinction between :ref:`interpreter-level and app-level `. From noreply at buildbot.pypy.org Sun Feb 15 09:46:38 2015 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 15 Feb 2015 09:46:38 +0100 (CET) Subject: [pypy-commit] pypy default: Merged in jstasiak/pypy (pull request #304) Message-ID: <20150215084638.C17EC1C009F@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r75884:53f81286eb86 Date: 2015-02-15 10:46 +0200 http://bitbucket.org/pypy/pypy/changeset/53f81286eb86/ Log: Merged in jstasiak/pypy (pull request #304) Mention --withmod-... options in the documentation diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst --- a/pypy/doc/getting-started-dev.rst +++ b/pypy/doc/getting-started-dev.rst @@ -188,6 +188,10 @@ it runs on CPython level. If you want to return to PyPy, press (under Linux) or , (under Windows). +Also note that not all modules are available by default in this mode (for +example: ``_continuation`` needed by ``greenlet``) , you may need to use one of +``--withmod-...`` command line options. + You may be interested in reading more about the distinction between :ref:`interpreter-level and app-level `. From noreply at buildbot.pypy.org Sun Feb 15 10:13:52 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 15 Feb 2015 10:13:52 +0100 (CET) Subject: [pypy-commit] pypy default: Compile the shared libs used for tests with "-g -O0". Message-ID: <20150215091352.C4A5A1C03B3@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75885:1abbb0a9835d Date: 2015-02-15 10:03 +0100 http://bitbucket.org/pypy/pypy/changeset/1abbb0a9835d/ Log: Compile the shared libs used for tests with "-g -O0". diff --git a/rpython/translator/tool/cbuild.py b/rpython/translator/tool/cbuild.py --- a/rpython/translator/tool/cbuild.py +++ b/rpython/translator/tool/cbuild.py @@ -293,7 +293,8 @@ d['separate_module_files'] = () return files, ExternalCompilationInfo(**d) - def compile_shared_lib(self, outputfilename=None, ignore_a_files=False): + def compile_shared_lib(self, outputfilename=None, ignore_a_files=False, + debug_mode=True): self = self.convert_sources_to_files() if ignore_a_files: if not [fn for fn in self.link_files if fn.endswith('.a')]: @@ -318,6 +319,8 @@ if ignore_a_files: d['link_files'] = [fn for fn in d['link_files'] if not fn.endswith('.a')] + if debug_mode and sys.platform != 'win32': + d['compile_extra'] = d['compile_extra'] + ('-g', '-O0') d['compile_extra'] = d['compile_extra'] + ( '-DRPY_EXTERN=RPY_EXPORTED',) self = ExternalCompilationInfo(**d) From noreply at buildbot.pypy.org Sun Feb 15 10:13:53 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 15 Feb 2015 10:13:53 +0100 (CET) Subject: [pypy-commit] pypy default: issue #1984 fix Message-ID: <20150215091353.EBF5C1C03B3@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75886:cce58dd9c317 Date: 2015-02-15 10:13 +0100 http://bitbucket.org/pypy/pypy/changeset/cce58dd9c317/ Log: issue #1984 fix diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py --- a/pypy/module/_continuation/interp_continuation.py +++ b/pypy/module/_continuation/interp_continuation.py @@ -49,9 +49,6 @@ def switch(self, w_to): sthread = self.sthread - if sthread is not None and sthread.is_empty_handle(self.h): - global_state.clear() - raise geterror(self.space, "continulet already finished") to = self.space.interp_w(W_Continulet, w_to, can_be_None=True) if to is not None and to.sthread is None: to = None @@ -62,6 +59,9 @@ to = None else: return get_result() # else: no-op + if sthread is not None and sthread.is_empty_handle(self.h): + global_state.clear() + raise geterror(self.space, "continulet already finished") if to is not None: if to.sthread is not sthread: global_state.clear() diff --git a/pypy/module/_continuation/test/test_stacklet.py b/pypy/module/_continuation/test/test_stacklet.py --- a/pypy/module/_continuation/test/test_stacklet.py +++ b/pypy/module/_continuation/test/test_stacklet.py @@ -698,3 +698,12 @@ except ZeroDivisionError: got = c1.switch() assert got == (None, None, None) + + def test_bug_issue1984(self): + from _continuation import continulet, error + + c1 = continulet.__new__(continulet) + c2 = continulet(lambda g: None) + + continulet.switch(c1, to=c2) + raises(error, continulet.switch, c1, to=c2) From noreply at buildbot.pypy.org Sun Feb 15 11:16:29 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 15 Feb 2015 11:16:29 +0100 (CET) Subject: [pypy-commit] pypy default: Test (thanks vpelletier on irc) and fix for a case where we'd propagate Message-ID: <20150215101629.E726C1C03B3@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75887:b1c4fcb04a42 Date: 2015-02-15 11:16 +0100 http://bitbucket.org/pypy/pypy/changeset/b1c4fcb04a42/ Log: Test (thanks vpelletier on irc) and fix for a case where we'd propagate EAGAIN to app-level even though the BufferingInputStream still has buffered data diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py --- a/pypy/module/_file/interp_file.py +++ b/pypy/module/_file/interp_file.py @@ -206,7 +206,12 @@ # a special-case only for read() (similar to CPython, which # also loses partial data with other methods): if we get # EAGAIN after already some data was received, return it. + # Note that we can get EAGAIN while there is buffered data + # waiting; read that too. if is_wouldblock_error(e): + pos, buf = stream.peek() + if len(buf) > pos: + result.append(stream.read(min(n, len(buf) - pos))) got = result.build() if len(got) > 0: return got diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py --- a/pypy/module/_file/test/test_file.py +++ b/pypy/module/_file/test/test_file.py @@ -302,15 +302,14 @@ if cls.runappdirect: py.test.skip("works with internals of _file impl on py.py") - state = [0] def read(fd, n=None): if fd != 424242: return cls.old_read(fd, n) - if state[0] == 0: - state[0] += 1 + if cls.state == 0: + cls.state += 1 return "xyz" - if state[0] < 3: - state[0] += 1 + if cls.state < 3: + cls.state += 1 raise OSError(errno.EAGAIN, "xyz") return '' os.read = read @@ -319,13 +318,20 @@ stdin.name = '' cls.w_stream = stdin + def setup_method(self, meth): + self.__class__.state = 0 + def teardown_class(cls): os.read = cls.old_read - def test_nonblocking_file(self): + def test_nonblocking_file_all(self): res = self.stream.read() assert res == 'xyz' + def test_nonblocking_file_max(self): + res = self.stream.read(100) + assert res == 'xyz' + class AppTestConcurrency(object): # these tests only really make sense on top of a translated pypy-c, # because on top of py.py the inner calls to os.write() don't From noreply at buildbot.pypy.org Sun Feb 15 11:18:52 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 15 Feb 2015 11:18:52 +0100 (CET) Subject: [pypy-commit] pypy default: Add a clearer interface Message-ID: <20150215101852.DD4011C03B3@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75888:af52d1553f8f Date: 2015-02-15 11:18 +0100 http://bitbucket.org/pypy/pypy/changeset/af52d1553f8f/ Log: Add a clearer interface diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py --- a/pypy/module/_file/interp_file.py +++ b/pypy/module/_file/interp_file.py @@ -209,9 +209,9 @@ # Note that we can get EAGAIN while there is buffered data # waiting; read that too. if is_wouldblock_error(e): - pos, buf = stream.peek() - if len(buf) > pos: - result.append(stream.read(min(n, len(buf) - pos))) + m = stream.count_buffered_bytes() + if m > 0: + result.append(stream.read(min(n, m))) got = result.build() if len(got) > 0: return got diff --git a/rpython/rlib/streamio.py b/rpython/rlib/streamio.py --- a/rpython/rlib/streamio.py +++ b/rpython/rlib/streamio.py @@ -295,6 +295,10 @@ def peek(self): return (0, '') + def count_buffered_bytes(self): + pos, buf = self.peek() + return len(buf) - pos + def try_to_find_file_descriptor(self): return -1 From noreply at buildbot.pypy.org Sun Feb 15 11:21:58 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 15 Feb 2015 11:21:58 +0100 (CET) Subject: [pypy-commit] pypy default: add mention of b1c4fcb04a42 Message-ID: <20150215102158.275A41C009F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75889:d093f1195396 Date: 2015-02-15 11:21 +0100 http://bitbucket.org/pypy/pypy/changeset/d093f1195396/ Log: add mention of b1c4fcb04a42 diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,6 +5,11 @@ .. this is a revision shortly after release-2.5.x .. startrev: 397b96217b85 + +Fix non-blocking file reads sometimes raising EAGAIN even though they +have buffered data waiting (b1c4fcb04a42) + + .. branch: vmprof .. branch: stackroot-speedup-2 From noreply at buildbot.pypy.org Sun Feb 15 11:50:55 2015 From: noreply at buildbot.pypy.org (jstasiak) Date: Sun, 15 Feb 2015 11:50:55 +0100 (CET) Subject: [pypy-commit] pypy gitignore: .gitignore build files Message-ID: <20150215105055.42CB81C009F@cobra.cs.uni-duesseldorf.de> Author: Jakub Stasiak Branch: gitignore Changeset: r75890:51d1666de67f Date: 2015-02-14 01:33 +0100 http://bitbucket.org/pypy/pypy/changeset/51d1666de67f/ Log: .gitignore build files diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,10 @@ bin/pypy-c include/*.h +include/numpy/ lib_pypy/ctypes_config_cache/_[^_]*_*.py +libpypy-c.dylib +pypy-c pypy/_cache pypy/doc/*.html pypy/doc/config/*.html @@ -18,4 +21,5 @@ pypy/translator/c/src/dtoa.o pypy/translator/goal/pypy-c pypy/translator/goal/target*-c -release/ \ No newline at end of file +release/ +rpython/_cache/ From noreply at buildbot.pypy.org Sun Feb 15 11:50:56 2015 From: noreply at buildbot.pypy.org (jstasiak) Date: Sun, 15 Feb 2015 11:50:56 +0100 (CET) Subject: [pypy-commit] pypy gitignore: .gitignore all library types, not only dylib Message-ID: <20150215105056.73B4B1C009F@cobra.cs.uni-duesseldorf.de> Author: Jakub Stasiak Branch: gitignore Changeset: r75891:e0034906c925 Date: 2015-02-15 10:02 +0000 http://bitbucket.org/pypy/pypy/changeset/e0034906c925/ Log: .gitignore all library types, not only dylib diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,7 @@ include/*.h include/numpy/ lib_pypy/ctypes_config_cache/_[^_]*_*.py -libpypy-c.dylib +libpypy-c.* pypy-c pypy/_cache pypy/doc/*.html From noreply at buildbot.pypy.org Sun Feb 15 11:50:57 2015 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 15 Feb 2015 11:50:57 +0100 (CET) Subject: [pypy-commit] pypy default: Merged in jstasiak/pypy/gitignore (pull request #302) Message-ID: <20150215105057.920EA1C009F@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r75892:ffe4c4278efd Date: 2015-02-15 12:51 +0200 http://bitbucket.org/pypy/pypy/changeset/ffe4c4278efd/ Log: Merged in jstasiak/pypy/gitignore (pull request #302) Add some build artifacts to .gitignore diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,10 @@ bin/pypy-c include/*.h +include/numpy/ lib_pypy/ctypes_config_cache/_[^_]*_*.py +libpypy-c.* +pypy-c pypy/_cache pypy/doc/*.html pypy/doc/config/*.html @@ -18,4 +21,5 @@ pypy/translator/c/src/dtoa.o pypy/translator/goal/pypy-c pypy/translator/goal/target*-c -release/ \ No newline at end of file +release/ +rpython/_cache/ From noreply at buildbot.pypy.org Sun Feb 15 11:51:55 2015 From: noreply at buildbot.pypy.org (jstasiak) Date: Sun, 15 Feb 2015 11:51:55 +0100 (CET) Subject: [pypy-commit] pypy fix-kqueue-error2: Fix exception being raised by kqueue.control (CPython compatibility) Message-ID: <20150215105155.F38291C009F@cobra.cs.uni-duesseldorf.de> Author: Jakub Stasiak Branch: fix-kqueue-error2 Changeset: r75893:de656c8e19b6 Date: 2015-02-15 10:51 +0100 http://bitbucket.org/pypy/pypy/changeset/de656c8e19b6/ Log: Fix exception being raised by kqueue.control (CPython compatibility) diff --git a/lib-python/2.7/test/test_kqueue.py b/lib-python/2.7/test/test_kqueue.py --- a/lib-python/2.7/test/test_kqueue.py +++ b/lib-python/2.7/test/test_kqueue.py @@ -205,6 +205,17 @@ b.close() kq.close() + def test_control_raises_oserror(self): + kq = select.kqueue() + event = select.kevent(123456, select.KQ_FILTER_READ, select.KQ_EV_DELETE) + with self.assertRaises(OSError) as cm: + kq.control([event], 0, 0) + + self.assertEqual(cm.exception.args[0], errno.EBADF) + self.assertEqual(cm.exception.errno, errno.EBADF) + + kq.close() + def test_main(): test_support.run_unittest(TestKQueue) diff --git a/pypy/module/select/interp_kqueue.py b/pypy/module/select/interp_kqueue.py --- a/pypy/module/select/interp_kqueue.py +++ b/pypy/module/select/interp_kqueue.py @@ -201,7 +201,7 @@ max_events, ptimeout) if nfds < 0: - raise exception_from_saved_errno(space, space.w_IOError) + raise exception_from_saved_errno(space, space.w_OSError) else: elist_w = [None] * nfds for i in xrange(nfds): From noreply at buildbot.pypy.org Sun Feb 15 11:51:57 2015 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 15 Feb 2015 11:51:57 +0100 (CET) Subject: [pypy-commit] pypy default: Merged in jstasiak/pypy/fix-kqueue-error2 (pull request #305) Message-ID: <20150215105157.2EA711C009F@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r75894:3e90693d2d35 Date: 2015-02-15 12:52 +0200 http://bitbucket.org/pypy/pypy/changeset/3e90693d2d35/ Log: Merged in jstasiak/pypy/fix-kqueue-error2 (pull request #305) Fix exception being raised by kqueue.control (CPython compatibility) diff --git a/lib-python/2.7/test/test_kqueue.py b/lib-python/2.7/test/test_kqueue.py --- a/lib-python/2.7/test/test_kqueue.py +++ b/lib-python/2.7/test/test_kqueue.py @@ -205,6 +205,17 @@ b.close() kq.close() + def test_control_raises_oserror(self): + kq = select.kqueue() + event = select.kevent(123456, select.KQ_FILTER_READ, select.KQ_EV_DELETE) + with self.assertRaises(OSError) as cm: + kq.control([event], 0, 0) + + self.assertEqual(cm.exception.args[0], errno.EBADF) + self.assertEqual(cm.exception.errno, errno.EBADF) + + kq.close() + def test_main(): test_support.run_unittest(TestKQueue) diff --git a/pypy/module/select/interp_kqueue.py b/pypy/module/select/interp_kqueue.py --- a/pypy/module/select/interp_kqueue.py +++ b/pypy/module/select/interp_kqueue.py @@ -201,7 +201,7 @@ max_events, ptimeout) if nfds < 0: - raise exception_from_saved_errno(space, space.w_IOError) + raise exception_from_saved_errno(space, space.w_OSError) else: elist_w = [None] * nfds for i in xrange(nfds): From noreply at buildbot.pypy.org Sun Feb 15 15:40:02 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 15 Feb 2015 15:40:02 +0100 (CET) Subject: [pypy-commit] stmgc default: probably? quite unsure about this (see demo/demo_random) Message-ID: <20150215144002.ABC681C009F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1622:60f7ccae893c Date: 2015-02-15 14:25 +0100 http://bitbucket.org/pypy/stmgc/changeset/60f7ccae893c/ Log: probably? quite unsure about this (see demo/demo_random) diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -96,12 +96,9 @@ object_t *_stm_allocate_old(ssize_t size_rounded_up) { - /* only for tests xxx but stm_setup_prebuilt() uses this now too */ + /* this is for tests, and for stm_setup_prebuilt() */ stm_char *p = allocate_outside_nursery_large(size_rounded_up); object_t *o = (object_t *)p; - - // sharing seg0 needs to be current: - assert(STM_SEGMENT->segment_num == 0); memset(REAL_ADDRESS(STM_SEGMENT->segment_base, o), 0, size_rounded_up); o->stm_flags = GCFLAG_WRITE_BARRIER; From noreply at buildbot.pypy.org Sun Feb 15 15:40:03 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 15 Feb 2015 15:40:03 +0100 (CET) Subject: [pypy-commit] stmgc default: another quite unsure attempt Message-ID: <20150215144003.C25681C009F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1623:7540c1f155d0 Date: 2015-02-15 14:35 +0100 http://bitbucket.org/pypy/stmgc/changeset/7540c1f155d0/ Log: another quite unsure attempt diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -563,10 +563,13 @@ struct stm_commit_log_entry_s *cl, *next; #ifndef NDEBUG - /* check that all segments are at the same revision: */ + /* check that all segments are at the same revision (or not running + a transaction at all): */ cl = get_priv_segment(0)->last_commit_log_entry; for (long i = 1; i < NB_SEGMENTS; i++) { - assert(get_priv_segment(i)->last_commit_log_entry == cl); + if (get_priv_segment(i)->transaction_state != TS_NONE) { + assert(get_priv_segment(i)->last_commit_log_entry == cl); + } } #endif From noreply at buildbot.pypy.org Sun Feb 15 15:49:30 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 15 Feb 2015 15:49:30 +0100 (CET) Subject: [pypy-commit] pypy default: Simplify the logic in rweaklist. Message-ID: <20150215144930.83C5E1C009F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75895:5eb6bfa9ec8e Date: 2015-02-15 15:49 +0100 http://bitbucket.org/pypy/pypy/changeset/5eb6bfa9ec8e/ Log: Simplify the logic in rweaklist. diff --git a/rpython/rlib/rweaklist.py b/rpython/rlib/rweaklist.py --- a/rpython/rlib/rweaklist.py +++ b/rpython/rlib/rweaklist.py @@ -1,50 +1,40 @@ import weakref from rpython.rlib.rweakref import dead_ref - -def _reduced_value(s): - while True: - divide = s & 1 - s >>= 1 - if not divide: - return s +INITIAL_SIZE = 4 class RWeakListMixin(object): _mixin_ = True def initialize(self): - self.handles = [] - self.look_distance = 0 + self.handles = [dead_ref] * INITIAL_SIZE + self.free_list = range(INITIAL_SIZE) def get_all_handles(self): return self.handles def reserve_next_handle_index(self): - # The reservation ordering done here is tweaked for pypy's - # memory allocator. We look from index 'look_distance'. - # Look_distance increases from 0. But we also look at - # "look_distance/2" or "/4" or "/8", etc. If we find that one - # of these secondary locations is free, we assume it's because - # there was recently a minor collection; so we reset - # look_distance to 0 and start again from the lowest locations. - length = len(self.handles) - for d in range(self.look_distance, length): - if self.handles[d]() is None: - self.look_distance = d + 1 - return d - s = _reduced_value(d) - if self.handles[s]() is None: - break - # restart from the beginning - for d in range(0, length): - if self.handles[d]() is None: - self.look_distance = d + 1 - return d - # full! extend, but don't use '+=' here - self.handles = self.handles + [dead_ref] * (length // 3 + 5) - self.look_distance = length + 1 - return length + # (this algorithm should be amortized constant-time) + # get the next 'free_list' entry, if any + free_list = self.free_list + try: + return free_list.pop() + except IndexError: + pass + # slow path: collect all now-free handles in 'free_list' + handles = self.handles + for i in range(len(handles)): + if handles[i]() is None: + free_list.append(i) + # double the size of the self.handles list, but don't do that + # if there are more than 66% of handles free already + if len(free_list) * 3 < len(handles) * 2: + free_list.extend(range(len(handles), len(handles) * 2)) + # don't use '+=' on 'self.handles' + self.handles = handles = handles + [dead_ref] * len(handles) + # + return free_list.pop() def add_handle(self, content): index = self.reserve_next_handle_index() diff --git a/rpython/rlib/test/test_rweaklist.py b/rpython/rlib/test/test_rweaklist.py --- a/rpython/rlib/test/test_rweaklist.py +++ b/rpython/rlib/test/test_rweaklist.py @@ -1,20 +1,5 @@ import gc -from rpython.rlib.rweaklist import RWeakListMixin, _reduced_value as reduced_value - - -def test_reduced_value(): - assert reduced_value(0) == 0 - assert reduced_value(1) == 0 - assert reduced_value(2) == 1 - assert reduced_value(3) == 0 - assert reduced_value(4) == 2 - assert reduced_value(5) == 1 - assert reduced_value(6) == 3 - assert reduced_value(7) == 0 - assert reduced_value(8) == 4 - assert reduced_value(9) == 2 - assert reduced_value(10) == 5 - assert reduced_value(11) == 1 +from rpython.rlib.rweaklist import RWeakListMixin, INITIAL_SIZE class A(object): @@ -25,29 +10,30 @@ a1 = A(); a2 = A() wlist = RWeakListMixin(); wlist.initialize() i = wlist.add_handle(a1) - assert i == 0 + assert i == INITIAL_SIZE - 1 i = wlist.reserve_next_handle_index() - assert i == 1 + assert i == INITIAL_SIZE - 2 wlist.store_handle(i, a2) - assert wlist.fetch_handle(0) is a1 - assert wlist.fetch_handle(1) is a2 + assert wlist.fetch_handle(INITIAL_SIZE - 1) is a1 + assert wlist.fetch_handle(INITIAL_SIZE - 2) is a2 # del a2 for i in range(5): gc.collect() - if wlist.fetch_handle(1) is None: + if wlist.fetch_handle(INITIAL_SIZE - 2) is None: break else: - raise AssertionError("handle(1) did not disappear") - assert wlist.fetch_handle(0) is a1 + raise AssertionError("second handle() did not disappear") + assert wlist.fetch_handle(INITIAL_SIZE - 1) is a1 def test_reuse(): alist = [A() for i in range(200)] wlist = RWeakListMixin(); wlist.initialize() + mapping = [] for i in range(200): j = wlist.reserve_next_handle_index() - assert j == i - wlist.store_handle(i, alist[i]) + mapping.append(j) + wlist.store_handle(j, alist[i]) # del alist[1::2] del alist[1::2] @@ -57,8 +43,8 @@ for i in range(5): gc.collect() # - for i in range(200): - a = wlist.fetch_handle(i) + for i, j in enumerate(mapping): + a = wlist.fetch_handle(j) if i % 32 == 0: assert a is alist[i // 32] else: @@ -67,6 +53,7 @@ maximum = -1 for i in range(200): j = wlist.reserve_next_handle_index() + assert wlist.fetch_handle(j) is None maximum = max(maximum, j) wlist.store_handle(j, A()) - assert maximum <= 240 + assert maximum < 256 From noreply at buildbot.pypy.org Sun Feb 15 16:05:43 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 15 Feb 2015 16:05:43 +0100 (CET) Subject: [pypy-commit] pypy default: Fix Message-ID: <20150215150543.431EA1C009F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75896:1cd58e083148 Date: 2015-02-15 15:56 +0100 http://bitbucket.org/pypy/pypy/changeset/1cd58e083148/ Log: Fix diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py --- a/pypy/module/_io/interp_iobase.py +++ b/pypy/module/_io/interp_iobase.py @@ -385,20 +385,22 @@ def flush_all(self, space): while True: handles = self.get_all_handles() - if len(handles) == 0: - break self.initialize() # reset the state here + progress = False for wr in handles: w_iobase = wr() if w_iobase is None: continue + progress = True try: space.call_method(w_iobase, 'flush') except OperationError: # Silencing all errors is bad, but getting randomly # interrupted here is equally as bad, and potentially # more frequent (because of shutdown issues). - pass + pass + if not progress: + break def get_autoflusher(space): return space.fromcache(AutoFlusher) From noreply at buildbot.pypy.org Sun Feb 15 16:05:44 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 15 Feb 2015 16:05:44 +0100 (CET) Subject: [pypy-commit] pypy default: Test and fix (discovered by 5eb6bfa9ec8e) Message-ID: <20150215150544.601801C009F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75897:6b4ac7be0d78 Date: 2015-02-15 16:05 +0100 http://bitbucket.org/pypy/pypy/changeset/6b4ac7be0d78/ Log: Test and fix (discovered by 5eb6bfa9ec8e) diff --git a/rpython/annotator/listdef.py b/rpython/annotator/listdef.py --- a/rpython/annotator/listdef.py +++ b/rpython/annotator/listdef.py @@ -158,6 +158,10 @@ s_other_value = other.read_item() self.generalize(s_other_value) other.generalize(s_self_value) + if self.listitem.range_step is not None: + self.generalize_range_step(other.listitem.range_step) + if other.listitem.range_step is not None: + other.generalize_range_step(self.listitem.range_step) def offspring(self, *others): s_self_value = self.read_item() diff --git a/rpython/rtyper/test/test_rrange.py b/rpython/rtyper/test/test_rrange.py --- a/rpython/rtyper/test/test_rrange.py +++ b/rpython/rtyper/test/test_rrange.py @@ -184,3 +184,11 @@ return 5 res = self.interpret(fn, [1]) assert res == 20 + + def test_extend_range(self): + def fn(n): + lst = [n, n, n] + lst.extend(range(n)) + return len(lst) + res = self.interpret(fn, [5]) + assert res == 8 From noreply at buildbot.pypy.org Sun Feb 15 16:13:38 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 15 Feb 2015 16:13:38 +0100 (CET) Subject: [pypy-commit] pypy default: fix test Message-ID: <20150215151338.849111C009F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75898:956c35f6c0e4 Date: 2015-02-15 16:12 +0100 http://bitbucket.org/pypy/pypy/changeset/956c35f6c0e4/ Log: fix test diff --git a/pypy/module/_cffi_backend/test/test_handle.py b/pypy/module/_cffi_backend/test/test_handle.py --- a/pypy/module/_cffi_backend/test/test_handle.py +++ b/pypy/module/_cffi_backend/test/test_handle.py @@ -19,7 +19,7 @@ pwr = PseudoWeakRef() expected_content[index] = pwr ch.handles[index] = pwr - assert len(ch.handles) < 13500 + assert len(ch.handles) <= 16384 for index, pwr in expected_content.items(): assert ch.handles[index] is pwr From noreply at buildbot.pypy.org Sun Feb 15 17:44:52 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 15 Feb 2015 17:44:52 +0100 (CET) Subject: [pypy-commit] pypy default: Fix test_audioop Message-ID: <20150215164452.B3BAB1C1155@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r75899:858a789cd41f Date: 2015-02-15 17:41 +0100 http://bitbucket.org/pypy/pypy/changeset/858a789cd41f/ Log: Fix test_audioop diff --git a/lib_pypy/audioop.py b/lib_pypy/audioop.py --- a/lib_pypy/audioop.py +++ b/lib_pypy/audioop.py @@ -480,6 +480,7 @@ state_d, prev_i, cur_i, weightA, weightB) result = ffi.buffer(rv)[:trim_index] + d = state_d[0] samps = zip(prev_i, cur_i) return (result, (d, tuple(samps))) From noreply at buildbot.pypy.org Sun Feb 15 18:07:04 2015 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Sun, 15 Feb 2015 18:07:04 +0100 (CET) Subject: [pypy-commit] lang-smalltalk default: README.md edited online with Bitbucket Message-ID: <20150215170704.47A001C1155@cobra.cs.uni-duesseldorf.de> Author: anton_gulenko Branch: Changeset: r1072:8497f404f0ec Date: 2015-02-15 17:07 +0000 http://bitbucket.org/pypy/lang-smalltalk/changeset/8497f404f0ec/ Log: README.md edited online with Bitbucket diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -1,93 +1,98 @@ -RSqueak -========= - -A Squeak VM written in RPython. - -Setup ----- - -### Required Projects - -You need three repositories: - -* [This one](https://bitbucket.org/pypy/lang-smalltalk) -* [pypy/pypy](https://bitbucket.org/pypy/pypy) -* [pypy/rsdl](https://bitbucket.org/pypy/rsdl) - * Alternatively download RSDL package from [PYPI](https://pypi.python.org/pypi/rsdl) - * Then unpack and install it using ```python setup.py install``` or ```pypy setup.py install``` - -### Required packages - -You need the following packages on your OS. Install with your favorite package manager: - -* pypy - * For faster translation of the RSqueak VM. Alternatively use default Python. -* libsdl-dev -* libffi-dev - -### Adjusting the PYTHONPATH -In order to allow the RPython toolchain to find the rsdl and pypy packages you have to add the two folders to the PYTHONPATH. - -If you are using the rsdl *repository*, you have to add the rsdl subfolder of the rsdl repository to the PYTHONPATH. - -``` -export PYTHONPATH=${PYTHONPATH}:.:[path/to/pypy]:[path/to/rsdl] -``` - -### Setting the SDL Driver -For testing the basic functionality of the VM you might want to disable the UI. You can do so by setting the SDL_VIDEODRIVER environment variable to dummy. - -``` -export SDL_VIDEODRIVER=dummy -``` - -### Building & Tests -Execute the following commands inside the main directory of this repository. - -To build the VM: - -``` -[path to pypy repository]/rpython/bin/rpython targetimageloadingsmalltalk.py -``` - -To build the VM with enabled just-in-time compiler: - -``` -[path to pypy repository]/rpython/bin/rpython -Ojit targetimageloadingsmalltalk.py -``` - -To run the tests (most do not require building): - -``` -[path to pypy repository]/pytest.py [--slow|--quick] spyvm/test -``` - -### Starting an image -The build process will produce an executable called rsqueak. -The ```image/``` directory contains two images you can open with the following. -Use ```--help``` to see command line switches. - -``` -./rsqueak images/mini.image -./rsqueak images/Squeak4.5-noBitBlt.image -``` - - - - -STM-enabled Rsqueak -=== -This is a branch of RSqueak which incorporates the RPython STM transformation. Most of the initial code base comes from the results of a project seminar (https://bitbucket.org/amintos/lang-smalltalk). The stmgc-c7 branch is based on this version and the 64bit branch. - -Setup for stm-enabled RSqueak ---- -You can see the current state of the integration of the RPython STM in our stmgc-c7 branch. -Beware that you can only build this branch if you have a 64-bit linux. To build this branch you have to setup several things: - -1. Change your local pypy repository to the stm-gc7 branch, commit dd3c06b -2. Get a clang which has the patches from ([Clang patches](https://bitbucket.org/pypy/stmgc/src/d164a5bcad5e7615b4362b6a1a49d51e2e06de0c/c7/llvmfix/?at=default)). If you have a Debian-based OS you can use the following package: [llvm-pypystm](https://launchpad.net/~malte.swart/+archive/ubuntu/llvm-pypystm). - -To build, use the following command: -``` -[path to pypy repository]/rpython/bin/rpython --gc=stmgc targetimageloadingsmalltalk.py -``` +#**This Project has moved**# +##Please visit [RSqueak on Github](https://github.com/HPI-SWA-Lab/RSqueak)## + +---- + +RSqueak +========= + +A Squeak VM written in RPython. + +Setup +---- + +### Required Projects + +You need three repositories: + +* [This one](https://bitbucket.org/pypy/lang-smalltalk) +* [pypy/pypy](https://bitbucket.org/pypy/pypy) +* [pypy/rsdl](https://bitbucket.org/pypy/rsdl) + * Alternatively download RSDL package from [PYPI](https://pypi.python.org/pypi/rsdl) + * Then unpack and install it using ```python setup.py install``` or ```pypy setup.py install``` + +### Required packages + +You need the following packages on your OS. Install with your favorite package manager: + +* pypy + * For faster translation of the RSqueak VM. Alternatively use default Python. +* libsdl-dev +* libffi-dev + +### Adjusting the PYTHONPATH +In order to allow the RPython toolchain to find the rsdl and pypy packages you have to add the two folders to the PYTHONPATH. + +If you are using the rsdl *repository*, you have to add the rsdl subfolder of the rsdl repository to the PYTHONPATH. + +``` +export PYTHONPATH=${PYTHONPATH}:.:[path/to/pypy]:[path/to/rsdl] +``` + +### Setting the SDL Driver +For testing the basic functionality of the VM you might want to disable the UI. You can do so by setting the SDL_VIDEODRIVER environment variable to dummy. + +``` +export SDL_VIDEODRIVER=dummy +``` + +### Building & Tests +Execute the following commands inside the main directory of this repository. + +To build the VM: + +``` +[path to pypy repository]/rpython/bin/rpython targetimageloadingsmalltalk.py +``` + +To build the VM with enabled just-in-time compiler: + +``` +[path to pypy repository]/rpython/bin/rpython -Ojit targetimageloadingsmalltalk.py +``` + +To run the tests (most do not require building): + +``` +[path to pypy repository]/pytest.py [--slow|--quick] spyvm/test +``` + +### Starting an image +The build process will produce an executable called rsqueak. +The ```image/``` directory contains two images you can open with the following. +Use ```--help``` to see command line switches. + +``` +./rsqueak images/mini.image +./rsqueak images/Squeak4.5-noBitBlt.image +``` + + + + +STM-enabled Rsqueak +=== +This is a branch of RSqueak which incorporates the RPython STM transformation. Most of the initial code base comes from the results of a project seminar (https://bitbucket.org/amintos/lang-smalltalk). The stmgc-c7 branch is based on this version and the 64bit branch. + +Setup for stm-enabled RSqueak +--- +You can see the current state of the integration of the RPython STM in our stmgc-c7 branch. +Beware that you can only build this branch if you have a 64-bit linux. To build this branch you have to setup several things: + +1. Change your local pypy repository to the stm-gc7 branch, commit dd3c06b +2. Get a clang which has the patches from ([Clang patches](https://bitbucket.org/pypy/stmgc/src/d164a5bcad5e7615b4362b6a1a49d51e2e06de0c/c7/llvmfix/?at=default)). If you have a Debian-based OS you can use the following package: [llvm-pypystm](https://launchpad.net/~malte.swart/+archive/ubuntu/llvm-pypystm). + +To build, use the following command: +``` +[path to pypy repository]/rpython/bin/rpython --gc=stmgc targetimageloadingsmalltalk.py +``` \ No newline at end of file From noreply at buildbot.pypy.org Sun Feb 15 19:36:39 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 15 Feb 2015 19:36:39 +0100 (CET) Subject: [pypy-commit] pypy default: Rename function unicodehelper.encode_error_handler(), it is used in many places in py3k, Message-ID: <20150215183639.464D01C11B7@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r75900:9f349766cc7d Date: 2015-02-15 19:36 +0100 http://bitbucket.org/pypy/pypy/changeset/9f349766cc7d/ Log: Rename function unicodehelper.encode_error_handler(), it is used in many places in py3k, and it's too dangerous to change the raised exception. diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -25,8 +25,8 @@ self.reason = reason @specialize.memo() -def encode_error_handler(space): - # Fast version of the "strict" errors handler. +def rpy_encode_error_handler(): + # A RPython version of the "strict" error handler. def raise_unicode_exception_encode(errors, encoding, msg, u, startingpos, endingpos): raise RUnicodeEncodeError(encoding, u, startingpos, endingpos, msg) diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -439,12 +439,12 @@ try: if encoding == 'ascii': u = space.unicode_w(w_object) - eh = unicodehelper.encode_error_handler(space) + eh = unicodehelper.rpy_encode_error_handler() return space.wrap(unicode_encode_ascii( u, len(u), None, errorhandler=eh)) if encoding == 'utf-8': u = space.unicode_w(w_object) - eh = unicodehelper.encode_error_handler(space) + eh = unicodehelper.rpy_encode_error_handler() return space.wrap(unicode_encode_utf_8( u, len(u), None, errorhandler=eh, allow_surrogates=True)) From noreply at buildbot.pypy.org Sun Feb 15 19:43:20 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 15 Feb 2015 19:43:20 +0100 (CET) Subject: [pypy-commit] pypy py3k: Fix a Rpython crash in test_socket Message-ID: <20150215184320.C848D1C11B7@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r75901:62c9e1cde45a Date: 2015-02-15 17:49 +0100 http://bitbucket.org/pypy/pypy/changeset/62c9e1cde45a/ Log: Fix a Rpython crash in test_socket diff --git a/pypy/bin/pyinteractive.py b/pypy/bin/pyinteractive.py --- a/pypy/bin/pyinteractive.py +++ b/pypy/bin/pyinteractive.py @@ -10,7 +10,8 @@ import sys import time -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) +pypy_path = os.path.join(os.path.dirname(__file__), '..', '..') +sys.path.insert(0, os.path.abspath(pypy_path)) from pypy.tool import option from pypy.interpreter import main, interactive, error, gateway diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -67,8 +67,11 @@ return identifier u = self._value eh = unicodehelper.encode_error_handler(space) - identifier = unicode_encode_utf_8(u, len(u), None, - errorhandler=eh) + try: + identifier = unicode_encode_utf_8(u, len(u), None, + errorhandler=eh) + except unicodehelper.RUnicodeEncodeError, ue: + raise wrap_encode_error(space, ue) self._utf8 = identifier return identifier @@ -499,13 +502,7 @@ return space.wrapbytes(unicode_encode_utf_8( u, len(u), None, errorhandler=eh)) except unicodehelper.RUnicodeEncodeError, ue: - raise OperationError(space.w_UnicodeEncodeError, - space.newtuple([ - space.wrap(ue.encoding), - space.wrap(ue.object), - space.wrap(ue.start), - space.wrap(ue.end), - space.wrap(ue.reason)])) + raise wrap_encode_error(space, ue) from pypy.module._codecs.interp_codecs import lookup_codec w_encoder = space.getitem(lookup_codec(space, encoding), space.wrap(0)) if errors is None: @@ -521,6 +518,16 @@ return w_retval +def wrap_encode_error(space, ue): + raise OperationError(space.w_UnicodeEncodeError, + space.newtuple([ + space.wrap(ue.encoding), + space.wrap(ue.object), + space.wrap(ue.start), + space.wrap(ue.end), + space.wrap(ue.reason)])) + + def decode_object(space, w_obj, encoding, errors): if encoding is None: encoding = getdefaultencoding(space) From noreply at buildbot.pypy.org Sun Feb 15 19:43:22 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 15 Feb 2015 19:43:22 +0100 (CET) Subject: [pypy-commit] pypy py3k: Rename function unicodehelper.encode_error_handler(), it is used in many places in py3k, Message-ID: <20150215184322.178861C11B7@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r75902:ab4e84ecd24c Date: 2015-02-15 19:36 +0100 http://bitbucket.org/pypy/pypy/changeset/ab4e84ecd24c/ Log: Rename function unicodehelper.encode_error_handler(), it is used in many places in py3k, and it's too dangerous to change the raised exception. diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -24,6 +24,19 @@ space.wrap(msg)])) return raise_unicode_exception_decode + at specialize.memo() +def encode_error_handler(space): + # Fast version of the "strict" errors handler. + def raise_unicode_exception_encode(errors, encoding, msg, u, + startingpos, endingpos): + raise OperationError(space.w_UnicodeEncodeError, + space.newtuple([space.wrap(encoding), + space.wrap(u), + space.wrap(startingpos), + space.wrap(endingpos), + space.wrap(msg)])) + return raise_unicode_exception_encode + class RUnicodeEncodeError(Exception): def __init__(self, encoding, object, start, end, reason): self.encoding = encoding @@ -33,8 +46,8 @@ self.reason = reason @specialize.memo() -def encode_error_handler(space): - # Fast version of the "strict" errors handler. +def rpy_encode_error_handler(): + # A RPython version of the "strict" error handler. def raise_unicode_exception_encode(errors, encoding, msg, u, startingpos, endingpos): raise RUnicodeEncodeError(encoding, u, startingpos, endingpos, msg) diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -66,7 +66,7 @@ if identifier is not None: return identifier u = self._value - eh = unicodehelper.encode_error_handler(space) + eh = unicodehelper.rpy_encode_error_handler() try: identifier = unicode_encode_utf_8(u, len(u), None, errorhandler=eh) @@ -493,12 +493,12 @@ try: if encoding == 'ascii': u = space.unicode_w(w_object) - eh = unicodehelper.encode_error_handler(space) + eh = unicodehelper.rpy_encode_error_handler() return space.wrapbytes(unicode_encode_ascii( u, len(u), None, errorhandler=eh)) if encoding == 'utf-8': u = space.unicode_w(w_object) - eh = unicodehelper.encode_error_handler(space) + eh = unicodehelper.rpy_encode_error_handler() return space.wrapbytes(unicode_encode_utf_8( u, len(u), None, errorhandler=eh)) except unicodehelper.RUnicodeEncodeError, ue: From noreply at buildbot.pypy.org Sun Feb 15 19:46:38 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 15 Feb 2015 19:46:38 +0100 (CET) Subject: [pypy-commit] pypy default: Issue #1985: SSL_OP_NO_COMPRESSION is not always defined. Message-ID: <20150215184638.9E5921C11B7@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r75903:c550e14a63bb Date: 2015-02-15 19:46 +0100 http://bitbucket.org/pypy/pypy/changeset/c550e14a63bb/ Log: Issue #1985: SSL_OP_NO_COMPRESSION is not always defined. Should help translation on OS/X. diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -79,7 +79,8 @@ constants["OP_CIPHER_SERVER_PREFERENCE"] = SSL_OP_CIPHER_SERVER_PREFERENCE constants["OP_SINGLE_DH_USE"] = SSL_OP_SINGLE_DH_USE constants["OP_SINGLE_ECDH_USE"] = SSL_OP_SINGLE_ECDH_USE -constants["OP_NO_COMPRESSION"] = SSL_OP_NO_COMPRESSION +if SSL_NO_COMPRESSION is not None: + constants["OP_NO_COMPRESSION"] = SSL_OP_NO_COMPRESSION constants["OPENSSL_VERSION_NUMBER"] = OPENSSL_VERSION_NUMBER ver = OPENSSL_VERSION_NUMBER diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py --- a/rpython/rlib/ropenssl.py +++ b/rpython/rlib/ropenssl.py @@ -96,7 +96,7 @@ "SSL_OP_SINGLE_DH_USE") SSL_OP_SINGLE_ECDH_USE = rffi_platform.ConstantInteger( "SSL_OP_SINGLE_ECDH_USE") - SSL_OP_NO_COMPRESSION = rffi_platform.ConstantInteger( + SSL_OP_NO_COMPRESSION = rffi_platform.DefinedConstantInteger( "SSL_OP_NO_COMPRESSION") SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS = rffi_platform.ConstantInteger( "SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS") From noreply at buildbot.pypy.org Sun Feb 15 20:00:52 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 15 Feb 2015 20:00:52 +0100 (CET) Subject: [pypy-commit] pypy default: SSL: Fix crldp for older versions of openssl. Message-ID: <20150215190052.01EBE1C123B@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r75904:ad7f26afe1b2 Date: 2015-02-15 20:00 +0100 http://bitbucket.org/pypy/pypy/changeset/ad7f26afe1b2/ Log: SSL: Fix crldp for older versions of openssl. diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -79,7 +79,7 @@ constants["OP_CIPHER_SERVER_PREFERENCE"] = SSL_OP_CIPHER_SERVER_PREFERENCE constants["OP_SINGLE_DH_USE"] = SSL_OP_SINGLE_DH_USE constants["OP_SINGLE_ECDH_USE"] = SSL_OP_SINGLE_ECDH_USE -if SSL_NO_COMPRESSION is not None: +if SSL_OP_NO_COMPRESSION is not None: constants["OP_NO_COMPRESSION"] = SSL_OP_NO_COMPRESSION constants["OPENSSL_VERSION_NUMBER"] = OPENSSL_VERSION_NUMBER @@ -887,9 +887,13 @@ libssl_AUTHORITY_INFO_ACCESS_free(info) def _get_crl_dp(space, certificate): - # Calls x509v3_cache_extensions and sets up crldp - libssl_X509_check_ca(certificate) - dps = certificate[0].c_crldp + if OPENSSL_VERSION_NUMBER >= 0x10001000: + # Calls x509v3_cache_extensions and sets up crldp + libssl_X509_check_ca(certificate) + dps = certificate[0].c_crldp + else: + dps = rffi.cast(stack_st_DIST_POINT, libssl_X509_get_ext_d2i( + certificate, NID_crl_distribution_points, None, None)) if not dps: return None diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py --- a/rpython/rlib/ropenssl.py +++ b/rpython/rlib/ropenssl.py @@ -68,16 +68,19 @@ _compilation_info_ = eci OPENSSL_EXPORT_VAR_AS_FUNCTION = rffi_platform.Defined( "OPENSSL_EXPORT_VAR_AS_FUNCTION") -if rffi_platform.configure(CConfigBootstrap)["OPENSSL_EXPORT_VAR_AS_FUNCTION"]: + OPENSSL_VERSION_NUMBER = rffi_platform.ConstantInteger( + "OPENSSL_VERSION_NUMBER") + +cconfig = rffi_platform.configure(CConfigBootstrap) +if cconfig["OPENSSL_EXPORT_VAR_AS_FUNCTION"]: ASN1_ITEM_EXP = lltype.Ptr(lltype.FuncType([], ASN1_ITEM)) else: ASN1_ITEM_EXP = ASN1_ITEM +OPENSSL_VERSION_NUMBER = cconfig["OPENSSL_VERSION_NUMBER"] class CConfig: _compilation_info_ = eci - OPENSSL_VERSION_NUMBER = rffi_platform.ConstantInteger( - "OPENSSL_VERSION_NUMBER") SSLEAY_VERSION = rffi_platform.DefinedConstantString( "SSLEAY_VERSION", "SSLeay_version(SSLEAY_VERSION)") OPENSSL_NO_SSL2 = rffi_platform.Defined("OPENSSL_NO_SSL2") @@ -147,6 +150,7 @@ NID_ad_ca_issuers = rffi_platform.ConstantInteger("NID_ad_ca_issuers") NID_info_access = rffi_platform.ConstantInteger("NID_info_access") NID_X9_62_prime256v1 = rffi_platform.ConstantInteger("NID_X9_62_prime256v1") + NID_crl_distribution_points = rffi_platform.ConstantInteger("NID_crl_distribution_points") GEN_DIRNAME = rffi_platform.ConstantInteger("GEN_DIRNAME") GEN_EMAIL = rffi_platform.ConstantInteger("GEN_EMAIL") GEN_DNS = rffi_platform.ConstantInteger("GEN_DNS") @@ -162,9 +166,10 @@ OBJ_NAME_TYPE_MD_METH = rffi_platform.ConstantInteger( "OBJ_NAME_TYPE_MD_METH") - X509_st = rffi_platform.Struct( - 'struct x509_st', - [('crldp', stack_st_DIST_POINT)]) + if OPENSSL_VERSION_NUMBER >= 0x10001000: + X509_st = rffi_platform.Struct( + 'struct x509_st', + [('crldp', stack_st_DIST_POINT)]) # Some structures, with only the fields used in the _ssl module X509_name_entry_st = rffi_platform.Struct('struct X509_name_entry_st', @@ -226,7 +231,10 @@ SSL_CIPHER = rffi.COpaquePtr('SSL_CIPHER') SSL = rffi.COpaquePtr('SSL') BIO = rffi.COpaquePtr('BIO') -X509 = rffi.CArrayPtr(X509_st) +if OPENSSL_VERSION_NUMBER >= 0x10001000: + X509 = rffi.CArrayPtr(X509_st) +else: + X509 = rffi.COpaquePtr('X509') X509_NAME_ENTRY = rffi.CArrayPtr(X509_name_entry_st) X509_EXTENSION = rffi.CArrayPtr(X509_extension_st) X509_STORE = rffi.CArrayPtr(x509_store_st) From noreply at buildbot.pypy.org Sun Feb 15 20:04:21 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 15 Feb 2015 20:04:21 +0100 (CET) Subject: [pypy-commit] pypy py3k: py3-ify another test. Message-ID: <20150215190421.C40031C123B@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r75905:bed47579375c Date: 2015-02-15 20:04 +0100 http://bitbucket.org/pypy/pypy/changeset/bed47579375c/ Log: py3-ify another test. diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py --- a/pypy/objspace/std/test/test_memoryobject.py +++ b/pypy/objspace/std/test/test_memoryobject.py @@ -129,7 +129,7 @@ assert "released memory" in repr(v) def test_pypy_raw_address_base(self): - raises(ValueError, memoryview("foobar")._pypy_raw_address) - e = raises(ValueError, memoryview(bytearray("foobar"))._pypy_raw_address) + raises(ValueError, memoryview(b"foobar")._pypy_raw_address) + e = raises(ValueError, memoryview(bytearray(b"foobar"))._pypy_raw_address) assert 'BytearrayBuffer' in str(e.value) From noreply at buildbot.pypy.org Sun Feb 15 20:13:53 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 15 Feb 2015 20:13:53 +0100 (CET) Subject: [pypy-commit] pypy default: Oops, fix translation Message-ID: <20150215191353.A85921C009F@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r75906:fd175f59c77f Date: 2015-02-15 20:13 +0100 http://bitbucket.org/pypy/pypy/changeset/fd175f59c77f/ Log: Oops, fix translation diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -62,7 +62,10 @@ return result def encode_utf8(space, uni): + # Note that this function never raises UnicodeEncodeError, + # since surrogate pairs are allowed. + # This is not the case with Python3. return runicode.unicode_encode_utf_8( uni, len(uni), "strict", - errorhandler=encode_error_handler(space), + errorhandler=rpy_encode_error_handler(), allow_surrogates=True) From noreply at buildbot.pypy.org Sun Feb 15 20:37:19 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 15 Feb 2015 20:37:19 +0100 (CET) Subject: [pypy-commit] pypy default: Skip undefined error codes. Message-ID: <20150215193719.8BD1E1C123E@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r75907:b6651f030e15 Date: 2015-02-15 20:37 +0100 http://bitbucket.org/pypy/pypy/changeset/b6651f030e15/ Log: Skip undefined error codes. diff --git a/pypy/module/_ssl/ssl_data.py b/pypy/module/_ssl/ssl_data.py --- a/pypy/module/_ssl/ssl_data.py +++ b/pypy/module/_ssl/ssl_data.py @@ -369,7 +369,8 @@ LIBRARY_CODES_TO_NAMES[cconfig[code]] = code ERROR_CODES_TO_NAMES = {} for lib, code in error_codes: - ERROR_CODES_TO_NAMES[cconfig[lib], cconfig[code]] = code + if cconfig[code] is not None: + ERROR_CODES_TO_NAMES[cconfig[lib], cconfig[code]] = code ALERT_DESCRIPTION_CODES = {} for name in AD_NAMES: From noreply at buildbot.pypy.org Mon Feb 16 01:05:42 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 16 Feb 2015 01:05:42 +0100 (CET) Subject: [pypy-commit] pypy py3k: hg merge default Message-ID: <20150216000542.7CAD71C03B3@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r75908:37fec51281b6 Date: 2015-02-15 22:47 +0100 http://bitbucket.org/pypy/pypy/changeset/37fec51281b6/ Log: hg merge default diff too long, truncating to 2000 out of 28392 lines diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,10 @@ bin/pypy-c include/*.h +include/numpy/ lib_pypy/ctypes_config_cache/_[^_]*_*.py +libpypy-c.* +pypy-c pypy/_cache pypy/doc/*.html pypy/doc/config/*.html @@ -18,4 +21,5 @@ pypy/translator/c/src/dtoa.o pypy/translator/goal/pypy-c pypy/translator/goal/target*-c -release/ \ No newline at end of file +release/ +rpython/_cache/ diff --git a/lib-python/2.7/CGIHTTPServer.py b/lib-python/2.7/CGIHTTPServer.py --- a/lib-python/2.7/CGIHTTPServer.py +++ b/lib-python/2.7/CGIHTTPServer.py @@ -106,16 +106,16 @@ def run_cgi(self): """Execute a CGI script.""" dir, rest = self.cgi_info - - i = rest.find('/') + path = dir + '/' + rest + i = path.find('/', len(dir)+1) while i >= 0: - nextdir = rest[:i] - nextrest = rest[i+1:] + nextdir = path[:i] + nextrest = path[i+1:] scriptdir = self.translate_path(nextdir) if os.path.isdir(scriptdir): dir, rest = nextdir, nextrest - i = rest.find('/') + i = path.find('/', len(dir)+1) else: break diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py --- a/lib-python/2.7/Cookie.py +++ b/lib-python/2.7/Cookie.py @@ -56,7 +56,7 @@ >>> C = Cookie.SmartCookie() [Note: Long-time users of Cookie.py will remember using -Cookie.Cookie() to create an Cookie object. Although deprecated, it +Cookie.Cookie() to create a Cookie object. Although deprecated, it is still supported by the code. See the Backward Compatibility notes for more information.] @@ -426,6 +426,8 @@ "version" : "Version", } + _flags = {'secure', 'httponly'} + def __init__(self): # Set defaults self.key = self.value = self.coded_value = None @@ -529,9 +531,11 @@ _LegalCharsPatt = r"[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=]" _CookiePattern = re.compile( r"(?x)" # This is a Verbose pattern + r"\s*" # Optional whitespace at start of cookie r"(?P" # Start of group 'key' ""+ _LegalCharsPatt +"+?" # Any word of at least one letter, nongreedy r")" # End of group 'key' + r"(" # Optional group: there may not be a value. r"\s*=\s*" # Equal Sign r"(?P" # Start of group 'val' r'"(?:[^\\"]|\\.)*"' # Any doublequoted string @@ -540,7 +544,9 @@ r"|" # or ""+ _LegalCharsPatt +"*" # Any word or empty string r")" # End of group 'val' - r"\s*;?" # Probably ending in a semi-colon + r")?" # End of optional value group + r"\s*" # Any number of spaces. + r"(\s+|;|$)" # Ending either at space, semicolon, or EOS. ) @@ -585,8 +591,12 @@ def __setitem__(self, key, value): """Dictionary style assignment.""" - rval, cval = self.value_encode(value) - self.__set(key, rval, cval) + if isinstance(value, Morsel): + # allow assignment of constructed Morsels (e.g. for pickling) + dict.__setitem__(self, key, value) + else: + rval, cval = self.value_encode(value) + self.__set(key, rval, cval) # end __setitem__ def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"): @@ -641,7 +651,7 @@ while 0 <= i < n: # Start looking for a cookie - match = patt.search(str, i) + match = patt.match(str, i) if not match: break # No more cookies K,V = match.group("key"), match.group("val") @@ -656,8 +666,12 @@ M[ K[1:] ] = V elif K.lower() in Morsel._reserved: if M: - M[ K ] = _unquote(V) - else: + if V is None: + if K.lower() in Morsel._flags: + M[K] = True + else: + M[K] = _unquote(V) + elif V is not None: rval, cval = self.value_decode(V) self.__set(K, rval, cval) M = self[K] diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py --- a/lib-python/2.7/SocketServer.py +++ b/lib-python/2.7/SocketServer.py @@ -416,8 +416,12 @@ self.socket = socket.socket(self.address_family, self.socket_type) if bind_and_activate: - self.server_bind() - self.server_activate() + try: + self.server_bind() + self.server_activate() + except: + self.server_close() + raise def server_bind(self): """Called by constructor to bind the socket. diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py --- a/lib-python/2.7/_abcoll.py +++ b/lib-python/2.7/_abcoll.py @@ -143,7 +143,7 @@ methods except for __contains__, __iter__ and __len__. To override the comparisons (presumably for speed, as the - semantics are fixed), all you have to do is redefine __le__ and + semantics are fixed), redefine __le__ and __ge__, then the other operations will automatically follow suit. """ diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py --- a/lib-python/2.7/argparse.py +++ b/lib-python/2.7/argparse.py @@ -1089,7 +1089,14 @@ # parse all the remaining options into the namespace # store any unrecognized options on the object, so that the top # level parser can decide what to do with them - namespace, arg_strings = parser.parse_known_args(arg_strings, namespace) + + # In case this subparser defines new defaults, we parse them + # in a new namespace object and then update the original + # namespace for the relevant parts. + subnamespace, arg_strings = parser.parse_known_args(arg_strings, None) + for key, value in vars(subnamespace).items(): + setattr(namespace, key, value) + if arg_strings: vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, []) getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings) diff --git a/lib-python/2.7/asynchat.py b/lib-python/2.7/asynchat.py --- a/lib-python/2.7/asynchat.py +++ b/lib-python/2.7/asynchat.py @@ -46,12 +46,17 @@ you - by calling your self.found_terminator() method. """ +import asyncore +import errno import socket -import asyncore from collections import deque from sys import py3kwarning from warnings import filterwarnings, catch_warnings +_BLOCKING_IO_ERRORS = (errno.EAGAIN, errno.EALREADY, errno.EINPROGRESS, + errno.EWOULDBLOCK) + + class async_chat (asyncore.dispatcher): """This is an abstract class. You must derive from this class, and add the two methods collect_incoming_data() and found_terminator()""" @@ -109,6 +114,8 @@ try: data = self.recv (self.ac_in_buffer_size) except socket.error, why: + if why.args[0] in _BLOCKING_IO_ERRORS: + return self.handle_error() return diff --git a/lib-python/2.7/bsddb/test/test_queue.py b/lib-python/2.7/bsddb/test/test_queue.py --- a/lib-python/2.7/bsddb/test/test_queue.py +++ b/lib-python/2.7/bsddb/test/test_queue.py @@ -10,6 +10,7 @@ #---------------------------------------------------------------------- + at unittest.skip("fails on Windows; see issue 22943") class SimpleQueueTestCase(unittest.TestCase): def setUp(self): self.filename = get_new_database_path() diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py --- a/lib-python/2.7/cookielib.py +++ b/lib-python/2.7/cookielib.py @@ -1719,12 +1719,12 @@ def __repr__(self): r = [] for cookie in self: r.append(repr(cookie)) - return "<%s[%s]>" % (self.__class__, ", ".join(r)) + return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r)) def __str__(self): r = [] for cookie in self: r.append(str(cookie)) - return "<%s[%s]>" % (self.__class__, ", ".join(r)) + return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r)) # derives from IOError for backwards-compatibility with Python 2.4.0 diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py --- a/lib-python/2.7/ctypes/test/test_pointers.py +++ b/lib-python/2.7/ctypes/test/test_pointers.py @@ -7,6 +7,8 @@ c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float] python_types = [int, int, int, int, int, long, int, long, long, long, float, float] +LargeNamedType = type('T' * 2 ** 25, (Structure,), {}) +large_string = 'T' * 2 ** 25 class PointersTestCase(unittest.TestCase): @@ -188,5 +190,11 @@ mth = WINFUNCTYPE(None)(42, "name", (), None) self.assertEqual(bool(mth), True) + def test_pointer_type_name(self): + self.assertTrue(POINTER(LargeNamedType)) + + def test_pointer_type_str_name(self): + self.assertTrue(POINTER(large_string)) + if __name__ == '__main__': unittest.main() diff --git a/lib-python/2.7/ctypes/test/test_python_api.py b/lib-python/2.7/ctypes/test/test_python_api.py --- a/lib-python/2.7/ctypes/test/test_python_api.py +++ b/lib-python/2.7/ctypes/test/test_python_api.py @@ -46,8 +46,8 @@ # This test is unreliable, because it is possible that code in # unittest changes the refcount of the '42' integer. So, it # is disabled by default. - @requires("refcount") def test_PyInt_Long(self): + requires("refcount") ref42 = grc(42) pythonapi.PyInt_FromLong.restype = py_object self.assertEqual(pythonapi.PyInt_FromLong(42), 42) diff --git a/lib-python/2.7/ctypes/test/test_win32.py b/lib-python/2.7/ctypes/test/test_win32.py --- a/lib-python/2.7/ctypes/test/test_win32.py +++ b/lib-python/2.7/ctypes/test/test_win32.py @@ -38,8 +38,11 @@ @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') class FunctionCallTestCase(unittest.TestCase): - @requires("SEH") + @unittest.skipUnless('MSC' in sys.version, "SEH only supported by MSC") + @unittest.skipIf(sys.executable.endswith('_d.exe'), + "SEH not enabled in debug builds") def test_SEH(self): + requires("SEH") # Call functions with invalid arguments, and make sure # that access violations are trapped and raise an # exception. @@ -87,9 +90,29 @@ dll = CDLL(_ctypes_test.__file__) - pt = POINT(10, 10) - rect = RECT(0, 0, 20, 20) - self.assertEqual(1, dll.PointInRect(byref(rect), pt)) + pt = POINT(15, 25) + left = c_long.in_dll(dll, 'left') + top = c_long.in_dll(dll, 'top') + right = c_long.in_dll(dll, 'right') + bottom = c_long.in_dll(dll, 'bottom') + rect = RECT(left, top, right, bottom) + PointInRect = dll.PointInRect + PointInRect.argtypes = [POINTER(RECT), POINT] + self.assertEqual(1, PointInRect(byref(rect), pt)) + + ReturnRect = dll.ReturnRect + ReturnRect.argtypes = [c_int, RECT, POINTER(RECT), POINT, RECT, + POINTER(RECT), POINT, RECT] + ReturnRect.restype = RECT + for i in range(4): + ret = ReturnRect(i, rect, pointer(rect), pt, rect, + byref(rect), pt, rect) + # the c function will check and modify ret if something is + # passed in improperly + self.assertEqual(ret.left, left.value) + self.assertEqual(ret.right, right.value) + self.assertEqual(ret.top, top.value) + self.assertEqual(ret.bottom, bottom.value) if __name__ == '__main__': unittest.main() diff --git a/lib-python/2.7/decimal.py b/lib-python/2.7/decimal.py --- a/lib-python/2.7/decimal.py +++ b/lib-python/2.7/decimal.py @@ -136,7 +136,6 @@ __version__ = '1.70' # Highest version of the spec this complies with -import copy as _copy import math as _math import numbers as _numbers @@ -3665,6 +3664,8 @@ if self._is_special: sign = _format_sign(self._sign, spec) body = str(self.copy_abs()) + if spec['type'] == '%': + body += '%' return _format_align(sign, body, spec) # a type of None defaults to 'g' or 'G', depending on context @@ -6033,7 +6034,10 @@ format_dict['decimal_point'] = '.' # record whether return type should be str or unicode - format_dict['unicode'] = isinstance(format_spec, unicode) + try: + format_dict['unicode'] = isinstance(format_spec, unicode) + except NameError: + format_dict['unicode'] = False return format_dict diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py --- a/lib-python/2.7/distutils/__init__.py +++ b/lib-python/2.7/distutils/__init__.py @@ -15,5 +15,5 @@ # Updated automatically by the Python release process. # #--start constants-- -__version__ = "2.7.8" +__version__ = "2.7.9" #--end constants-- diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py --- a/lib-python/2.7/distutils/command/build_ext.py +++ b/lib-python/2.7/distutils/command/build_ext.py @@ -245,7 +245,7 @@ # Python's library directory must be appended to library_dirs # See Issues: #1600860, #4366 if (sysconfig.get_config_var('Py_ENABLE_SHARED')): - if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): + if not sysconfig.python_build: # building third party extensions self.library_dirs.append(sysconfig.get_config_var('LIBDIR')) else: diff --git a/lib-python/2.7/distutils/command/upload.py b/lib-python/2.7/distutils/command/upload.py --- a/lib-python/2.7/distutils/command/upload.py +++ b/lib-python/2.7/distutils/command/upload.py @@ -136,8 +136,8 @@ # Build up the MIME payload for the POST data boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254' - sep_boundary = '\n--' + boundary - end_boundary = sep_boundary + '--' + sep_boundary = '\r\n--' + boundary + end_boundary = sep_boundary + '--\r\n' body = StringIO.StringIO() for key, value in data.items(): # handle multiple entries for the same name @@ -151,14 +151,13 @@ fn = "" body.write(sep_boundary) - body.write('\nContent-Disposition: form-data; name="%s"'%key) + body.write('\r\nContent-Disposition: form-data; name="%s"' % key) body.write(fn) - body.write("\n\n") + body.write("\r\n\r\n") body.write(value) if value and value[-1] == '\r': body.write('\n') # write an extra newline (lurve Macs) body.write(end_boundary) - body.write("\n") body = body.getvalue() self.announce("Submitting %s to %s" % (filename, self.repository), log.INFO) diff --git a/lib-python/2.7/distutils/file_util.py b/lib-python/2.7/distutils/file_util.py --- a/lib-python/2.7/distutils/file_util.py +++ b/lib-python/2.7/distutils/file_util.py @@ -85,7 +85,8 @@ (os.symlink) instead of copying: set it to "hard" or "sym"; if it is None (the default), files are copied. Don't set 'link' on systems that don't support it: 'copy_file()' doesn't check if hard or symbolic - linking is available. + linking is available. If hardlink fails, falls back to + _copy_file_contents(). Under Mac OS, uses the native file copy function in macostools; on other systems, uses '_copy_file_contents()' to copy file contents. @@ -137,24 +138,31 @@ # (Unix only, of course, but that's the caller's responsibility) if link == 'hard': if not (os.path.exists(dst) and os.path.samefile(src, dst)): - os.link(src, dst) + try: + os.link(src, dst) + return (dst, 1) + except OSError: + # If hard linking fails, fall back on copying file + # (some special filesystems don't support hard linking + # even under Unix, see issue #8876). + pass elif link == 'sym': if not (os.path.exists(dst) and os.path.samefile(src, dst)): os.symlink(src, dst) + return (dst, 1) # Otherwise (non-Mac, not linking), copy the file contents and # (optionally) copy the times and mode. - else: - _copy_file_contents(src, dst) - if preserve_mode or preserve_times: - st = os.stat(src) + _copy_file_contents(src, dst) + if preserve_mode or preserve_times: + st = os.stat(src) - # According to David Ascher , utime() should be done - # before chmod() (at least under NT). - if preserve_times: - os.utime(dst, (st[ST_ATIME], st[ST_MTIME])) - if preserve_mode: - os.chmod(dst, S_IMODE(st[ST_MODE])) + # According to David Ascher , utime() should be done + # before chmod() (at least under NT). + if preserve_times: + os.utime(dst, (st[ST_ATIME], st[ST_MTIME])) + if preserve_mode: + os.chmod(dst, S_IMODE(st[ST_MODE])) return (dst, 1) diff --git a/lib-python/2.7/distutils/sysconfig_cpython.py b/lib-python/2.7/distutils/sysconfig_cpython.py --- a/lib-python/2.7/distutils/sysconfig_cpython.py +++ b/lib-python/2.7/distutils/sysconfig_cpython.py @@ -165,7 +165,8 @@ # version and build tools may not support the same set # of CPU architectures for universal builds. global _config_vars - if not _config_vars.get('CUSTOMIZED_OSX_COMPILER', ''): + # Use get_config_var() to ensure _config_vars is initialized. + if not get_config_var('CUSTOMIZED_OSX_COMPILER'): import _osx_support _osx_support.customize_compiler(_config_vars) _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True' diff --git a/lib-python/2.7/distutils/tests/test_bdist_rpm.py b/lib-python/2.7/distutils/tests/test_bdist_rpm.py --- a/lib-python/2.7/distutils/tests/test_bdist_rpm.py +++ b/lib-python/2.7/distutils/tests/test_bdist_rpm.py @@ -25,6 +25,7 @@ """ class BuildRpmTestCase(support.TempdirManager, + support.EnvironGuard, support.LoggingSilencer, unittest.TestCase): @@ -50,6 +51,7 @@ def test_quiet(self): # let's create a package tmp_dir = self.mkdtemp() + os.environ['HOME'] = tmp_dir # to confine dir '.rpmdb' creation pkg_dir = os.path.join(tmp_dir, 'foo') os.mkdir(pkg_dir) self.write_file((pkg_dir, 'setup.py'), SETUP_PY) @@ -92,6 +94,7 @@ def test_no_optimize_flag(self): # let's create a package that brakes bdist_rpm tmp_dir = self.mkdtemp() + os.environ['HOME'] = tmp_dir # to confine dir '.rpmdb' creation pkg_dir = os.path.join(tmp_dir, 'foo') os.mkdir(pkg_dir) self.write_file((pkg_dir, 'setup.py'), SETUP_PY) diff --git a/lib-python/2.7/distutils/tests/test_dist.py b/lib-python/2.7/distutils/tests/test_dist.py --- a/lib-python/2.7/distutils/tests/test_dist.py +++ b/lib-python/2.7/distutils/tests/test_dist.py @@ -11,7 +11,7 @@ from distutils.dist import Distribution, fix_help_options from distutils.cmd import Command import distutils.dist -from test.test_support import TESTFN, captured_stdout, run_unittest +from test.test_support import TESTFN, captured_stdout, run_unittest, unlink from distutils.tests import support @@ -64,6 +64,7 @@ with open(TESTFN, "w") as f: f.write("[global]\n") f.write("command_packages = foo.bar, splat") + self.addCleanup(unlink, TESTFN) files = [TESTFN] sys.argv.append("build") diff --git a/lib-python/2.7/distutils/tests/test_file_util.py b/lib-python/2.7/distutils/tests/test_file_util.py --- a/lib-python/2.7/distutils/tests/test_file_util.py +++ b/lib-python/2.7/distutils/tests/test_file_util.py @@ -8,6 +8,11 @@ from distutils.tests import support from test.test_support import run_unittest + +requires_os_link = unittest.skipUnless(hasattr(os, "link"), + "test requires os.link()") + + class FileUtilTestCase(support.TempdirManager, unittest.TestCase): def _log(self, msg, *args): @@ -74,6 +79,44 @@ copy_file(foo, dst_dir) self.assertTrue(os.path.exists(os.path.join(dst_dir, 'foo'))) + @requires_os_link + def test_copy_file_hard_link(self): + with open(self.source, 'w') as f: + f.write('some content') + st = os.stat(self.source) + copy_file(self.source, self.target, link='hard') + st2 = os.stat(self.source) + st3 = os.stat(self.target) + self.assertTrue(os.path.samestat(st, st2), (st, st2)) + self.assertTrue(os.path.samestat(st2, st3), (st2, st3)) + with open(self.source, 'r') as f: + self.assertEqual(f.read(), 'some content') + + @requires_os_link + def test_copy_file_hard_link_failure(self): + # If hard linking fails, copy_file() falls back on copying file + # (some special filesystems don't support hard linking even under + # Unix, see issue #8876). + with open(self.source, 'w') as f: + f.write('some content') + st = os.stat(self.source) + def _os_link(*args): + raise OSError(0, "linking unsupported") + old_link = os.link + os.link = _os_link + try: + copy_file(self.source, self.target, link='hard') + finally: + os.link = old_link + st2 = os.stat(self.source) + st3 = os.stat(self.target) + self.assertTrue(os.path.samestat(st, st2), (st, st2)) + self.assertFalse(os.path.samestat(st2, st3), (st2, st3)) + for fn in (self.source, self.target): + with open(fn, 'r') as f: + self.assertEqual(f.read(), 'some content') + + def test_suite(): return unittest.makeSuite(FileUtilTestCase) diff --git a/lib-python/2.7/distutils/tests/test_sysconfig.py b/lib-python/2.7/distutils/tests/test_sysconfig.py --- a/lib-python/2.7/distutils/tests/test_sysconfig.py +++ b/lib-python/2.7/distutils/tests/test_sysconfig.py @@ -3,6 +3,9 @@ import test import unittest import shutil +import subprocess +import sys +import textwrap from distutils import sysconfig from distutils.tests import support @@ -99,6 +102,24 @@ self.assertEqual(global_sysconfig.get_config_var('LDSHARED'), sysconfig.get_config_var('LDSHARED')) self.assertEqual(global_sysconfig.get_config_var('CC'), sysconfig.get_config_var('CC')) + def test_customize_compiler_before_get_config_vars(self): + # Issue #21923: test that a Distribution compiler + # instance can be called without an explicit call to + # get_config_vars(). + with open(TESTFN, 'w') as f: + f.writelines(textwrap.dedent('''\ + from distutils.core import Distribution + config = Distribution().get_command_obj('config') + # try_compile may pass or it may fail if no compiler + # is found but it should not raise an exception. + rc = config.try_compile('int x;') + ''')) + p = subprocess.Popen([str(sys.executable), TESTFN], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True) + outs, errs = p.communicate() + self.assertEqual(0, p.returncode, "Subprocess failed: " + outs) def test_suite(): diff --git a/lib-python/2.7/distutils/tests/test_upload.py b/lib-python/2.7/distutils/tests/test_upload.py --- a/lib-python/2.7/distutils/tests/test_upload.py +++ b/lib-python/2.7/distutils/tests/test_upload.py @@ -119,7 +119,7 @@ # what did we send ? self.assertIn('dédé', self.last_open.req.data) headers = dict(self.last_open.req.headers) - self.assertEqual(headers['Content-length'], '2085') + self.assertEqual(headers['Content-length'], '2159') self.assertTrue(headers['Content-type'].startswith('multipart/form-data')) self.assertEqual(self.last_open.req.get_method(), 'POST') self.assertEqual(self.last_open.req.get_full_url(), diff --git a/lib-python/2.7/doctest.py b/lib-python/2.7/doctest.py --- a/lib-python/2.7/doctest.py +++ b/lib-python/2.7/doctest.py @@ -216,7 +216,7 @@ # get_data() opens files as 'rb', so one must do the equivalent # conversion as universal newlines would do. return file_contents.replace(os.linesep, '\n'), filename - with open(filename) as f: + with open(filename, 'U') as f: return f.read(), filename # Use sys.stdout encoding for ouput. diff --git a/lib-python/2.7/email/feedparser.py b/lib-python/2.7/email/feedparser.py --- a/lib-python/2.7/email/feedparser.py +++ b/lib-python/2.7/email/feedparser.py @@ -49,8 +49,8 @@ simple abstraction -- it parses until EOF closes the current message. """ def __init__(self): - # The last partial line pushed into this object. - self._partial = '' + # Chunks of the last partial line pushed into this object. + self._partial = [] # The list of full, pushed lines, in reverse order self._lines = [] # The stack of false-EOF checking predicates. @@ -66,8 +66,8 @@ def close(self): # Don't forget any trailing partial line. - self._lines.append(self._partial) - self._partial = '' + self.pushlines(''.join(self._partial).splitlines(True)) + self._partial = [] self._closed = True def readline(self): @@ -95,8 +95,29 @@ def push(self, data): """Push some new data into this object.""" - # Handle any previous leftovers - data, self._partial = self._partial + data, '' + # Crack into lines, but preserve the linesep characters on the end of each + parts = data.splitlines(True) + + if not parts or not parts[0].endswith(('\n', '\r')): + # No new complete lines, so just accumulate partials + self._partial += parts + return + + if self._partial: + # If there are previous leftovers, complete them now + self._partial.append(parts[0]) + parts[0:1] = ''.join(self._partial).splitlines(True) + del self._partial[:] + + # If the last element of the list does not end in a newline, then treat + # it as a partial line. We only check for '\n' here because a line + # ending with '\r' might be a line that was split in the middle of a + # '\r\n' sequence (see bugs 1555570 and 1721862). + if not parts[-1].endswith('\n'): + self._partial = [parts.pop()] + self.pushlines(parts) + + def pushlines(self, lines): # Crack into lines, but preserve the newlines on the end of each parts = NLCRE_crack.split(data) # The *ahem* interesting behaviour of re.split when supplied grouping diff --git a/lib-python/2.7/email/mime/nonmultipart.py b/lib-python/2.7/email/mime/nonmultipart.py --- a/lib-python/2.7/email/mime/nonmultipart.py +++ b/lib-python/2.7/email/mime/nonmultipart.py @@ -12,7 +12,7 @@ class MIMENonMultipart(MIMEBase): - """Base class for MIME multipart/* type messages.""" + """Base class for MIME non-multipart type messages.""" def attach(self, payload): # The public API prohibits attaching multiple subparts to MIMEBase diff --git a/lib-python/2.7/email/test/test_email.py b/lib-python/2.7/email/test/test_email.py --- a/lib-python/2.7/email/test/test_email.py +++ b/lib-python/2.7/email/test/test_email.py @@ -11,6 +11,7 @@ import warnings import textwrap from cStringIO import StringIO +from random import choice import email @@ -2578,16 +2579,64 @@ bsf.push(il) nt += n n1 = 0 - while True: - ol = bsf.readline() - if ol == NeedMoreData: - break + for ol in iter(bsf.readline, NeedMoreData): om.append(ol) n1 += 1 self.assertEqual(n, n1) self.assertEqual(len(om), nt) self.assertEqual(''.join([il for il, n in imt]), ''.join(om)) + def test_push_random(self): + from email.feedparser import BufferedSubFile, NeedMoreData + + n = 10000 + chunksize = 5 + chars = 'abcd \t\r\n' + + s = ''.join(choice(chars) for i in range(n)) + '\n' + target = s.splitlines(True) + + bsf = BufferedSubFile() + lines = [] + for i in range(0, len(s), chunksize): + chunk = s[i:i+chunksize] + bsf.push(chunk) + lines.extend(iter(bsf.readline, NeedMoreData)) + self.assertEqual(lines, target) + + +class TestFeedParsers(TestEmailBase): + + def parse(self, chunks): + from email.feedparser import FeedParser + feedparser = FeedParser() + for chunk in chunks: + feedparser.feed(chunk) + return feedparser.close() + + def test_newlines(self): + m = self.parse(['a:\nb:\rc:\r\nd:\n']) + self.assertEqual(m.keys(), ['a', 'b', 'c', 'd']) + m = self.parse(['a:\nb:\rc:\r\nd:']) + self.assertEqual(m.keys(), ['a', 'b', 'c', 'd']) + m = self.parse(['a:\rb', 'c:\n']) + self.assertEqual(m.keys(), ['a', 'bc']) + m = self.parse(['a:\r', 'b:\n']) + self.assertEqual(m.keys(), ['a', 'b']) + m = self.parse(['a:\r', '\nb:\n']) + self.assertEqual(m.keys(), ['a', 'b']) + + def test_long_lines(self): + # Expected peak memory use on 32-bit platform: 4*N*M bytes. + M, N = 1000, 20000 + m = self.parse(['a:b\n\n'] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', 'b')]) + self.assertEqual(m.get_payload(), 'x'*M*N) + m = self.parse(['a:b\r\r'] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', 'b')]) + self.assertEqual(m.get_payload(), 'x'*M*N) + m = self.parse(['a:\r', 'b: '] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', ''), ('b', 'x'*M*N)]) class TestParsers(TestEmailBase): @@ -3180,7 +3229,6 @@ self.assertEqual(res, '=?iso-8859-2?q?abc?=') self.assertIsInstance(res, str) - # Test RFC 2231 header parameters (en/de)coding class TestRFC2231(TestEmailBase): def test_get_param(self): diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/__init__.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python2 +from __future__ import print_function + +import os +import os.path +import pkgutil +import shutil +import sys +import tempfile + + +__all__ = ["version", "bootstrap"] + + +_SETUPTOOLS_VERSION = "7.0" + +_PIP_VERSION = "1.5.6" + +# pip currently requires ssl support, so we try to provide a nicer +# error message when that is missing (http://bugs.python.org/issue19744) +_MISSING_SSL_MESSAGE = ("pip {} requires SSL/TLS".format(_PIP_VERSION)) +try: + import ssl +except ImportError: + ssl = None + + def _require_ssl_for_pip(): + raise RuntimeError(_MISSING_SSL_MESSAGE) +else: + def _require_ssl_for_pip(): + pass + +_PROJECTS = [ + ("setuptools", _SETUPTOOLS_VERSION), + ("pip", _PIP_VERSION), +] + + +def _run_pip(args, additional_paths=None): + # Add our bundled software to the sys.path so we can import it + if additional_paths is not None: + sys.path = additional_paths + sys.path + + # Install the bundled software + import pip + pip.main(args) + + +def version(): + """ + Returns a string specifying the bundled version of pip. + """ + return _PIP_VERSION + + +def _disable_pip_configuration_settings(): + # We deliberately ignore all pip environment variables + # when invoking pip + # See http://bugs.python.org/issue19734 for details + keys_to_remove = [k for k in os.environ if k.startswith("PIP_")] + for k in keys_to_remove: + del os.environ[k] + # We also ignore the settings in the default pip configuration file + # See http://bugs.python.org/issue20053 for details + os.environ['PIP_CONFIG_FILE'] = os.devnull + + +def bootstrap(root=None, upgrade=False, user=False, + altinstall=False, default_pip=True, + verbosity=0): + """ + Bootstrap pip into the current Python installation (or the given root + directory). + + Note that calling this function will alter both sys.path and os.environ. + """ + if altinstall and default_pip: + raise ValueError("Cannot use altinstall and default_pip together") + + _require_ssl_for_pip() + _disable_pip_configuration_settings() + + # By default, installing pip and setuptools installs all of the + # following scripts (X.Y == running Python version): + # + # pip, pipX, pipX.Y, easy_install, easy_install-X.Y + # + # pip 1.5+ allows ensurepip to request that some of those be left out + if altinstall: + # omit pip, pipX and easy_install + os.environ["ENSUREPIP_OPTIONS"] = "altinstall" + elif not default_pip: + # omit pip and easy_install + os.environ["ENSUREPIP_OPTIONS"] = "install" + + tmpdir = tempfile.mkdtemp() + try: + # Put our bundled wheels into a temporary directory and construct the + # additional paths that need added to sys.path + additional_paths = [] + for project, version in _PROJECTS: + wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version) + whl = pkgutil.get_data( + "ensurepip", + "_bundled/{}".format(wheel_name), + ) + with open(os.path.join(tmpdir, wheel_name), "wb") as fp: + fp.write(whl) + + additional_paths.append(os.path.join(tmpdir, wheel_name)) + + # Construct the arguments to be passed to the pip command + args = ["install", "--no-index", "--find-links", tmpdir] + if root: + args += ["--root", root] + if upgrade: + args += ["--upgrade"] + if user: + args += ["--user"] + if verbosity: + args += ["-" + "v" * verbosity] + + _run_pip(args + [p[0] for p in _PROJECTS], additional_paths) + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def _uninstall_helper(verbosity=0): + """Helper to support a clean default uninstall process on Windows + + Note that calling this function may alter os.environ. + """ + # Nothing to do if pip was never installed, or has been removed + try: + import pip + except ImportError: + return + + # If the pip version doesn't match the bundled one, leave it alone + if pip.__version__ != _PIP_VERSION: + msg = ("ensurepip will only uninstall a matching version " + "({!r} installed, {!r} bundled)") + print(msg.format(pip.__version__, _PIP_VERSION), file=sys.stderr) + return + + _require_ssl_for_pip() + _disable_pip_configuration_settings() + + # Construct the arguments to be passed to the pip command + args = ["uninstall", "-y"] + if verbosity: + args += ["-" + "v" * verbosity] + + _run_pip(args + [p[0] for p in reversed(_PROJECTS)]) + + +def _main(argv=None): + if ssl is None: + print("Ignoring ensurepip failure: {}".format(_MISSING_SSL_MESSAGE), + file=sys.stderr) + return + + import argparse + parser = argparse.ArgumentParser(prog="python -m ensurepip") + parser.add_argument( + "--version", + action="version", + version="pip {}".format(version()), + help="Show the version of pip that is bundled with this Python.", + ) + parser.add_argument( + "-v", "--verbose", + action="count", + default=0, + dest="verbosity", + help=("Give more output. Option is additive, and can be used up to 3 " + "times."), + ) + parser.add_argument( + "-U", "--upgrade", + action="store_true", + default=False, + help="Upgrade pip and dependencies, even if already installed.", + ) + parser.add_argument( + "--user", + action="store_true", + default=False, + help="Install using the user scheme.", + ) + parser.add_argument( + "--root", + default=None, + help="Install everything relative to this alternate root directory.", + ) + parser.add_argument( + "--altinstall", + action="store_true", + default=False, + help=("Make an alternate install, installing only the X.Y versioned" + "scripts (Default: pipX, pipX.Y, easy_install-X.Y)"), + ) + parser.add_argument( + "--default-pip", + action="store_true", + default=True, + dest="default_pip", + help=argparse.SUPPRESS, + ) + parser.add_argument( + "--no-default-pip", + action="store_false", + dest="default_pip", + help=("Make a non default install, installing only the X and X.Y " + "versioned scripts."), + ) + + args = parser.parse_args(argv) + + bootstrap( + root=args.root, + upgrade=args.upgrade, + user=args.user, + verbosity=args.verbosity, + altinstall=args.altinstall, + default_pip=args.default_pip, + ) diff --git a/lib-python/2.7/ensurepip/__main__.py b/lib-python/2.7/ensurepip/__main__.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/__main__.py @@ -0,0 +1,4 @@ +import ensurepip + +if __name__ == "__main__": + ensurepip._main() diff --git a/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..097ab43430d4c1302b0be353a8c16407c370693b GIT binary patch [cut] diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..fa1d1054da1dab98f8906555d31a9fda271b3a85 GIT binary patch [cut] diff --git a/lib-python/2.7/ensurepip/_uninstall.py b/lib-python/2.7/ensurepip/_uninstall.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/_uninstall.py @@ -0,0 +1,30 @@ +"""Basic pip uninstallation support, helper for the Windows uninstaller""" + +import argparse +import ensurepip + + +def _main(argv=None): + parser = argparse.ArgumentParser(prog="python -m ensurepip._uninstall") + parser.add_argument( + "--version", + action="version", + version="pip {}".format(ensurepip.version()), + help="Show the version of pip this will attempt to uninstall.", + ) + parser.add_argument( + "-v", "--verbose", + action="count", + default=0, + dest="verbosity", + help=("Give more output. Option is additive, and can be used up to 3 " + "times."), + ) + + args = parser.parse_args(argv) + + ensurepip._uninstall_helper(verbosity=args.verbosity) + + +if __name__ == "__main__": + _main() diff --git a/lib-python/2.7/glob.py b/lib-python/2.7/glob.py --- a/lib-python/2.7/glob.py +++ b/lib-python/2.7/glob.py @@ -35,11 +35,16 @@ patterns. """ + dirname, basename = os.path.split(pathname) if not has_magic(pathname): - if os.path.lexists(pathname): - yield pathname + if basename: + if os.path.lexists(pathname): + yield pathname + else: + # Patterns ending with a slash should match only directories + if os.path.isdir(dirname): + yield pathname return - dirname, basename = os.path.split(pathname) if not dirname: for name in glob1(os.curdir, basename): yield name diff --git a/lib-python/2.7/gzip.py b/lib-python/2.7/gzip.py --- a/lib-python/2.7/gzip.py +++ b/lib-python/2.7/gzip.py @@ -164,9 +164,16 @@ def _write_gzip_header(self): self.fileobj.write('\037\213') # magic header self.fileobj.write('\010') # compression method - fname = os.path.basename(self.name) - if fname.endswith(".gz"): - fname = fname[:-3] + try: + # RFC 1952 requires the FNAME field to be Latin-1. Do not + # include filenames that cannot be represented that way. + fname = os.path.basename(self.name) + if not isinstance(fname, str): + fname = fname.encode('latin-1') + if fname.endswith('.gz'): + fname = fname[:-3] + except UnicodeEncodeError: + fname = '' flags = 0 if fname: flags = FNAME diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py --- a/lib-python/2.7/hashlib.py +++ b/lib-python/2.7/hashlib.py @@ -15,8 +15,9 @@ md5(), sha1(), sha224(), sha256(), sha384(), and sha512() -More algorithms may be available on your platform but the above are -guaranteed to exist. +More algorithms may be available on your platform but the above are guaranteed +to exist. See the algorithms_guaranteed and algorithms_available attributes +to find out what algorithm names can be passed to new(). NOTE: If you want the adler32 or crc32 hash functions they are available in the zlib module. @@ -58,9 +59,14 @@ # always available algorithm is added. __always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512') +algorithms_guaranteed = set(__always_supported) +algorithms_available = set(__always_supported) + algorithms = __always_supported -__all__ = __always_supported + ('new', 'algorithms', 'pbkdf2_hmac') +__all__ = __always_supported + ('new', 'algorithms_guaranteed', + 'algorithms_available', 'algorithms', + 'pbkdf2_hmac') def __get_builtin_constructor(name): @@ -128,6 +134,8 @@ import _hashlib new = __hash_new __get_hash = __get_openssl_constructor + algorithms_available = algorithms_available.union( + _hashlib.openssl_md_meth_names) except ImportError: new = __py_new __get_hash = __get_builtin_constructor diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py --- a/lib-python/2.7/httplib.py +++ b/lib-python/2.7/httplib.py @@ -215,6 +215,10 @@ # maximal line length when calling readline(). _MAXLINE = 65536 +# maximum amount of headers accepted +_MAXHEADERS = 100 + + class HTTPMessage(mimetools.Message): def addheader(self, key, value): @@ -271,6 +275,8 @@ elif self.seekable: tell = self.fp.tell while True: + if len(hlist) > _MAXHEADERS: + raise HTTPException("got more than %d headers" % _MAXHEADERS) if tell: try: startofline = tell() @@ -1185,21 +1191,29 @@ def __init__(self, host, port=None, key_file=None, cert_file=None, strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - source_address=None): + source_address=None, context=None): HTTPConnection.__init__(self, host, port, strict, timeout, source_address) self.key_file = key_file self.cert_file = cert_file + if context is None: + context = ssl._create_default_https_context() + if key_file or cert_file: + context.load_cert_chain(cert_file, key_file) + self._context = context def connect(self): "Connect to a host on a given (SSL) port." - sock = self._create_connection((self.host, self.port), - self.timeout, self.source_address) + HTTPConnection.connect(self) + if self._tunnel_host: - self.sock = sock - self._tunnel() - self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file) + server_hostname = self._tunnel_host + else: + server_hostname = self.host + + self.sock = self._context.wrap_socket(self.sock, + server_hostname=server_hostname) __all__.append("HTTPSConnection") @@ -1214,14 +1228,15 @@ _connection_class = HTTPSConnection def __init__(self, host='', port=None, key_file=None, cert_file=None, - strict=None): + strict=None, context=None): # provide a default host, pass the X509 cert info # urf. compensate for bad input. if port == 0: port = None self._setup(self._connection_class(host, port, key_file, - cert_file, strict)) + cert_file, strict, + context=context)) # we never actually use these for anything, but we keep them # here for compatibility with post-1.5.2 CVS. diff --git a/lib-python/2.7/idlelib/Bindings.py b/lib-python/2.7/idlelib/Bindings.py --- a/lib-python/2.7/idlelib/Bindings.py +++ b/lib-python/2.7/idlelib/Bindings.py @@ -75,7 +75,8 @@ ('!_Auto-open Stack Viewer', '<>'), ]), ('options', [ - ('_Configure IDLE...', '<>'), + ('Configure _IDLE', '<>'), + ('Configure _Extensions', '<>'), None, ]), ('help', [ diff --git a/lib-python/2.7/idlelib/CallTipWindow.py b/lib-python/2.7/idlelib/CallTipWindow.py --- a/lib-python/2.7/idlelib/CallTipWindow.py +++ b/lib-python/2.7/idlelib/CallTipWindow.py @@ -2,9 +2,8 @@ After ToolTip.py, which uses ideas gleaned from PySol Used by the CallTips IDLE extension. - """ -from Tkinter import * +from Tkinter import Toplevel, Label, LEFT, SOLID, TclError HIDE_VIRTUAL_EVENT_NAME = "<>" HIDE_SEQUENCES = ("", "") @@ -133,35 +132,28 @@ return bool(self.tipwindow) -def _calltip_window(parent): - root = Tk() - root.title("Test calltips") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) +def _calltip_window(parent): # htest # + from Tkinter import Toplevel, Text, LEFT, BOTH - class MyEditWin: # comparenceptually an editor_window - def __init__(self): - text = self.text = Text(root) - text.pack(side=LEFT, fill=BOTH, expand=1) - text.insert("insert", "string.split") - root.update() - self.calltip = CallTip(text) + top = Toplevel(parent) + top.title("Test calltips") + top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, + parent.winfo_rooty() + 150)) + text = Text(top) + text.pack(side=LEFT, fill=BOTH, expand=1) + text.insert("insert", "string.split") + top.update() + calltip = CallTip(text) - text.event_add("<>", "(") - text.event_add("<>", ")") - text.bind("<>", self.calltip_show) - text.bind("<>", self.calltip_hide) - - text.focus_set() - root.mainloop() - - def calltip_show(self, event): - self.calltip.showtip("Hello world", "insert", "end") - - def calltip_hide(self, event): - self.calltip.hidetip() - - editwin = MyEditWin() + def calltip_show(event): + calltip.showtip("(s=Hello world)", "insert", "end") + def calltip_hide(event): + calltip.hidetip() + text.event_add("<>", "(") + text.event_add("<>", ")") + text.bind("<>", calltip_show) + text.bind("<>", calltip_hide) + text.focus_set() if __name__=='__main__': from idlelib.idle_test.htest import run diff --git a/lib-python/2.7/idlelib/ClassBrowser.py b/lib-python/2.7/idlelib/ClassBrowser.py --- a/lib-python/2.7/idlelib/ClassBrowser.py +++ b/lib-python/2.7/idlelib/ClassBrowser.py @@ -19,6 +19,9 @@ from idlelib.TreeWidget import TreeNode, TreeItem, ScrolledCanvas from idlelib.configHandler import idleConf +file_open = None # Method...Item and Class...Item use this. +# Normally PyShell.flist.open, but there is no PyShell.flist for htest. + class ClassBrowser: def __init__(self, flist, name, path, _htest=False): @@ -27,6 +30,9 @@ """ _htest - bool, change box when location running htest. """ + global file_open + if not _htest: + file_open = PyShell.flist.open self.name = name self.file = os.path.join(path[0], self.name + ".py") self._htest = _htest @@ -101,7 +107,7 @@ return [] try: dict = pyclbr.readmodule_ex(name, [dir] + sys.path) - except ImportError, msg: + except ImportError: return [] items = [] self.classes = {} @@ -170,7 +176,7 @@ def OnDoubleClick(self): if not os.path.exists(self.file): return - edit = PyShell.flist.open(self.file) + edit = file_open(self.file) if hasattr(self.cl, 'lineno'): lineno = self.cl.lineno edit.gotoline(lineno) @@ -206,7 +212,7 @@ def OnDoubleClick(self): if not os.path.exists(self.file): return - edit = PyShell.flist.open(self.file) + edit = file_open(self.file) edit.gotoline(self.cl.methods[self.name]) def _class_browser(parent): #Wrapper for htest @@ -221,8 +227,9 @@ dir, file = os.path.split(file) name = os.path.splitext(file)[0] flist = PyShell.PyShellFileList(parent) + global file_open + file_open = flist.open ClassBrowser(flist, name, [dir], _htest=True) - parent.mainloop() if __name__ == "__main__": from idlelib.idle_test.htest import run diff --git a/lib-python/2.7/idlelib/ColorDelegator.py b/lib-python/2.7/idlelib/ColorDelegator.py --- a/lib-python/2.7/idlelib/ColorDelegator.py +++ b/lib-python/2.7/idlelib/ColorDelegator.py @@ -2,7 +2,6 @@ import re import keyword import __builtin__ -from Tkinter import * from idlelib.Delegator import Delegator from idlelib.configHandler import idleConf @@ -34,7 +33,6 @@ prog = re.compile(make_pat(), re.S) idprog = re.compile(r"\s+(\w+)", re.S) -asprog = re.compile(r".*?\b(as)\b") class ColorDelegator(Delegator): @@ -42,7 +40,6 @@ Delegator.__init__(self) self.prog = prog self.idprog = idprog - self.asprog = asprog self.LoadTagDefs() def setdelegate(self, delegate): @@ -74,7 +71,6 @@ "DEFINITION": idleConf.GetHighlight(theme, "definition"), "SYNC": {'background':None,'foreground':None}, "TODO": {'background':None,'foreground':None}, - "BREAK": idleConf.GetHighlight(theme, "break"), "ERROR": idleConf.GetHighlight(theme, "error"), # The following is used by ReplaceDialog: "hit": idleConf.GetHighlight(theme, "hit"), @@ -216,22 +212,6 @@ self.tag_add("DEFINITION", head + "+%dc" % a, head + "+%dc" % b) - elif value == "import": - # color all the "as" words on same line, except - # if in a comment; cheap approximation to the - # truth - if '#' in chars: - endpos = chars.index('#') - else: - endpos = len(chars) - while True: - m1 = self.asprog.match(chars, b, endpos) - if not m1: - break - a, b = m1.span(1) - self.tag_add("KEYWORD", - head + "+%dc" % a, - head + "+%dc" % b) m = self.prog.search(chars, m.end()) if "SYNC" in self.tag_names(next + "-1c"): head = next @@ -255,20 +235,23 @@ for tag in self.tagdefs.keys(): self.tag_remove(tag, "1.0", "end") -def _color_delegator(parent): +def _color_delegator(parent): # htest # + from Tkinter import Toplevel, Text from idlelib.Percolator import Percolator - root = Tk() - root.title("Test ColorDelegator") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) - source = "if somename: x = 'abc' # comment\nprint" - text = Text(root, background="white") + + top = Toplevel(parent) + top.title("Test ColorDelegator") + top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, + parent.winfo_rooty() + 150)) + source = "if somename: x = 'abc' # comment\nprint\n" + text = Text(top, background="white") + text.pack(expand=1, fill="both") text.insert("insert", source) - text.pack(expand=1, fill="both") + text.focus_set() + p = Percolator(text) d = ColorDelegator() p.insertfilter(d) - root.mainloop() if __name__ == "__main__": from idlelib.idle_test.htest import run diff --git a/lib-python/2.7/idlelib/Debugger.py b/lib-python/2.7/idlelib/Debugger.py --- a/lib-python/2.7/idlelib/Debugger.py +++ b/lib-python/2.7/idlelib/Debugger.py @@ -1,6 +1,5 @@ import os import bdb -import types from Tkinter import * from idlelib.WindowList import ListedToplevel from idlelib.ScrolledList import ScrolledList diff --git a/lib-python/2.7/idlelib/EditorWindow.py b/lib-python/2.7/idlelib/EditorWindow.py --- a/lib-python/2.7/idlelib/EditorWindow.py +++ b/lib-python/2.7/idlelib/EditorWindow.py @@ -1,6 +1,6 @@ import sys import os -from platform import python_version +import platform import re import imp from Tkinter import * @@ -22,6 +22,8 @@ # The default tab setting for a Text widget, in average-width characters. TK_TABWIDTH_DEFAULT = 8 +_py_version = ' (%s)' % platform.python_version() + def _sphinx_version(): "Format sys.version_info to produce the Sphinx version string used to install the chm docs" major, minor, micro, level, serial = sys.version_info @@ -151,7 +153,7 @@ # Safari requires real file:-URLs EditorWindow.help_url = 'file://' + EditorWindow.help_url else: - EditorWindow.help_url = "http://docs.python.org/%d.%d" % sys.version_info[:2] + EditorWindow.help_url = "https://docs.python.org/%d.%d/" % sys.version_info[:2] currentTheme=idleConf.CurrentTheme() self.flist = flist root = root or flist.root @@ -214,6 +216,8 @@ text.bind("<>", self.python_docs) text.bind("<>", self.about_dialog) text.bind("<>", self.config_dialog) + text.bind("<>", + self.config_extensions_dialog) text.bind("<>", self.open_module) text.bind("<>", lambda event: "break") text.bind("<>", self.select_all) @@ -568,6 +572,8 @@ def config_dialog(self, event=None): configDialog.ConfigDialog(self.top,'Settings') + def config_extensions_dialog(self, event=None): + configDialog.ConfigExtensionsDialog(self.top) def help_dialog(self, event=None): if self.root: @@ -691,30 +697,29 @@ return # XXX Ought to insert current file's directory in front of path try: - (f, file, (suffix, mode, type)) = _find_module(name) + (f, file_path, (suffix, mode, mtype)) = _find_module(name) except (NameError, ImportError) as msg: tkMessageBox.showerror("Import error", str(msg), parent=self.text) return - if type != imp.PY_SOURCE: + if mtype != imp.PY_SOURCE: tkMessageBox.showerror("Unsupported type", "%s is not a source module" % name, parent=self.text) return if f: f.close() if self.flist: - self.flist.open(file) + self.flist.open(file_path) else: - self.io.loadfile(file) + self.io.loadfile(file_path) + return file_path def open_class_browser(self, event=None): filename = self.io.filename - if not filename: - tkMessageBox.showerror( - "No filename", - "This buffer has no associated filename", - master=self.text) - self.text.focus_set() - return None + if not (self.__class__.__name__ == 'PyShellEditorWindow' + and filename): + filename = self.open_module() + if filename is None: + return head, tail = os.path.split(filename) base, ext = os.path.splitext(tail) from idlelib import ClassBrowser @@ -779,7 +784,7 @@ self.color = None def ResetColorizer(self): - "Update the colour theme" + "Update the color theme" # Called from self.filename_change_hook and from configDialog.py self._rmcolorizer() self._addcolorizer() @@ -944,7 +949,7 @@ short = self.short_title() long = self.long_title() if short and long: - title = short + " - " + long + title = short + " - " + long + _py_version elif short: title = short elif long: @@ -968,14 +973,13 @@ self.undo.reset_undo() def short_title(self): - pyversion = "Python " + python_version() + ": " filename = self.io.filename if filename: filename = os.path.basename(filename) else: filename = "Untitled" # return unicode string to display non-ASCII chars correctly - return pyversion + self._filename_to_unicode(filename) + return self._filename_to_unicode(filename) def long_title(self): # return unicode string to display non-ASCII chars correctly @@ -1711,7 +1715,8 @@ tk.call('set', 'tcl_nonwordchars', '[^a-zA-Z0-9_]') -def _editor_window(parent): +def _editor_window(parent): # htest # + # error if close master window first - timer event, after script root = parent fixwordbreaks(root) if sys.argv[1:]: @@ -1721,7 +1726,8 @@ macosxSupport.setupApp(root, None) edit = EditorWindow(root=root, filename=filename) edit.text.bind("<>", edit.close_event) - parent.mainloop() + # Does not stop error, neither does following + # edit.text.bind("<>", edit.close_event) if __name__ == '__main__': diff --git a/lib-python/2.7/idlelib/GrepDialog.py b/lib-python/2.7/idlelib/GrepDialog.py --- a/lib-python/2.7/idlelib/GrepDialog.py +++ b/lib-python/2.7/idlelib/GrepDialog.py @@ -45,10 +45,10 @@ def create_entries(self): SearchDialogBase.create_entries(self) - self.globent = self.make_entry("In files:", self.globvar) + self.globent = self.make_entry("In files:", self.globvar)[0] def create_other_buttons(self): - f = self.make_frame() + f = self.make_frame()[0] btn = Checkbutton(f, anchor="w", variable=self.recvar, @@ -131,7 +131,7 @@ self.top.withdraw() -def _grep_dialog(parent): # for htest +def _grep_dialog(parent): # htest # from idlelib.PyShell import PyShellFileList root = Tk() root.title("Test GrepDialog") diff --git a/lib-python/2.7/idlelib/IOBinding.py b/lib-python/2.7/idlelib/IOBinding.py --- a/lib-python/2.7/idlelib/IOBinding.py +++ b/lib-python/2.7/idlelib/IOBinding.py @@ -19,11 +19,7 @@ from idlelib.configHandler import idleConf -try: - from codecs import BOM_UTF8 -except ImportError: - # only available since Python 2.3 - BOM_UTF8 = '\xef\xbb\xbf' +from codecs import BOM_UTF8 # Try setting the locale, so that we can find out # what encoding to use @@ -72,6 +68,7 @@ encoding = encoding.lower() coding_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)') +blank_re = re.compile(r'^[ \t\f]*(?:[#\r\n]|$)') class EncodingMessage(SimpleDialog): "Inform user that an encoding declaration is needed." @@ -130,6 +127,8 @@ match = coding_re.match(line) if match is not None: break + if not blank_re.match(line): + return None else: return None name = match.group(1) @@ -529,6 +528,8 @@ ("All files", "*"), ] + defaultextension = '.py' if sys.platform == 'darwin' else '' + def askopenfile(self): dir, base = self.defaultfilename("open") if not self.opendialog: @@ -554,8 +555,10 @@ def asksavefile(self): dir, base = self.defaultfilename("save") if not self.savedialog: - self.savedialog = tkFileDialog.SaveAs(master=self.text, - filetypes=self.filetypes) + self.savedialog = tkFileDialog.SaveAs( + master=self.text, + filetypes=self.filetypes, + defaultextension=self.defaultextension) filename = self.savedialog.show(initialdir=dir, initialfile=base) if isinstance(filename, unicode): filename = filename.encode(filesystemencoding) diff --git a/lib-python/2.7/idlelib/NEWS.txt b/lib-python/2.7/idlelib/NEWS.txt --- a/lib-python/2.7/idlelib/NEWS.txt +++ b/lib-python/2.7/idlelib/NEWS.txt @@ -1,6 +1,183 @@ +What's New in IDLE 2.7.9? +========================= + +*Release data: 2014-12-07* (projected) + +- Issue #16893: Update Idle doc chapter to match current Idle and add new + information. + +- Issue #3068: Add Idle extension configuration dialog to Options menu. + Changes are written to HOME/.idlerc/config-extensions.cfg. + Original patch by Tal Einat. + +- Issue #16233: A module browser (File : Class Browser, Alt+C) requires a + editor window with a filename. When Class Browser is requested otherwise, + from a shell, output window, or 'Untitled' editor, Idle no longer displays + an error box. It now pops up an Open Module box (Alt+M). If a valid name + is entered and a module is opened, a corresponding browser is also opened. + +- Issue #4832: Save As to type Python files automatically adds .py to the + name you enter (even if your system does not display it). Some systems + automatically add .txt when type is Text files. + +- Issue #21986: Code objects are not normally pickled by the pickle module. + To match this, they are no longer pickled when running under Idle. + +- Issue #22221: IDLE now ignores the source encoding declaration on the second + line if the first line contains anything except a comment. + +- Issue #17390: Adjust Editor window title; remove 'Python', + move version to end. + +- Issue #14105: Idle debugger breakpoints no longer disappear + when inseting or deleting lines. + + +What's New in IDLE 2.7.8? +========================= + +*Release date: 2014-06-29* + +- Issue #21940: Add unittest for WidgetRedirector. Initial patch by Saimadhav + Heblikar. + +- Issue #18592: Add unittest for SearchDialogBase. Patch by Phil Webster. + +- Issue #21694: Add unittest for ParenMatch. Patch by Saimadhav Heblikar. + +- Issue #21686: add unittest for HyperParser. Original patch by Saimadhav + Heblikar. + +- Issue #12387: Add missing upper(lower)case versions of default Windows key + bindings for Idle so Caps Lock does not disable them. Patch by Roger Serwy. + +- Issue #21695: Closing a Find-in-files output window while the search is + still in progress no longer closes Idle. + +- Issue #18910: Add unittest for textView. Patch by Phil Webster. + +- Issue #18292: Add unittest for AutoExpand. Patch by Saihadhav Heblikar. + +- Issue #18409: Add unittest for AutoComplete. Patch by Phil Webster. + + +What's New in IDLE 2.7.7? +========================= + +*Release date: 2014-05-31* + +- Issue #18104: Add idlelib/idle_test/htest.py with a few sample tests to begin + consolidating and improving human-validated tests of Idle. Change other files + as needed to work with htest. Running the module as __main__ runs all tests. + +- Issue #21139: Change default paragraph width to 72, the PEP 8 recommendation. + +- Issue #21284: Paragraph reformat test passes after user changes reformat width. + +- Issue #20406: Use Python application icons for Idle window title bars. + Patch mostly by Serhiy Storchaka. + +- Issue #21029: Occurrences of "print" are now consistently colored as + being a keyword (the colorizer doesn't know if print functions are + enabled in the source). + +- Issue #17721: Remove non-functional configuration dialog help button until we + make it actually gives some help when clicked. Patch by Guilherme Sim�es. + +- Issue #17390: Add Python version to Idle editor window title bar. + Original patches by Edmond Burnett and Kent Johnson. + +- Issue #20058: sys.stdin.readline() in IDLE now always returns only one line. + +- Issue #19481: print() of unicode, str or bytearray subclass instance in IDLE + no more hangs. + +- Issue #18270: Prevent possible IDLE AttributeError on OS X when no initial + shell window is present. + +- Issue #17654: Ensure IDLE menus are customized properly on OS X for + non-framework builds and for all variants of Tk. + + +What's New in IDLE 2.7.6? +========================= + +*Release date: 2013-11-10* + +- Issue #19426: Fixed the opening of Python source file with specified encoding. + +- Issue #18873: IDLE now detects Python source code encoding only in comment + lines. + +- Issue #18988: The "Tab" key now works when a word is already autocompleted. + +- Issue #18489: Add tests for SearchEngine. Original patch by Phil Webster. + +- Issue #18429: Format / Format Paragraph, now works when comment blocks + are selected. As with text blocks, this works best when the selection + only includes complete lines. + +- Issue #18226: Add docstrings and unittests for FormatParagraph.py. + Original patches by Todd Rovito and Phil Webster. + +- Issue #18279: Format - Strip trailing whitespace no longer marks a file as + changed when it has not been changed. This fix followed the addition of a + test file originally written by Phil Webster (the issue's main goal). + +- Issue #18539: Calltips now work for float default arguments. + +- Issue #7136: In the Idle File menu, "New Window" is renamed "New File". + Patch by Tal Einat, Roget Serwy, and Todd Rovito. + +- Issue #8515: Set __file__ when run file in IDLE. + Initial patch by Bruce Frederiksen. + +- Issue #5492: Avoid traceback when exiting IDLE caused by a race condition. + +- Issue #17511: Keep IDLE find dialog open after clicking "Find Next". + Original patch by Sarah K. + +- Issue #15392: Create a unittest framework for IDLE. + Preliminary patch by Rajagopalasarma Jayakrishnan + See Lib/idlelib/idle_test/README.txt for how to run Idle tests. + +- Issue #14146: Highlight source line while debugging on Windows. + +- Issue #17532: Always include Options menu for IDLE on OS X. + Patch by Guilherme Sim�es. + + What's New in IDLE 2.7.5? ========================= +*Release date: 2013-05-12* + +- Issue #17838: Allow sys.stdin to be reassigned. + +- Issue #14735: Update IDLE docs to omit "Control-z on Windows". + +- Issue #17585: Fixed IDLE regression. Now closes when using exit() or quit(). + +- Issue #17657: Show full Tk version in IDLE's about dialog. + Patch by Todd Rovito. + +- Issue #17613: Prevent traceback when removing syntax colorizer in IDLE. + +- Issue #1207589: Backwards-compatibility patch for right-click menu in IDLE. + +- Issue #16887: IDLE now accepts Cancel in tabify/untabify dialog box. + +- Issue #14254: IDLE now handles readline correctly across shell restarts. + +- Issue #17614: IDLE no longer raises exception when quickly closing a file. + +- Issue #6698: IDLE now opens just an editor window when configured to do so. + +- Issue #8900: Using keyboard shortcuts in IDLE to open a file no longer + raises an exception. + +- Issue #6649: Fixed missing exit status in IDLE. Patch by Guilherme Polo. + - Issue #17390: Display Python version on Idle title bar. Initial patch by Edmond Burnett. @@ -8,17 +185,67 @@ What's New in IDLE 2.7.4? ========================= +*Release date: 2013-04-06* + +- Issue #17625: In IDLE, close the replace dialog after it is used. + +- IDLE was displaying spurious SystemExit tracebacks when running scripts + that terminated by raising SystemExit (i.e. unittest and turtledemo). + +- Issue #9290: In IDLE the sys.std* streams now implement io.TextIOBase + interface and support all mandatory methods and properties. + +- Issue #16829: IDLE printing no longer fails if there are spaces or other + special characters in the file path. + +- Issue #16819: IDLE method completion now correctly works for unicode literals. + +- Issue #16504: IDLE now catches SyntaxErrors raised by tokenizer. Patch by + Roger Serwy. + +- Issue #1207589: Add Cut/Copy/Paste items to IDLE right click Context Menu + Patch by Todd Rovito. + +- Issue #13052: Fix IDLE crashing when replace string in Search/Replace dialog + ended with '\'. Patch by Roger Serwy. + +- Issue #9803: Don't close IDLE on saving if breakpoint is open. + Patch by Roger Serwy. + +- Issue #14958: Change IDLE systax highlighting to recognize all string and byte + literals currently supported in Python 2.7. + +- Issue #14962: Update text coloring in IDLE shell window after changing + options. Patch by Roger Serwy. + +- Issue #10997: Prevent a duplicate entry in IDLE's "Recent Files" menu. + +- Issue #12510: Attempting to get invalid tooltip no longer closes IDLE. + Original patch by Roger Serwy. + +- Issue #10365: File open dialog now works instead of crashing + even when parent window is closed. Patch by Roger Serwy. + +- Issue #14876: Use user-selected font for highlight configuration. + Patch by Roger Serwy. + +- Issue #14409: IDLE now properly executes commands in the Shell window + when it cannot read the normal config files on startup and + has to use the built-in default key bindings. + There was previously a bug in one of the defaults. + +- Issue #3573: IDLE hangs when passing invalid command line args + (directory(ies) instead of file(s)) (Patch by Guilherme Polo) + +- Issue #5219: Prevent event handler cascade in IDLE. + - Issue #15318: Prevent writing to sys.stdin. - Issue #13532, #15319: Check that arguments to sys.stdout.write are strings. -- Issue # 12510: Attempt to get certain tool tips no longer crashes IDLE. - -- Issue10365: File open dialog now works instead of crashing even when +- Issue #10365: File open dialog now works instead of crashing even when parent window is closed while dialog is open. -- Issue 14876: use user-selected font for highlight configuration. - - Issue #14018: Update checks for unstable system Tcl/Tk versions on OS X to include versions shipped with OS X 10.7 and 10.8 in addition to 10.6. @@ -29,6 +256,27 @@ What's New in IDLE 2.7.3? ========================= +*Release date: 2012-04-09* + +- Issue #964437 Make IDLE help window non-modal. + Patch by Guilherme Polo and Roger Serwy. + +- Issue #13933: IDLE auto-complete did not work with some imported + module, like hashlib. (Patch by Roger Serwy) + +- Issue #13506: Add '' to path for IDLE Shell when started and restarted with Restart Shell. + Original patches by Marco Scataglini and Roger Serwy. + +- Issue #4625: If IDLE cannot write to its recent file or breakpoint From noreply at buildbot.pypy.org Mon Feb 16 01:05:43 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 16 Feb 2015 01:05:43 +0100 (CET) Subject: [pypy-commit] pypy py3k: Fix ssl module Message-ID: <20150216000543.B59271C03B3@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r75909:3896a15828da Date: 2015-02-16 01:05 +0100 http://bitbucket.org/pypy/pypy/changeset/3896a15828da/ Log: Fix ssl module diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -1,4 +1,3 @@ -<<<<<<< local import weakref from rpython.rlib import rpoll, rsocket @@ -14,6 +13,7 @@ from pypy.interpreter.error import OperationError, oefmt, wrap_oserror from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.interpreter.unicodehelper import fsdecode from pypy.module._ssl.ssl_data import ( LIBRARY_CODES_TO_NAMES, ERROR_CODES_TO_NAMES) from pypy.module._socket import interp_socket @@ -177,202 +177,6 @@ SOCKET_STORAGE = RWeakValueDictionary(int, W_Root) -class SSLContext(W_Root): - ctx = lltype.nullptr(SSL_CTX.TO) - - def __init__(self, space, protocol): - if protocol == PY_SSL_VERSION_TLS1: - method = libssl_TLSv1_method() - elif protocol == PY_SSL_VERSION_SSL3 and not OPENSSL_NO_SSL3: - method = libssl_SSLv3_method() - elif protocol == PY_SSL_VERSION_SSL2 and not OPENSSL_NO_SSL2: - method = libssl_SSLv2_method() - elif protocol == PY_SSL_VERSION_SSL23: - method = libssl_SSLv23_method() - else: - raise oefmt(space.w_ValueError, "invalid protocol version") - self.ctx = libssl_SSL_CTX_new(method) - - # Defaults - libssl_SSL_CTX_set_verify(self.ctx, SSL_VERIFY_NONE, None) - options = SSL_OP_ALL & ~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS - if protocol != PY_SSL_VERSION_SSL2: - options |= SSL_OP_NO_SSLv2 - libssl_SSL_CTX_set_options(self.ctx, options) - libssl_SSL_CTX_set_session_id_context(self.ctx, "Python", len("Python")) - - def __del__(self): - if self.ctx: - libssl_SSL_CTX_free(self.ctx) - - @unwrap_spec(protocol=int) - def descr_new(space, w_subtype, protocol=PY_SSL_VERSION_SSL23): - self = space.allocate_instance(SSLContext, w_subtype) - self.__init__(space, protocol) - if not self.ctx: - raise ssl_error(space, "failed to allocate SSL context") - return space.wrap(self) - - @unwrap_spec(cipherlist=str) - def set_ciphers_w(self, space, cipherlist): - ret = libssl_SSL_CTX_set_cipher_list(self.ctx, cipherlist) - if ret == 0: - # Clearing the error queue is necessary on some OpenSSL - # versions, otherwise the error will be reported again - # when another SSL call is done. - libssl_ERR_clear_error() - raise ssl_error(space, "No cipher can be selected.") - - def get_verify_mode_w(self, space): - verify_mode = libssl_SSL_CTX_get_verify_mode(self.ctx) - if verify_mode == SSL_VERIFY_NONE: - return space.wrap(PY_SSL_CERT_NONE) - elif verify_mode == SSL_VERIFY_PEER: - return space.wrap(PY_SSL_CERT_OPTIONAL) - elif verify_mode == (SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT): - return space.wrap(PY_SSL_CERT_REQUIRED) - else: - raise ssl_error( - space, "invalid return value from SSL_CTX_get_verify_mode") - - def set_verify_mode_w(self, space, w_mode): - mode = space.int_w(w_mode) - if mode == PY_SSL_CERT_NONE: - verify_mode = SSL_VERIFY_NONE - elif mode == PY_SSL_CERT_OPTIONAL: - verify_mode = SSL_VERIFY_PEER - elif mode == PY_SSL_CERT_REQUIRED: - verify_mode = SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT - else: - raise OperationError(space.w_ValueError, space.wrap( - "invalid value for verify_mode")) - libssl_SSL_CTX_set_verify(self.ctx, verify_mode, None) - - def get_options_w(self, space): - return space.wrap(libssl_SSL_CTX_get_options(self.ctx)) - - def set_options_w(self, space, w_value): - value = space.int_w(w_value) - opts = libssl_SSL_CTX_get_options(self.ctx) - clear = opts & ~value - set = ~opts & value - if clear: - if HAVE_SSL_CTX_CLEAR_OPTIONS: - libssl_SSL_CTX_clear_options(self.ctx, clear) - else: - raise OperationError(space.w_ValueError, space.wrap( - "can't clear options before OpenSSL 0.9.8m")) - if set: - libssl_SSL_CTX_set_options(self.ctx, set) - - def load_cert_chain_w(self, space, w_certfile, w_keyfile=None): - if space.is_none(w_certfile): - certfile = None - else: - certfile = space.str_w(w_certfile) - if space.is_none(w_keyfile): - keyfile = certfile - else: - keyfile = space.str_w(w_keyfile) - - ret = libssl_SSL_CTX_use_certificate_chain_file(self.ctx, certfile) - if ret != 1: - errno = get_saved_errno() - if errno: - libssl_ERR_clear_error() - raise wrap_oserror(space, OSError(errno, ''), - exception_name = 'w_IOError') - else: - raise _ssl_seterror(space, None, -1) - - ret = libssl_SSL_CTX_use_PrivateKey_file(self.ctx, keyfile, - SSL_FILETYPE_PEM) - if ret != 1: - errno = get_saved_errno() - if errno: - libssl_ERR_clear_error() - raise wrap_oserror(space, OSError(errno, ''), - exception_name = 'w_IOError') - else: - raise _ssl_seterror(space, None, -1) - - ret = libssl_SSL_CTX_check_private_key(self.ctx) - if ret != 1: - raise _ssl_seterror(space, None, -1) - - def load_verify_locations_w(self, space, w_cafile=None, w_capath=None): - if space.is_none(w_cafile): - cafile = None - else: - cafile = space.str_w(w_cafile) - if space.is_none(w_capath): - capath = None - else: - capath = space.str_w(w_capath) - if cafile is None and capath is None: - raise OperationError(space.w_TypeError, space.wrap( - "cafile and capath cannot be both omitted")) - ret = libssl_SSL_CTX_load_verify_locations( - self.ctx, cafile, capath) - if ret != 1: - errno = get_saved_errno() - if errno: - libssl_ERR_clear_error() - raise wrap_oserror(space, OSError(errno, ''), - exception_name = 'w_IOError') - else: - raise _ssl_seterror(space, None, -1) - - @unwrap_spec(server_side=int) - def wrap_socket_w(self, space, w_sock, server_side, - w_server_hostname=None): - assert w_sock is not None - # server_hostname is either None (or absent), or to be encoded - # using the idna encoding. - if space.is_none(w_server_hostname): - hostname = None - else: - hostname = space.bytes_w( - space.call_method(w_server_hostname, - "encode", space.wrap("idna"))) - - if hostname and not HAS_SNI: - raise OperationError(space.w_ValueError, - space.wrap("server_hostname is not supported " - "by your OpenSSL library")) - - return new_sslobject(space, self.ctx, w_sock, server_side, hostname) - - def session_stats_w(self, space): - w_stats = space.newdict() - for name, ssl_func in SSL_CTX_STATS: - w_value = space.wrap(ssl_func(self.ctx)) - space.setitem_str(w_stats, name, w_value) - return w_stats - - def set_default_verify_paths_w(self, space): - ret = libssl_SSL_CTX_set_default_verify_paths(self.ctx) - if ret != 1: - raise _ssl_seterror(space, None, -1) - - -SSLContext.typedef = TypeDef( - "_SSLContext", - __new__ = interp2app(SSLContext.descr_new.im_func), - options = GetSetProperty(SSLContext.get_options_w, - SSLContext.set_options_w), - verify_mode = GetSetProperty(SSLContext.get_verify_mode_w, - SSLContext.set_verify_mode_w), - _wrap_socket = interp2app(SSLContext.wrap_socket_w), - set_ciphers = interp2app(SSLContext.set_ciphers_w), - load_cert_chain = interp2app(SSLContext.load_cert_chain_w), - load_verify_locations = interp2app(SSLContext.load_verify_locations_w), - session_stats = interp2app(SSLContext.session_stats_w), - set_default_verify_paths=interp2app(SSLContext.set_default_verify_paths_w), -) - - - if HAVE_OPENSSL_RAND: # helper routines for seeding the SSL PRNG @unwrap_spec(string=str, entropy=float) @@ -499,7 +303,7 @@ "Underlying socket too large for select().") elif sockstate == SOCKET_HAS_BEEN_CLOSED: if libssl_SSL_get_shutdown(self.ssl) == SSL_RECEIVED_SHUTDOWN: - if space.is_none(w_buf): + if space.is_none(w_buffer): return space.wrapbytes('') else: return space.wrap(0) @@ -525,7 +329,7 @@ sockstate = checkwait(space, w_socket, True) elif (err == SSL_ERROR_ZERO_RETURN and libssl_SSL_get_shutdown(self.ssl) == SSL_RECEIVED_SHUTDOWN): - if space.is_none(w_buf): + if space.is_none(w_buffer): return space.wrapbytes('') else: return space.wrap(0) @@ -761,13 +565,13 @@ length = libssl_SSL_get_peer_finished(self.ssl, buf, CB_MAXLEN) if length > 0: - return space.wrap(rffi.charpsize2str(buf, intmask(length))) + return space.wrapbytes(rffi.charpsize2str(buf, intmask(length))) def descr_get_context(self, space): return self.w_ctx def descr_set_context(self, space, w_ctx): - ctx = space.interp_w(_SSLContext, w_ctx) + ctx = space.interp_w(SSLContext, w_ctx) if not HAS_SNI: raise oefmt(space.w_NotImplementedError, "setting a socket's context " @@ -776,22 +580,20 @@ libssl_SSL_set_SSL_CTX(self.ssl, ctx.ctx) -_SSLSocket.typedef = TypeDef( - "_ssl._SSLSocket", - - do_handshake=interp2app(_SSLSocket.do_handshake), - write=interp2app(_SSLSocket.write), - read=interp2app(_SSLSocket.read), - pending=interp2app(_SSLSocket.pending), - peer_certificate=interp2app(_SSLSocket.peer_certificate), - cipher=interp2app(_SSLSocket.cipher), - shutdown=interp2app(_SSLSocket.shutdown), - selected_npn_protocol = interp2app(_SSLSocket.selected_npn_protocol), - compression = interp2app(_SSLSocket.compression_w), - version = interp2app(_SSLSocket.version_w), - tls_unique_cb = interp2app(_SSLSocket.tls_unique_cb_w), - context=GetSetProperty(_SSLSocket.descr_get_context, - _SSLSocket.descr_set_context), +SSLSocket.typedef = TypeDef("_ssl._SSLSocket", + write = interp2app(SSLSocket.write), + pending = interp2app(SSLSocket.pending), + read = interp2app(SSLSocket.read), + do_handshake = interp2app(SSLSocket.do_handshake), + shutdown = interp2app(SSLSocket.shutdown), + cipher = interp2app(SSLSocket.cipher), + peer_certificate = interp2app(SSLSocket.peer_certificate), + selected_npn_protocol = interp2app(SSLSocket.selected_npn_protocol), + compression = interp2app(SSLSocket.compression_w), + version = interp2app(SSLSocket.version_w), + tls_unique_cb = interp2app(SSLSocket.tls_unique_cb_w), + context=GetSetProperty(SSLSocket.descr_get_context, + SSLSocket.descr_set_context), ) def _certificate_to_der(space, certificate): @@ -801,7 +603,7 @@ if length < 0: raise _ssl_seterror(space, None, 0) try: - return space.wrap(rffi.charpsize2str(buf_ptr[0], length)) + return space.wrapbytes(rffi.charpsize2str(buf_ptr[0], length)) finally: libssl_OPENSSL_free(buf_ptr[0]) @@ -1021,16 +823,6 @@ return space.newtuple([w_name, w_value]) -SSLSocket.typedef = TypeDef("_SSLSocket", - write = interp2app(SSLSocket.write), - pending = interp2app(SSLSocket.pending), - read = interp2app(SSLSocket.read), - do_handshake = interp2app(SSLSocket.do_handshake), - shutdown = interp2app(SSLSocket.shutdown), - cipher = interp2app(SSLSocket.cipher), - peer_certificate = interp2app(SSLSocket.peer_certificate), -) - def _get_aia_uri(space, certificate, nid): info = rffi.cast(AUTHORITY_INFO_ACCESS, libssl_X509_get_ext_d2i( certificate, NID_info_access, None, None)) @@ -1109,6 +901,7 @@ libssl_SSL_set_connect_state(ss.ssl) else: libssl_SSL_set_accept_state(ss.ssl) + ss.socket_type = side ss.w_socket = weakref.ref(w_sock) return ss @@ -1339,11 +1132,8 @@ # The high-level ssl.SSLSocket object index = rffi.cast(lltype.Signed, libssl_SSL_get_app_data(ssl)) w_ssl = SOCKET_STORAGE.get(index) - assert isinstance(w_ssl, _SSLSocket) - if w_ssl.ssl_sock_weakref_w is not None: - w_ssl_socket = w_ssl.ssl_sock_weakref_w() - else: - w_ssl_socket = space.w_None + assert isinstance(w_ssl, SSLSocket) + w_ssl_socket = w_ssl # So far. Need to change in 3.3. if space.is_none(w_ssl_socket): ad[0] = rffi.cast(rffi.INT, SSL_AD_INTERNAL_ERROR) return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_ALERT_FATAL) @@ -1383,10 +1173,10 @@ return rffi.cast(rffi.INT, SSL_TLSEXT_ERR_ALERT_FATAL) -class _SSLContext(W_Root): - @staticmethod - @unwrap_spec(protocol=int) - def descr_new(space, w_subtype, protocol): +class SSLContext(W_Root): + ctx = lltype.nullptr(SSL_CTX.TO) + + def __init__(self, space, protocol): if protocol == PY_SSL_VERSION_TLS1: method = libssl_TLSv1_method() elif protocol == PY_SSL_VERSION_SSL3 and not OPENSSL_NO_SSL3: @@ -1397,17 +1187,19 @@ method = libssl_SSLv23_method() else: raise oefmt(space.w_ValueError, "invalid protocol version") - ctx = libssl_SSL_CTX_new(method) - if not ctx: + self.ctx = libssl_SSL_CTX_new(method) + if not self.ctx: raise ssl_error(space, "failed to allocate SSL context") - self = space.allocate_instance(_SSLContext, w_subtype) - self.ctx = ctx self.check_hostname = False + + # Defaults + libssl_SSL_CTX_set_verify(self.ctx, SSL_VERIFY_NONE, None) options = SSL_OP_ALL & ~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS if protocol != PY_SSL_VERSION_SSL2: options |= SSL_OP_NO_SSLv2 - libssl_SSL_CTX_set_options(ctx, options) + libssl_SSL_CTX_set_options(self.ctx, options) + libssl_SSL_CTX_set_session_id_context(self.ctx, "Python", len("Python")) if not OPENSSL_NO_ECDH: # Allow automatic ECDH curve selection (on @@ -1425,18 +1217,46 @@ finally: libssl_EC_KEY_free(key) - return self + def __del__(self): + if self.ctx: + libssl_SSL_CTX_free(self.ctx) + + @staticmethod + @unwrap_spec(protocol=int) + def descr_new(space, w_subtype, protocol=PY_SSL_VERSION_SSL23): + self = space.allocate_instance(SSLContext, w_subtype) + self.__init__(space, protocol) + return space.wrap(self) + + @unwrap_spec(cipherlist=str) + def set_ciphers_w(self, space, cipherlist): + ret = libssl_SSL_CTX_set_cipher_list(self.ctx, cipherlist) + if ret == 0: + # Clearing the error queue is necessary on some OpenSSL + # versions, otherwise the error will be reported again + # when another SSL call is done. + libssl_ERR_clear_error() + raise ssl_error(space, "No cipher can be selected.") @unwrap_spec(server_side=int) - def descr_wrap_socket(self, space, w_sock, server_side, w_server_hostname=None, w_ssl_sock=None): - return _SSLSocket.descr_new(space, self, w_sock, server_side, w_server_hostname, w_ssl_sock) + def wrap_socket_w(self, space, w_sock, server_side, + w_server_hostname=None): + assert w_sock is not None + # server_hostname is either None (or absent), or to be encoded + # using the idna encoding. + if space.is_none(w_server_hostname): + hostname = None + else: + hostname = space.bytes_w( + space.call_method(w_server_hostname, + "encode", space.wrap("idna"))) - @unwrap_spec(cipherlist=str) - def descr_set_ciphers(self, space, cipherlist): - ret = libssl_SSL_CTX_set_cipher_list(self.ctx, cipherlist) - if ret == 0: - libssl_ERR_clear_error() - raise ssl_error(space, "No cipher can be selected.") + if hostname and not HAS_SNI: + raise OperationError(space.w_ValueError, + space.wrap("server_hostname is not supported " + "by your OpenSSL library")) + + return new_sslobject(space, self.ctx, w_sock, server_side, hostname) def session_stats_w(self, space): w_stats = space.newdict() @@ -1806,30 +1626,30 @@ libssl_SSL_CTX_set_tlsext_servername_arg(self.ctx, rffi.cast(rffi.VOIDP, index)) -_SSLContext.typedef = TypeDef( +SSLContext.typedef = TypeDef( "_ssl._SSLContext", - __new__=interp2app(_SSLContext.descr_new), - _wrap_socket=interp2app(_SSLContext.descr_wrap_socket), - set_ciphers=interp2app(_SSLContext.descr_set_ciphers), - cert_store_stats=interp2app(_SSLContext.cert_store_stats_w), - load_cert_chain=interp2app(_SSLContext.load_cert_chain_w), - load_dh_params=interp2app(_SSLContext.load_dh_params_w), - load_verify_locations=interp2app(_SSLContext.load_verify_locations_w), - session_stats = interp2app(_SSLContext.session_stats_w), - set_default_verify_paths=interp2app(_SSLContext.descr_set_default_verify_paths), - _set_npn_protocols=interp2app(_SSLContext.set_npn_protocols_w), - get_ca_certs=interp2app(_SSLContext.get_ca_certs_w), - set_ecdh_curve=interp2app(_SSLContext.set_ecdh_curve_w), - set_servername_callback=interp2app(_SSLContext.set_servername_callback_w), + __new__ = interp2app(SSLContext.descr_new), + _wrap_socket = interp2app(SSLContext.wrap_socket_w), + set_ciphers = interp2app(SSLContext.set_ciphers_w), + load_cert_chain = interp2app(SSLContext.load_cert_chain_w), + load_verify_locations = interp2app(SSLContext.load_verify_locations_w), + session_stats = interp2app(SSLContext.session_stats_w), + cert_store_stats=interp2app(SSLContext.cert_store_stats_w), + load_dh_params=interp2app(SSLContext.load_dh_params_w), + set_default_verify_paths=interp2app(SSLContext.descr_set_default_verify_paths), + _set_npn_protocols=interp2app(SSLContext.set_npn_protocols_w), + get_ca_certs=interp2app(SSLContext.get_ca_certs_w), + set_ecdh_curve=interp2app(SSLContext.set_ecdh_curve_w), + set_servername_callback=interp2app(SSLContext.set_servername_callback_w), - options=GetSetProperty(_SSLContext.descr_get_options, - _SSLContext.descr_set_options), - verify_mode=GetSetProperty(_SSLContext.descr_get_verify_mode, - _SSLContext.descr_set_verify_mode), - verify_flags=GetSetProperty(_SSLContext.descr_get_verify_flags, - _SSLContext.descr_set_verify_flags), - check_hostname=GetSetProperty(_SSLContext.descr_get_check_hostname, - _SSLContext.descr_set_check_hostname), + options=GetSetProperty(SSLContext.descr_get_options, + SSLContext.descr_set_options), + verify_mode=GetSetProperty(SSLContext.descr_get_verify_mode, + SSLContext.descr_set_verify_mode), + verify_flags=GetSetProperty(SSLContext.descr_get_verify_flags, + SSLContext.descr_set_verify_flags), + check_hostname=GetSetProperty(SSLContext.descr_get_check_hostname, + SSLContext.descr_set_check_hostname), ) @@ -1880,7 +1700,7 @@ if not path: return space.w_None else: - return space.wrapbytes(rffi.charp2str(path)) + return fsdecode(space, space.wrapbytes(rffi.charp2str(path))) def get_default_verify_paths(space): return space.newtuple([ diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py --- a/pypy/module/_ssl/test/test_ssl.py +++ b/pypy/module/_ssl/test/test_ssl.py @@ -73,28 +73,6 @@ s = _socket.socket() ss = ssl.wrap_socket(s) - exc = raises(_socket.error, ss.do_handshake) - if sys.platform == 'win32': - assert exc.value.errno == 10057 # WSAENOTCONN - else: - assert exc.value.errno == 32 # Broken pipe - del exc, ss, s - gc.collect() # force the destructor() to be called now - - def test_async_closed(self): - import _ssl, _socket, sys, gc - s = _socket.socket() - s.settimeout(3) - if sys.version_info < (2, 7, 9): - ss = _ssl.sslwrap(s, 0) - else: - ss = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)._wrap_socket(s, 0) - s.close() - exc = raises(_ssl.SSLError, ss.write, "data") - assert exc.value.message == 'Underlying socket has been closed.' - del exc, ss, s - gc.collect() # force the destructor() to be called now - def test_test_decode_nullbytecert(self): import _ssl p = _ssl._test_decode_cert(self.nullbytecert) @@ -119,7 +97,7 @@ s = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1) raises(ValueError, _ssl._SSLContext, -1) - assert type(s.options) is long + assert type(s.options) is int assert s.options & _ssl.OP_NO_SSLv2 s.options &= ~_ssl.OP_NO_SSLv2 assert not s.options & _ssl.OP_NO_SSLv2 @@ -136,7 +114,7 @@ exc = raises(ValueError, "s.verify_mode = 1234") assert str(exc.value) == "invalid value for verify_mode" - assert type(s.verify_flags) is long + assert type(s.verify_flags) is int assert s.verify_flags == _ssl.VERIFY_DEFAULT s.verify_flags = _ssl.VERIFY_CRL_CHECK_LEAF assert s.verify_flags == _ssl.VERIFY_CRL_CHECK_LEAF @@ -260,7 +238,7 @@ import socket, _ssl, gc ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1) ctx._set_npn_protocols(b'\x08http/1.1\x06spdy/2') - ss = ctx._wrap_socket(self.s._sock, True, + ss = ctx._wrap_socket(self.s, True, server_hostname="svn.python.org") self.s.close() del ss; gc.collect() @@ -269,7 +247,7 @@ import ssl, sys, gc ss = ssl.wrap_socket(self.s) ss.do_handshake() - assert isinstance(ss.get_channel_binding(), bytes) + assert isinstance(ss._sslobj.tls_unique_cb(), bytes) self.s.close() del ss; gc.collect() @@ -277,7 +255,7 @@ import ssl, sys, gc ss = ssl.wrap_socket(self.s) ss.do_handshake() - assert ss.compression() in [None, 'ZLIB', 'RLE'] + assert ss._sslobj.compression() in [None, 'ZLIB', 'RLE'] self.s.close() del ss; gc.collect() @@ -351,7 +329,7 @@ ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1) with open(self.keycert) as f: - cacert_pem = f.read().decode('ascii') + cacert_pem = f.read() ctx.load_verify_locations(cadata=cacert_pem) assert ctx.cert_store_stats()["x509_ca"] == 0 @@ -450,32 +428,6 @@ s = str(exc.value) assert s.startswith("[PEM: NO_START_LINE] no start line") - def test_subclass(self): - # Check that the appropriate SSLError subclass is raised - # (this only tests one of them) - import _ssl, _socket - ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1) - s = _socket.socket() - try: - s.bind(("127.0.0.1", 0)) - s.listen(5) - c = _socket.socket() - c.connect(s.getsockname()) - c.setblocking(False) - - c = ctx._wrap_socket(c, False) - try: - exc = raises(_ssl.SSLWantReadError, c.do_handshake) - msg= str(exc.value) - assert msg.startswith("The operation did not complete (read)") - # For compatibility - assert exc.value.errno == _ssl.SSL_ERROR_WANT_READ - finally: - c.shutdown() - finally: - s.close() ->>>>>>> other - SSL_CERTIFICATE = """ -----BEGIN CERTIFICATE----- MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py --- a/pypy/objspace/std/objspace.py +++ b/pypy/objspace/std/objspace.py @@ -155,7 +155,7 @@ # poor man's x.decode('ascii', 'replace'), since it's not # supported by RPython if not we_are_translated(): - print 'WARNING: space.str() called on a non-ascii byte string: %r' % x + print 'WARNING: space.wrap() called on a non-ascii byte string: %r' % x lst = [] for ch in x: ch = ord(ch) From noreply at buildbot.pypy.org Mon Feb 16 07:04:03 2015 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 16 Feb 2015 07:04:03 +0100 (CET) Subject: [pypy-commit] pypy default: fix failing whatsnew-head and test_version tests Message-ID: <20150216060403.415CC1C1155@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r75910:427ccadfbbd6 Date: 2015-02-16 08:04 +0200 http://bitbucket.org/pypy/pypy/changeset/427ccadfbbd6/ Log: fix failing whatsnew-head and test_version tests diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -6,8 +6,8 @@ .. startrev: 397b96217b85 -Fix non-blocking file reads sometimes raising EAGAIN even though they -have buffered data waiting (b1c4fcb04a42) +Non-blocking file reads sometimes raised EAGAIN even though they +had buffered data waiting, fixed in b1c4fcb04a42 .. branch: vmprof @@ -18,3 +18,8 @@ .. branch: stdlib-2.7.9 Update stdlib to version 2.7.9 + +.. branch: fix-kqueue-error2 +Fix exception being raised by kqueue.control (CPython compatibility) + +.. branch: gitignore diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -29,7 +29,7 @@ #define PY_VERSION "2.7.9" /* PyPy version as a string */ -#define PYPY_VERSION "2.6.0" +#define PYPY_VERSION "2.6.0-alpha0" /* Subversion Revision number of this file (not of the repository). * Empty since Mercurial migration. */ From noreply at buildbot.pypy.org Mon Feb 16 07:24:28 2015 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 16 Feb 2015 07:24:28 +0100 (CET) Subject: [pypy-commit] pypy default: add test that fails on win32, save_err is wrong on SSL_CTX_use_certificate_chain_file in rpython/rlib/ropenssl? Message-ID: <20150216062428.541241C1155@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r75911:37183f083a3a Date: 2015-02-16 08:25 +0200 http://bitbucket.org/pypy/pypy/changeset/37183f083a3a/ Log: add test that fails on win32, save_err is wrong on SSL_CTX_use_certificate_chain_file in rpython/rlib/ropenssl? diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py --- a/pypy/module/_ssl/test/test_ssl.py +++ b/pypy/module/_ssl/test/test_ssl.py @@ -307,12 +307,13 @@ os.path.dirname(__file__), 'dh512.pem')) def test_load_cert_chain(self): - import _ssl + import _ssl, errno ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1) ctx.load_cert_chain(self.keycert) ctx.load_cert_chain(self.cert, self.key) - raises(IOError, ctx.load_cert_chain, "inexistent.pem") - raises(_ssl.SSLError, ctx.load_cert_chain, self.badcert) + exc = raises(IOError, ctx.load_cert_chain, "inexistent.pem") + assert exc.value.errno == errno.ENOENT + exc = raises(_ssl.SSLError, ctx.load_cert_chain, self.badcert) raises(_ssl.SSLError, ctx.load_cert_chain, self.emptycert) # Password protected key and cert raises(_ssl.SSLError, ctx.load_cert_chain, self.cert_protected, From noreply at buildbot.pypy.org Mon Feb 16 08:37:37 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 16 Feb 2015 08:37:37 +0100 (CET) Subject: [pypy-commit] pypy py3k: Remove space.bytes_w alias Message-ID: <20150216073737.BC10E1C1155@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r75912:c0c26bc17d9d Date: 2015-02-16 08:37 +0100 http://bitbucket.org/pypy/pypy/changeset/c0c26bc17d9d/ Log: Remove space.bytes_w alias diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1484,8 +1484,6 @@ def bytes_w(self, w_obj): return w_obj.bytes_w(self) - bytes_w = str_w # Python2 - def str0_w(self, w_obj): "Like str_w, but rejects strings with NUL bytes." from rpython.rlib import rstring From noreply at buildbot.pypy.org Mon Feb 16 10:41:31 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 16 Feb 2015 10:41:31 +0100 (CET) Subject: [pypy-commit] pypy py3k: SSL passwords can be unicode. Message-ID: <20150216094131.303181C034E@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r75913:2cf9d354d605 Date: 2015-02-16 09:52 +0100 http://bitbucket.org/pypy/pypy/changeset/2cf9d354d605/ Log: SSL passwords can be unicode. diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -1094,13 +1094,16 @@ if pw_info.w_callable: try: w_result = space.call_function(pw_info.w_callable) - try: - password = pw_info.space.bufferstr_w(w_result) - except OperationError as e: - if not e.match(space, space.w_TypeError): - raise - raise oefmt(space.w_TypeError, - "password callback must return a string") + if space.isinstance_w(w_result, space.w_unicode): + password = space.str_w(w_result) + else: + try: + password = pw_info.space.bufferstr_w(w_result) + except OperationError as e: + if not e.match(space, space.w_TypeError): + raise + raise oefmt(space.w_TypeError, + "password callback must return a string") except OperationError as e: pw_info.operationerror = e return rffi.cast(rffi.INT, -1) @@ -1362,13 +1365,16 @@ if space.is_true(space.callable(w_password)): pw_info.w_callable = w_password else: - try: - pw_info.password = space.bufferstr_w(w_password) - except OperationError as e: - if not e.match(space, space.w_TypeError): - raise - raise oefmt(space.w_TypeError, - "password should be a string or callable") + if space.isinstance_w(w_password, space.w_unicode): + pw_info.password = space.str_w(w_password) + else: + try: + pw_info.password = space.bufferstr_w(w_password) + except OperationError as e: + if not e.match(space, space.w_TypeError): + raise + raise oefmt(space.w_TypeError, + "password should be a string or callable") libssl_SSL_CTX_set_default_passwd_cb( self.ctx, _password_callback) From noreply at buildbot.pypy.org Mon Feb 16 10:41:32 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 16 Feb 2015 10:41:32 +0100 (CET) Subject: [pypy-commit] pypy py3k: Fix syntax error in test Message-ID: <20150216094132.66B901C034E@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r75914:e8654d723f68 Date: 2015-02-16 09:54 +0100 http://bitbucket.org/pypy/pypy/changeset/e8654d723f68/ Log: Fix syntax error in test diff --git a/pypy/module/_sre/test/test_app_sre.py b/pypy/module/_sre/test/test_app_sre.py --- a/pypy/module/_sre/test/test_app_sre.py +++ b/pypy/module/_sre/test/test_app_sre.py @@ -206,8 +206,8 @@ import sys if sys.version_info < (2, 7, 9): skip() - assert re.match("(foo)", "foo").group(1L) == "foo" - exc = raises(IndexError, re.match("", "").group, sys.maxint + 1) + assert re.match("(foo)", "foo").group(1) == "foo" + exc = raises(IndexError, re.match("", "").group, sys.maxsize + 1) assert str(exc.value) == "no such group" def test_expand(self): From noreply at buildbot.pypy.org Mon Feb 16 10:41:33 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 16 Feb 2015 10:41:33 +0100 (CET) Subject: [pypy-commit] pypy py3k: Adapt compiler test to py3k: coding cookie is only used if source is bytes. Message-ID: <20150216094133.8DA811C034E@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r75915:cb55f5432429 Date: 2015-02-16 10:16 +0100 http://bitbucket.org/pypy/pypy/changeset/cb55f5432429/ Log: Adapt compiler test to py3k: coding cookie is only used if source is bytes. diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py --- a/pypy/interpreter/test/test_compiler.py +++ b/pypy/interpreter/test/test_compiler.py @@ -1187,20 +1187,15 @@ def test_encoding(self): code = b'# -*- coding: badencoding -*-\npass\n' raises(SyntaxError, compile, code, 'tmp', 'exec') - code = u"# -*- coding: utf-8 -*-\npass\n" - raises(SyntaxError, compile, code, 'tmp', 'exec') code = 'u"\xc2\xa4"\n' assert eval(code) == u'\xc2\xa4' code = u'u"\xc2\xa4"\n' assert eval(code) == u'\xc2\xa4' - code = '# -*- coding: latin1 -*-\nu"\xc2\xa4"\n' + code = b'# -*- coding: latin1 -*-\nu"\xc2\xa4"\n' assert eval(code) == u'\xc2\xa4' - code = '# -*- coding: utf-8 -*-\nu"\xc2\xa4"\n' + code = b'# -*- coding: utf-8 -*-\nu"\xc2\xa4"\n' assert eval(code) == u'\xa4' - code = '# -*- coding: iso8859-15 -*-\nu"\xc2\xa4"\n' + code = b'# -*- coding: iso8859-15 -*-\nu"\xc2\xa4"\n' assert eval(code) == u'\xc2\u20ac' - import sys - if sys.version_info < (2, 7, 9): - skip() - code = 'u"""\\\n# -*- coding: utf-8 -*-\n\xc2\xa4"""\n' - assert eval(code) == u'# -*- coding: utf-8 -*-\n\xc2\xa4' + code = b'u"""\\\n# -*- coding: ascii -*-\n\xc2\xa4"""\n' + assert eval(code) == u'# -*- coding: ascii -*-\n\xa4' From noreply at buildbot.pypy.org Mon Feb 16 10:41:34 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 16 Feb 2015 10:41:34 +0100 (CET) Subject: [pypy-commit] pypy py3k: Fix refcount issue in cpython3 test: this object's reference was Message-ID: <20150216094134.C4D931C034E@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r75916:745619bed083 Date: 2015-02-16 10:40 +0100 http://bitbucket.org/pypy/pypy/changeset/745619bed083/ Log: Fix refcount issue in cpython3 test: this object's reference was already "stolen" by a PyTuple_SET_ITEM above. diff --git a/lib_pypy/_testcapimodule.c b/lib_pypy/_testcapimodule.c --- a/lib_pypy/_testcapimodule.c +++ b/lib_pypy/_testcapimodule.c @@ -808,7 +808,7 @@ return raiseTestError("test_L_code", "L code returned wrong value for long 42"); - Py_DECREF(num); + /* Py_DECREF(num); <== CPython bug */ num = PyLong_FromLong(42); if (num == NULL) return NULL; @@ -990,7 +990,7 @@ return raiseTestError("test_k_code", "k code returned wrong value for long 0xFFF...FFF"); - Py_DECREF(num); + /* Py_DECREF(num); <== CPython bug */ num = PyLong_FromString("-FFFFFFFF000000000000000042", NULL, 16); if (num == NULL) return NULL; From noreply at buildbot.pypy.org Mon Feb 16 12:45:52 2015 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 16 Feb 2015 12:45:52 +0100 (CET) Subject: [pypy-commit] pypy vmprof: Somewhat controversial strategy - store info about code objects in a tempfile(). Message-ID: <20150216114552.7496D1C12DF@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r75917:ef83e90448b8 Date: 2015-02-16 13:41 +0200 http://bitbucket.org/pypy/pypy/changeset/ef83e90448b8/ Log: Somewhat controversial strategy - store info about code objects in a tempfile(). Let's see how it goes diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -38,8 +38,8 @@ "_csv", "cppyy", "_pypyjson" ]) -if sys.platform.startswith('linux') and sys.maxint > 2147483647: - working_modules.add('_vmprof') +#if sys.platform.startswith('linux') and sys.maxint > 2147483647: +# working_modules.add('_vmprof') translation_modules = default_modules.copy() translation_modules.update([ diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1,15 +1,14 @@ -import sys +import sys, os from rpython.rlib.cache import Cache from rpython.tool.uid import HUGEVAL_BYTES -from rpython.rlib import jit, types +from rpython.rlib import jit, types, rfile from rpython.rlib.debug import make_sure_not_resized from rpython.rlib.objectmodel import (we_are_translated, newlist_hint, compute_unique_id, specialize) from rpython.rlib.signature import signature from rpython.rlib.rarithmetic import r_uint, SHRT_MIN, SHRT_MAX, \ INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX -from rpython.rlib.rweaklist import RWeakListMixin from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag, UserDelAction) @@ -367,10 +366,6 @@ # ____________________________________________________________ -class CodeObjWeakList(RWeakListMixin): - def __init__(self): - self.initialize() - class ObjSpace(object): """Base class for the interpreter-level implementations of object spaces. http://pypy.readthedocs.org/en/latest/objspace.html""" @@ -394,7 +389,6 @@ self.check_signal_action = None # changed by the signal module self.user_del_action = UserDelAction(self) self._code_of_sys_exc_info = None - self.all_code_objs = CodeObjWeakList() # can be overridden to a subclass self.initialize() @@ -672,16 +666,31 @@ assert ec is not None return ec + def _open_code_info_file(self): + ec = self.getexecutioncontext() + try: + ec.code_info_file = os.tmpfile() + except: + ec.code_info_file_present = False # we failed to open it + def register_code_object(self, pycode): - callback = self.getexecutioncontext().register_code_callback - if callback is not None: - callback(self, pycode) - self.all_code_objs.add_handle(pycode) - - def set_code_callback(self, callback): ec = self.getexecutioncontext() - ec.register_code_callback = callback - + if ec.code_info_file is None: + self._open_code_info_file() + if not ec.code_info_file_present: + return + try: + rfile.write_int(ec.code_info_file, pycode._unique_id) + s = pycode._get_full_name() + rfile.write_int(ec.code_info_file, len(s)) + ec.code_info_file.write(s) + except OSError: + ec.code_info_file_present = False + try: + ec.code_info_file.close() + except: + pass # can fail, ignore + def _freeze_(self): return True diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -33,7 +33,9 @@ self.profilefunc = None self.w_profilefuncarg = None self.thread_disappeared = False # might be set to True after os.fork() - self.register_code_callback = None + self.code_info_file = None + self.code_info_file_present = True + if sys.maxint == 2147483647: self._code_unique_id = 0 # XXX this is wrong, it won't work on 32bit else: diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -70,6 +70,7 @@ 'debug_stop' : 'interp_debug.debug_stop', 'debug_print_once' : 'interp_debug.debug_print_once', 'debug_flush' : 'interp_debug.debug_flush', + 'all_code_info' : 'interp_magic.all_code_info', 'builtinify' : 'interp_magic.builtinify', 'lookup_special' : 'interp_magic.lookup_special', 'do_what_I_mean' : 'interp_magic.do_what_I_mean', diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -130,3 +130,15 @@ def locals_to_fast(space, w_frame): assert isinstance(w_frame, PyFrame) w_frame.locals2fast() + +def all_code_info(space): + ec = space.getexecutioncontext() + if not ec.code_info_file_present or ec.code_info_file is None: + raise OperationError(space.w_RuntimeError, space.wrap( + "Info file not present, error writing it")) + ec.code_info_file.flush() + ec.code_info_file.seek(0, 0) + try: + return space.wrap(ec.code_info_file.read()) + finally: + ec.code_info_file.seek(2, 0) diff --git a/pypy/module/__pypy__/test/test_special.py b/pypy/module/__pypy__/test/test_special.py --- a/pypy/module/__pypy__/test/test_special.py +++ b/pypy/module/__pypy__/test/test_special.py @@ -83,6 +83,13 @@ s = set([2, 3, 4]) assert strategy(s) == "IntegerSetStrategy" + def test_code_info_file(self): + from __pypy__ import all_code_info + + assert len(all_code_info()) > 0 # eh, what else? + exec """def function_of_a_relatively_unique_name(): + pass""" + assert 'function_of_a_relatively_unique_name' in all_code_info() class AppTestJitFeatures(object): spaceconfig = {"translation.jit": True} diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -138,6 +138,7 @@ self.fileno = -1 def enable(self, space, fileno, period): + xxx if self.is_enabled: raise oefmt(space.w_ValueError, "_vmprof already enabled") self.fileno = fileno diff --git a/rpython/rlib/rfile.py b/rpython/rlib/rfile.py --- a/rpython/rlib/rfile.py +++ b/rpython/rlib/rfile.py @@ -3,7 +3,7 @@ python builtin open() """ -import os, stat, errno +import os, stat, errno, sys from rpython.rlib import rposix from rpython.rlib.objectmodel import enforceargs from rpython.rlib.rarithmetic import intmask @@ -221,6 +221,22 @@ return stdin, stdout, stderr +def write_int(f, l): + if sys.maxint == 2147483647: + f.write(chr(l & 0xff) + + chr((l >> 8) & 0xff) + + chr((l >> 16) & 0xff) + + chr((l >> 24) & 0xff)) + else: + f.write(chr(l & 0xff) + + chr((l >> 8) & 0xff) + + chr((l >> 16) & 0xff) + + chr((l >> 24) & 0xff) + + chr((l >> 32) & 0xff) + + chr((l >> 40) & 0xff) + + chr((l >> 48) & 0xff) + + chr((l >> 56) & 0xff)) + class RFile(object): _setbuf = lltype.nullptr(rffi.CCHARP.TO) _univ_newline = False From noreply at buildbot.pypy.org Mon Feb 16 12:45:53 2015 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 16 Feb 2015 12:45:53 +0100 (CET) Subject: [pypy-commit] pypy vmprof: merge Message-ID: <20150216114553.CA0211C12DF@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r75918:c48ac690c37b Date: 2015-02-16 13:45 +0200 http://bitbucket.org/pypy/pypy/changeset/c48ac690c37b/ Log: merge diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py --- a/rpython/jit/backend/llsupport/asmmemmgr.py +++ b/rpython/jit/backend/llsupport/asmmemmgr.py @@ -5,7 +5,7 @@ from rpython.rlib.debug import debug_start, debug_print, debug_stop from rpython.rlib.debug import have_debug_prints from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.rlib.rbisect import bisect, bisect_tuple +from rpython.rlib.rbisect import bisect_left, bisect_left_tuple _memmngr = None # global reference so we can use @entrypoint :/ @@ -55,19 +55,16 @@ self.total_mallocs -= r_uint(stop - start) self._add_free_block(start, stop) # fix up jit_addr_map - jit_adr_start = bisect(self.jit_addr_map, start) - jit_adr_stop = bisect(self.jit_addr_map, stop) - self.jit_addr_map = (self.jit_addr_map[:jit_adr_start] + - self.jit_addr_map[jit_adr_stop:]) - self.jit_frame_depth_map = (self.jit_frame_depth_map[:jit_adr_start] + - self.jit_frame_depth_map[jit_adr_stop:]) + jit_adr_start = bisect_left(self.jit_addr_map, start) + jit_adr_stop = bisect_left(self.jit_addr_map, stop) + del self.jit_addr_map[jit_adr_start:jit_adr_stop] + del self.jit_frame_depth_map[jit_adr_start:jit_adr_stop] # fix up codemap # (there should only be zero or one codemap entry in that range, # but still we use a range to distinguish between zero and one) - codemap_adr_start = bisect_tuple(self.jit_codemap, start) - codemap_adr_stop = bisect_tuple(self.jit_codemap, stop) - self.jit_codemap = (self.jit_codemap[:codemap_adr_start] + - self.jit_codemap[codemap_adr_stop:]) + codemap_adr_start = bisect_left_tuple(self.jit_codemap, start) + codemap_adr_stop = bisect_left_tuple(self.jit_codemap, stop) + del self.jit_codemap[codemap_adr_start:codemap_adr_stop] def open_malloc(self, minsize): """Allocate at least minsize bytes. Returns (start, stop).""" @@ -183,7 +180,7 @@ self.jit_addr_map += [0] * len(frame_positions) self.jit_frame_depth_map += [0] * len(frame_positions) else: - start = bisect(self.jit_addr_map, rawstart) + start = bisect_left(self.jit_addr_map, rawstart) self.jit_addr_map = (self.jit_addr_map[:start] + [0] * len(frame_positions) + self.jit_addr_map[start:]) @@ -196,12 +193,8 @@ def register_codemap(self, codemap): start = codemap[0] - pos = bisect_tuple(self.jit_codemap, start) - if pos == len(self.jit_codemap): # common case - self.jit_codemap.append(codemap) - else: - self.jit_codemap = (self.jit_codemap[:pos] + [codemap] + - self.jit_codemap[pos:]) + pos = bisect_left_tuple(self.jit_codemap, start) + self.jit_codemap.insert(pos, codemap) def _delete(self): "NOT_RPYTHON" diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py --- a/rpython/jit/backend/llsupport/codemap.py +++ b/rpython/jit/backend/llsupport/codemap.py @@ -12,7 +12,7 @@ from rpython.rlib import rgc from rpython.rlib.entrypoint import jit_entrypoint from rpython.jit.backend.llsupport import asmmemmgr -from rpython.rlib.rbisect import bisect, bisect_tuple +from rpython.rlib.rbisect import bisect_right, bisect_right_tuple from rpython.rtyper.lltypesystem import lltype, rffi @jit_entrypoint([lltype.Signed], lltype.Signed, @@ -21,7 +21,7 @@ def stack_depth_at_loc(loc): _memmngr = asmmemmgr._memmngr - pos = bisect(_memmngr.jit_addr_map, loc + 1) + pos = bisect_right(_memmngr.jit_addr_map, loc) if pos == 0 or pos == len(_memmngr.jit_addr_map): return -1 return _memmngr.jit_frame_depth_map[pos - 1] @@ -43,9 +43,7 @@ def find_codemap_at_addr(addr): _memmngr = asmmemmgr._memmngr - res = bisect_tuple(_memmngr.jit_codemap, addr) - 1 - if res == len(_memmngr.jit_codemap): - return -1 + res = bisect_right_tuple(_memmngr.jit_codemap, addr) - 1 return res @jit_entrypoint([lltype.Signed, lltype.Signed, diff --git a/rpython/rlib/rbisect.py b/rpython/rlib/rbisect.py --- a/rpython/rlib/rbisect.py +++ b/rpython/rlib/rbisect.py @@ -1,21 +1,39 @@ -def bisect(a, x): +def bisect_left(a, x): """Return the index in the sorted list 'a' of 'x'. If 'x' is not in 'a', return the index where it can be inserted.""" lo = 0 hi = len(a) while lo < hi: mid = (lo+hi)//2 - if x <= a[mid]: hi = mid + if a[mid] < x: lo = mid+1 + else: hi = mid + return lo + +def bisect_right(a, x): + lo = 0 + hi = len(a) + while lo < hi: + mid = (lo+hi)//2 + if x < a[mid]: hi = mid else: lo = mid+1 return lo # a copy of the above, but compares the first item of a tuple only -def bisect_tuple(a, x): +def bisect_left_tuple(a, x): lo = 0 hi = len(a) while lo < hi: mid = (lo+hi)//2 - if x <= a[mid][0]: hi = mid + if a[mid][0] < x: lo = mid+1 + else: hi = mid + return lo + +def bisect_right_tuple(a, x): + lo = 0 + hi = len(a) + while lo < hi: + mid = (lo+hi)//2 + if x < a[mid][0]: hi = mid else: lo = mid+1 return lo diff --git a/rpython/rlib/test/test_rbisect.py b/rpython/rlib/test/test_rbisect.py --- a/rpython/rlib/test/test_rbisect.py +++ b/rpython/rlib/test/test_rbisect.py @@ -1,7 +1,7 @@ -from rpython.rlib.rbisect import bisect +from rpython.rlib.rbisect import bisect_left, bisect_right -def test_bisect(): +def test_bisect_left(): cases = [ ([], 1, 0), ([1], 0, 0), @@ -44,4 +44,50 @@ ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 5, 10), ] for lst, elem, exp in cases: - assert bisect(lst, elem) == exp + assert bisect_left(lst, elem) == exp + +def test_bisect_right(): + cases = [ + + ([], 1, 0), + ([1], 0, 0), + ([1], 1, 1), + ([1], 2, 1), + ([1, 1], 0, 0), + ([1, 1], 1, 2), + ([1, 1], 2, 2), + ([1, 1, 1], 0, 0), + ([1, 1, 1], 1, 3), + ([1, 1, 1], 2, 3), + ([1, 1, 1, 1], 0, 0), + ([1, 1, 1, 1], 1, 4), + ([1, 1, 1, 1], 2, 4), + ([1, 2], 0, 0), + ([1, 2], 1, 1), + ([1, 2], 1.5, 1), + ([1, 2], 2, 2), + ([1, 2], 3, 2), + ([1, 1, 2, 2], 0, 0), + ([1, 1, 2, 2], 1, 2), + ([1, 1, 2, 2], 1.5, 2), + ([1, 1, 2, 2], 2, 4), + ([1, 1, 2, 2], 3, 4), + ([1, 2, 3], 0, 0), + ([1, 2, 3], 1, 1), + ([1, 2, 3], 1.5, 1), + ([1, 2, 3], 2, 2), + ([1, 2, 3], 2.5, 2), + ([1, 2, 3], 3, 3), + ([1, 2, 3], 4, 3), + ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 0, 0), + ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 1, 1), + ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 1.5, 1), + ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 2, 3), + ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 2.5, 3), + ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 3, 6), + ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 3.5, 6), + ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 4, 10), + ([1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 5, 10), + ] + for lst, elem, exp in cases: + assert bisect_right(lst, elem) == exp From noreply at buildbot.pypy.org Mon Feb 16 15:10:37 2015 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 16 Feb 2015 15:10:37 +0100 (CET) Subject: [pypy-commit] pypy dtrace-support: start writing dtrace support for basic events Message-ID: <20150216141037.BD56F1C06B9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: dtrace-support Changeset: r75919:1c6e04d35be2 Date: 2015-02-16 16:09 +0200 http://bitbucket.org/pypy/pypy/changeset/1c6e04d35be2/ Log: start writing dtrace support for basic events diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py --- a/rpython/config/translationoption.py +++ b/rpython/config/translationoption.py @@ -189,6 +189,9 @@ BoolOption("lldebug0", "If true, makes an lldebug0 build", default=False, cmdline="--lldebug0"), + BoolOption("dtrace", + "If true, enable emitting dtrace debug probes", default=False, + cmdline='--enable-dtrace'), OptionDescription("backendopt", "Backend Optimization Options", [ # control inlining diff --git a/rpython/translator/c/database.py b/rpython/translator/c/database.py --- a/rpython/translator/c/database.py +++ b/rpython/translator/c/database.py @@ -16,6 +16,7 @@ from rpython.translator.c.extfunc import do_the_getting from rpython.translator.c import gc from rpython.tool.identity_dict import identity_dict +from rpython.flowspace.model import Constant class NoCorrespondingNode(Exception): @@ -47,6 +48,7 @@ self.containerstats = {} self.externalfuncs = {} self.helper2ptr = {} + self.debug_nodes = set() # late_initializations is for when the value you want to # assign to a constant object is something C doesn't think is @@ -232,6 +234,13 @@ else: raise Exception("don't know about %r" % (obj,)) + def seen_debug_start(self, op): + arg = op.args[0] + if not isinstance(arg, Constant): + return + name = ''.join(arg.value.chars) + self.debug_nodes.add(name) + def complete(self, show_progress=True): assert not self.completed if self.translator and self.translator.rtyper: diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -792,6 +792,7 @@ return x def OP_DEBUG_START(self, op): + self.db.seen_debug_start(op) return self._op_debug('PYPY_DEBUG_START', op.args[0]) def OP_DEBUG_STOP(self, op): diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -246,8 +246,22 @@ self.extrafiles = self.eventually_copy(extra) self.gen_makefile(targetdir, exe_name=exe_name, headers_to_precompile=headers_to_precompile) + if self.config.translation.dtrace: + self._generate_dtrace_probe_file(db.debug_nodes) return cfile + def _generate_dtrace_probe_file(self, debug_nodes): + name = self.targetdir.join('pypy.p') + f = name.open('w') + f.write('provider pypy_probes {\n') + for debug_node in debug_nodes: + debug_node = debug_node.replace('-', '_') + f.write(' probe %s__start(void);' % debug_node) + f.write(' probe %s__done(void);' % debug_node) + f.write('};') + f.close() + # XXX run dtrace + def eventually_copy(self, cfiles): extrafiles = [] for fn in cfiles: diff --git a/rpython/translator/c/test/test_dtrace.py b/rpython/translator/c/test/test_dtrace.py new file mode 100644 --- /dev/null +++ b/rpython/translator/c/test/test_dtrace.py @@ -0,0 +1,19 @@ + +from rpython.translator.c.test.test_standalone import StandaloneTests +from rpython.rlib.debug import debug_start, debug_stop +from rpython.config.translationoption import get_combined_translation_config + +class TestDTrace(StandaloneTests): + config = get_combined_translation_config(translating=True) + config.translation.dtrace = True + + def test_dtrace_probes(self): + def f(argv): + debug_start("x") + for i in range(10000000): + pass + debug_stop("x") + return 0 + + _, cbuilder = self.compile(f) + cbuilder.cmdexec('') diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -1390,7 +1390,6 @@ and result.count('a') == 1 and result.count('d') == 6) - class TestShared(StandaloneTests): def test_entrypoint(self): From noreply at buildbot.pypy.org Mon Feb 16 16:32:53 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 16 Feb 2015 16:32:53 +0100 (CET) Subject: [pypy-commit] stmgc default: Backed out changeset: 60f7ccae893c Message-ID: <20150216153253.B182D1C034E@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1624:edb90028410a Date: 2015-02-16 15:53 +0100 http://bitbucket.org/pypy/stmgc/changeset/edb90028410a/ Log: Backed out changeset: 60f7ccae893c diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -96,9 +96,12 @@ object_t *_stm_allocate_old(ssize_t size_rounded_up) { - /* this is for tests, and for stm_setup_prebuilt() */ + /* only for tests xxx but stm_setup_prebuilt() uses this now too */ stm_char *p = allocate_outside_nursery_large(size_rounded_up); object_t *o = (object_t *)p; + + // sharing seg0 needs to be current: + assert(STM_SEGMENT->segment_num == 0); memset(REAL_ADDRESS(STM_SEGMENT->segment_base, o), 0, size_rounded_up); o->stm_flags = GCFLAG_WRITE_BARRIER; From noreply at buildbot.pypy.org Mon Feb 16 16:32:54 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 16 Feb 2015 16:32:54 +0100 (CET) Subject: [pypy-commit] stmgc default: Backed out changeset: 7540c1f155d0 Message-ID: <20150216153254.CA25E1C034E@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1625:689a9663d13f Date: 2015-02-16 15:55 +0100 http://bitbucket.org/pypy/stmgc/changeset/689a9663d13f/ Log: Backed out changeset: 7540c1f155d0 diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -566,13 +566,10 @@ struct stm_commit_log_entry_s *cl, *next; #ifndef NDEBUG - /* check that all segments are at the same revision (or not running - a transaction at all): */ + /* check that all segments are at the same revision: */ cl = get_priv_segment(0)->last_commit_log_entry; for (long i = 1; i < NB_SEGMENTS; i++) { - if (get_priv_segment(i)->transaction_state != TS_NONE) { - assert(get_priv_segment(i)->last_commit_log_entry == cl); - } + assert(get_priv_segment(i)->last_commit_log_entry == cl); } #endif From noreply at buildbot.pypy.org Mon Feb 16 16:32:55 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 16 Feb 2015 16:32:55 +0100 (CET) Subject: [pypy-commit] stmgc default: fix demo_random.c Message-ID: <20150216153255.C3BDD1C034E@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1626:1df0ed9de072 Date: 2015-02-16 16:34 +0100 http://bitbucket.org/pypy/stmgc/changeset/1df0ed9de072/ Log: fix demo_random.c diff --git a/c8/demo/demo_random.c b/c8/demo/demo_random.c --- a/c8/demo/demo_random.c +++ b/c8/demo/demo_random.c @@ -9,12 +9,12 @@ #include "stmgc.h" -#define NUMTHREADS 3 +#define NUMTHREADS 4 #define STEPS_PER_THREAD 500 #define THREAD_STARTS 1000 // how many restarts of threads #define PREBUILT_ROOTS 3 #define MAXROOTS 1000 -#define FORKS 3 +#define FORKS 0 // SUPPORT struct node_s; @@ -438,7 +438,7 @@ .next = NULL }; - stm_start_inevitable_transaction(&stm_thread_local); + //stm_start_inevitable_transaction(&stm_thread_local); for (i = 0; i < PREBUILT_ROOTS; i++) { void* new_templ = malloc(sizeof(struct node_s)); memcpy(new_templ, &prebuilt_template, sizeof(struct node_s)); @@ -451,7 +451,7 @@ ((nodeptr_t)prebuilt_roots[i])->my_hash = hash; } } - stm_commit_transaction(); + //stm_commit_transaction(); } int main(void) @@ -470,10 +470,11 @@ stm_setup(); + setup_globals(); + stm_register_thread_local(&stm_thread_local); stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf); - setup_globals(); int thread_starts = NUMTHREADS * THREAD_STARTS; for (i = 0; i < NUMTHREADS; i++) { From noreply at buildbot.pypy.org Mon Feb 16 16:32:56 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 16 Feb 2015 16:32:56 +0100 (CET) Subject: [pypy-commit] stmgc default: fix for on-major-gc validation not doing anything when a transaction thought it was still uncommitted and inevitable Message-ID: <20150216153256.BFC2E1C034E@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1627:30791bd74ef2 Date: 2015-02-16 16:34 +0100 http://bitbucket.org/pypy/stmgc/changeset/30791bd74ef2/ Log: fix for on-major-gc validation not doing anything when a transaction thought it was still uncommitted and inevitable diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -179,7 +179,7 @@ /* increment_total_allocated(4096); */ if (copy_from_segnum == -1) { - /* this page is only accessible in the sharing segment so far (new + /* this page is only accessible in the sharing segment seg0 so far (new allocation). We can thus simply mark it accessible here. */ pagecopy(get_virtual_page(my_segnum, pagenum), get_virtual_page(0, pagenum)); @@ -288,7 +288,8 @@ { /* returns true if we reached a valid state, or false if we need to abort now */ - dprintf(("_stm_validate()\n")); + dprintf(("_stm_validate() at cl=%p, rev=%lu\n", STM_PSEGMENT->last_commit_log_entry, + STM_PSEGMENT->last_commit_log_entry->rev_num)); /* go from last known entry in commit log to the most current one and apply all changes done by other transactions. Abort if we have read one of @@ -388,6 +389,7 @@ !needs_abort); /* if we abort, we still want to copy everything */ } + dprintf(("_stm_validate() to cl=%p, rev=%lu\n", cl, cl->rev_num)); /* last fully validated entry */ STM_PSEGMENT->last_commit_log_entry = cl; if (cl == last_cl) @@ -531,6 +533,9 @@ _validate_and_attach(new); } + STM_PSEGMENT->transaction_state = TS_NONE; + STM_PSEGMENT->safe_point = SP_NO_TRANSACTION; + acquire_modification_lock(STM_SEGMENT->segment_num); list_clear(STM_PSEGMENT->modified_old_objects); STM_PSEGMENT->last_commit_log_entry = new; @@ -843,6 +848,7 @@ push_new_objects_to_other_segments(); /* push before validate. otherwise they are reachable too early */ + bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE; _validate_and_add_to_commit_log(); invoke_and_clear_user_callbacks(0); /* for commit */ @@ -854,7 +860,7 @@ stm_rewind_jmp_forget(STM_SEGMENT->running_thread); - if (globally_unique_transaction && STM_PSEGMENT->transaction_state == TS_INEVITABLE) { + if (globally_unique_transaction && was_inev) { committed_globally_unique_transaction(); } diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -100,7 +100,9 @@ stm_char *p = allocate_outside_nursery_large(size_rounded_up); object_t *o = (object_t *)p; - // sharing seg0 needs to be current: + /* Sharing seg0 needs to be current, because in core.c handle_segfault_in_page, + we depend on simply copying the page from seg0 if it was never accessed by + anyone so far (we only run in seg1 <= seg < NB_SEGMENT). */ assert(STM_SEGMENT->segment_num == 0); memset(REAL_ADDRESS(STM_SEGMENT->segment_base, o), 0, size_rounded_up); o->stm_flags = GCFLAG_WRITE_BARRIER; From noreply at buildbot.pypy.org Mon Feb 16 18:51:47 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 16 Feb 2015 18:51:47 +0100 (CET) Subject: [pypy-commit] pypy py3.3: Fixes Message-ID: <20150216175147.683A01C1155@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3.3 Changeset: r75921:36e9b041cbb7 Date: 2015-02-16 18:46 +0100 http://bitbucket.org/pypy/pypy/changeset/36e9b041cbb7/ Log: Fixes diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py --- a/pypy/module/_ssl/test/test_ssl.py +++ b/pypy/module/_ssl/test/test_ssl.py @@ -411,6 +411,7 @@ cls.w_keycert = cls.space.wrap(str(tmpfile)) def test_str(self): + import _ssl # The str() of a SSLError doesn't include the errno e = _ssl.SSLError(1, "foo") assert str(e) == "foo" diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -335,7 +335,7 @@ "Send a signal to a thread." ret = c_pthread_kill(tid, signum) if widen(ret) < 0: - raise exception_from_errno(space, space.w_OSError) + raise exception_from_saved_errno(space, space.w_OSError) # the signal may have been send to the current thread space.getexecutioncontext().checksignals() @@ -379,7 +379,7 @@ with lltype.scoped_alloc(rffi.INTP.TO, 1) as signum_ptr: ret = c_sigwait(sigset, signum_ptr) if ret != 0: - raise exception_from_errno(space, space.w_OSError) + raise exception_from_saved_errno(space, space.w_OSError) signum = signum_ptr[0] return space.wrap(signum) @@ -387,7 +387,7 @@ with lltype.scoped_alloc(c_sigset_t.TO) as mask: ret = c_sigpending(mask) if ret != 0: - raise exception_from_errno(space, space.w_OSError) + raise exception_from_saved_errno(space, space.w_OSError) return _sigset_to_signals(space, mask) @unwrap_spec(how=int) @@ -396,7 +396,7 @@ with lltype.scoped_alloc(c_sigset_t.TO) as previous: ret = c_pthread_sigmask(how, sigset, previous) if ret != 0: - raise exception_from_errno(space, space.w_OSError) + raise exception_from_saved_errno(space, space.w_OSError) # if signals was unblocked, signal handlers have been called space.getexecutioncontext().checksignals() return _sigset_to_signals(space, previous) diff --git a/rpython/rlib/rsignal.py b/rpython/rlib/rsignal.py --- a/rpython/rlib/rsignal.py +++ b/rpython/rlib/rsignal.py @@ -102,7 +102,8 @@ save_err=rffi.RFFI_SAVE_ERRNO) c_getitimer = external('getitimer', [rffi.INT, itimervalP], rffi.INT) -c_pthread_kill = external('pthread_kill', [lltype.Signed, rffi.INT], rffi.INT) +c_pthread_kill = external('pthread_kill', [lltype.Signed, rffi.INT], rffi.INT, + save_err=rffi.RFFI_SAVE_ERRNO) if sys.platform != 'win32': c_sigset_t = rffi.COpaquePtr('sigset_t', compilation_info=eci) @@ -110,7 +111,10 @@ c_sigaddset = external('sigaddset', [c_sigset_t, rffi.INT], rffi.INT) c_sigismember = external('sigismember', [c_sigset_t, rffi.INT], rffi.INT) c_sigwait = external('sigwait', [c_sigset_t, rffi.INTP], rffi.INT, - releasegil=True) - c_sigpending = external('sigpending', [c_sigset_t], rffi.INT) + releasegil=True, + save_err=rffi.RFFI_SAVE_ERRNO) + c_sigpending = external('sigpending', [c_sigset_t], rffi.INT, + save_err=rffi.RFFI_SAVE_ERRNO) c_pthread_sigmask = external('pthread_sigmask', - [rffi.INT, c_sigset_t, c_sigset_t], rffi.INT) + [rffi.INT, c_sigset_t, c_sigset_t], rffi.INT, + save_err=rffi.RFFI_SAVE_ERRNO) From noreply at buildbot.pypy.org Mon Feb 16 18:51:46 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 16 Feb 2015 18:51:46 +0100 (CET) Subject: [pypy-commit] pypy py3.3: hg merge py3k Message-ID: <20150216175146.3AF721C1155@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3.3 Changeset: r75920:2f8ea7e067df Date: 2015-02-16 15:41 +0100 http://bitbucket.org/pypy/pypy/changeset/2f8ea7e067df/ Log: hg merge py3k Lots of conflicts... diff too long, truncating to 2000 out of 50524 lines diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,10 @@ bin/pypy-c include/*.h +include/numpy/ lib_pypy/ctypes_config_cache/_[^_]*_*.py +libpypy-c.* +pypy-c pypy/_cache pypy/doc/*.html pypy/doc/config/*.html @@ -18,4 +21,5 @@ pypy/translator/c/src/dtoa.o pypy/translator/goal/pypy-c pypy/translator/goal/target*-c -release/ \ No newline at end of file +release/ +rpython/_cache/ diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -7,10 +7,7 @@ 9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm 20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0 -20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0 -0000000000000000000000000000000000000000 release-2.3.0 394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1 -32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 -0000000000000000000000000000000000000000 release-2.2=3.1 +10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -42,19 +42,19 @@ Amaury Forgeot d'Arc Samuele Pedroni Alex Gaynor + Brian Kearns + Matti Picus + Philip Jenvey Michael Hudson David Schneider - Matti Picus - Brian Kearns - Philip Jenvey Holger Krekel Christian Tismer Hakan Ardo Benjamin Peterson Manuel Jacob + Ronan Lamy Anders Chrigstrom Eric van Riet Paap - Ronan Lamy Wim Lavrijsen Richard Emslie Alexander Schremmer @@ -68,9 +68,9 @@ Camillo Bruni Laura Creighton Toon Verwaest + Romain Guillebert Leonardo Santagada Seo Sanghyeon - Romain Guillebert Justin Peel Ronny Pfannschmidt David Edelsohn @@ -91,15 +91,16 @@ Michal Bendowski Jan de Mooij stian + Tyler Wade Michael Foord Stephan Diehl - Tyler Wade Stefan Schwarzer Valentino Volonghi Tomek Meka Patrick Maupin Bob Ippolito Bruno Gola + David Malcolm Jean-Paul Calderone Timo Paulssen Squeaky @@ -108,18 +109,19 @@ Marius Gedminas Martin Matusiak Konstantin Lopuhin + Wenzhu Man John Witulski - Wenzhu Man + Laurence Tratt + Ivan Sichmann Freitas Greg Price Dario Bertini Mark Pearse Simon Cross - Ivan Sichmann Freitas Andreas Stührk + Stefano Rivera Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov - Stefano Rivera Paweł Piotr Przeradowski Paul deGrandis Ilya Osadchiy @@ -129,7 +131,6 @@ tav Taavi Burns Georg Brandl - Laurence Tratt Bert Freudenberg Stian Andreassen Wanja Saatkamp @@ -141,13 +142,12 @@ Jeremy Thurgood Rami Chowdhury Tobias Pape - David Malcolm Eugene Oden Henry Mason Vasily Kuznetsov Preston Timmons + David Ripton Jeff Terrace - David Ripton Dusty Phillips Lukas Renggli Guenter Jantzen @@ -166,13 +166,16 @@ Gintautas Miliauskas Michael Twomey Lucian Branescu Mihaila + Yichao Yu Gabriel Lavoie Olivier Dormond Jared Grubb Karl Bartel + Wouter van Heyst Brian Dorsey Victor Stinner Andrews Medina + anatoly techtonik Stuart Williams Jasper Schulz Christian Hudon @@ -182,12 +185,11 @@ Michael Cheng Justas Sadzevicius Gasper Zejn - anatoly techtonik Neil Shepperd + Stanislaw Halik Mikael Schönenberg Elmo M?ntynen Jonathan David Riehl - Stanislaw Halik Anders Qvist Corbin Simpson Chirag Jadwani @@ -196,10 +198,13 @@ Vincent Legoll Alan McIntyre Alexander Sedov + Attila Gobi Christopher Pope Christian Tismer Marc Abramowitz Dan Stromberg + Arjun Naik + Valentina Mukhamedzhanova Stefano Parmesan Alexis Daboville Jens-Uwe Mager @@ -213,8 +218,6 @@ Sylvain Thenault Nathan Taylor Vladimir Kryachko - Arjun Naik - Attila Gobi Jacek Generowicz Alejandro J. Cura Jacob Oscarson @@ -222,22 +225,23 @@ Ryan Gonzalez Ian Foote Kristjan Valur Jonsson + David Lievens Neil Blakey-Milner Lutz Paelike Lucio Torre Lars Wassermann - Valentina Mukhamedzhanova Henrik Vendelbo Dan Buch Miguel de Val Borro Artur Lisiecki Sergey Kishchenko - Yichao Yu Ignas Mikalajunas Christoph Gerum Martin Blais Lene Wagner Tomo Cocoa + Toni Mattis + Lucas Stadler roberto at goyle Yury V. Zaytsev Anna Katrina Dominguez @@ -265,23 +269,30 @@ Stephan Busemann Rafał Gałczyński Christian Muirhead + Berker Peksag James Lan shoma hosaka - Daniel Neuh?user - Matthew Miller + Daniel Neuhäuser + Ben Mather + halgari + Boglarka Vezer + Chris Pressey Buck Golemon Konrad Delong Dinu Gherman Chris Lambacher coolbutuseless at gmail.com + Jim Baker Rodrigo Araújo - Jim Baker + Nikolaos-Digenis Karagiannis James Robert Armin Ronacher Brett Cannon + Donald Stufft yrttyr aliceinwire OlivierBlanvillain + Dan Sanders Zooko Wilcox-O Hearn Tomer Chachamu Christopher Groskopf @@ -295,6 +306,7 @@ Markus Unterwaditzer Even Wiik Thomassen jbs + squeaky soareschen Kurt Griffiths Mike Bayer @@ -306,6 +318,7 @@ Anna Ravencroft Dan Crosta Julien Phalip + Roman Podoliaka Dan Loewenherz Heinrich-Heine University, Germany diff --git a/lib-python/2.7/CGIHTTPServer.py b/lib-python/2.7/CGIHTTPServer.py --- a/lib-python/2.7/CGIHTTPServer.py +++ b/lib-python/2.7/CGIHTTPServer.py @@ -106,16 +106,16 @@ def run_cgi(self): """Execute a CGI script.""" dir, rest = self.cgi_info - - i = rest.find('/') + path = dir + '/' + rest + i = path.find('/', len(dir)+1) while i >= 0: - nextdir = rest[:i] - nextrest = rest[i+1:] + nextdir = path[:i] + nextrest = path[i+1:] scriptdir = self.translate_path(nextdir) if os.path.isdir(scriptdir): dir, rest = nextdir, nextrest - i = rest.find('/') + i = path.find('/', len(dir)+1) else: break diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py --- a/lib-python/2.7/Cookie.py +++ b/lib-python/2.7/Cookie.py @@ -56,7 +56,7 @@ >>> C = Cookie.SmartCookie() [Note: Long-time users of Cookie.py will remember using -Cookie.Cookie() to create an Cookie object. Although deprecated, it +Cookie.Cookie() to create a Cookie object. Although deprecated, it is still supported by the code. See the Backward Compatibility notes for more information.] @@ -426,6 +426,8 @@ "version" : "Version", } + _flags = {'secure', 'httponly'} + def __init__(self): # Set defaults self.key = self.value = self.coded_value = None @@ -529,9 +531,11 @@ _LegalCharsPatt = r"[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=]" _CookiePattern = re.compile( r"(?x)" # This is a Verbose pattern + r"\s*" # Optional whitespace at start of cookie r"(?P" # Start of group 'key' ""+ _LegalCharsPatt +"+?" # Any word of at least one letter, nongreedy r")" # End of group 'key' + r"(" # Optional group: there may not be a value. r"\s*=\s*" # Equal Sign r"(?P" # Start of group 'val' r'"(?:[^\\"]|\\.)*"' # Any doublequoted string @@ -540,7 +544,9 @@ r"|" # or ""+ _LegalCharsPatt +"*" # Any word or empty string r")" # End of group 'val' - r"\s*;?" # Probably ending in a semi-colon + r")?" # End of optional value group + r"\s*" # Any number of spaces. + r"(\s+|;|$)" # Ending either at space, semicolon, or EOS. ) @@ -585,8 +591,12 @@ def __setitem__(self, key, value): """Dictionary style assignment.""" - rval, cval = self.value_encode(value) - self.__set(key, rval, cval) + if isinstance(value, Morsel): + # allow assignment of constructed Morsels (e.g. for pickling) + dict.__setitem__(self, key, value) + else: + rval, cval = self.value_encode(value) + self.__set(key, rval, cval) # end __setitem__ def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"): @@ -641,7 +651,7 @@ while 0 <= i < n: # Start looking for a cookie - match = patt.search(str, i) + match = patt.match(str, i) if not match: break # No more cookies K,V = match.group("key"), match.group("val") @@ -656,8 +666,12 @@ M[ K[1:] ] = V elif K.lower() in Morsel._reserved: if M: - M[ K ] = _unquote(V) - else: + if V is None: + if K.lower() in Morsel._flags: + M[K] = True + else: + M[K] = _unquote(V) + elif V is not None: rval, cval = self.value_decode(V) self.__set(K, rval, cval) M = self[K] diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py --- a/lib-python/2.7/SocketServer.py +++ b/lib-python/2.7/SocketServer.py @@ -416,8 +416,12 @@ self.socket = socket.socket(self.address_family, self.socket_type) if bind_and_activate: - self.server_bind() - self.server_activate() + try: + self.server_bind() + self.server_activate() + except: + self.server_close() + raise def server_bind(self): """Called by constructor to bind the socket. diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py --- a/lib-python/2.7/_abcoll.py +++ b/lib-python/2.7/_abcoll.py @@ -143,7 +143,7 @@ methods except for __contains__, __iter__ and __len__. To override the comparisons (presumably for speed, as the - semantics are fixed), all you have to do is redefine __le__ and + semantics are fixed), redefine __le__ and __ge__, then the other operations will automatically follow suit. """ diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py --- a/lib-python/2.7/argparse.py +++ b/lib-python/2.7/argparse.py @@ -1089,7 +1089,14 @@ # parse all the remaining options into the namespace # store any unrecognized options on the object, so that the top # level parser can decide what to do with them - namespace, arg_strings = parser.parse_known_args(arg_strings, namespace) + + # In case this subparser defines new defaults, we parse them + # in a new namespace object and then update the original + # namespace for the relevant parts. + subnamespace, arg_strings = parser.parse_known_args(arg_strings, None) + for key, value in vars(subnamespace).items(): + setattr(namespace, key, value) + if arg_strings: vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, []) getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings) diff --git a/lib-python/2.7/asynchat.py b/lib-python/2.7/asynchat.py --- a/lib-python/2.7/asynchat.py +++ b/lib-python/2.7/asynchat.py @@ -46,12 +46,17 @@ you - by calling your self.found_terminator() method. """ +import asyncore +import errno import socket -import asyncore from collections import deque from sys import py3kwarning from warnings import filterwarnings, catch_warnings +_BLOCKING_IO_ERRORS = (errno.EAGAIN, errno.EALREADY, errno.EINPROGRESS, + errno.EWOULDBLOCK) + + class async_chat (asyncore.dispatcher): """This is an abstract class. You must derive from this class, and add the two methods collect_incoming_data() and found_terminator()""" @@ -109,6 +114,8 @@ try: data = self.recv (self.ac_in_buffer_size) except socket.error, why: + if why.args[0] in _BLOCKING_IO_ERRORS: + return self.handle_error() return diff --git a/lib-python/2.7/bsddb/test/test_queue.py b/lib-python/2.7/bsddb/test/test_queue.py --- a/lib-python/2.7/bsddb/test/test_queue.py +++ b/lib-python/2.7/bsddb/test/test_queue.py @@ -10,6 +10,7 @@ #---------------------------------------------------------------------- + at unittest.skip("fails on Windows; see issue 22943") class SimpleQueueTestCase(unittest.TestCase): def setUp(self): self.filename = get_new_database_path() diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py --- a/lib-python/2.7/collections.py +++ b/lib-python/2.7/collections.py @@ -17,6 +17,10 @@ except ImportError: assert '__pypy__' not in _sys.builtin_module_names newdict = lambda _ : {} +try: + from __pypy__ import reversed_dict +except ImportError: + reversed_dict = lambda d: reversed(d.keys()) try: from thread import get_ident as _get_ident @@ -29,142 +33,35 @@ ################################################################################ class OrderedDict(dict): - 'Dictionary that remembers insertion order' - # An inherited dict maps keys to values. - # The inherited dict provides __getitem__, __len__, __contains__, and get. - # The remaining methods are order-aware. - # Big-O running times for all methods are the same as regular dictionaries. + '''Dictionary that remembers insertion order. - # The internal self.__map dict maps keys to links in a doubly linked list. - # The circular doubly linked list starts and ends with a sentinel element. - # The sentinel element never gets deleted (this simplifies the algorithm). - # Each link is stored as a list of length three: [PREV, NEXT, KEY]. + In PyPy all dicts are ordered anyway. This is mostly useful as a + placeholder to mean "this dict must be ordered even on CPython". - def __init__(self, *args, **kwds): - '''Initialize an ordered dictionary. The signature is the same as - regular dictionaries, but keyword arguments are not recommended because - their insertion order is arbitrary. - - ''' - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__root - except AttributeError: - self.__root = root = [] # sentinel node - root[:] = [root, root, None] - self.__map = {} - self.__update(*args, **kwds) - - def __setitem__(self, key, value, dict_setitem=dict.__setitem__): - 'od.__setitem__(i, y) <==> od[i]=y' - # Setting a new item creates a new link at the end of the linked list, - # and the inherited dictionary is updated with the new key/value pair. - if key not in self: - root = self.__root - last = root[0] - last[1] = root[0] = self.__map[key] = [last, root, key] - return dict_setitem(self, key, value) - - def __delitem__(self, key, dict_delitem=dict.__delitem__): - 'od.__delitem__(y) <==> del od[y]' - # Deleting an existing item uses self.__map to find the link which gets - # removed by updating the links in the predecessor and successor nodes. - dict_delitem(self, key) - link_prev, link_next, _ = self.__map.pop(key) - link_prev[1] = link_next # update link_prev[NEXT] - link_next[0] = link_prev # update link_next[PREV] - - def __iter__(self): - 'od.__iter__() <==> iter(od)' - # Traverse the linked list in order. - root = self.__root - curr = root[1] # start at the first node - while curr is not root: - yield curr[2] # yield the curr[KEY] - curr = curr[1] # move to next node + Known difference: iterating over an OrderedDict which is being + concurrently modified raises RuntimeError in PyPy. In CPython + instead we get some behavior that appears reasonable in some + cases but is nonsensical in other cases. This is officially + forbidden by the CPython docs, so we forbid it explicitly for now. + ''' def __reversed__(self): - 'od.__reversed__() <==> reversed(od)' - # Traverse the linked list in reverse order. - root = self.__root - curr = root[0] # start at the last node - while curr is not root: - yield curr[2] # yield the curr[KEY] - curr = curr[0] # move to previous node - - def clear(self): - 'od.clear() -> None. Remove all items from od.' - root = self.__root - root[:] = [root, root, None] - self.__map.clear() - dict.clear(self) - - # -- the following methods do not depend on the internal structure -- - - def keys(self): - 'od.keys() -> list of keys in od' - return list(self) - - def values(self): - 'od.values() -> list of values in od' - return [self[key] for key in self] - - def items(self): - 'od.items() -> list of (key, value) pairs in od' - return [(key, self[key]) for key in self] - - def iterkeys(self): - 'od.iterkeys() -> an iterator over the keys in od' - return iter(self) - - def itervalues(self): - 'od.itervalues -> an iterator over the values in od' - for k in self: - yield self[k] - - def iteritems(self): - 'od.iteritems -> an iterator over the (key, value) pairs in od' - for k in self: - yield (k, self[k]) - - update = MutableMapping.update - - __update = update # let subclasses override update without breaking __init__ - - __marker = object() - - def pop(self, key, default=__marker): - '''od.pop(k[,d]) -> v, remove specified key and return the corresponding - value. If key is not found, d is returned if given, otherwise KeyError - is raised. - - ''' - if key in self: - result = self[key] - del self[key] - return result - if default is self.__marker: - raise KeyError(key) - return default - - def setdefault(self, key, default=None): - 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' - if key in self: - return self[key] - self[key] = default - return default + return reversed_dict(self) def popitem(self, last=True): '''od.popitem() -> (k, v), return and remove a (key, value) pair. Pairs are returned in LIFO order if last is true or FIFO order if false. ''' - if not self: - raise KeyError('dictionary is empty') - key = next(reversed(self) if last else iter(self)) - value = self.pop(key) - return key, value + if last: + return dict.popitem(self) + else: + it = dict.__iter__(self) + try: + k = it.next() + except StopIteration: + raise KeyError('dictionary is empty') + return (k, self.pop(k)) def __repr__(self, _repr_running={}): 'od.__repr__() <==> repr(od)' @@ -183,8 +80,6 @@ 'Return state information for pickling' items = [[k, self[k]] for k in self] inst_dict = vars(self).copy() - for k in vars(OrderedDict()): - inst_dict.pop(k, None) if inst_dict: return (self.__class__, (items,), inst_dict) return self.__class__, (items,) @@ -193,17 +88,6 @@ 'od.copy() -> a shallow copy of od' return self.__class__(self) - @classmethod - def fromkeys(cls, iterable, value=None): - '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S. - If not specified, the value defaults to None. - - ''' - self = cls() - for key in iterable: - self[key] = value - return self - def __eq__(self, other): '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive while comparison to a regular mapping is order-insensitive. diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py --- a/lib-python/2.7/cookielib.py +++ b/lib-python/2.7/cookielib.py @@ -1719,12 +1719,12 @@ def __repr__(self): r = [] for cookie in self: r.append(repr(cookie)) - return "<%s[%s]>" % (self.__class__, ", ".join(r)) + return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r)) def __str__(self): r = [] for cookie in self: r.append(str(cookie)) - return "<%s[%s]>" % (self.__class__, ", ".join(r)) + return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r)) # derives from IOError for backwards-compatibility with Python 2.4.0 diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py --- a/lib-python/2.7/ctypes/test/test_frombuffer.py +++ b/lib-python/2.7/ctypes/test/test_frombuffer.py @@ -2,7 +2,6 @@ import array import gc import unittest -from ctypes.test import xfail class X(Structure): _fields_ = [("c_int", c_int)] @@ -11,7 +10,6 @@ self._init_called = True class Test(unittest.TestCase): - @xfail def test_fom_buffer(self): a = array.array("i", range(16)) x = (c_int * 16).from_buffer(a) @@ -34,10 +32,9 @@ del a; gc.collect(); gc.collect(); gc.collect() self.assertEqual(x[:], expected) - self.assertRaises(TypeError, + self.assertRaises((TypeError, ValueError), (c_char * 16).from_buffer, "a" * 16) - @xfail def test_fom_buffer_with_offset(self): a = array.array("i", range(16)) x = (c_int * 15).from_buffer(a, sizeof(c_int)) @@ -46,7 +43,6 @@ self.assertRaises(ValueError, lambda: (c_int * 16).from_buffer(a, sizeof(c_int))) self.assertRaises(ValueError, lambda: (c_int * 1).from_buffer(a, 16 * sizeof(c_int))) - @xfail def test_from_buffer_copy(self): a = array.array("i", range(16)) x = (c_int * 16).from_buffer_copy(a) @@ -71,7 +67,6 @@ x = (c_char * 16).from_buffer_copy("a" * 16) self.assertEqual(x[:], "a" * 16) - @xfail def test_fom_buffer_copy_with_offset(self): a = array.array("i", range(16)) x = (c_int * 15).from_buffer_copy(a, sizeof(c_int)) diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py --- a/lib-python/2.7/ctypes/test/test_pointers.py +++ b/lib-python/2.7/ctypes/test/test_pointers.py @@ -7,6 +7,8 @@ c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float] python_types = [int, int, int, int, int, long, int, long, long, long, float, float] +LargeNamedType = type('T' * 2 ** 25, (Structure,), {}) +large_string = 'T' * 2 ** 25 class PointersTestCase(unittest.TestCase): @@ -188,5 +190,11 @@ mth = WINFUNCTYPE(None)(42, "name", (), None) self.assertEqual(bool(mth), True) + def test_pointer_type_name(self): + self.assertTrue(POINTER(LargeNamedType)) + + def test_pointer_type_str_name(self): + self.assertTrue(POINTER(large_string)) + if __name__ == '__main__': unittest.main() diff --git a/lib-python/2.7/ctypes/test/test_python_api.py b/lib-python/2.7/ctypes/test/test_python_api.py --- a/lib-python/2.7/ctypes/test/test_python_api.py +++ b/lib-python/2.7/ctypes/test/test_python_api.py @@ -46,8 +46,8 @@ # This test is unreliable, because it is possible that code in # unittest changes the refcount of the '42' integer. So, it # is disabled by default. - @requires("refcount") def test_PyInt_Long(self): + requires("refcount") ref42 = grc(42) pythonapi.PyInt_FromLong.restype = py_object self.assertEqual(pythonapi.PyInt_FromLong(42), 42) diff --git a/lib-python/2.7/ctypes/test/test_win32.py b/lib-python/2.7/ctypes/test/test_win32.py --- a/lib-python/2.7/ctypes/test/test_win32.py +++ b/lib-python/2.7/ctypes/test/test_win32.py @@ -38,8 +38,11 @@ @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') class FunctionCallTestCase(unittest.TestCase): - @requires("SEH") + @unittest.skipUnless('MSC' in sys.version, "SEH only supported by MSC") + @unittest.skipIf(sys.executable.endswith('_d.exe'), + "SEH not enabled in debug builds") def test_SEH(self): + requires("SEH") # Call functions with invalid arguments, and make sure # that access violations are trapped and raise an # exception. @@ -87,9 +90,29 @@ dll = CDLL(_ctypes_test.__file__) - pt = POINT(10, 10) - rect = RECT(0, 0, 20, 20) - self.assertEqual(1, dll.PointInRect(byref(rect), pt)) + pt = POINT(15, 25) + left = c_long.in_dll(dll, 'left') + top = c_long.in_dll(dll, 'top') + right = c_long.in_dll(dll, 'right') + bottom = c_long.in_dll(dll, 'bottom') + rect = RECT(left, top, right, bottom) + PointInRect = dll.PointInRect + PointInRect.argtypes = [POINTER(RECT), POINT] + self.assertEqual(1, PointInRect(byref(rect), pt)) + + ReturnRect = dll.ReturnRect + ReturnRect.argtypes = [c_int, RECT, POINTER(RECT), POINT, RECT, + POINTER(RECT), POINT, RECT] + ReturnRect.restype = RECT + for i in range(4): + ret = ReturnRect(i, rect, pointer(rect), pt, rect, + byref(rect), pt, rect) + # the c function will check and modify ret if something is + # passed in improperly + self.assertEqual(ret.left, left.value) + self.assertEqual(ret.right, right.value) + self.assertEqual(ret.top, top.value) + self.assertEqual(ret.bottom, bottom.value) if __name__ == '__main__': unittest.main() diff --git a/lib-python/2.7/decimal.py b/lib-python/2.7/decimal.py --- a/lib-python/2.7/decimal.py +++ b/lib-python/2.7/decimal.py @@ -136,7 +136,6 @@ __version__ = '1.70' # Highest version of the spec this complies with -import copy as _copy import math as _math import numbers as _numbers @@ -3665,6 +3664,8 @@ if self._is_special: sign = _format_sign(self._sign, spec) body = str(self.copy_abs()) + if spec['type'] == '%': + body += '%' return _format_align(sign, body, spec) # a type of None defaults to 'g' or 'G', depending on context @@ -6033,7 +6034,10 @@ format_dict['decimal_point'] = '.' # record whether return type should be str or unicode - format_dict['unicode'] = isinstance(format_spec, unicode) + try: + format_dict['unicode'] = isinstance(format_spec, unicode) + except NameError: + format_dict['unicode'] = False return format_dict diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py --- a/lib-python/2.7/distutils/__init__.py +++ b/lib-python/2.7/distutils/__init__.py @@ -15,5 +15,5 @@ # Updated automatically by the Python release process. # #--start constants-- -__version__ = "2.7.8" +__version__ = "2.7.9" #--end constants-- diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py --- a/lib-python/2.7/distutils/command/build_ext.py +++ b/lib-python/2.7/distutils/command/build_ext.py @@ -245,7 +245,7 @@ # Python's library directory must be appended to library_dirs # See Issues: #1600860, #4366 if (sysconfig.get_config_var('Py_ENABLE_SHARED')): - if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): + if not sysconfig.python_build: # building third party extensions self.library_dirs.append(sysconfig.get_config_var('LIBDIR')) else: diff --git a/lib-python/2.7/distutils/command/upload.py b/lib-python/2.7/distutils/command/upload.py --- a/lib-python/2.7/distutils/command/upload.py +++ b/lib-python/2.7/distutils/command/upload.py @@ -136,8 +136,8 @@ # Build up the MIME payload for the POST data boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254' - sep_boundary = '\n--' + boundary - end_boundary = sep_boundary + '--' + sep_boundary = '\r\n--' + boundary + end_boundary = sep_boundary + '--\r\n' body = StringIO.StringIO() for key, value in data.items(): # handle multiple entries for the same name @@ -151,14 +151,13 @@ fn = "" body.write(sep_boundary) - body.write('\nContent-Disposition: form-data; name="%s"'%key) + body.write('\r\nContent-Disposition: form-data; name="%s"' % key) body.write(fn) - body.write("\n\n") + body.write("\r\n\r\n") body.write(value) if value and value[-1] == '\r': body.write('\n') # write an extra newline (lurve Macs) body.write(end_boundary) - body.write("\n") body = body.getvalue() self.announce("Submitting %s to %s" % (filename, self.repository), log.INFO) diff --git a/lib-python/2.7/distutils/file_util.py b/lib-python/2.7/distutils/file_util.py --- a/lib-python/2.7/distutils/file_util.py +++ b/lib-python/2.7/distutils/file_util.py @@ -85,7 +85,8 @@ (os.symlink) instead of copying: set it to "hard" or "sym"; if it is None (the default), files are copied. Don't set 'link' on systems that don't support it: 'copy_file()' doesn't check if hard or symbolic - linking is available. + linking is available. If hardlink fails, falls back to + _copy_file_contents(). Under Mac OS, uses the native file copy function in macostools; on other systems, uses '_copy_file_contents()' to copy file contents. @@ -137,24 +138,31 @@ # (Unix only, of course, but that's the caller's responsibility) if link == 'hard': if not (os.path.exists(dst) and os.path.samefile(src, dst)): - os.link(src, dst) + try: + os.link(src, dst) + return (dst, 1) + except OSError: + # If hard linking fails, fall back on copying file + # (some special filesystems don't support hard linking + # even under Unix, see issue #8876). + pass elif link == 'sym': if not (os.path.exists(dst) and os.path.samefile(src, dst)): os.symlink(src, dst) + return (dst, 1) # Otherwise (non-Mac, not linking), copy the file contents and # (optionally) copy the times and mode. - else: - _copy_file_contents(src, dst) - if preserve_mode or preserve_times: - st = os.stat(src) + _copy_file_contents(src, dst) + if preserve_mode or preserve_times: + st = os.stat(src) - # According to David Ascher , utime() should be done - # before chmod() (at least under NT). - if preserve_times: - os.utime(dst, (st[ST_ATIME], st[ST_MTIME])) - if preserve_mode: - os.chmod(dst, S_IMODE(st[ST_MODE])) + # According to David Ascher , utime() should be done + # before chmod() (at least under NT). + if preserve_times: + os.utime(dst, (st[ST_ATIME], st[ST_MTIME])) + if preserve_mode: + os.chmod(dst, S_IMODE(st[ST_MODE])) return (dst, 1) diff --git a/lib-python/2.7/distutils/sysconfig_cpython.py b/lib-python/2.7/distutils/sysconfig_cpython.py --- a/lib-python/2.7/distutils/sysconfig_cpython.py +++ b/lib-python/2.7/distutils/sysconfig_cpython.py @@ -165,7 +165,8 @@ # version and build tools may not support the same set # of CPU architectures for universal builds. global _config_vars - if not _config_vars.get('CUSTOMIZED_OSX_COMPILER', ''): + # Use get_config_var() to ensure _config_vars is initialized. + if not get_config_var('CUSTOMIZED_OSX_COMPILER'): import _osx_support _osx_support.customize_compiler(_config_vars) _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True' diff --git a/lib-python/2.7/distutils/tests/test_bdist_rpm.py b/lib-python/2.7/distutils/tests/test_bdist_rpm.py --- a/lib-python/2.7/distutils/tests/test_bdist_rpm.py +++ b/lib-python/2.7/distutils/tests/test_bdist_rpm.py @@ -25,6 +25,7 @@ """ class BuildRpmTestCase(support.TempdirManager, + support.EnvironGuard, support.LoggingSilencer, unittest.TestCase): @@ -50,6 +51,7 @@ def test_quiet(self): # let's create a package tmp_dir = self.mkdtemp() + os.environ['HOME'] = tmp_dir # to confine dir '.rpmdb' creation pkg_dir = os.path.join(tmp_dir, 'foo') os.mkdir(pkg_dir) self.write_file((pkg_dir, 'setup.py'), SETUP_PY) @@ -92,6 +94,7 @@ def test_no_optimize_flag(self): # let's create a package that brakes bdist_rpm tmp_dir = self.mkdtemp() + os.environ['HOME'] = tmp_dir # to confine dir '.rpmdb' creation pkg_dir = os.path.join(tmp_dir, 'foo') os.mkdir(pkg_dir) self.write_file((pkg_dir, 'setup.py'), SETUP_PY) diff --git a/lib-python/2.7/distutils/tests/test_dist.py b/lib-python/2.7/distutils/tests/test_dist.py --- a/lib-python/2.7/distutils/tests/test_dist.py +++ b/lib-python/2.7/distutils/tests/test_dist.py @@ -11,7 +11,7 @@ from distutils.dist import Distribution, fix_help_options from distutils.cmd import Command import distutils.dist -from test.test_support import TESTFN, captured_stdout, run_unittest +from test.test_support import TESTFN, captured_stdout, run_unittest, unlink from distutils.tests import support @@ -64,6 +64,7 @@ with open(TESTFN, "w") as f: f.write("[global]\n") f.write("command_packages = foo.bar, splat") + self.addCleanup(unlink, TESTFN) files = [TESTFN] sys.argv.append("build") diff --git a/lib-python/2.7/distutils/tests/test_file_util.py b/lib-python/2.7/distutils/tests/test_file_util.py --- a/lib-python/2.7/distutils/tests/test_file_util.py +++ b/lib-python/2.7/distutils/tests/test_file_util.py @@ -8,6 +8,11 @@ from distutils.tests import support from test.test_support import run_unittest + +requires_os_link = unittest.skipUnless(hasattr(os, "link"), + "test requires os.link()") + + class FileUtilTestCase(support.TempdirManager, unittest.TestCase): def _log(self, msg, *args): @@ -74,6 +79,44 @@ copy_file(foo, dst_dir) self.assertTrue(os.path.exists(os.path.join(dst_dir, 'foo'))) + @requires_os_link + def test_copy_file_hard_link(self): + with open(self.source, 'w') as f: + f.write('some content') + st = os.stat(self.source) + copy_file(self.source, self.target, link='hard') + st2 = os.stat(self.source) + st3 = os.stat(self.target) + self.assertTrue(os.path.samestat(st, st2), (st, st2)) + self.assertTrue(os.path.samestat(st2, st3), (st2, st3)) + with open(self.source, 'r') as f: + self.assertEqual(f.read(), 'some content') + + @requires_os_link + def test_copy_file_hard_link_failure(self): + # If hard linking fails, copy_file() falls back on copying file + # (some special filesystems don't support hard linking even under + # Unix, see issue #8876). + with open(self.source, 'w') as f: + f.write('some content') + st = os.stat(self.source) + def _os_link(*args): + raise OSError(0, "linking unsupported") + old_link = os.link + os.link = _os_link + try: + copy_file(self.source, self.target, link='hard') + finally: + os.link = old_link + st2 = os.stat(self.source) + st3 = os.stat(self.target) + self.assertTrue(os.path.samestat(st, st2), (st, st2)) + self.assertFalse(os.path.samestat(st2, st3), (st2, st3)) + for fn in (self.source, self.target): + with open(fn, 'r') as f: + self.assertEqual(f.read(), 'some content') + + def test_suite(): return unittest.makeSuite(FileUtilTestCase) diff --git a/lib-python/2.7/distutils/tests/test_sysconfig.py b/lib-python/2.7/distutils/tests/test_sysconfig.py --- a/lib-python/2.7/distutils/tests/test_sysconfig.py +++ b/lib-python/2.7/distutils/tests/test_sysconfig.py @@ -3,6 +3,9 @@ import test import unittest import shutil +import subprocess +import sys +import textwrap from distutils import sysconfig from distutils.tests import support @@ -99,6 +102,24 @@ self.assertEqual(global_sysconfig.get_config_var('LDSHARED'), sysconfig.get_config_var('LDSHARED')) self.assertEqual(global_sysconfig.get_config_var('CC'), sysconfig.get_config_var('CC')) + def test_customize_compiler_before_get_config_vars(self): + # Issue #21923: test that a Distribution compiler + # instance can be called without an explicit call to + # get_config_vars(). + with open(TESTFN, 'w') as f: + f.writelines(textwrap.dedent('''\ + from distutils.core import Distribution + config = Distribution().get_command_obj('config') + # try_compile may pass or it may fail if no compiler + # is found but it should not raise an exception. + rc = config.try_compile('int x;') + ''')) + p = subprocess.Popen([str(sys.executable), TESTFN], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True) + outs, errs = p.communicate() + self.assertEqual(0, p.returncode, "Subprocess failed: " + outs) def test_suite(): diff --git a/lib-python/2.7/distutils/tests/test_upload.py b/lib-python/2.7/distutils/tests/test_upload.py --- a/lib-python/2.7/distutils/tests/test_upload.py +++ b/lib-python/2.7/distutils/tests/test_upload.py @@ -119,7 +119,7 @@ # what did we send ? self.assertIn('dédé', self.last_open.req.data) headers = dict(self.last_open.req.headers) - self.assertEqual(headers['Content-length'], '2085') + self.assertEqual(headers['Content-length'], '2159') self.assertTrue(headers['Content-type'].startswith('multipart/form-data')) self.assertEqual(self.last_open.req.get_method(), 'POST') self.assertEqual(self.last_open.req.get_full_url(), diff --git a/lib-python/2.7/doctest.py b/lib-python/2.7/doctest.py --- a/lib-python/2.7/doctest.py +++ b/lib-python/2.7/doctest.py @@ -216,7 +216,7 @@ # get_data() opens files as 'rb', so one must do the equivalent # conversion as universal newlines would do. return file_contents.replace(os.linesep, '\n'), filename - with open(filename) as f: + with open(filename, 'U') as f: return f.read(), filename # Use sys.stdout encoding for ouput. diff --git a/lib-python/2.7/email/feedparser.py b/lib-python/2.7/email/feedparser.py --- a/lib-python/2.7/email/feedparser.py +++ b/lib-python/2.7/email/feedparser.py @@ -49,8 +49,8 @@ simple abstraction -- it parses until EOF closes the current message. """ def __init__(self): - # The last partial line pushed into this object. - self._partial = '' + # Chunks of the last partial line pushed into this object. + self._partial = [] # The list of full, pushed lines, in reverse order self._lines = [] # The stack of false-EOF checking predicates. @@ -66,8 +66,8 @@ def close(self): # Don't forget any trailing partial line. - self._lines.append(self._partial) - self._partial = '' + self.pushlines(''.join(self._partial).splitlines(True)) + self._partial = [] self._closed = True def readline(self): @@ -95,8 +95,29 @@ def push(self, data): """Push some new data into this object.""" - # Handle any previous leftovers - data, self._partial = self._partial + data, '' + # Crack into lines, but preserve the linesep characters on the end of each + parts = data.splitlines(True) + + if not parts or not parts[0].endswith(('\n', '\r')): + # No new complete lines, so just accumulate partials + self._partial += parts + return + + if self._partial: + # If there are previous leftovers, complete them now + self._partial.append(parts[0]) + parts[0:1] = ''.join(self._partial).splitlines(True) + del self._partial[:] + + # If the last element of the list does not end in a newline, then treat + # it as a partial line. We only check for '\n' here because a line + # ending with '\r' might be a line that was split in the middle of a + # '\r\n' sequence (see bugs 1555570 and 1721862). + if not parts[-1].endswith('\n'): + self._partial = [parts.pop()] + self.pushlines(parts) + + def pushlines(self, lines): # Crack into lines, but preserve the newlines on the end of each parts = NLCRE_crack.split(data) # The *ahem* interesting behaviour of re.split when supplied grouping diff --git a/lib-python/2.7/email/mime/nonmultipart.py b/lib-python/2.7/email/mime/nonmultipart.py --- a/lib-python/2.7/email/mime/nonmultipart.py +++ b/lib-python/2.7/email/mime/nonmultipart.py @@ -12,7 +12,7 @@ class MIMENonMultipart(MIMEBase): - """Base class for MIME multipart/* type messages.""" + """Base class for MIME non-multipart type messages.""" def attach(self, payload): # The public API prohibits attaching multiple subparts to MIMEBase diff --git a/lib-python/2.7/email/test/test_email.py b/lib-python/2.7/email/test/test_email.py --- a/lib-python/2.7/email/test/test_email.py +++ b/lib-python/2.7/email/test/test_email.py @@ -11,6 +11,7 @@ import warnings import textwrap from cStringIO import StringIO +from random import choice import email @@ -2578,16 +2579,64 @@ bsf.push(il) nt += n n1 = 0 - while True: - ol = bsf.readline() - if ol == NeedMoreData: - break + for ol in iter(bsf.readline, NeedMoreData): om.append(ol) n1 += 1 self.assertEqual(n, n1) self.assertEqual(len(om), nt) self.assertEqual(''.join([il for il, n in imt]), ''.join(om)) + def test_push_random(self): + from email.feedparser import BufferedSubFile, NeedMoreData + + n = 10000 + chunksize = 5 + chars = 'abcd \t\r\n' + + s = ''.join(choice(chars) for i in range(n)) + '\n' + target = s.splitlines(True) + + bsf = BufferedSubFile() + lines = [] + for i in range(0, len(s), chunksize): + chunk = s[i:i+chunksize] + bsf.push(chunk) + lines.extend(iter(bsf.readline, NeedMoreData)) + self.assertEqual(lines, target) + + +class TestFeedParsers(TestEmailBase): + + def parse(self, chunks): + from email.feedparser import FeedParser + feedparser = FeedParser() + for chunk in chunks: + feedparser.feed(chunk) + return feedparser.close() + + def test_newlines(self): + m = self.parse(['a:\nb:\rc:\r\nd:\n']) + self.assertEqual(m.keys(), ['a', 'b', 'c', 'd']) + m = self.parse(['a:\nb:\rc:\r\nd:']) + self.assertEqual(m.keys(), ['a', 'b', 'c', 'd']) + m = self.parse(['a:\rb', 'c:\n']) + self.assertEqual(m.keys(), ['a', 'bc']) + m = self.parse(['a:\r', 'b:\n']) + self.assertEqual(m.keys(), ['a', 'b']) + m = self.parse(['a:\r', '\nb:\n']) + self.assertEqual(m.keys(), ['a', 'b']) + + def test_long_lines(self): + # Expected peak memory use on 32-bit platform: 4*N*M bytes. + M, N = 1000, 20000 + m = self.parse(['a:b\n\n'] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', 'b')]) + self.assertEqual(m.get_payload(), 'x'*M*N) + m = self.parse(['a:b\r\r'] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', 'b')]) + self.assertEqual(m.get_payload(), 'x'*M*N) + m = self.parse(['a:\r', 'b: '] + ['x'*M] * N) + self.assertEqual(m.items(), [('a', ''), ('b', 'x'*M*N)]) class TestParsers(TestEmailBase): @@ -3180,7 +3229,6 @@ self.assertEqual(res, '=?iso-8859-2?q?abc?=') self.assertIsInstance(res, str) - # Test RFC 2231 header parameters (en/de)coding class TestRFC2231(TestEmailBase): def test_get_param(self): diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/__init__.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python2 +from __future__ import print_function + +import os +import os.path +import pkgutil +import shutil +import sys +import tempfile + + +__all__ = ["version", "bootstrap"] + + +_SETUPTOOLS_VERSION = "7.0" + +_PIP_VERSION = "1.5.6" + +# pip currently requires ssl support, so we try to provide a nicer +# error message when that is missing (http://bugs.python.org/issue19744) +_MISSING_SSL_MESSAGE = ("pip {} requires SSL/TLS".format(_PIP_VERSION)) +try: + import ssl +except ImportError: + ssl = None + + def _require_ssl_for_pip(): + raise RuntimeError(_MISSING_SSL_MESSAGE) +else: + def _require_ssl_for_pip(): + pass + +_PROJECTS = [ + ("setuptools", _SETUPTOOLS_VERSION), + ("pip", _PIP_VERSION), +] + + +def _run_pip(args, additional_paths=None): + # Add our bundled software to the sys.path so we can import it + if additional_paths is not None: + sys.path = additional_paths + sys.path + + # Install the bundled software + import pip + pip.main(args) + + +def version(): + """ + Returns a string specifying the bundled version of pip. + """ + return _PIP_VERSION + + +def _disable_pip_configuration_settings(): + # We deliberately ignore all pip environment variables + # when invoking pip + # See http://bugs.python.org/issue19734 for details + keys_to_remove = [k for k in os.environ if k.startswith("PIP_")] + for k in keys_to_remove: + del os.environ[k] + # We also ignore the settings in the default pip configuration file + # See http://bugs.python.org/issue20053 for details + os.environ['PIP_CONFIG_FILE'] = os.devnull + + +def bootstrap(root=None, upgrade=False, user=False, + altinstall=False, default_pip=True, + verbosity=0): + """ + Bootstrap pip into the current Python installation (or the given root + directory). + + Note that calling this function will alter both sys.path and os.environ. + """ + if altinstall and default_pip: + raise ValueError("Cannot use altinstall and default_pip together") + + _require_ssl_for_pip() + _disable_pip_configuration_settings() + + # By default, installing pip and setuptools installs all of the + # following scripts (X.Y == running Python version): + # + # pip, pipX, pipX.Y, easy_install, easy_install-X.Y + # + # pip 1.5+ allows ensurepip to request that some of those be left out + if altinstall: + # omit pip, pipX and easy_install + os.environ["ENSUREPIP_OPTIONS"] = "altinstall" + elif not default_pip: + # omit pip and easy_install + os.environ["ENSUREPIP_OPTIONS"] = "install" + + tmpdir = tempfile.mkdtemp() + try: + # Put our bundled wheels into a temporary directory and construct the + # additional paths that need added to sys.path + additional_paths = [] + for project, version in _PROJECTS: + wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version) + whl = pkgutil.get_data( + "ensurepip", + "_bundled/{}".format(wheel_name), + ) + with open(os.path.join(tmpdir, wheel_name), "wb") as fp: + fp.write(whl) + + additional_paths.append(os.path.join(tmpdir, wheel_name)) + + # Construct the arguments to be passed to the pip command + args = ["install", "--no-index", "--find-links", tmpdir] + if root: + args += ["--root", root] + if upgrade: + args += ["--upgrade"] + if user: + args += ["--user"] + if verbosity: + args += ["-" + "v" * verbosity] + + _run_pip(args + [p[0] for p in _PROJECTS], additional_paths) + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def _uninstall_helper(verbosity=0): + """Helper to support a clean default uninstall process on Windows + + Note that calling this function may alter os.environ. + """ + # Nothing to do if pip was never installed, or has been removed + try: + import pip + except ImportError: + return + + # If the pip version doesn't match the bundled one, leave it alone + if pip.__version__ != _PIP_VERSION: + msg = ("ensurepip will only uninstall a matching version " + "({!r} installed, {!r} bundled)") + print(msg.format(pip.__version__, _PIP_VERSION), file=sys.stderr) + return + + _require_ssl_for_pip() + _disable_pip_configuration_settings() + + # Construct the arguments to be passed to the pip command + args = ["uninstall", "-y"] + if verbosity: + args += ["-" + "v" * verbosity] + + _run_pip(args + [p[0] for p in reversed(_PROJECTS)]) + + +def _main(argv=None): + if ssl is None: + print("Ignoring ensurepip failure: {}".format(_MISSING_SSL_MESSAGE), + file=sys.stderr) + return + + import argparse + parser = argparse.ArgumentParser(prog="python -m ensurepip") + parser.add_argument( + "--version", + action="version", + version="pip {}".format(version()), + help="Show the version of pip that is bundled with this Python.", + ) + parser.add_argument( + "-v", "--verbose", + action="count", + default=0, + dest="verbosity", + help=("Give more output. Option is additive, and can be used up to 3 " + "times."), + ) + parser.add_argument( + "-U", "--upgrade", + action="store_true", + default=False, + help="Upgrade pip and dependencies, even if already installed.", + ) + parser.add_argument( + "--user", + action="store_true", + default=False, + help="Install using the user scheme.", + ) + parser.add_argument( + "--root", + default=None, + help="Install everything relative to this alternate root directory.", + ) + parser.add_argument( + "--altinstall", + action="store_true", + default=False, + help=("Make an alternate install, installing only the X.Y versioned" + "scripts (Default: pipX, pipX.Y, easy_install-X.Y)"), + ) + parser.add_argument( + "--default-pip", + action="store_true", + default=True, + dest="default_pip", + help=argparse.SUPPRESS, + ) + parser.add_argument( + "--no-default-pip", + action="store_false", + dest="default_pip", + help=("Make a non default install, installing only the X and X.Y " + "versioned scripts."), + ) + + args = parser.parse_args(argv) + + bootstrap( + root=args.root, + upgrade=args.upgrade, + user=args.user, + verbosity=args.verbosity, + altinstall=args.altinstall, + default_pip=args.default_pip, + ) diff --git a/lib-python/2.7/ensurepip/__main__.py b/lib-python/2.7/ensurepip/__main__.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/__main__.py @@ -0,0 +1,4 @@ +import ensurepip + +if __name__ == "__main__": + ensurepip._main() diff --git a/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..097ab43430d4c1302b0be353a8c16407c370693b GIT binary patch [cut] diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..fa1d1054da1dab98f8906555d31a9fda271b3a85 GIT binary patch [cut] diff --git a/lib-python/2.7/ensurepip/_uninstall.py b/lib-python/2.7/ensurepip/_uninstall.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/ensurepip/_uninstall.py @@ -0,0 +1,30 @@ +"""Basic pip uninstallation support, helper for the Windows uninstaller""" + +import argparse +import ensurepip + + +def _main(argv=None): + parser = argparse.ArgumentParser(prog="python -m ensurepip._uninstall") + parser.add_argument( + "--version", + action="version", + version="pip {}".format(ensurepip.version()), + help="Show the version of pip this will attempt to uninstall.", + ) + parser.add_argument( + "-v", "--verbose", + action="count", + default=0, + dest="verbosity", + help=("Give more output. Option is additive, and can be used up to 3 " + "times."), + ) + + args = parser.parse_args(argv) + + ensurepip._uninstall_helper(verbosity=args.verbosity) + + +if __name__ == "__main__": + _main() diff --git a/lib-python/2.7/glob.py b/lib-python/2.7/glob.py --- a/lib-python/2.7/glob.py +++ b/lib-python/2.7/glob.py @@ -35,11 +35,16 @@ patterns. """ + dirname, basename = os.path.split(pathname) if not has_magic(pathname): - if os.path.lexists(pathname): - yield pathname + if basename: + if os.path.lexists(pathname): + yield pathname + else: + # Patterns ending with a slash should match only directories + if os.path.isdir(dirname): + yield pathname return - dirname, basename = os.path.split(pathname) if not dirname: for name in glob1(os.curdir, basename): yield name diff --git a/lib-python/2.7/gzip.py b/lib-python/2.7/gzip.py --- a/lib-python/2.7/gzip.py +++ b/lib-python/2.7/gzip.py @@ -164,9 +164,16 @@ def _write_gzip_header(self): self.fileobj.write('\037\213') # magic header self.fileobj.write('\010') # compression method - fname = os.path.basename(self.name) - if fname.endswith(".gz"): - fname = fname[:-3] + try: + # RFC 1952 requires the FNAME field to be Latin-1. Do not + # include filenames that cannot be represented that way. + fname = os.path.basename(self.name) + if not isinstance(fname, str): + fname = fname.encode('latin-1') + if fname.endswith('.gz'): + fname = fname[:-3] + except UnicodeEncodeError: + fname = '' flags = 0 if fname: flags = FNAME diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py --- a/lib-python/2.7/hashlib.py +++ b/lib-python/2.7/hashlib.py @@ -15,8 +15,9 @@ md5(), sha1(), sha224(), sha256(), sha384(), and sha512() -More algorithms may be available on your platform but the above are -guaranteed to exist. +More algorithms may be available on your platform but the above are guaranteed +to exist. See the algorithms_guaranteed and algorithms_available attributes +to find out what algorithm names can be passed to new(). NOTE: If you want the adler32 or crc32 hash functions they are available in the zlib module. @@ -58,9 +59,14 @@ # always available algorithm is added. __always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512') +algorithms_guaranteed = set(__always_supported) +algorithms_available = set(__always_supported) + algorithms = __always_supported -__all__ = __always_supported + ('new', 'algorithms', 'pbkdf2_hmac') +__all__ = __always_supported + ('new', 'algorithms_guaranteed', + 'algorithms_available', 'algorithms', + 'pbkdf2_hmac') def __get_builtin_constructor(name): @@ -128,6 +134,8 @@ import _hashlib new = __hash_new __get_hash = __get_openssl_constructor + algorithms_available = algorithms_available.union( + _hashlib.openssl_md_meth_names) except ImportError: new = __py_new __get_hash = __get_builtin_constructor diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py --- a/lib-python/2.7/httplib.py +++ b/lib-python/2.7/httplib.py @@ -215,6 +215,10 @@ # maximal line length when calling readline(). _MAXLINE = 65536 +# maximum amount of headers accepted +_MAXHEADERS = 100 + + class HTTPMessage(mimetools.Message): def addheader(self, key, value): @@ -271,6 +275,8 @@ elif self.seekable: tell = self.fp.tell while True: + if len(hlist) > _MAXHEADERS: + raise HTTPException("got more than %d headers" % _MAXHEADERS) if tell: try: startofline = tell() @@ -1185,21 +1191,29 @@ def __init__(self, host, port=None, key_file=None, cert_file=None, strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - source_address=None): + source_address=None, context=None): HTTPConnection.__init__(self, host, port, strict, timeout, source_address) self.key_file = key_file self.cert_file = cert_file + if context is None: + context = ssl._create_default_https_context() + if key_file or cert_file: + context.load_cert_chain(cert_file, key_file) + self._context = context def connect(self): "Connect to a host on a given (SSL) port." - sock = self._create_connection((self.host, self.port), - self.timeout, self.source_address) + HTTPConnection.connect(self) + if self._tunnel_host: - self.sock = sock - self._tunnel() - self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file) + server_hostname = self._tunnel_host + else: + server_hostname = self.host + + self.sock = self._context.wrap_socket(self.sock, + server_hostname=server_hostname) __all__.append("HTTPSConnection") @@ -1214,14 +1228,15 @@ _connection_class = HTTPSConnection def __init__(self, host='', port=None, key_file=None, cert_file=None, - strict=None): + strict=None, context=None): # provide a default host, pass the X509 cert info # urf. compensate for bad input. if port == 0: port = None self._setup(self._connection_class(host, port, key_file, - cert_file, strict)) + cert_file, strict, + context=context)) # we never actually use these for anything, but we keep them # here for compatibility with post-1.5.2 CVS. diff --git a/lib-python/2.7/idlelib/Bindings.py b/lib-python/2.7/idlelib/Bindings.py --- a/lib-python/2.7/idlelib/Bindings.py +++ b/lib-python/2.7/idlelib/Bindings.py @@ -75,7 +75,8 @@ ('!_Auto-open Stack Viewer', '<>'), ]), ('options', [ - ('_Configure IDLE...', '<>'), + ('Configure _IDLE', '<>'), + ('Configure _Extensions', '<>'), None, ]), ('help', [ diff --git a/lib-python/2.7/idlelib/CallTipWindow.py b/lib-python/2.7/idlelib/CallTipWindow.py --- a/lib-python/2.7/idlelib/CallTipWindow.py +++ b/lib-python/2.7/idlelib/CallTipWindow.py @@ -2,9 +2,8 @@ After ToolTip.py, which uses ideas gleaned from PySol Used by the CallTips IDLE extension. - """ -from Tkinter import * +from Tkinter import Toplevel, Label, LEFT, SOLID, TclError HIDE_VIRTUAL_EVENT_NAME = "<>" HIDE_SEQUENCES = ("", "") @@ -133,35 +132,28 @@ return bool(self.tipwindow) -def _calltip_window(parent): - root = Tk() - root.title("Test calltips") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) +def _calltip_window(parent): # htest # + from Tkinter import Toplevel, Text, LEFT, BOTH - class MyEditWin: # comparenceptually an editor_window - def __init__(self): - text = self.text = Text(root) - text.pack(side=LEFT, fill=BOTH, expand=1) - text.insert("insert", "string.split") - root.update() - self.calltip = CallTip(text) + top = Toplevel(parent) + top.title("Test calltips") + top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, + parent.winfo_rooty() + 150)) + text = Text(top) + text.pack(side=LEFT, fill=BOTH, expand=1) + text.insert("insert", "string.split") + top.update() + calltip = CallTip(text) - text.event_add("<>", "(") - text.event_add("<>", ")") - text.bind("<>", self.calltip_show) - text.bind("<>", self.calltip_hide) - - text.focus_set() - root.mainloop() - - def calltip_show(self, event): - self.calltip.showtip("Hello world", "insert", "end") - - def calltip_hide(self, event): - self.calltip.hidetip() - - editwin = MyEditWin() + def calltip_show(event): + calltip.showtip("(s=Hello world)", "insert", "end") + def calltip_hide(event): + calltip.hidetip() + text.event_add("<>", "(") + text.event_add("<>", ")") + text.bind("<>", calltip_show) + text.bind("<>", calltip_hide) + text.focus_set() if __name__=='__main__': from idlelib.idle_test.htest import run diff --git a/lib-python/2.7/idlelib/ClassBrowser.py b/lib-python/2.7/idlelib/ClassBrowser.py --- a/lib-python/2.7/idlelib/ClassBrowser.py +++ b/lib-python/2.7/idlelib/ClassBrowser.py @@ -19,6 +19,9 @@ from idlelib.TreeWidget import TreeNode, TreeItem, ScrolledCanvas from idlelib.configHandler import idleConf +file_open = None # Method...Item and Class...Item use this. +# Normally PyShell.flist.open, but there is no PyShell.flist for htest. + class ClassBrowser: def __init__(self, flist, name, path, _htest=False): @@ -27,6 +30,9 @@ """ _htest - bool, change box when location running htest. """ + global file_open + if not _htest: + file_open = PyShell.flist.open self.name = name self.file = os.path.join(path[0], self.name + ".py") self._htest = _htest @@ -101,7 +107,7 @@ return [] try: dict = pyclbr.readmodule_ex(name, [dir] + sys.path) - except ImportError, msg: + except ImportError: return [] items = [] self.classes = {} @@ -170,7 +176,7 @@ def OnDoubleClick(self): if not os.path.exists(self.file): return - edit = PyShell.flist.open(self.file) + edit = file_open(self.file) if hasattr(self.cl, 'lineno'): lineno = self.cl.lineno edit.gotoline(lineno) @@ -206,7 +212,7 @@ def OnDoubleClick(self): if not os.path.exists(self.file): return - edit = PyShell.flist.open(self.file) + edit = file_open(self.file) edit.gotoline(self.cl.methods[self.name]) def _class_browser(parent): #Wrapper for htest @@ -221,8 +227,9 @@ dir, file = os.path.split(file) name = os.path.splitext(file)[0] flist = PyShell.PyShellFileList(parent) + global file_open + file_open = flist.open ClassBrowser(flist, name, [dir], _htest=True) - parent.mainloop() if __name__ == "__main__": from idlelib.idle_test.htest import run diff --git a/lib-python/2.7/idlelib/ColorDelegator.py b/lib-python/2.7/idlelib/ColorDelegator.py --- a/lib-python/2.7/idlelib/ColorDelegator.py +++ b/lib-python/2.7/idlelib/ColorDelegator.py @@ -2,7 +2,6 @@ import re import keyword import __builtin__ -from Tkinter import * from idlelib.Delegator import Delegator from idlelib.configHandler import idleConf @@ -34,7 +33,6 @@ prog = re.compile(make_pat(), re.S) idprog = re.compile(r"\s+(\w+)", re.S) -asprog = re.compile(r".*?\b(as)\b") class ColorDelegator(Delegator): @@ -42,7 +40,6 @@ Delegator.__init__(self) self.prog = prog self.idprog = idprog - self.asprog = asprog self.LoadTagDefs() def setdelegate(self, delegate): @@ -74,7 +71,6 @@ "DEFINITION": idleConf.GetHighlight(theme, "definition"), "SYNC": {'background':None,'foreground':None}, "TODO": {'background':None,'foreground':None}, - "BREAK": idleConf.GetHighlight(theme, "break"), "ERROR": idleConf.GetHighlight(theme, "error"), # The following is used by ReplaceDialog: "hit": idleConf.GetHighlight(theme, "hit"), @@ -216,22 +212,6 @@ self.tag_add("DEFINITION", head + "+%dc" % a, head + "+%dc" % b) - elif value == "import": - # color all the "as" words on same line, except - # if in a comment; cheap approximation to the - # truth - if '#' in chars: - endpos = chars.index('#') - else: - endpos = len(chars) - while True: - m1 = self.asprog.match(chars, b, endpos) - if not m1: - break - a, b = m1.span(1) - self.tag_add("KEYWORD", - head + "+%dc" % a, - head + "+%dc" % b) m = self.prog.search(chars, m.end()) if "SYNC" in self.tag_names(next + "-1c"): head = next @@ -255,20 +235,23 @@ for tag in self.tagdefs.keys(): self.tag_remove(tag, "1.0", "end") -def _color_delegator(parent): +def _color_delegator(parent): # htest # + from Tkinter import Toplevel, Text from idlelib.Percolator import Percolator - root = Tk() - root.title("Test ColorDelegator") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) - source = "if somename: x = 'abc' # comment\nprint" - text = Text(root, background="white") + + top = Toplevel(parent) + top.title("Test ColorDelegator") + top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, + parent.winfo_rooty() + 150)) + source = "if somename: x = 'abc' # comment\nprint\n" + text = Text(top, background="white") + text.pack(expand=1, fill="both") text.insert("insert", source) - text.pack(expand=1, fill="both") + text.focus_set() + p = Percolator(text) d = ColorDelegator() p.insertfilter(d) - root.mainloop() if __name__ == "__main__": from idlelib.idle_test.htest import run diff --git a/lib-python/2.7/idlelib/Debugger.py b/lib-python/2.7/idlelib/Debugger.py --- a/lib-python/2.7/idlelib/Debugger.py +++ b/lib-python/2.7/idlelib/Debugger.py @@ -1,6 +1,5 @@ import os import bdb -import types from Tkinter import * from idlelib.WindowList import ListedToplevel from idlelib.ScrolledList import ScrolledList diff --git a/lib-python/2.7/idlelib/EditorWindow.py b/lib-python/2.7/idlelib/EditorWindow.py From noreply at buildbot.pypy.org Mon Feb 16 20:41:49 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Mon, 16 Feb 2015 20:41:49 +0100 (CET) Subject: [pypy-commit] pypy framestate: handle interaction of return with finally: in analyze_signals() Message-ID: <20150216194149.95DE81C1155@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: framestate Changeset: r75922:e6c7a78a35e5 Date: 2015-02-14 23:11 +0000 http://bitbucket.org/pypy/pypy/changeset/e6c7a78a35e5/ Log: handle interaction of return with finally: in analyze_signals() diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py --- a/rpython/flowspace/bytecode.py +++ b/rpython/flowspace/bytecode.py @@ -618,8 +618,7 @@ block.operations.append(POP_BLOCK(offset=self.offset)) elif isinstance(context, FinallyBlock): block.operations.append(POP_BLOCK(offset=self.offset)) - reader.splice_finally_handler(block, context) - block = context.handler_end + block = reader.splice_finally_handler(block, context) else: # LoopBlock reader.blockstack.append(context) block.set_exits([self.target]) @@ -645,9 +644,20 @@ class RETURN(NullaryOpcode): num = name = 'RETURN' arg = NO_ARG + def do_signals(self, reader): + block = reader.curr_block + assert block.operations[-1] is self + del block.operations[-1] + from rpython.flowspace.flowcontext import FinallyBlock + while reader.blockstack: + context = reader.blockstack.pop() + block.operations.append(POP_BLOCK(offset=self.offset)) + if isinstance(context, FinallyBlock): + block = reader.splice_finally_handler(block, context) + block.operations.append(self) + def eval(self, ctx): - from rpython.flowspace.flowcontext import Return - raise Return() + ctx.do_return() @bc_reader.register_opcode class END_FINALLY(NullaryOpcode): diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py --- a/rpython/flowspace/flowcontext.py +++ b/rpython/flowspace/flowcontext.py @@ -1012,9 +1012,6 @@ class Return(FlowSignal): """Signals a 'return' statement. """ - def nomoreblocks(self, ctx): - ctx.do_return() - @property def args(self): return [] From noreply at buildbot.pypy.org Mon Feb 16 20:41:56 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Mon, 16 Feb 2015 20:41:56 +0100 (CET) Subject: [pypy-commit] pypy framestate: deal with unreachable END_FINALLY Message-ID: <20150216194156.1485D1C1155@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: framestate Changeset: r75927:feaa2a862f65 Date: 2015-02-16 19:21 +0000 http://bitbucket.org/pypy/pypy/changeset/feaa2a862f65/ Log: deal with unreachable END_FINALLY diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py --- a/rpython/flowspace/bytecode.py +++ b/rpython/flowspace/bytecode.py @@ -235,7 +235,10 @@ def splice_finally_handler(self, block, context): cell = [] + copied = {} def copy_block(handler): + if handler in copied: + return copied[handler] b = handler.copy() if handler is context.handler_end: instr = b.operations.pop() @@ -244,13 +247,17 @@ else: b.set_exits([copy_block(child) for child in handler._exits]) self.blocks.append(b) + copied[handler] = b return b block.set_exits([copy_block(context.handler)]) - copy_of_handler_end, = cell - return copy_of_handler_end + if cell: + copy_of_handler_end, = cell + return copy_of_handler_end + else: # END_FINALLY is unreachable + return None def check_graph(self): - for b in self.blocks: + for b in self.graph.iterblocks(): if not b._exits: instr = b.operations[-1] assert instr.name in ( @@ -670,7 +677,9 @@ block = reader.splice_finally_handler(block, context.block) assert len(block.operations) == 1 block.operations = [PUSH_NONE()] + block.operations + [POP_TOP()] - block.operations.append(self) + if block is not None: + block.operations.append(self) + block.set_exits([]) def eval(self, ctx): ctx.do_return() diff --git a/rpython/flowspace/test/test_bytecode.py b/rpython/flowspace/test/test_bytecode.py --- a/rpython/flowspace/test/test_bytecode.py +++ b/rpython/flowspace/test/test_bytecode.py @@ -12,7 +12,7 @@ else: return 0 bc_graph = make_graph(f) - assert [lst[0].offset for lst in bc_graph.dump()] == [0, 6, 10, 14] + assert [lst[0].offset for lst in bc_graph.dump()] == [0, 6, 10] assert bc_graph.dump()[0][0] == bc_reader.new_instr('LOAD_FAST', 0) def test_blockstack(): diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py --- a/rpython/flowspace/test/test_objspace.py +++ b/rpython/flowspace/test/test_objspace.py @@ -263,6 +263,19 @@ def test_finallys(self): x = self.codetest(self.finallys) + def test_branching_in_finally(self): + def f(x, y): + try: + return x + finally: + if x: + x = 0 + if y > 0: + y -= 1 + return y + self.codetest(f) + + #__________________________________________________________ def const_pow(): return 2 ** 5 From noreply at buildbot.pypy.org Mon Feb 16 20:41:51 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Mon, 16 Feb 2015 20:41:51 +0100 (CET) Subject: [pypy-commit] pypy framestate: delete unused FlowSignals Message-ID: <20150216194151.10BB21C1155@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: framestate Changeset: r75923:dae9c828b3d3 Date: 2015-02-15 02:40 +0000 http://bitbucket.org/pypy/pypy/changeset/dae9c828b3d3/ Log: delete unused FlowSignals diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py --- a/rpython/flowspace/bytecode.py +++ b/rpython/flowspace/bytecode.py @@ -138,7 +138,6 @@ except KeyError: return GenericOpcode(self.opnames[opnum], opnum, arg, offset) - def _iter_instr(self, code): self.offset = 0 i = 0 diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py --- a/rpython/flowspace/flowcontext.py +++ b/rpython/flowspace/flowcontext.py @@ -1010,17 +1010,6 @@ return type(other) is type(self) and other.args == self.args -class Return(FlowSignal): - """Signals a 'return' statement. """ - @property - def args(self): - return [] - - @staticmethod - def rebuild(): - return Return() - - class Raise(FlowSignal): """Signals an application-level exception (i.e. an OperationException).""" @@ -1061,35 +1050,6 @@ raise StopFlowing -class Break(FlowSignal): - """Signals a 'break' statement.""" - - @property - def args(self): - return [] - - @staticmethod - def rebuild(): - return Break.singleton - -Break.singleton = Break() - -class Continue(FlowSignal): - """Signals a 'continue' statement. - Argument is the bytecode position of the beginning of the loop.""" - - def __init__(self, jump_to): - self.jump_to = jump_to - - @property - def args(self): - return [const(self.jump_to)] - - @staticmethod - def rebuild(w_jump_to): - return Continue(w_jump_to.value) - - class FrameBlock(object): """Abstract base class for frame blocks from the blockstack, used by the SETUP_XXX and POP_BLOCK opcodes.""" @@ -1117,7 +1077,7 @@ class LoopBlock(FrameBlock): """A loop block. Stores the end-of-loop pointer in case of 'break'.""" - handles = (Break, Continue) + handles = type(None) class ExceptBlock(FrameBlock): """An try:except: block. Stores the position of the exception handler.""" diff --git a/rpython/flowspace/test/test_flowcontext.py b/rpython/flowspace/test/test_flowcontext.py --- a/rpython/flowspace/test/test_flowcontext.py +++ b/rpython/flowspace/test/test_flowcontext.py @@ -2,14 +2,11 @@ import pytest from rpython.flowspace.model import Variable, FSException from rpython.flowspace.flowcontext import ( - Return, Raise, RaiseImplicit, Continue, Break) + Raise, RaiseImplicit) @pytest.mark.parametrize('signal', [ - Return(), Raise(FSException(Variable(), Variable())), RaiseImplicit(FSException(Variable(), Variable())), - Break(), - Continue(42), ]) def test_signals(signal): assert signal.rebuild(*signal.args) == signal From noreply at buildbot.pypy.org Mon Feb 16 20:41:52 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Mon, 16 Feb 2015 20:41:52 +0100 (CET) Subject: [pypy-commit] pypy framestate: add 2 tests for with blocks Message-ID: <20150216194152.51B001C1155@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: framestate Changeset: r75924:42000e974b69 Date: 2015-02-15 04:19 +0000 http://bitbucket.org/pypy/pypy/changeset/42000e974b69/ Log: add 2 tests for with blocks diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py --- a/rpython/flowspace/test/test_objspace.py +++ b/rpython/flowspace/test/test_objspace.py @@ -960,6 +960,20 @@ 'simple_call': 4, # __enter__, g and 2 possible calls to __exit__ } + def test_return_in_with(self): + def f(x): + with x: + return 1 + self.codetest(f) + + def test_break_in_with(self): + def f(n, x): + for i in range(n): + with x: + break + return 1 + self.codetest(f) + def monkey_patch_code(self, code, stacksize, flags, codestring, names, varnames): c = code return types.CodeType(c.co_argcount, c.co_nlocals, stacksize, flags, From noreply at buildbot.pypy.org Mon Feb 16 20:41:53 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Mon, 16 Feb 2015 20:41:53 +0100 (CET) Subject: [pypy-commit] pypy framestate: Use SETUP_ instructions rather than FrameBlocks in do_signals() Message-ID: <20150216194153.880A41C1155@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: framestate Changeset: r75925:53b3547f5829 Date: 2015-02-15 17:55 +0000 http://bitbucket.org/pypy/pypy/changeset/53b3547f5829/ Log: Use SETUP_ instructions rather than FrameBlocks in do_signals() diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py --- a/rpython/flowspace/bytecode.py +++ b/rpython/flowspace/bytecode.py @@ -584,17 +584,15 @@ block = reader.curr_block assert block.operations[-1] is self del block.operations[-1] - from rpython.flowspace.flowcontext import ExceptBlock, FinallyBlock while reader.blockstack: context = reader.blockstack.pop() block.operations.append(POP_BLOCK(offset=self.offset)) - if isinstance(context, ExceptBlock): + if isinstance(context, SETUP_EXCEPT): pass - elif isinstance(context, FinallyBlock): - reader.splice_finally_handler(block, context) - block = context.handler_end + elif isinstance(context, (SETUP_WITH, SETUP_FINALLY)): + block = reader.splice_finally_handler(block, context.block) else: # LoopBlock - block.set_exits([context.handler]) + block.set_exits([context.target]) return raise BytecodeCorruption( "A break statement should not escape from the function") @@ -610,15 +608,14 @@ block = reader.curr_block assert block.operations[-1] is self del block.operations[-1] - from rpython.flowspace.flowcontext import ExceptBlock, FinallyBlock while reader.blockstack: context = reader.blockstack.pop() - if isinstance(context, ExceptBlock): + if isinstance(context, SETUP_EXCEPT): block.operations.append(POP_BLOCK(offset=self.offset)) - elif isinstance(context, FinallyBlock): + elif isinstance(context, (SETUP_FINALLY, SETUP_WITH)): block.operations.append(POP_BLOCK(offset=self.offset)) - block = reader.splice_finally_handler(block, context) - else: # LoopBlock + block = reader.splice_finally_handler(block, context.block) + else: # SETUP_LOOP reader.blockstack.append(context) block.set_exits([self.target]) return @@ -647,12 +644,11 @@ block = reader.curr_block assert block.operations[-1] is self del block.operations[-1] - from rpython.flowspace.flowcontext import FinallyBlock while reader.blockstack: context = reader.blockstack.pop() block.operations.append(POP_BLOCK(offset=self.offset)) - if isinstance(context, FinallyBlock): - block = reader.splice_finally_handler(block, context) + if isinstance(context, SETUP_FINALLY): + block = reader.splice_finally_handler(block, context.block) block.operations.append(self) def eval(self, ctx): @@ -706,7 +702,7 @@ def context_effect(self, reader): self.target.set_blockstack(reader.blockstack) - reader.blockstack.append(self.block) + reader.blockstack.append(self) def eval(self, ctx): self.block.stackdepth = ctx.stackdepth diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py --- a/rpython/flowspace/test/test_objspace.py +++ b/rpython/flowspace/test/test_objspace.py @@ -964,7 +964,9 @@ def f(x): with x: return 1 - self.codetest(f) + graph = self.codetest(f) + simplify_graph(graph) + assert self.all_operations(graph) == {'getattr': 2, 'simple_call': 2} def test_break_in_with(self): def f(n, x): From noreply at buildbot.pypy.org Mon Feb 16 20:41:54 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Mon, 16 Feb 2015 20:41:54 +0100 (CET) Subject: [pypy-commit] pypy framestate: put a dummy unroller on the stack for WITH_CLEANUP Message-ID: <20150216194154.BF1ED1C1155@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: framestate Changeset: r75926:3838b73fb409 Date: 2015-02-16 16:57 +0000 http://bitbucket.org/pypy/pypy/changeset/3838b73fb409/ Log: put a dummy unroller on the stack for WITH_CLEANUP diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py --- a/rpython/flowspace/bytecode.py +++ b/rpython/flowspace/bytecode.py @@ -12,6 +12,8 @@ CO_VARARGS = 0x0004 CO_VARKEYWORDS = 0x0008 +w_None = const(None) + def cpython_code_signature(code): "([list-of-arg-names], vararg-name-or-None, kwarg-name-or-None)." argcount = code.co_argcount @@ -589,8 +591,12 @@ block.operations.append(POP_BLOCK(offset=self.offset)) if isinstance(context, SETUP_EXCEPT): pass - elif isinstance(context, (SETUP_WITH, SETUP_FINALLY)): + elif isinstance(context, SETUP_FINALLY): block = reader.splice_finally_handler(block, context.block) + elif isinstance(context, SETUP_WITH): + block = reader.splice_finally_handler(block, context.block) + assert len(block.operations) == 1 + block.operations = [PUSH_NONE()] + block.operations + [POP_TOP()] else: # LoopBlock block.set_exits([context.target]) return @@ -612,9 +618,14 @@ context = reader.blockstack.pop() if isinstance(context, SETUP_EXCEPT): block.operations.append(POP_BLOCK(offset=self.offset)) - elif isinstance(context, (SETUP_FINALLY, SETUP_WITH)): + elif isinstance(context, SETUP_FINALLY): block.operations.append(POP_BLOCK(offset=self.offset)) block = reader.splice_finally_handler(block, context.block) + elif isinstance(context, SETUP_WITH): + block.operations.append(POP_BLOCK(offset=self.offset)) + block = reader.splice_finally_handler(block, context.block) + assert len(block.operations) == 1 + block.operations = [PUSH_NONE()] + block.operations + [POP_TOP()] else: # SETUP_LOOP reader.blockstack.append(context) block.set_exits([self.target]) @@ -637,6 +648,12 @@ w_value = ctx.popvalue() ctx.w_return_value = w_value +class PUSH_NONE(NullaryOpcode): + num = name = 'PUSH_NONE' + arg = NO_ARG + def eval(self, ctx): + ctx.pushvalue(w_None) + class RETURN(NullaryOpcode): num = name = 'RETURN' arg = NO_ARG @@ -649,6 +666,10 @@ block.operations.append(POP_BLOCK(offset=self.offset)) if isinstance(context, SETUP_FINALLY): block = reader.splice_finally_handler(block, context.block) + elif isinstance(context, SETUP_WITH): + block = reader.splice_finally_handler(block, context.block) + assert len(block.operations) == 1 + block.operations = [PUSH_NONE()] + block.operations + [POP_TOP()] block.operations.append(self) def eval(self, ctx): From noreply at buildbot.pypy.org Mon Feb 16 21:48:00 2015 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 16 Feb 2015 21:48:00 +0100 (CET) Subject: [pypy-commit] pypy py3.3: Fix typo in cffi Message-ID: <20150216204800.68F1C1C06B9@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3.3 Changeset: r75928:3c7542345bfa Date: 2015-02-16 21:47 +0100 http://bitbucket.org/pypy/pypy/changeset/3c7542345bfa/ Log: Fix typo in cffi diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py --- a/lib_pypy/cffi/verifier.py +++ b/lib_pypy/cffi/verifier.py @@ -4,7 +4,7 @@ if sys.version_info >= (3, 3): import importlib.machinery - def extension_suffixes(): + def _extension_suffixes(): return importlib.machinery.EXTENSION_SUFFIXES[:] else: import imp From noreply at buildbot.pypy.org Mon Feb 16 23:15:00 2015 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 16 Feb 2015 23:15:00 +0100 (CET) Subject: [pypy-commit] pypy default: test, fix load_dh_params get_saved_errno for win32 Message-ID: <20150216221500.1F8861C034E@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r75929:1bf192278b3f Date: 2015-02-17 00:11 +0200 http://bitbucket.org/pypy/pypy/changeset/1bf192278b3f/ Log: test, fix load_dh_params get_saved_errno for win32 diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py --- a/pypy/module/_ssl/test/test_ssl.py +++ b/pypy/module/_ssl/test/test_ssl.py @@ -361,12 +361,14 @@ assert ctx.cert_store_stats() == {'x509_ca': 0, 'crl': 0, 'x509': 1} def test_load_dh_params(self): - import _ssl + import _ssl, errno ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1) ctx.load_dh_params(self.dh512) raises(TypeError, ctx.load_dh_params) raises(TypeError, ctx.load_dh_params, None) raises(_ssl.SSLError, ctx.load_dh_params, self.keycert) + exc = raises(IOError, ctx.load_dh_params, "inexistent.pem") + assert exc.value.errno == errno.ENOENT def test_set_ecdh_curve(self): import _ssl diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py --- a/rpython/rlib/ropenssl.py +++ b/rpython/rlib/ropenssl.py @@ -5,7 +5,7 @@ from rpython.translator.platform import platform from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.rlib.unroll import unrolling_iterable -from rpython.rlib._rsocket_rffi import MAX_FD_SIZE, SAVE_ERR +from rpython.rlib._rsocket_rffi import SAVE_ERR if sys.platform == 'win32' and platform.name != 'mingw32': @@ -467,7 +467,7 @@ ssl_external('BIO_new', [BIO_METHOD], BIO) ssl_external('BIO_set_nbio', [BIO, rffi.INT], rffi.INT, macro=True) ssl_external('BIO_new_file', [rffi.CCHARP, rffi.CCHARP], BIO, - save_err=SAVE_ERR) + save_err=rffi.RFFI_FULL_ERRNO_ZERO) ssl_external('BIO_new_mem_buf', [rffi.VOIDP, rffi.INT], BIO) ssl_external('BIO_free', [BIO], rffi.INT) ssl_external('BIO_reset', [BIO], rffi.INT, macro=True) From noreply at buildbot.pypy.org Tue Feb 17 07:01:11 2015 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 17 Feb 2015 07:01:11 +0100 (CET) Subject: [pypy-commit] pypy default: import * masked unused import from 1bf192278b3f, fix Message-ID: <20150217060111.463D91C014B@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r75930:3ffd8f58f1b5 Date: 2015-02-17 08:01 +0200 http://bitbucket.org/pypy/pypy/changeset/3ffd8f58f1b5/ Log: import * masked unused import from 1bf192278b3f, fix diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -1,6 +1,8 @@ from rpython.rlib import rpoll, rsocket, rthread, rweakref from rpython.rlib.rarithmetic import intmask, widen, r_uint from rpython.rlib.ropenssl import * +from pypy.module._socket import interp_socket +from rpython.rlib._rsocket_rffi import MAX_FD_SIZE from rpython.rlib.rposix import get_saved_errno from rpython.rlib.rweakref import RWeakValueDictionary from rpython.rlib.objectmodel import specialize, compute_unique_id @@ -12,7 +14,6 @@ from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.module._ssl.ssl_data import ( LIBRARY_CODES_TO_NAMES, ERROR_CODES_TO_NAMES) -from pypy.module._socket import interp_socket # user defined constants From noreply at buildbot.pypy.org Tue Feb 17 14:08:34 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 17 Feb 2015 14:08:34 +0100 (CET) Subject: [pypy-commit] pypy nonquadratic-heapcache: start refactoring the heapcache to not be quadratic Message-ID: <20150217130834.D79C81C0455@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: nonquadratic-heapcache Changeset: r75932:8b248206d983 Date: 2015-02-17 09:01 +0100 http://bitbucket.org/pypy/pypy/changeset/8b248206d983/ Log: start refactoring the heapcache to not be quadratic diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py --- a/rpython/jit/metainterp/heapcache.py +++ b/rpython/jit/metainterp/heapcache.py @@ -1,14 +1,19 @@ from rpython.jit.metainterp.history import ConstInt from rpython.jit.metainterp.resoperation import rop +class HeapCacheValue(object): + def __init__(self, box): + self.box = box + self.known_class = False + self.nonstandard_virtualizable = False class HeapCache(object): def __init__(self): self.reset() def reset(self, reset_virtuals=True, trace_branch=True): - # contains boxes where the class is already known - self.known_class_boxes = {} + # maps boxes to values + self.values = {} # store the boxes that contain newly allocated objects, this maps the # boxes to a bool, the bool indicates whether or not the object has # escaped the trace or not (True means the box never escaped, False @@ -51,6 +56,12 @@ self.input_indirections = {} self.output_indirections = {} + def get_value(self, box): + value = self.values.get(box, None) + if not value: + value = self.values[box] = HeapCacheValue(box) + return value + def _input_indirection(self, box): return self.input_indirections.get(box, box) @@ -215,16 +226,22 @@ self.reset(reset_virtuals=False, trace_branch=False) def is_class_known(self, box): - return box in self.known_class_boxes + value = self.values.get(box, None) + if value: + return value.known_class + return False def class_now_known(self, box): - self.known_class_boxes[box] = None + self.get_value(box).known_class = True def is_nonstandard_virtualizable(self, box): - return box in self.nonstandard_virtualizables + value = self.values.get(box, None) + if value: + return value.nonstandard_virtualizable + return False def nonstandard_virtualizables_now_known(self, box): - self.nonstandard_virtualizables[box] = None + self.get_value(box).nonstandard_virtualizable = True def is_unescaped(self, box): return self.new_boxes.get(box, False) @@ -276,6 +293,7 @@ if frombox in self.new_boxes: new_d[frombox] = tobox new_d[box] = fieldbox + print len(self.new_boxes), len(d), len(new_d) return new_d def getarrayitem(self, box, indexbox, descr): From noreply at buildbot.pypy.org Tue Feb 17 14:08:33 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 17 Feb 2015 14:08:33 +0100 (CET) Subject: [pypy-commit] pypy global-cell-cache: nonsense, need to cache the cells if any Message-ID: <20150217130833.B9AFD1C0455@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: global-cell-cache Changeset: r75931:c68ac70e39f2 Date: 2015-02-06 13:56 +0100 http://bitbucket.org/pypy/pypy/changeset/c68ac70e39f2/ Log: nonsense, need to cache the cells if any diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py --- a/pypy/objspace/std/celldict.py +++ b/pypy/objspace/std/celldict.py @@ -199,7 +199,7 @@ result = entry_value else: # need to fill the cache - result = strategy.getitem_str( + result = strategy.getdictvalue_no_unwrapping( w_dict, frame.getname_u(nameindex)) entry = pycode._celldict_cache[nameindex] if entry is INVALID_CACHE_ENTRY: diff --git a/pypy/objspace/std/test/test_celldict.py b/pypy/objspace/std/test/test_celldict.py --- a/pypy/objspace/std/test/test_celldict.py +++ b/pypy/objspace/std/test/test_celldict.py @@ -1,6 +1,7 @@ import py -from pypy.objspace.std.celldict import ModuleDictStrategy +from pypy.objspace.std.celldict import ( + ModuleDictStrategy, init_celldict_cache, _finditem_with_cache) from pypy.objspace.std.dictmultiobject import W_DictMultiObject from pypy.objspace.std.test.test_dictmultiobject import ( BaseTestRDictImplementation, BaseTestDevolvedDictImplementation, FakeSpace, @@ -57,6 +58,71 @@ v3 = strategy.version assert v2 is v3 + def test_global_cache(self): + class FakePycode(object): + co_names_w = ['a', 'b'] + class FakeFrame(object): + def getname_u(self, i): + return FakePycode.co_names_w[i] + pycode = FakePycode() + frame = FakeFrame() + init_celldict_cache(pycode) + assert pycode._celldict_cache + + strategy = ModuleDictStrategy(space) + storage = strategy.get_empty_storage() + d = W_DictMultiObject(space, strategy, storage) + + v1 = strategy.version + + # fill cache + assert _finditem_with_cache(space, frame, 0, pycode, d, None, None) == None + assert pycode._celldict_cache[0].version is v1 + assert pycode._celldict_cache[0].value is None + assert pycode._celldict_cache[1].version is None + assert pycode._celldict_cache[1].value is None + # does not need the pycode any more + assert _finditem_with_cache(space, frame, 0, None, d, v1, None) == None + + # insert a key + key = "a" + w_key = self.FakeString(key) + d.setitem(w_key, 1) + v2 = strategy.version + + # fill cache + assert _finditem_with_cache(space, frame, 0, pycode, d, v1, None) == 1 + assert pycode._celldict_cache[0].version is v2 + assert pycode._celldict_cache[0].value == 1 + assert pycode._celldict_cache[1].version is None + assert pycode._celldict_cache[1].value is None + # does not need the pycode any more + assert _finditem_with_cache(space, frame, 0, None, d, v2, 1) == 1 + + # overwrite key, now it's a cell + d.setitem(w_key, 2) + v3 = strategy.version + # the cell is cached + cell = _finditem_with_cache(space, frame, 0, pycode, d, v2, None) + assert cell.w_value == 2 + assert pycode._celldict_cache[0].version is v3 + assert pycode._celldict_cache[0].value is cell + # does not need the pycode any more + assert _finditem_with_cache(space, frame, 0, None, d, v3, 2) == 2 + + d.setitem(w_key, 3) + v4 = strategy.version + assert v3 is v4 + assert d.getitem(w_key) == 3 + assert d.strategy.getdictvalue_no_unwrapping(d, key).w_value == 3 + + d.delitem(w_key) + v5 = strategy.version + assert v5 is not v4 + assert d.getitem(w_key) is None + assert d.strategy.getdictvalue_no_unwrapping(d, key) is None + + class AppTestModuleDict(object): spaceconfig = {"objspace.std.withcelldict": True} From noreply at buildbot.pypy.org Tue Feb 17 14:08:36 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 17 Feb 2015 14:08:36 +0100 (CET) Subject: [pypy-commit] pypy nonquadratic-heapcache: refactor heap cache to use HeapCacheValues everywhere Message-ID: <20150217130836.19A1B1C0455@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: nonquadratic-heapcache Changeset: r75933:33600f2c12d8 Date: 2015-02-17 11:36 +0100 http://bitbucket.org/pypy/pypy/changeset/33600f2c12d8/ Log: refactor heap cache to use HeapCacheValues everywhere diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py --- a/rpython/jit/metainterp/heapcache.py +++ b/rpython/jit/metainterp/heapcache.py @@ -5,13 +5,22 @@ def __init__(self, box): self.box = box self.known_class = False + # did we see the allocation during tracing? + self.seen_allocation = False + self.is_unescaped = False + self.likely_virtual = False self.nonstandard_virtualizable = False + self.lengthvalue = None + self.dependencies = None + + def __repr__(self): + return 'HeapCacheValue(%s)' % (self.box, ) class HeapCache(object): def __init__(self): self.reset() - def reset(self, reset_virtuals=True, trace_branch=True): + def reset(self): # maps boxes to values self.values = {} # store the boxes that contain newly allocated objects, this maps the @@ -19,54 +28,48 @@ # escaped the trace or not (True means the box never escaped, False # means it did escape), its presences in the mapping shows that it was # allocated inside the trace - if trace_branch: - self.new_boxes = {} - else: - for box in self.new_boxes: - self.new_boxes[box] = False - if reset_virtuals: - self.likely_virtuals = {} # only for jit.isvirtual() + #if trace_branch: + #self.new_boxes = {} + # pass + #else: + #for box in self.new_boxes: + # self.new_boxes[box] = False + # pass + #if reset_virtuals: + # self.likely_virtuals = {} # only for jit.isvirtual() # Tracks which boxes should be marked as escaped when the key box # escapes. - self.dependencies = {} - # contains frame boxes that are not virtualizables - if trace_branch: - self.nonstandard_virtualizables = {} + #self.dependencies = {} # heap cache - # maps descrs to {from_box, to_box} dicts + # maps descrs to {from_value, to_value} dicts self.heap_cache = {} # heap array cache - # maps descrs to {index: {from_box: to_box}} dicts + # maps descrs to {index: {from_value: to_value}} dicts self.heap_array_cache = {} - # cache the length of arrays - self.length_cache = {} - # replace_box is called surprisingly often, therefore it's not efficient - # to go over all the dicts and fix them. - # instead, these two dicts are kept, and a replace_box adds an entry to - # each of them. - # every time one of the dicts heap_cache, heap_array_cache, length_cache - # is accessed, suitable indirections need to be performed + def reset_keep_likely_virtuals(self): + for value in self.values.itervalues(): + value.is_unescaped = False + self.heap_cache = {} + self.heap_array_cache = {} - # this looks all very subtle, but in practice the patterns of - # replacements should not be that complex. Usually a box is replaced by - # a const, once. Also, if something goes wrong, the effect is that less - # caching than possible is done, which is not a huge problem. - self.input_indirections = {} - self.output_indirections = {} - - def get_value(self, box): + def getvalue(self, box): value = self.values.get(box, None) if not value: value = self.values[box] = HeapCacheValue(box) return value + def getvalues(self, boxes): + return [self.getvalue(box) for box in boxes] + def _input_indirection(self, box): - return self.input_indirections.get(box, box) + value = self.values.get(box, None) + if value is None: + return box + return value.box - def _output_indirection(self, box): - return self.output_indirections.get(box, box) + _output_indirection = _input_indirection def invalidate_caches(self, opnum, descr, argboxes): self.mark_escaped(opnum, descr, argboxes) @@ -75,18 +78,22 @@ def mark_escaped(self, opnum, descr, argboxes): if opnum == rop.SETFIELD_GC: assert len(argboxes) == 2 - box, valuebox = argboxes - if self.is_unescaped(box) and self.is_unescaped(valuebox): - self.dependencies.setdefault(box, []).append(valuebox) + value, fieldvalue = self.getvalues(argboxes) + if value.is_unescaped and fieldvalue.is_unescaped: + if value.dependencies is None: + value.dependencies = [] + value.dependencies.append(fieldvalue) else: - self._escape(valuebox) + self._escape(fieldvalue) elif opnum == rop.SETARRAYITEM_GC: assert len(argboxes) == 3 - box, indexbox, valuebox = argboxes - if self.is_unescaped(box) and self.is_unescaped(valuebox): - self.dependencies.setdefault(box, []).append(valuebox) + value, indexvalue, fieldvalue = self.getvalues(argboxes) + if value.is_unescaped and fieldvalue.is_unescaped: + if value.dependencies is None: + value.dependencies = [] + value.dependencies.append(fieldvalue) else: - self._escape(valuebox) + self._escape(fieldvalue) elif (opnum == rop.CALL and descr.get_extra_info().oopspecindex == descr.get_extra_info().OS_ARRAYCOPY and isinstance(argboxes[3], ConstInt) and @@ -106,25 +113,20 @@ opnum != rop.INSTANCE_PTR_EQ and opnum != rop.INSTANCE_PTR_NE): for box in argboxes: - self._escape(box) + self._escape_box(box) - def _escape(self, box): - try: - unescaped = self.new_boxes[box] - except KeyError: - pass - else: - if unescaped: - self.new_boxes[box] = False - try: - del self.likely_virtuals[box] - except KeyError: - pass - try: - deps = self.dependencies.pop(box) - except KeyError: - pass - else: + def _escape_box(self, box): + value = self.values.get(box, None) + if not value: + return + self._escape(value) + + def _escape(self, value): + value.is_unescaped = False + value.likely_virtual = False + deps = value.dependencies + value.dependencies = None + if deps is not None: for dep in deps: self._escape(dep) @@ -157,6 +159,7 @@ # A special case for ll_arraycopy, because it is so common, and its # effects are so well defined. elif effectinfo.oopspecindex == effectinfo.OS_ARRAYCOPY: + seen_allocation_of_target = self.getvalue(argboxes[2]).seen_allocation if ( isinstance(argboxes[3], ConstInt) and isinstance(argboxes[4], ConstInt) and @@ -187,43 +190,43 @@ except KeyError: pass else: - if argboxes[2] in self.new_boxes: - for frombox in idx_cache.keys(): - if not self.is_unescaped(frombox): - del idx_cache[frombox] + if seen_allocation_of_target: + for fromvalue in idx_cache.keys(): + if not fromvalue.seen_allocation: + del idx_cache[fromvalue] else: idx_cache.clear() return elif ( - argboxes[2] in self.new_boxes and + seen_allocation_of_target and len(effectinfo.write_descrs_arrays) == 1 ): # Fish the descr out of the effectinfo cache = self.heap_array_cache.get(effectinfo.write_descrs_arrays[0], None) if cache is not None: for idx, cache in cache.iteritems(): - for frombox in cache.keys(): - if not self.is_unescaped(frombox): - del cache[frombox] + for fromvalue in cache.keys(): + if not fromvalue.seen_allocation: + del cache[fromvalue] return else: # Only invalidate things that are either escaped or arguments - for descr, boxes in self.heap_cache.iteritems(): - for box in boxes.keys(): - if not self.is_unescaped(box) or box in argboxes: - del boxes[box] + for descr, values in self.heap_cache.iteritems(): + for value in values.keys(): + if not self.is_unescaped(value.box) or value.box in argboxes: + del values[value] for descr, indices in self.heap_array_cache.iteritems(): - for boxes in indices.itervalues(): - for box in boxes.keys(): - if not self.is_unescaped(box) or box in argboxes: - del boxes[box] + for values in indices.itervalues(): + for value in values.keys(): + if not self.is_unescaped(value.box) or value.box in argboxes: + del values[value] return # XXX not completely sure, but I *think* it is needed to reset() the # state at least in the 'CALL_*' operations that release the GIL. We # tried to do only the kind of resetting done by the two loops just # above, but hit an assertion in "pypy test_multiprocessing.py". - self.reset(reset_virtuals=False, trace_branch=False) + self.reset_keep_likely_virtuals() def is_class_known(self, box): value = self.values.get(box, None) @@ -232,7 +235,7 @@ return False def class_now_known(self, box): - self.get_value(box).known_class = True + self.getvalue(box).known_class = True def is_nonstandard_virtualizable(self, box): value = self.values.get(box, None) @@ -241,34 +244,44 @@ return False def nonstandard_virtualizables_now_known(self, box): - self.get_value(box).nonstandard_virtualizable = True + self.getvalue(box).nonstandard_virtualizable = True def is_unescaped(self, box): - return self.new_boxes.get(box, False) + value = self.values.get(box, None) + if value: + return value.is_unescaped + return False def is_likely_virtual(self, box): - return box in self.likely_virtuals + value = self.values.get(box, None) + if value: + return value.likely_virtual + return False def new(self, box): - self.new_boxes[box] = True - self.likely_virtuals[box] = None + value = self.getvalue(box) + value.is_unescaped = True + value.likely_virtual = True + value.seen_allocation = True def new_array(self, box, lengthbox): self.new(box) self.arraylen_now_known(box, lengthbox) def getfield(self, box, descr): - box = self._input_indirection(box) - d = self.heap_cache.get(descr, None) - if d: - tobox = d.get(box, None) - return self._output_indirection(tobox) + value = self.values.get(box, None) + if value: + d = self.heap_cache.get(descr, None) + if d: + tovalue = d.get(value, None) + if tovalue: + return tovalue.box return None def getfield_now_known(self, box, descr, fieldbox): - box = self._input_indirection(box) - fieldbox = self._input_indirection(fieldbox) - self.heap_cache.setdefault(descr, {})[box] = fieldbox + value = self.getvalue(box) + fieldvalue = self.getvalue(fieldbox) + self.heap_cache.setdefault(descr, {})[value] = fieldvalue def setfield(self, box, fieldbox, descr): d = self.heap_cache.get(descr, None) @@ -276,51 +289,55 @@ self.heap_cache[descr] = new_d def _do_write_with_aliasing(self, d, box, fieldbox): - box = self._input_indirection(box) - fieldbox = self._input_indirection(fieldbox) + value = self.getvalue(box) + fieldvalue = self.getvalue(fieldbox) # slightly subtle logic here - # a write to an arbitrary box, all other boxes can alias this one - if not d or box not in self.new_boxes: + # a write to an arbitrary value, all other valuees can alias this one + if not d or not value.seen_allocation: # therefore we throw away the cache - return {box: fieldbox} + return {value: fieldvalue} # the object we are writing to is freshly allocated - # only remove some boxes from the cache + # only remove some valuees from the cache new_d = {} - for frombox, tobox in d.iteritems(): - # the other box is *also* freshly allocated - # therefore frombox and box *must* contain different objects + for fromvalue, tovalue in d.iteritems(): + # the other value is *also* one where we saw the allocation + # therefore fromvalue and value *must* contain different objects # thus we can keep it in the cache - if frombox in self.new_boxes: - new_d[frombox] = tobox - new_d[box] = fieldbox - print len(self.new_boxes), len(d), len(new_d) + if fromvalue.seen_allocation: + new_d[fromvalue] = tovalue + new_d[value] = fieldvalue return new_d def getarrayitem(self, box, indexbox, descr): if not isinstance(indexbox, ConstInt): - return - box = self._input_indirection(box) + return None + value = self.values.get(box, None) + if value is None: + return None index = indexbox.getint() cache = self.heap_array_cache.get(descr, None) if cache: indexcache = cache.get(index, None) if indexcache is not None: - return self._output_indirection(indexcache.get(box, None)) + resvalue = indexcache.get(value, None) + if resvalue: + return resvalue.box + return None - def getarrayitem_now_known(self, box, indexbox, valuebox, descr): + def getarrayitem_now_known(self, box, indexbox, fieldbox, descr): if not isinstance(indexbox, ConstInt): return - box = self._input_indirection(box) - valuebox = self._input_indirection(valuebox) + value = self.getvalue(box) + fieldvalue = self.getvalue(fieldbox) index = indexbox.getint() cache = self.heap_array_cache.setdefault(descr, {}) indexcache = cache.get(index, None) if indexcache is not None: - indexcache[box] = valuebox + indexcache[value] = fieldvalue else: - cache[index] = {box: valuebox} + cache[index] = {value: fieldvalue} - def setarrayitem(self, box, indexbox, valuebox, descr): + def setarrayitem(self, box, indexbox, fieldbox, descr): if not isinstance(indexbox, ConstInt): cache = self.heap_array_cache.get(descr, None) if cache is not None: @@ -329,16 +346,21 @@ index = indexbox.getint() cache = self.heap_array_cache.setdefault(descr, {}) indexcache = cache.get(index, None) - cache[index] = self._do_write_with_aliasing(indexcache, box, valuebox) + cache[index] = self._do_write_with_aliasing(indexcache, box, fieldbox) def arraylen(self, box): - box = self._input_indirection(box) - return self._output_indirection(self.length_cache.get(box, None)) + value = self.values.get(box, None) + if value and value.length: + return value.length.box + return None def arraylen_now_known(self, box, lengthbox): - box = self._input_indirection(box) - self.length_cache[box] = self._input_indirection(lengthbox) + value = self.getvalue(box) + value.length = self.getvalue(lengthbox) def replace_box(self, oldbox, newbox): - self.input_indirections[self._output_indirection(newbox)] = self._input_indirection(oldbox) - self.output_indirections[self._input_indirection(oldbox)] = self._output_indirection(newbox) + value = self.values.get(oldbox, None) + if value is None: + return + value.box = newbox + self.values[newbox] = value diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -2192,7 +2192,8 @@ duplicates[box] = None def reached_loop_header(self, greenboxes, redboxes): - self.heapcache.reset(reset_virtuals=False) + #self.heapcache.reset(reset_virtuals=False) + self.heapcache.reset_keep_likely_virtuals() duplicates = {} self.remove_consts_and_duplicates(redboxes, len(redboxes), diff --git a/rpython/jit/metainterp/test/test_heapcache.py b/rpython/jit/metainterp/test/test_heapcache.py --- a/rpython/jit/metainterp/test/test_heapcache.py +++ b/rpython/jit/metainterp/test/test_heapcache.py @@ -594,9 +594,9 @@ h.new(box1) assert h.is_unescaped(box1) assert h.is_likely_virtual(box1) - h.reset(reset_virtuals=False) + h.reset_keep_likely_virtuals() assert not h.is_unescaped(box1) assert h.is_likely_virtual(box1) - h._escape(box1) + h._escape_box(box1) assert not h.is_unescaped(box1) assert not h.is_likely_virtual(box1) From noreply at buildbot.pypy.org Tue Feb 17 14:08:37 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 17 Feb 2015 14:08:37 +0100 (CET) Subject: [pypy-commit] pypy nonquadratic-heapcache: typo Message-ID: <20150217130837.4AAAF1C0455@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: nonquadratic-heapcache Changeset: r75934:efa6184e075b Date: 2015-02-17 11:45 +0100 http://bitbucket.org/pypy/pypy/changeset/efa6184e075b/ Log: typo diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py --- a/rpython/jit/metainterp/heapcache.py +++ b/rpython/jit/metainterp/heapcache.py @@ -10,7 +10,7 @@ self.is_unescaped = False self.likely_virtual = False self.nonstandard_virtualizable = False - self.lengthvalue = None + self.length = None self.dependencies = None def __repr__(self): From noreply at buildbot.pypy.org Tue Feb 17 14:08:38 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 17 Feb 2015 14:08:38 +0100 (CET) Subject: [pypy-commit] pypy nonquadratic-heapcache: not sure this is true Message-ID: <20150217130838.701911C0455@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: nonquadratic-heapcache Changeset: r75935:04307b13db83 Date: 2015-02-17 11:45 +0100 http://bitbucket.org/pypy/pypy/changeset/04307b13db83/ Log: not sure this is true diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py --- a/rpython/jit/metainterp/heapcache.py +++ b/rpython/jit/metainterp/heapcache.py @@ -102,6 +102,7 @@ len(descr.get_extra_info().write_descrs_arrays) == 1): # ARRAYCOPY with constant starts and constant length doesn't escape # its argument + # XXX really? pass # GETFIELD_GC, MARK_OPAQUE_PTR, PTR_EQ, and PTR_NE don't escape their # arguments From noreply at buildbot.pypy.org Wed Feb 18 10:06:14 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 18 Feb 2015 10:06:14 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: mention revision Message-ID: <20150218090614.BFEBF1C150C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r75974:6478158e32cd Date: 2015-02-18 10:05 +0100 http://bitbucket.org/pypy/pypy/changeset/6478158e32cd/ Log: mention revision diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -185,7 +185,7 @@ if level == LEVEL_KNOWNCLASS: return self.known_class # stmgc: added "and not self.is_null()" here, it's also in default - # but the method was moved. Ignore the conflict. + # but the method was moved. Ignore the conflict. (97c2a0a39120) elif level == LEVEL_CONSTANT and not self.is_null(): return cpu.ts.cls_of_box(self.box) else: From noreply at buildbot.pypy.org Wed Feb 18 12:34:13 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 18 Feb 2015 12:34:13 +0100 (CET) Subject: [pypy-commit] pypy default: Performance: avoid releasing the GIL around thread.lock.release(), Message-ID: <20150218113413.3F8481C1197@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75975:b190e63c3c3c Date: 2015-02-18 11:41 +0100 http://bitbucket.org/pypy/pypy/changeset/b190e63c3c3c/ Log: Performance: avoid releasing the GIL around thread.lock.release(), as well as around uncontended thread.lock.acquire(). diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -59,16 +59,18 @@ rffi.INT, releasegil=True) # release the GIL c_thread_releaselock = llexternal('RPyThreadReleaseLock', [TLOCKP], lltype.Void, - releasegil=True) # release the GIL + _nowrapper=True) # *don't* release the GIL # another set of functions, this time in versions that don't cause the -# GIL to be released. To use to handle the GIL lock itself. +# GIL to be released. Used to be there to handle the GIL lock itself, +# but that was changed (see rgil.py). Now here for performance only. c_thread_acquirelock_NOAUTO = llexternal('RPyThreadAcquireLock', [TLOCKP, rffi.INT], rffi.INT, _nowrapper=True) -c_thread_releaselock_NOAUTO = llexternal('RPyThreadReleaseLock', - [TLOCKP], lltype.Void, - _nowrapper=True) +c_thread_acquirelock_timed_NOAUTO = llexternal('RPyThreadAcquireLockTimed', + [TLOCKP, rffi.LONGLONG, rffi.INT], + rffi.INT, _nowrapper=True) +c_thread_releaselock_NOAUTO = c_thread_releaselock def allocate_lock(): @@ -131,9 +133,21 @@ self._lock = ll_lock def acquire(self, flag): - res = c_thread_acquirelock(self._lock, int(flag)) + # fast-path: try to acquire the lock without releasing the GIL + res = c_thread_acquirelock_timed_NOAUTO( + self._lock, + rffi.cast(rffi.LONGLONG, 0), + rffi.cast(rffi.INT, 0)) res = rffi.cast(lltype.Signed, res) - return bool(res) + if not flag: + # acquire(False): return a boolean that says if it worked + return res != 0 + else: + # acquire(True): if res == 0, we must invoke the slow-path + # releasing the GIL. This is done with conditional_call() to + # avoid JIT branches. + jit.conditional_call(res == 0, c_thread_acquirelock, self._lock, 1) + return True def acquire_timed(self, timeout): """Timeout is in microseconds. Returns 0 in case of failure, From noreply at buildbot.pypy.org Wed Feb 18 12:34:14 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 18 Feb 2015 12:34:14 +0100 (CET) Subject: [pypy-commit] pypy default: improve the test. it fails Message-ID: <20150218113414.6BED91C1197@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75976:c84c4faec46d Date: 2015-02-18 12:34 +0100 http://bitbucket.org/pypy/pypy/changeset/c84c4faec46d/ Log: improve the test. it fails diff --git a/rpython/jit/metainterp/test/test_recursive.py b/rpython/jit/metainterp/test/test_recursive.py --- a/rpython/jit/metainterp/test/test_recursive.py +++ b/rpython/jit/metainterp/test/test_recursive.py @@ -774,8 +774,8 @@ def main(codeno): frame = Frame() frame.thing = Thing(0) - portal(codeno, frame) - return frame.thing.val + result = portal(codeno, frame) + return result def portal(codeno, frame): i = 0 @@ -791,7 +791,7 @@ s += subframe.thing.val frame.thing = Thing(nextval + 1) i += 1 - return frame.thing.val + return frame.thing.val + s res = self.meta_interp(main, [0], inline=True) self.check_resops(call=0, cond_call=0) # got removed by optimization From noreply at buildbot.pypy.org Wed Feb 18 12:39:59 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 18 Feb 2015 12:39:59 +0100 (CET) Subject: [pypy-commit] pypy default: fix? Message-ID: <20150218113959.2D80B1C0382@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75977:c1f14f44bdec Date: 2015-02-18 12:39 +0100 http://bitbucket.org/pypy/pypy/changeset/c1f14f44bdec/ Log: fix? diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -144,9 +144,11 @@ return res != 0 else: # acquire(True): if res == 0, we must invoke the slow-path - # releasing the GIL. This is done with conditional_call() to - # avoid JIT branches. - jit.conditional_call(res == 0, c_thread_acquirelock, self._lock, 1) + # releasing the GIL. Too bad we can't use jit.conditional_call() + # here, because it can't be used with functions that can release + # the GIL... + if res == 0: + c_thread_acquirelock(self._lock, 1) return True def acquire_timed(self, timeout): From noreply at buildbot.pypy.org Wed Feb 18 12:46:35 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 18 Feb 2015 12:46:35 +0100 (CET) Subject: [pypy-commit] pypy default: Revert the two-step acquire. I think there is little point if Message-ID: <20150218114635.195C21C12F8@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75978:41092ff4edad Date: 2015-02-18 12:46 +0100 http://bitbucket.org/pypy/pypy/changeset/41092ff4edad/ Log: Revert the two-step acquire. I think there is little point if it creates JIT branches... diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -133,23 +133,16 @@ self._lock = ll_lock def acquire(self, flag): - # fast-path: try to acquire the lock without releasing the GIL - res = c_thread_acquirelock_timed_NOAUTO( - self._lock, - rffi.cast(rffi.LONGLONG, 0), - rffi.cast(rffi.INT, 0)) - res = rffi.cast(lltype.Signed, res) - if not flag: - # acquire(False): return a boolean that says if it worked - return res != 0 + if flag: + c_thread_acquirelock(self._lock, 1) + return True else: - # acquire(True): if res == 0, we must invoke the slow-path - # releasing the GIL. Too bad we can't use jit.conditional_call() - # here, because it can't be used with functions that can release - # the GIL... - if res == 0: - c_thread_acquirelock(self._lock, 1) - return True + res = c_thread_acquirelock_timed_NOAUTO( + self._lock, + rffi.cast(rffi.LONGLONG, 0), + rffi.cast(rffi.INT, 0)) + res = rffi.cast(lltype.Signed, res) + return bool(res) def acquire_timed(self, timeout): """Timeout is in microseconds. Returns 0 in case of failure, From noreply at buildbot.pypy.org Wed Feb 18 15:33:52 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 18 Feb 2015 15:33:52 +0100 (CET) Subject: [pypy-commit] stmgc default: possibly fix a memory leak Message-ID: <20150218143352.71B9A1C026B@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1635:19500805ccb8 Date: 2015-02-18 15:23 +0100 http://bitbucket.org/pypy/stmgc/changeset/19500805ccb8/ Log: possibly fix a memory leak diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -62,6 +62,10 @@ size, (char*)(addr - stm_object_pages), (uintptr_t)(addr - stm_object_pages) / 4096UL)); + /* set stm_flags to 0 in seg0 so that major gc will see them + as not visited during sweeping */ + ((struct object_s*)addr)->stm_flags = 0; + return (stm_char*)(addr - stm_object_pages); } @@ -89,6 +93,8 @@ size, (char*)(addr - stm_object_pages), (uintptr_t)(addr - stm_object_pages) / 4096UL)); + ((struct object_s*)addr)->stm_flags = 0; + spinlock_release(lock_growth_large); return (stm_char*)(addr - stm_object_pages); } diff --git a/c8/stm/smallmalloc.c b/c8/stm/smallmalloc.c --- a/c8/stm/smallmalloc.c +++ b/c8/stm/smallmalloc.c @@ -175,13 +175,17 @@ increment_total_allocated(size); - if (UNLIKELY(result == NULL)) + if (UNLIKELY(result == NULL)) { + char *addr = _allocate_small_slowpath(size); + ((struct object_s*)addr)->stm_flags = 0; return (stm_char*) - (_allocate_small_slowpath(size) - stm_object_pages); + (addr - stm_object_pages); + } *fl = result->next; /* dprintf(("allocate_outside_nursery_small(%lu): %p\n", */ /* size, (char*)((char *)result - stm_object_pages))); */ + ((struct object_s*)result)->stm_flags = 0; return (stm_char*) ((char *)result - stm_object_pages); } From noreply at buildbot.pypy.org Wed Feb 18 15:33:51 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 18 Feb 2015 15:33:51 +0100 (CET) Subject: [pypy-commit] stmgc default: reduce diff between c7 and c8 a bit Message-ID: <20150218143351.6D4261C026B@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1634:b57570d33a43 Date: 2015-02-18 11:43 +0100 http://bitbucket.org/pypy/stmgc/changeset/b57570d33a43/ Log: reduce diff between c7 and c8 a bit diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -774,9 +774,15 @@ void stm_start_inevitable_transaction(stm_thread_local_t *tl) { - s_mutex_lock(); - _stm_start_transaction(tl); - _stm_become_inevitable("stm_start_inevitable_transaction"); + /* used to be more efficient, starting directly an inevitable transaction, + but there is no real point any more, I believe */ + rewind_jmp_buf rjbuf; + stm_rewind_jmp_enterframe(tl, &rjbuf); + + stm_start_transaction(tl); + stm_become_inevitable(tl, "start_inevitable_transaction"); + + stm_rewind_jmp_leaveframe(tl, &rjbuf); } #ifdef STM_NO_AUTOMATIC_SETJMP @@ -858,15 +864,18 @@ bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE; _validate_and_add_to_commit_log(); - invoke_and_clear_user_callbacks(0); /* for commit */ + /* XXX do we still need a s_mutex_lock() section here? */ s_mutex_lock(); + enter_safe_point_if_requested(); assert(STM_SEGMENT->nursery_end == NURSERY_END); stm_rewind_jmp_forget(STM_SEGMENT->running_thread); + invoke_and_clear_user_callbacks(0); /* for commit */ + if (globally_unique_transaction && was_inev) { committed_globally_unique_transaction(); } diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -402,6 +402,7 @@ /* also visit all objs in the rewind-shadowstack */ long i; + assert(get_priv_segment(0)->transaction_state == TS_NONE); for (i = 1; i < NB_SEGMENTS; i++) { if (get_priv_segment(i)->transaction_state != TS_NONE) { mark_visit_possibly_new_object( From noreply at buildbot.pypy.org Wed Feb 18 15:33:50 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 18 Feb 2015 15:33:50 +0100 (CET) Subject: [pypy-commit] stmgc default: fix to make small old objs survive major gc Message-ID: <20150218143350.5B13C1C026B@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1633:dbbeeda92d3e Date: 2015-02-18 10:30 +0100 http://bitbucket.org/pypy/stmgc/changeset/dbbeeda92d3e/ Log: fix to make small old objs survive major gc diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -228,14 +228,14 @@ addr >= stm_object_pages+TOTAL_MEMORY) { /* actual segfault, unrelated to stmgc */ fprintf(stderr, "Segmentation fault: accessing %p\n", addr); - abort(); + raise(SIGINT); } int segnum = get_segment_of_linear_address(addr); if (segnum != STM_SEGMENT->segment_num) { fprintf(stderr, "Segmentation fault: accessing %p (seg %d) from" " seg %d\n", addr, segnum, STM_SEGMENT->segment_num); - abort(); + raise(SIGINT); } dprintf(("-> segment: %d\n", segnum)); @@ -244,7 +244,7 @@ if (pagenum < END_NURSERY_PAGE) { fprintf(stderr, "Segmentation fault: accessing %p (seg %d " "page %lu)\n", addr, segnum, pagenum); - abort(); + raise(SIGINT); } DEBUG_EXPECT_SEGFAULT(false); diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -2,7 +2,6 @@ # error "must be compiled via stmgc.c" #endif -static struct list_s *testing_prebuilt_objs = NULL; static struct tree_s *tree_prebuilt_objs = NULL; /* XXX refactor */ @@ -446,7 +445,7 @@ assert(realobj = (struct object_s*)REAL_ADDRESS(pseg->pub.segment_base, item)); assert(realobj->stm_flags & GCFLAG_WB_EXECUTED); - /* clear VISITED and ensure WB_EXECUTED in seg0 */ + /* clear VISITED (garbage) and ensure WB_EXECUTED in seg0 */ mark_visited_test_and_clear(item); realobj = (struct object_s*)REAL_ADDRESS(stm_object_pages, item); realobj->stm_flags |= GCFLAG_WB_EXECUTED; diff --git a/c8/stm/gcpage.h b/c8/stm/gcpage.h --- a/c8/stm/gcpage.h +++ b/c8/stm/gcpage.h @@ -8,7 +8,7 @@ #define GC_MAJOR_COLLECT 1.82 - +static struct list_s *testing_prebuilt_objs; static char *uninitialized_page_start; /* within segment 0 */ static char *uninitialized_page_stop; diff --git a/c8/stm/smallmalloc.c b/c8/stm/smallmalloc.c --- a/c8/stm/smallmalloc.c +++ b/c8/stm/smallmalloc.c @@ -196,6 +196,10 @@ memset(REAL_ADDRESS(STM_SEGMENT->segment_base, o), 0, size_rounded_up); o->stm_flags = GCFLAG_WRITE_BARRIER; + if (testing_prebuilt_objs == NULL) + testing_prebuilt_objs = list_create(); + LIST_APPEND(testing_prebuilt_objs, o); + dprintf(("_stm_allocate_old_small(%lu): %p, seg=%d, page=%lu\n", size_rounded_up, p, get_segment_of_linear_address(stm_object_pages + (uintptr_t)p), @@ -204,6 +208,7 @@ return o; } + /************************************************************/ static inline bool _smallmalloc_sweep_keep(char *p) diff --git a/c8/test/test_gcpage.py b/c8/test/test_gcpage.py --- a/c8/test/test_gcpage.py +++ b/c8/test/test_gcpage.py @@ -466,3 +466,10 @@ stm_major_collect() stm_major_collect() self.commit_transaction() + + def test_small_old_surives_major(self): + s = stm_allocate_old_small(16) + self.start_transaction() + stm_major_collect() + assert stm_get_char(s) == '\0' + self.commit_transaction() From noreply at buildbot.pypy.org Wed Feb 18 16:03:13 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Wed, 18 Feb 2015 16:03:13 +0100 (CET) Subject: [pypy-commit] stmgc default: phew, ensure that nobody applies our old backup copies after we just committed Message-ID: <20150218150313.38BF41C06A3@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1636:49aadef69fd0 Date: 2015-02-18 16:05 +0100 http://bitbucket.org/pypy/stmgc/changeset/49aadef69fd0/ Log: phew, ensure that nobody applies our old backup copies after we just committed diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -79,9 +79,9 @@ assert((get_page_status_in(STM_SEGMENT->segment_num, current_page_num) != PAGE_NO_ACCESS)); - dprintf(("import slice seg=%d obj=%p off=%lu sz=%d pg=%lu\n", - from_segnum, obj, SLICE_OFFSET(undo->slice), - SLICE_SIZE(undo->slice), current_page_num)); + /* dprintf(("import slice seg=%d obj=%p off=%lu sz=%d pg=%lu\n", */ + /* from_segnum, obj, SLICE_OFFSET(undo->slice), */ + /* SLICE_SIZE(undo->slice), current_page_num)); */ char *src, *dst; if (src_segment_base != NULL) src = REAL_ADDRESS(src_segment_base, oslice); @@ -232,6 +232,7 @@ } int segnum = get_segment_of_linear_address(addr); + OPT_ASSERT(segnum != 0); if (segnum != STM_SEGMENT->segment_num) { fprintf(stderr, "Segmentation fault: accessing %p (seg %d) from" " seg %d\n", addr, segnum, STM_SEGMENT->segment_num); @@ -263,11 +264,7 @@ struct stm_commit_log_entry_s *cl = &commit_log_root; fprintf(stderr, "commit log:\n"); - while ((cl = cl->next)) { - if (cl == INEV_RUNNING) { - fprintf(stderr, " INEVITABLE\n"); - return; - } + while (cl) { fprintf(stderr, " entry at %p: seg %d, rev %lu\n", cl, cl->segment_num, cl->rev_num); struct stm_undo_s *undo = cl->written; struct stm_undo_s *end = undo + cl->written_count; @@ -279,6 +276,12 @@ /* fprintf(stderr, " 0x%016lx", *(long *)(undo->backup + i)); */ fprintf(stderr, "\n"); } + + cl = cl->next; + if (cl == INEV_RUNNING) { + fprintf(stderr, " INEVITABLE\n"); + return; + } } } @@ -477,6 +480,12 @@ reset_wb_executed_flags(); check_all_write_barrier_flags(STM_SEGMENT->segment_base, STM_PSEGMENT->modified_old_objects); + + /* need to remove the entries in modified_old_objects "at the same + time" as the attach to commit log. Otherwise, another thread may + see the new CL entry, import it, look for backup copies in this + segment and find the old backup copies! */ + acquire_modification_lock(STM_SEGMENT->segment_num); } /* try to attach to commit log: */ @@ -493,6 +502,7 @@ } if (is_commit) { + release_modification_lock(STM_SEGMENT->segment_num); /* XXX: unfortunately, if we failed to attach our CL entry, we have to re-add the WB_EXECUTED flags before we try to validate again because of said condition (s.a) */ @@ -511,6 +521,16 @@ in major_do_validation_and_minor_collections, and don't free 'new' */ _stm_collectable_safe_point(); } + + if (is_commit) { + /* compare with _validate_and_add_to_commit_log */ + STM_PSEGMENT->transaction_state = TS_NONE; + STM_PSEGMENT->safe_point = SP_NO_TRANSACTION; + + list_clear(STM_PSEGMENT->modified_old_objects); + STM_PSEGMENT->last_commit_log_entry = new; + release_modification_lock(STM_SEGMENT->segment_num); + } } static void _validate_and_turn_inevitable(void) @@ -533,20 +553,19 @@ check_all_write_barrier_flags(STM_SEGMENT->segment_base, STM_PSEGMENT->modified_old_objects); + /* compare with _validate_and_attach: */ + STM_PSEGMENT->transaction_state = TS_NONE; + STM_PSEGMENT->safe_point = SP_NO_TRANSACTION; + list_clear(STM_PSEGMENT->modified_old_objects); + STM_PSEGMENT->last_commit_log_entry = new; + + /* do it: */ bool yes = __sync_bool_compare_and_swap(&old->next, INEV_RUNNING, new); OPT_ASSERT(yes); } else { _validate_and_attach(new); } - - STM_PSEGMENT->transaction_state = TS_NONE; - STM_PSEGMENT->safe_point = SP_NO_TRANSACTION; - - acquire_modification_lock(STM_SEGMENT->segment_num); - list_clear(STM_PSEGMENT->modified_old_objects); - STM_PSEGMENT->last_commit_log_entry = new; - release_modification_lock(STM_SEGMENT->segment_num); } /* ############# STM ############# */ @@ -1140,7 +1159,7 @@ if (get_page_status_in(i, page) != PAGE_NO_ACCESS) { /* shared or private, but never segfault */ char *dst = REAL_ADDRESS(get_segment_base(i), frag); - //dprintf(("-> flush %p to seg %lu, sz=%lu\n", frag, i, frag_size)); + dprintf(("-> flush %p to seg %lu, sz=%lu\n", frag, i, frag_size)); memcpy(dst, src, frag_size); } } diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -482,6 +482,7 @@ This is the case for transactions where MINOR_NOTHING_TO_DO() == true but they still did write-barriers on objects + (the objs are still in modified_old_objects list) */ lst = pseg->objects_pointing_to_nursery; if (!list_is_empty(lst)) { @@ -516,6 +517,7 @@ #pragma pop_macro("STM_PSEGMENT") } + static inline bool largemalloc_keep_object_at(char *data) { /* this is called by _stm_largemalloc_sweep() */ diff --git a/c8/stm/largemalloc.c b/c8/stm/largemalloc.c --- a/c8/stm/largemalloc.c +++ b/c8/stm/largemalloc.c @@ -616,6 +616,7 @@ /* use the callback to know if 'chunk' contains an object that survives or dies */ if (!_largemalloc_sweep_keep(chunk)) { + dprintf(("dies: %p\n", (char*)((char*)&chunk->d - stm_object_pages))); _large_free(chunk); /* dies */ } chunk = mnext; From noreply at buildbot.pypy.org Wed Feb 18 16:57:34 2015 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Wed, 18 Feb 2015 16:57:34 +0100 (CET) Subject: [pypy-commit] pypy default: Fixed SSL tests on OpenSSLs without npn Message-ID: <20150218155734.BCCD21C06A3@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r75979:9bd56d71be29 Date: 2015-02-18 07:57 -0800 http://bitbucket.org/pypy/pypy/changeset/9bd56d71be29/ Log: Fixed SSL tests on OpenSSLs without npn diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py --- a/pypy/module/_ssl/test/test_ssl.py +++ b/pypy/module/_ssl/test/test_ssl.py @@ -236,6 +236,9 @@ def test_npn_protocol(self): import socket, _ssl, gc + if not _ssl.HAS_NPN: + skip("NPN requires OpenSSL 1.0.1 or greater") + ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1) ctx._set_npn_protocols(b'\x08http/1.1\x06spdy/2') ss = ctx._wrap_socket(self.s._sock, True, From noreply at buildbot.pypy.org Wed Feb 18 17:29:03 2015 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Wed, 18 Feb 2015 17:29:03 +0100 (CET) Subject: [pypy-commit] pypy vendoring: start branch to autovendor tools Message-ID: <20150218162903.2D2D01C0471@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: vendoring Changeset: r75980:b0aa35281ea9 Date: 2015-01-31 16:00 +0100 http://bitbucket.org/pypy/pypy/changeset/b0aa35281ea9/ Log: start branch to autovendor tools From noreply at buildbot.pypy.org Wed Feb 18 17:29:04 2015 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Wed, 18 Feb 2015 17:29:04 +0100 (CET) Subject: [pypy-commit] pypy vendoring: docs Message-ID: <20150218162904.5BAA61C0471@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: vendoring Changeset: r75981:ff5dc0dc0699 Date: 2015-02-16 12:06 +0100 http://bitbucket.org/pypy/pypy/changeset/ff5dc0dc0699/ Log: docs diff --git a/site-packages/README b/site-packages/README --- a/site-packages/README +++ b/site-packages/README @@ -1,2 +1,14 @@ This directory exists so that 3rd party packages can be installed here. Read the source for site.py for more details. + + +For Convenience we include the following packages by default + + +* setuptools +* pip +* virtualenv +* prompt_toolkit - for implementing a readline +* ptpython - to give an alternative more awaesome python prompt +* pygments - highlighting in ptpython + From noreply at buildbot.pypy.org Wed Feb 18 18:14:44 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Wed, 18 Feb 2015 18:14:44 +0100 (CET) Subject: [pypy-commit] pypy framestate2: close branch before merging Message-ID: <20150218171444.C4A141C0382@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: framestate2 Changeset: r75983:83dceb08ef65 Date: 2015-02-18 17:10 +0000 http://bitbucket.org/pypy/pypy/changeset/83dceb08ef65/ Log: close branch before merging From noreply at buildbot.pypy.org Wed Feb 18 18:14:46 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Wed, 18 Feb 2015 18:14:46 +0100 (CET) Subject: [pypy-commit] pypy default: merge branch 'framestate2' Message-ID: <20150218171446.2722B1C0382@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r75984:1ac18445e487 Date: 2015-02-18 17:11 +0000 http://bitbucket.org/pypy/pypy/changeset/1ac18445e487/ Log: merge branch 'framestate2' Refactor FrameState. diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py --- a/rpython/flowspace/flowcontext.py +++ b/rpython/flowspace/flowcontext.py @@ -12,8 +12,7 @@ from rpython.flowspace.argument import CallSpec from rpython.flowspace.model import (Constant, Variable, Block, Link, c_last_exception, const, FSException) -from rpython.flowspace.framestate import (FrameState, recursively_unflatten, - recursively_flatten) +from rpython.flowspace.framestate import FrameState from rpython.flowspace.specialcase import (rpython_print_item, rpython_print_newline) from rpython.flowspace.operation import op @@ -278,6 +277,7 @@ "cmp_exc_match", ] + class FlowContext(object): def __init__(self, graph, code): self.graph = graph @@ -307,112 +307,91 @@ The locals are ordered according to self.pycode.signature. """ - self.valuestackdepth = code.co_nlocals - self.locals_stack_w = [None] * (code.co_stacksize + code.co_nlocals) + self.nlocals = code.co_nlocals + self.locals_w = [None] * code.co_nlocals + self.stack = [] + + @property + def stackdepth(self): + return len(self.stack) def pushvalue(self, w_object): - depth = self.valuestackdepth - self.locals_stack_w[depth] = w_object - self.valuestackdepth = depth + 1 + self.stack.append(w_object) def popvalue(self): - depth = self.valuestackdepth - 1 - assert depth >= self.pycode.co_nlocals, "pop from empty value stack" - w_object = self.locals_stack_w[depth] - self.locals_stack_w[depth] = None - self.valuestackdepth = depth - return w_object + return self.stack.pop() def peekvalue(self, index_from_top=0): # NOTE: top of the stack is peekvalue(0). - index = self.valuestackdepth + ~index_from_top - assert index >= self.pycode.co_nlocals, ( - "peek past the bottom of the stack") - return self.locals_stack_w[index] + index = ~index_from_top + return self.stack[index] def settopvalue(self, w_object, index_from_top=0): - index = self.valuestackdepth + ~index_from_top - assert index >= self.pycode.co_nlocals, ( - "settop past the bottom of the stack") - self.locals_stack_w[index] = w_object + index = ~index_from_top + self.stack[index] = w_object def popvalues(self, n): - values_w = [self.popvalue() for i in range(n)] - values_w.reverse() + if n == 0: + return [] + values_w = self.stack[-n:] + del self.stack[-n:] return values_w - def dropvalues(self, n): - finaldepth = self.valuestackdepth - n - for n in range(finaldepth, self.valuestackdepth): - self.locals_stack_w[n] = None - self.valuestackdepth = finaldepth - def dropvaluesuntil(self, finaldepth): - for n in range(finaldepth, self.valuestackdepth): - self.locals_stack_w[n] = None - self.valuestackdepth = finaldepth - - def save_locals_stack(self): - return self.locals_stack_w[:self.valuestackdepth] - - def restore_locals_stack(self, items_w): - self.locals_stack_w[:len(items_w)] = items_w - self.dropvaluesuntil(len(items_w)) + del self.stack[finaldepth:] def getstate(self, next_offset): - # getfastscope() can return real None, for undefined locals - data = self.save_locals_stack() - if self.last_exception is None: - data.append(Constant(None)) - data.append(Constant(None)) - else: - data.append(self.last_exception.w_type) - data.append(self.last_exception.w_value) - recursively_flatten(data) - return FrameState(data, self.blockstack[:], next_offset) + return FrameState(self.locals_w[:], self.stack[:], + self.last_exception, self.blockstack[:], next_offset) def setstate(self, state): """ Reset the context to the given frame state. """ - data = state.mergeable[:] - recursively_unflatten(data) - self.restore_locals_stack(data[:-2]) # Nones == undefined locals - if data[-2] == Constant(None): - assert data[-1] == Constant(None) - self.last_exception = None - else: - self.last_exception = FSException(data[-2], data[-1]) + self.locals_w = state.locals_w[:] + self.stack = state.stack[:] + self.last_exception = state.last_exception self.blockstack = state.blocklist[:] + self._normalize_raise_signals() + + def _normalize_raise_signals(self): + st = self.stack + for i in range(len(st)): + if isinstance(st[i], RaiseImplicit): + st[i] = Raise(st[i].w_exc) def guessbool(self, w_condition): if isinstance(w_condition, Constant): return w_condition.value return self.recorder.guessbool(self, w_condition) - def record(self, spaceop): + def maybe_merge(self): recorder = self.recorder if getattr(recorder, 'final_state', None) is not None: self.mergeblock(recorder.crnt_block, recorder.final_state) raise StopFlowing + + def record(self, spaceop): spaceop.offset = self.last_offset - recorder.append(spaceop) + self.recorder.append(spaceop) def do_op(self, op): + self.maybe_merge() self.record(op) self.guessexception(op.canraise) return op.result - def guessexception(self, exceptions, force=False): + def guessexception(self, exceptions): """ Catch possible exceptions implicitly. """ if not exceptions: return - if not force and not any(isinstance(block, (ExceptBlock, FinallyBlock)) - for block in self.blockstack): - # The implicit exception wouldn't be caught and would later get - # removed, so don't bother creating it. - return - self.recorder.guessexception(self, *exceptions) + # Implicit exceptions are ignored unless they are caught explicitly + if self.has_exc_handler(): + self.recorder.guessexception(self, *exceptions) + + def has_exc_handler(self): + return any(isinstance(block, (ExceptBlock, FinallyBlock)) + for block in self.blockstack) def build_flow(self): graph = self.graph @@ -430,35 +409,8 @@ while True: next_offset = self.handle_bytecode(next_offset) self.recorder.final_state = self.getstate(next_offset) - - except RaiseImplicit as e: - w_exc = e.w_exc - if isinstance(w_exc.w_type, Constant): - exc_cls = w_exc.w_type.value - else: - exc_cls = Exception - msg = "implicit %s shouldn't occur" % exc_cls.__name__ - w_type = Constant(AssertionError) - w_value = Constant(AssertionError(msg)) - link = Link([w_type, w_value], self.graph.exceptblock) - self.recorder.crnt_block.closeblock(link) - - except Raise as e: - w_exc = e.w_exc - if w_exc.w_type == const(ImportError): - msg = 'import statement always raises %s' % e - raise ImportError(msg) - link = Link([w_exc.w_type, w_exc.w_value], self.graph.exceptblock) - self.recorder.crnt_block.closeblock(link) - except StopFlowing: pass - - except Return as exc: - w_result = exc.w_value - link = Link([w_result], self.graph.returnblock) - self.recorder.crnt_block.closeblock(link) - except FlowingError as exc: if exc.ctx is None: exc.ctx = self @@ -476,14 +428,8 @@ if newstate is not None: break else: - newstate = currentstate.copy() - newblock = SpamBlock(newstate) - # unconditionally link the current block to the newblock - outputargs = currentstate.getoutputargs(newstate) - link = Link(outputargs, newblock) - currentblock.closeblock(link) + newblock = self.make_next_block(currentblock, currentstate) candidates.insert(0, newblock) - self.pendingblocks.append(newblock) return if newstate.matches(block.framestate): @@ -493,7 +439,7 @@ newblock = SpamBlock(newstate) varnames = self.pycode.co_varnames - for name, w_value in zip(varnames, newstate.mergeable): + for name, w_value in zip(varnames, newstate.locals_w): if isinstance(w_value, Variable): w_value.rename(name) # unconditionally link the current block to the newblock @@ -513,11 +459,21 @@ candidates.insert(0, newblock) self.pendingblocks.append(newblock) + def make_next_block(self, block, state): + newstate = state.copy() + newblock = SpamBlock(newstate) + # unconditionally link the current block to the newblock + outputargs = state.getoutputargs(newstate) + link = Link(outputargs, newblock) + block.closeblock(link) + self.pendingblocks.append(newblock) + return newblock + # hack for unrolling iterables, don't use this def replace_in_stack(self, oldvalue, newvalue): w_new = Constant(newvalue) - stack_items_w = self.locals_stack_w - for i in range(self.valuestackdepth - 1, self.pycode.co_nlocals - 1, -1): + stack_items_w = self.stack + for i in range(self.stackdepth - 1, - 1, -1): w_v = stack_items_w[i] if isinstance(w_v, Constant): if w_v.value is oldvalue: @@ -541,7 +497,7 @@ if isinstance(signal, block.handles): return block.handle(self, signal) block.cleanupstack(self) - return signal.nomoreblocks() + return signal.nomoreblocks(self) def getlocalvarname(self, index): return self.pycode.co_varnames[index] @@ -870,7 +826,7 @@ op.simple_call(w_exitfunc, w_None, w_None, w_None).eval(self) def LOAD_FAST(self, varindex): - w_value = self.locals_stack_w[varindex] + w_value = self.locals_w[varindex] if w_value is None: raise FlowingError("Local variable referenced before assignment") self.pushvalue(w_value) @@ -915,7 +871,7 @@ def STORE_FAST(self, varindex): w_newvalue = self.popvalue() assert w_newvalue is not None - self.locals_stack_w[varindex] = w_newvalue + self.locals_w[varindex] = w_newvalue if isinstance(w_newvalue, Variable): w_newvalue.rename(self.getlocalvarname(varindex)) @@ -1128,11 +1084,11 @@ op.simple_call(w_append_meth, w_value).eval(self) def DELETE_FAST(self, varindex): - if self.locals_stack_w[varindex] is None: + if self.locals_w[varindex] is None: varname = self.getlocalvarname(varindex) message = "local variable '%s' referenced before assignment" raise UnboundLocalError(message, varname) - self.locals_stack_w[varindex] = None + self.locals_w[varindex] = None def STORE_MAP(self, oparg): w_key = self.popvalue() @@ -1220,25 +1176,32 @@ WHY_CONTINUE, Continue WHY_YIELD not needed """ - def nomoreblocks(self): + def nomoreblocks(self, ctx): raise BytecodeCorruption("misplaced bytecode - should not return") + def __eq__(self, other): + return type(other) is type(self) and other.args == self.args + class Return(FlowSignal): """Signals a 'return' statement. - Argument is the wrapped object to return.""" - + Argument is the wrapped object to return. + """ def __init__(self, w_value): self.w_value = w_value - def nomoreblocks(self): - raise Return(self.w_value) + def nomoreblocks(self, ctx): + w_result = self.w_value + link = Link([w_result], ctx.graph.returnblock) + ctx.recorder.crnt_block.closeblock(link) + raise StopFlowing - def state_unpack_variables(self): + @property + def args(self): return [self.w_value] @staticmethod - def state_pack_variables(w_value): + def rebuild(w_value): return Return(w_value) class Raise(FlowSignal): @@ -1248,28 +1211,48 @@ def __init__(self, w_exc): self.w_exc = w_exc - def nomoreblocks(self): - raise self + def nomoreblocks(self, ctx): + w_exc = self.w_exc + if w_exc.w_type == const(ImportError): + msg = 'import statement always raises %s' % self + raise ImportError(msg) + link = Link([w_exc.w_type, w_exc.w_value], ctx.graph.exceptblock) + ctx.recorder.crnt_block.closeblock(link) + raise StopFlowing - def state_unpack_variables(self): + @property + def args(self): return [self.w_exc.w_type, self.w_exc.w_value] - @staticmethod - def state_pack_variables(w_type, w_value): - return Raise(FSException(w_type, w_value)) + @classmethod + def rebuild(cls, w_type, w_value): + return cls(FSException(w_type, w_value)) class RaiseImplicit(Raise): """Signals an exception raised implicitly""" + def nomoreblocks(self, ctx): + w_exc = self.w_exc + if isinstance(w_exc.w_type, Constant): + exc_cls = w_exc.w_type.value + else: + exc_cls = Exception + msg = "implicit %s shouldn't occur" % exc_cls.__name__ + w_type = Constant(AssertionError) + w_value = Constant(AssertionError(msg)) + link = Link([w_type, w_value], ctx.graph.exceptblock) + ctx.recorder.crnt_block.closeblock(link) + raise StopFlowing class Break(FlowSignal): """Signals a 'break' statement.""" - def state_unpack_variables(self): + @property + def args(self): return [] @staticmethod - def state_pack_variables(): + def rebuild(): return Break.singleton Break.singleton = Break() @@ -1281,11 +1264,12 @@ def __init__(self, jump_to): self.jump_to = jump_to - def state_unpack_variables(self): + @property + def args(self): return [const(self.jump_to)] @staticmethod - def state_pack_variables(w_jump_to): + def rebuild(w_jump_to): return Continue(w_jump_to.value) @@ -1295,21 +1279,21 @@ def __init__(self, ctx, handlerposition): self.handlerposition = handlerposition - self.valuestackdepth = ctx.valuestackdepth + self.stackdepth = ctx.stackdepth def __eq__(self, other): return (self.__class__ is other.__class__ and self.handlerposition == other.handlerposition and - self.valuestackdepth == other.valuestackdepth) + self.stackdepth == other.stackdepth) def __ne__(self, other): return not (self == other) def __hash__(self): - return hash((self.handlerposition, self.valuestackdepth)) + return hash((self.handlerposition, self.stackdepth)) def cleanupstack(self, ctx): - ctx.dropvaluesuntil(self.valuestackdepth) + ctx.dropvaluesuntil(self.stackdepth) def handle(self, ctx, unroller): raise NotImplementedError diff --git a/rpython/flowspace/framestate.py b/rpython/flowspace/framestate.py --- a/rpython/flowspace/framestate.py +++ b/rpython/flowspace/framestate.py @@ -1,21 +1,50 @@ -from rpython.flowspace.model import Variable, Constant +from rpython.flowspace.model import Variable, Constant, FSException from rpython.rlib.unroll import SpecTag +def _copy(v): + from rpython.flowspace.flowcontext import FlowSignal + if isinstance(v, Variable): + return Variable(v) + elif isinstance(v, FlowSignal): + vars = [_copy(var) for var in v.args] + return v.rebuild(*vars) + else: + return v + +def _union(seq1, seq2): + return [union(v1, v2) for v1, v2 in zip(seq1, seq2)] + class FrameState(object): - def __init__(self, mergeable, blocklist, next_offset): - self.mergeable = mergeable + def __init__(self, locals_w, stack, last_exception, blocklist, next_offset): + self.locals_w = locals_w + self.stack = stack + self.last_exception = last_exception self.blocklist = blocklist self.next_offset = next_offset + self._mergeable = None + + @property + def mergeable(self): + if self._mergeable is not None: + return self._mergeable + self._mergeable = data = self.locals_w + self.stack + if self.last_exception is None: + data.append(Constant(None)) + data.append(Constant(None)) + else: + data.append(self.last_exception.w_type) + data.append(self.last_exception.w_value) + recursively_flatten(data) + return data def copy(self): "Make a copy of this state in which all Variables are fresh." - newstate = [] - for w in self.mergeable: - if isinstance(w, Variable): - w = Variable(w) - newstate.append(w) - return FrameState(newstate, self.blocklist, self.next_offset) + exc = self.last_exception + if exc is not None: + exc = FSException(_copy(exc.w_type), _copy(exc.w_value)) + return FrameState(map(_copy, self.locals_w), map(_copy, self.stack), + exc, self.blocklist, self.next_offset) def getvariables(self): return [w for w in self.mergeable if isinstance(w, Variable)] @@ -33,18 +62,31 @@ return False return True + def _exc_args(self): + if self.last_exception is None: + return [Constant(None), Constant(None)] + else: + return [self.last_exception.w_type, + self.last_exception.w_value] + def union(self, other): """Compute a state that is at least as general as both self and other. A state 'a' is more general than a state 'b' if all Variables in 'b' are also Variables in 'a', but 'a' may have more Variables. """ - newstate = [] try: - for w1, w2 in zip(self.mergeable, other.mergeable): - newstate.append(union(w1, w2)) + locals = _union(self.locals_w, other.locals_w) + stack = _union(self.stack, other.stack) + if self.last_exception is None and other.last_exception is None: + exc = None + else: + args1 = self._exc_args() + args2 = other._exc_args() + exc = FSException(union(args1[0], args2[0]), + union(args1[1], args2[1])) except UnionError: return None - return FrameState(newstate, self.blocklist, self.next_offset) + return FrameState(locals, stack, exc, self.blocklist, self.next_offset) def getoutputargs(self, targetstate): "Return the output arguments needed to link self to targetstate." @@ -61,6 +103,7 @@ def union(w1, w2): "Union of two variables or constants." + from rpython.flowspace.flowcontext import FlowSignal if w1 == w2: return w1 if w1 is None or w2 is None: @@ -69,38 +112,21 @@ if isinstance(w1, Variable) or isinstance(w2, Variable): return Variable() # new fresh Variable if isinstance(w1, Constant) and isinstance(w2, Constant): - # FlowSignal represent stack unrollers in the stack. - # They should not be merged because they will be unwrapped. - # This is needed for try:except: and try:finally:, though - # it makes the control flow a bit larger by duplicating the - # handlers. - dont_merge_w1 = w1 in UNPICKLE_TAGS or isinstance(w1.value, SpecTag) - dont_merge_w2 = w2 in UNPICKLE_TAGS or isinstance(w2.value, SpecTag) - if dont_merge_w1 or dont_merge_w2: + if isinstance(w1.value, SpecTag) or isinstance(w2.value, SpecTag): raise UnionError else: return Variable() # generalize different constants + if isinstance(w1, FlowSignal) and isinstance(w2, FlowSignal): + if type(w1) is not type(w2): + raise UnionError + vars = [union(v1, v2) for v1, v2 in zip(w1.args, w2.args)] + return w1.rebuild(*vars) + if isinstance(w1, FlowSignal) or isinstance(w2, FlowSignal): + raise UnionError raise TypeError('union of %r and %r' % (w1.__class__.__name__, w2.__class__.__name__)) -# ____________________________________________________________ -# -# We have to flatten out the state of the frame into a list of -# Variables and Constants. This is done above by collecting the -# locals and the items on the value stack, but the latter may contain -# FlowSignal. We have to handle these specially, because -# some of them hide references to more Variables and Constants. -# The trick is to flatten ("pickle") them into the list so that the -# extra Variables show up directly in the list too. - -class PickleTag: - pass - -PICKLE_TAGS = {} -UNPICKLE_TAGS = {} - - def recursively_flatten(lst): from rpython.flowspace.flowcontext import FlowSignal i = 0 @@ -109,22 +135,4 @@ if not isinstance(unroller, FlowSignal): i += 1 else: - vars = unroller.state_unpack_variables() - key = unroller.__class__, len(vars) - try: - tag = PICKLE_TAGS[key] - except KeyError: - tag = PICKLE_TAGS[key] = Constant(PickleTag()) - UNPICKLE_TAGS[tag] = key - lst[i:i + 1] = [tag] + vars - - -def recursively_unflatten(lst): - for i in xrange(len(lst) - 1, -1, -1): - item = lst[i] - if item in UNPICKLE_TAGS: - unrollerclass, argcount = UNPICKLE_TAGS[item] - arguments = lst[i + 1:i + 1 + argcount] - del lst[i + 1:i + 1 + argcount] - unroller = unrollerclass.state_pack_variables(*arguments) - lst[i] = unroller + lst[i:i + 1] = unroller.args diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py --- a/rpython/flowspace/operation.py +++ b/rpython/flowspace/operation.py @@ -517,7 +517,7 @@ ctx.replace_in_stack(it, next_unroller) return const(v) w_item = ctx.do_op(self) - ctx.guessexception([StopIteration, RuntimeError], force=True) + ctx.recorder.guessexception(ctx, StopIteration, RuntimeError) return w_item class GetAttr(SingleDispatchMixin, HLOperation): diff --git a/rpython/flowspace/pygraph.py b/rpython/flowspace/pygraph.py --- a/rpython/flowspace/pygraph.py +++ b/rpython/flowspace/pygraph.py @@ -11,10 +11,10 @@ def __init__(self, func, code): from rpython.flowspace.flowcontext import SpamBlock - data = [None] * code.co_nlocals + locals = [None] * code.co_nlocals for i in range(code.formalargcount): - data[i] = Variable(code.co_varnames[i]) - state = FrameState(data + [Constant(None), Constant(None)], [], 0) + locals[i] = Variable(code.co_varnames[i]) + state = FrameState(locals, [], None, [], 0) initialblock = SpamBlock(state) super(PyGraph, self).__init__(self._sanitize_funcname(func), initialblock) self.func = func diff --git a/rpython/flowspace/test/test_flowcontext.py b/rpython/flowspace/test/test_flowcontext.py new file mode 100644 --- /dev/null +++ b/rpython/flowspace/test/test_flowcontext.py @@ -0,0 +1,15 @@ +""" Unit tests for flowcontext.py """ +import pytest +from rpython.flowspace.model import Variable, FSException +from rpython.flowspace.flowcontext import ( + Return, Raise, RaiseImplicit, Continue, Break) + + at pytest.mark.parametrize('signal', [ + Return(Variable()), + Raise(FSException(Variable(), Variable())), + RaiseImplicit(FSException(Variable(), Variable())), + Break(), + Continue(42), +]) +def test_signals(signal): + assert signal.rebuild(*signal.args) == signal diff --git a/rpython/flowspace/test/test_framestate.py b/rpython/flowspace/test/test_framestate.py --- a/rpython/flowspace/test/test_framestate.py +++ b/rpython/flowspace/test/test_framestate.py @@ -15,7 +15,7 @@ ctx = FlowContext(graph, code) # hack the frame ctx.setstate(graph.startblock.framestate) - ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Constant(None) + ctx.locals_w[-1] = Constant(None) return ctx def func_simple(x): @@ -31,7 +31,7 @@ def test_neq_hacked_framestate(self): ctx = self.get_context(self.func_simple) fs1 = ctx.getstate(0) - ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable() + ctx.locals_w[-1] = Variable() fs2 = ctx.getstate(0) assert not fs1.matches(fs2) @@ -44,7 +44,7 @@ def test_union_on_hacked_framestates(self): ctx = self.get_context(self.func_simple) fs1 = ctx.getstate(0) - ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable() + ctx.locals_w[-1] = Variable() fs2 = ctx.getstate(0) assert fs1.union(fs2).matches(fs2) # fs2 is more general assert fs2.union(fs1).matches(fs2) # fs2 is more general @@ -52,7 +52,7 @@ def test_restore_frame(self): ctx = self.get_context(self.func_simple) fs1 = ctx.getstate(0) - ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable() + ctx.locals_w[-1] = Variable() ctx.setstate(fs1) assert fs1.matches(ctx.getstate(0)) @@ -71,26 +71,25 @@ def test_getoutputargs(self): ctx = self.get_context(self.func_simple) fs1 = ctx.getstate(0) - ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable() + ctx.locals_w[-1] = Variable() fs2 = ctx.getstate(0) outputargs = fs1.getoutputargs(fs2) # 'x' -> 'x' is a Variable # locals_w[n-1] -> locals_w[n-1] is Constant(None) - assert outputargs == [ctx.locals_stack_w[0], Constant(None)] + assert outputargs == [ctx.locals_w[0], Constant(None)] def test_union_different_constants(self): ctx = self.get_context(self.func_simple) fs1 = ctx.getstate(0) - ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Constant(42) + ctx.locals_w[-1] = Constant(42) fs2 = ctx.getstate(0) fs3 = fs1.union(fs2) ctx.setstate(fs3) - assert isinstance(ctx.locals_stack_w[ctx.pycode.co_nlocals-1], - Variable) # generalized + assert isinstance(ctx.locals_w[-1], Variable) # generalized def test_union_spectag(self): ctx = self.get_context(self.func_simple) fs1 = ctx.getstate(0) - ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Constant(SpecTag()) + ctx.locals_w[-1] = Constant(SpecTag()) fs2 = ctx.getstate(0) assert fs1.union(fs2) is None # UnionError From noreply at buildbot.pypy.org Wed Feb 18 18:14:47 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Wed, 18 Feb 2015 18:14:47 +0100 (CET) Subject: [pypy-commit] pypy default: document branch Message-ID: <20150218171447.54E861C0382@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r75985:6bde3aefc8ba Date: 2015-02-18 17:13 +0000 http://bitbucket.org/pypy/pypy/changeset/6bde3aefc8ba/ Log: document branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -23,3 +23,6 @@ Fix exception being raised by kqueue.control (CPython compatibility) .. branch: gitignore + +.. branch: framestate2 +Refactor rpython.flowspace.framestate.FrameState. From noreply at buildbot.pypy.org Wed Feb 18 19:34:02 2015 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 18 Feb 2015 19:34:02 +0100 (CET) Subject: [pypy-commit] pypy default: Remove an exception never raised Message-ID: <20150218183402.117C91C0471@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75986:d7777f2ccd3f Date: 2015-02-18 19:33 +0100 http://bitbucket.org/pypy/pypy/changeset/d7777f2ccd3f/ Log: Remove an exception never raised diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -749,18 +749,14 @@ rstack._stack_criticalcode_stop() def handle_async_forcing(self, deadframe): - from rpython.jit.metainterp.resume import (force_from_resumedata, - AlreadyForced) + from rpython.jit.metainterp.resume import force_from_resumedata metainterp_sd = self.metainterp_sd vinfo = self.jitdriver_sd.virtualizable_info ginfo = self.jitdriver_sd.greenfield_info # there is some chance that this is already forced. In this case # the virtualizable would have a token = NULL - try: - all_virtuals = force_from_resumedata(metainterp_sd, self, deadframe, - vinfo, ginfo) - except AlreadyForced: - return + all_virtuals = force_from_resumedata(metainterp_sd, self, deadframe, + vinfo, ginfo) # The virtualizable data was stored on the real virtualizable above. # Handle all_virtuals: keep them for later blackholing from the # future failure of the GUARD_NOT_FORCED diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py --- a/rpython/jit/metainterp/resume.py +++ b/rpython/jit/metainterp/resume.py @@ -19,9 +19,6 @@ # because it needs to support optimize.py which encodes virtuals with # arbitrary cycles and also to compress the information -class AlreadyForced(Exception): - pass - class Snapshot(object): __slots__ = ('prev', 'boxes') From noreply at buildbot.pypy.org Wed Feb 18 20:50:01 2015 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 18 Feb 2015 20:50:01 +0100 (CET) Subject: [pypy-commit] pypy alt_errno: make alt_lasterror for win32 Message-ID: <20150218195001.724881C156B@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: alt_errno Changeset: r75987:6ecc88245506 Date: 2015-02-18 21:50 +0200 http://bitbucket.org/pypy/pypy/changeset/6ecc88245506/ Log: make alt_lasterror for win32 diff --git a/pypy/module/_cffi_backend/cerrno.py b/pypy/module/_cffi_backend/cerrno.py --- a/pypy/module/_cffi_backend/cerrno.py +++ b/pypy/module/_cffi_backend/cerrno.py @@ -23,8 +23,8 @@ @unwrap_spec(code=int) def getwinerror(space, code=-1): - from rpython.rlib.rwin32 import GetLastError_saved, FormatError + from rpython.rlib.rwin32 import GetLastError_alt_saved, FormatError if code == -1: - code = GetLastError_saved() + code = GetLastError_alt_saved() message = FormatError(code) return space.newtuple([space.wrap(code), space.wrap(message)]) diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py --- a/pypy/module/_rawffi/interp_rawffi.py +++ b/pypy/module/_rawffi/interp_rawffi.py @@ -617,10 +617,10 @@ # see also # https://bitbucket.org/pypy/pypy/issue/1944/ctypes-on-windows-getlasterror def get_last_error(space): - return space.wrap(rwin32.GetLastError_saved()) + return space.wrap(rwin32.GetLastError_alt_saved()) @unwrap_spec(error=int) def set_last_error(space, error): - rwin32.SetLastError_saved(error) + rwin32.SetLastError_alt_saved(error) else: # always have at least a dummy version of these functions # (https://bugs.pypy.org/issue1242) diff --git a/rpython/jit/backend/llsupport/llerrno.py b/rpython/jit/backend/llsupport/llerrno.py --- a/rpython/jit/backend/llsupport/llerrno.py +++ b/rpython/jit/backend/llsupport/llerrno.py @@ -47,6 +47,13 @@ else: return 5 * WORD +def get_alt_lasterror_offset(cpu): + if cpu.translate_support_code: + from rpython.rlib import rthread + return rthread.tlfield_alt_lasterror.getoffset() + else: + return 6 * WORD + def _fetch_addr_errno(): eci = ExternalCompilationInfo( diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -63,7 +63,7 @@ ad.lendescr, FLAG_FLOAT) self.setup() self._debug_errno_container = lltype.malloc( - rffi.CArray(lltype.Signed), 6, flavor='raw', zero=True, + rffi.CArray(lltype.Signed), 7, flavor='raw', zero=True, track_allocation=False) def getarraydescr_for_frame(self, type): diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py --- a/rpython/jit/backend/x86/callbuilder.py +++ b/rpython/jit/backend/x86/callbuilder.py @@ -196,10 +196,13 @@ SetLastError_addr = self.asm.cpu.cast_adr_to_int(adr) assert isinstance(self, CallBuilder32) # Windows 32-bit only # - rpy_lasterror = llerrno.get_rpy_lasterror_offset(self.asm.cpu) + if save_err & rffi.RFFI_ALT_ERRNO: + lasterror = llerrno.get_alt_lasterror_offset(self.asm.cpu) + else: + lasterror = llerrno.get_rpy_lasterror_offset(self.asm.cpu) tlofsreg = self.get_tlofs_reg() # => esi, callee-saved self.save_stack_position() # => edi, callee-saved - mc.PUSH_m((tlofsreg.value, rpy_lasterror)) + mc.PUSH_m((tlofsreg.value, lasterror)) mc.CALL(imm(SetLastError_addr)) # restore the stack position without assuming a particular # calling convention of _SetLastError() @@ -262,13 +265,16 @@ GetLastError_addr = self.asm.cpu.cast_adr_to_int(adr) assert isinstance(self, CallBuilder32) # Windows 32-bit only # - rpy_lasterror = llerrno.get_rpy_lasterror_offset(self.asm.cpu) + if save_err & rffi.RFFI_ALT_ERRNO: + lasterror = llerrno.get_alt_lasterror_offset(self.asm.cpu) + else: + lasterror = llerrno.get_rpy_lasterror_offset(self.asm.cpu) self.save_result_value(save_edx=True) # save eax/edx/xmm0 self.result_value_saved_early = True mc.CALL(imm(GetLastError_addr)) # tlofsreg = self.get_tlofs_reg() # => esi (possibly reused) - mc.MOV32_mr((tlofsreg.value, rpy_lasterror), eax.value) + mc.MOV32_mr((tlofsreg.value, lasterror), eax.value) def move_real_result_and_call_reacqgil_addr(self, fastgil): from rpython.jit.backend.x86 import rx86 diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -150,7 +150,10 @@ _set_errno(rffi.cast(rffi.INT, 0)) if WIN32 and (save_err & rffi.RFFI_READSAVED_LASTERROR): from rpython.rlib import rthread, rwin32 - err = rthread.tlfield_rpy_lasterror.getraw() + if save_err & rffi.RFFI_ALT_ERRNO: + err = rthread.tlfield_alt_lasterror.getraw() + else: + err = rthread.tlfield_rpy_lasterror.getraw() # careful, getraw() overwrites GetLastError. # We must assign it with _SetLastError() as the last # operation, i.e. after the errno handling. @@ -164,11 +167,17 @@ err = rwin32._GetLastError() # careful, setraw() overwrites GetLastError. # We must read it first, before the errno handling. - rthread.tlfield_rpy_lasterror.setraw(err) + if save_err & rffi.RFFI_ALT_ERRNO: + rthread.tlfield_alt_lasterror.setraw(err) + else: + rthread.tlfield_rpy_lasterror.setraw(err) elif save_err & rffi.RFFI_SAVE_WSALASTERROR: from rpython.rlib import rthread, _rsocket_rffi err = _rsocket_rffi._WSAGetLastError() - rthread.tlfield_rpy_lasterror.setraw(err) + if save_err & rffi.RFFI_ALT_ERRNO: + rthread.tlfield_alt_lasterror.setraw(err) + else: + rthread.tlfield_rpy_lasterror.setraw(err) if save_err & rffi.RFFI_SAVE_ERRNO: from rpython.rlib import rthread if save_err & rffi.RFFI_ALT_ERRNO: diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -374,6 +374,7 @@ if sys.platform == "win32": from rpython.rlib import rwin32 tlfield_rpy_lasterror = ThreadLocalField(rwin32.DWORD, "rpy_lasterror") + tlfield_alt_lasterror = ThreadLocalField(rwin32.DWORD, "alt_lasterror") def _threadlocalref_seeme(field): "NOT_RPYTHON" diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -145,6 +145,29 @@ from rpython.rlib import rthread rthread.tlfield_rpy_lasterror.setraw(rffi.cast(DWORD, err)) + def GetLastError_alt_saved(): + """Return the value of the "saved alt LastError". + The C-level GetLastError() is saved there after a call to a C + function, if that C function was declared with the flag + llexternal(..., save_err=RFFI_SAVE_LASTERROR | RFFI_ALT_ERRNO). + Functions without that flag don't change the saved LastError. + Alternatively, if the function was declared + RFFI_SAVE_WSALASTERROR | RFFI_ALT_ERRNO, + then the value of the C-level WSAGetLastError() is saved instead + (into the same "saved alt LastError" variable). + """ + from rpython.rlib import rthread + return rffi.cast(lltype.Signed, rthread.tlfield_alt_lasterror.getraw()) + + def SetLastError_alt_saved(err): + """Set the value of the saved alt LastError. This value will be used in + a call to the C-level SetLastError() just before calling the + following C function, provided it was declared + llexternal(..., save_err=RFFI_READSAVED_LASTERROR | RFFI_ALT_ERRNO). + """ + from rpython.rlib import rthread + rthread.tlfield_alt_lasterror.setraw(rffi.cast(DWORD, err)) + # In tests, the first call to _GetLastError() is always wrong, # because error is hidden by operations in ll2ctypes. Call it now. _GetLastError() From noreply at buildbot.pypy.org Thu Feb 19 00:40:25 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 19 Feb 2015 00:40:25 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: We don't really need rstm.allocate_preexisting(), so introduce and use Message-ID: <20150218234025.97FB31C009F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r75988:a0212f508821 Date: 2015-02-18 20:13 +0100 http://bitbucket.org/pypy/pypy/changeset/a0212f508821/ Log: We don't really need rstm.allocate_preexisting(), so introduce and use instead rstm.allocate_nonmovable(). There is an annoying rare bug that I've no clue about but could be related to allocate_preexisting(). diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -862,10 +862,7 @@ # produce a header that takes a STM_GUARD_FAILURE and jumps # to the target written there assert IS_X86_64 - p = lltype.malloc(STM_GUARD_FAILURE) - # xxx don't really need allocate_preexisting(). we only need - # some non-movable object - p = rstm.allocate_preexisting(p) + p = rstm.allocate_nonmovable(STM_GUARD_FAILURE) self.current_clt._stm_redirection = lltype.cast_opaque_ptr( llmemory.GCREF, p) addr = rffi.cast(lltype.Signed, p) @@ -1981,14 +1978,11 @@ def _generate_quick_failure_stm(self, fail_descr, target, guardtok): assert IS_X86_64 - p = lltype.malloc(STM_GUARD_FAILURE) + # we could maybe store the data directly on the faildescr. + p = rstm.allocate_nonmovable(STM_GUARD_FAILURE) p.fail_descr = fail_descr p.jump_target = target p.gcmap = guardtok.gcmap - # unclear if we really need a preexisting object here, or if we - # just need a regular but non-moving pointer. In the latter case - # we could maybe store the data directly on the faildescr. - p = rstm.allocate_preexisting(p) guardtok.faildescr._x86_stm_guard_failure = p addr = rffi.cast(lltype.Signed, p) addr += llmemory.offsetof(STM_GUARD_FAILURE, 'jump_target') diff --git a/rpython/memory/gctransform/stmframework.py b/rpython/memory/gctransform/stmframework.py --- a/rpython/memory/gctransform/stmframework.py +++ b/rpython/memory/gctransform/stmframework.py @@ -201,6 +201,23 @@ self.default(hop) self.pop_roots(hop, livevars) + def gct_stm_malloc_nonmovable(self, hop): + op = hop.spaceop + PTRTYPE = op.result.concretetype + TYPE = PTRTYPE.TO + type_id = self.get_type_id(TYPE) + + c_type_id = rmodel.inputconst(TYPE_ID, type_id) + info = self.layoutbuilder.get_info(type_id) + c_size = rmodel.inputconst(lltype.Signed, info.fixedsize) + + livevars = self.push_roots(hop) + v_result = hop.genop("stm_allocate_nonmovable", + [c_size, c_type_id], + resulttype=llmemory.GCREF) + self.pop_roots(hop, livevars) + hop.genop("cast_opaque_ptr", [v_result], resultvar=op.result) + # sync with lloperation.py gct_stm_become_inevitable = _gct_with_roots_pushed gct_stm_stop_all_other_threads = _gct_with_roots_pushed diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py --- a/rpython/rlib/rstm.py +++ b/rpython/rlib/rstm.py @@ -173,10 +173,15 @@ the current transaction can then proceed to change normally. This initial state must not contain GC pointers to any other uncommitted object.""" + # XXX is this buggy? TP = lltype.typeOf(p) size = llmemory.sizeof(TP.TO) return llop.stm_allocate_preexisting(TP, size, p) + at specialize.ll() +def allocate_nonmovable(GCTYPE): + return llop.stm_malloc_nonmovable(lltype.Ptr(GCTYPE)) + # ____________________________________________________________ class _Entry(ExtRegistryEntry): diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -419,6 +419,8 @@ 'stm_allocate_finalizer': LLOp(sideeffects=False, canmallocgc=True), 'stm_allocate_f_light': LLOp(sideeffects=False, canmallocgc=True), 'stm_allocate_preexisting':LLOp(sideeffects=False, canmallocgc=True), + 'stm_allocate_nonmovable':LLOp(sideeffects=False, canmallocgc=True), + 'stm_malloc_nonmovable': LLOp(sideeffects=False, canmallocgc=True), 'stm_get_from_obj': LLOp(sideeffects=False), 'stm_get_from_obj_const': LLOp(canfold=True), 'stm_set_into_obj': LLOp(), diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py --- a/rpython/translator/stm/funcgen.py +++ b/rpython/translator/stm/funcgen.py @@ -117,6 +117,15 @@ ' _stm_real_address((object_t *)%s));' % ( result, resulttype, arg_size, arg_idata)) +def stm_allocate_nonmovable(funcgen, op): + arg_size = funcgen.expr(op.args[0]) # <- could be smaller than 16 here + arg_type_id = funcgen.expr(op.args[1]) + result = funcgen.expr(op.result) + # XXX NULL returns? + return ('%s = (rpygcchar_t *)_stm_allocate_external(%s >= 16 ? %s : 16); ' % + (result, arg_size, arg_size) + + '((rpyobj_t *)%s)->tid = %s;' % (result, arg_type_id)) + def stm_get_from_obj(funcgen, op): assert op.args[0].concretetype == llmemory.GCREF arg_obj = funcgen.expr(op.args[0]) diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py --- a/rpython/translator/stm/test/test_ztranslated.py +++ b/rpython/translator/stm/test/test_ztranslated.py @@ -620,3 +620,19 @@ t, cbuilder = self.compile(main) data = cbuilder.cmdexec('') assert 'ok!\n' in data + + def test_allocate_nonmovable(self): + S = lltype.GcStruct('S', ('n', lltype.Signed)) + + def main(argv): + s1 = rstm.allocate_nonmovable(S) + s1.n = 42 + assert s1.n == 42 + assert not rgc.can_move(s1) + # + print "ok!" + return 0 + + t, cbuilder = self.compile(main) + data = cbuilder.cmdexec('') + assert 'ok!\n' in data From noreply at buildbot.pypy.org Thu Feb 19 00:40:26 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 19 Feb 2015 00:40:26 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Kill the mostly pointless sharing of PropagateExceptionDescr(). It is Message-ID: <20150218234026.C5E431C009F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r75989:92e947fa8014 Date: 2015-02-19 00:39 +0100 http://bitbucket.org/pypy/pypy/changeset/92e947fa8014/ Log: Kill the mostly pointless sharing of PropagateExceptionDescr(). It is wrong now! The backend assumes it can store _x86_stm_guard_failure on the guard's descrs. Check added. diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -1983,6 +1983,9 @@ p.fail_descr = fail_descr p.jump_target = target p.gcmap = guardtok.gcmap + # xxx this code is only translated, so the following check is + # good enough to say "nobody stored _x86_stm_guard_failure yet" + assert not guardtok.faildescr._x86_stm_guard_failure guardtok.faildescr._x86_stm_guard_failure = p addr = rffi.cast(lltype.Signed, p) addr += llmemory.offsetof(STM_GUARD_FAILURE, 'jump_target') diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -882,7 +882,7 @@ finishargs = [] # jd = jitdriver_sd - faildescr = jitdriver_sd.propagate_exc_descr + faildescr = PropagateExceptionDescr() operations = [ ResOperation(rop.CALL, callargs, result, descr=jd.portal_calldescr), ResOperation(rop.GUARD_NO_EXCEPTION, [], None, descr=faildescr), diff --git a/rpython/jit/metainterp/jitdriver.py b/rpython/jit/metainterp/jitdriver.py --- a/rpython/jit/metainterp/jitdriver.py +++ b/rpython/jit/metainterp/jitdriver.py @@ -20,7 +20,6 @@ # self.no_loop_header ... rpython.jit.metainterp.warmspot # self.stm_report_location.. rpython.jit.metainterp.warmspot # self.portal_finishtoken... rpython.jit.metainterp.pyjitpl - # self.propagate_exc_descr.. rpython.jit.metainterp.pyjitpl # self.index ... rpython.jit.codewriter.call # self.mainjitcode ... rpython.jit.codewriter.call diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -1628,7 +1628,6 @@ # # store this information for fastpath of call_assembler # (only the paths that can actually be taken) - exc_descr = compile.PropagateExceptionDescr() for jd in self.jitdrivers_sd: name = {history.INT: 'int', history.REF: 'ref', @@ -1636,9 +1635,6 @@ history.VOID: 'void'}[jd.result_type] tokens = getattr(self, 'loop_tokens_done_with_this_frame_%s' % name) jd.portal_finishtoken = tokens[0].finishdescr - jd.propagate_exc_descr = exc_descr - # - self.cpu.propagate_exception_descr = exc_descr # self.globaldata = MetaInterpGlobalData(self) diff --git a/rpython/jit/metainterp/test/test_compile.py b/rpython/jit/metainterp/test/test_compile.py --- a/rpython/jit/metainterp/test/test_compile.py +++ b/rpython/jit/metainterp/test/test_compile.py @@ -121,7 +121,6 @@ portal_runner_adr = llmemory.cast_ptr_to_adr(portal_runner_ptr) portal_calldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, None) portal_finishtoken = compile.DoneWithThisFrameDescrInt() - propagate_exc_descr = compile.PropagateExceptionDescr() num_red_args = 2 result_type = INT # From noreply at buildbot.pypy.org Thu Feb 19 01:21:26 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 19 Feb 2015 01:21:26 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: probable translation fix Message-ID: <20150219002126.C38F71C1356@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r75990:3b0f155eaa42 Date: 2015-02-19 01:21 +0100 http://bitbucket.org/pypy/pypy/changeset/3b0f155eaa42/ Log: probable translation fix diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -1983,8 +1983,6 @@ p.fail_descr = fail_descr p.jump_target = target p.gcmap = guardtok.gcmap - # xxx this code is only translated, so the following check is - # good enough to say "nobody stored _x86_stm_guard_failure yet" assert not guardtok.faildescr._x86_stm_guard_failure guardtok.faildescr._x86_stm_guard_failure = p addr = rffi.cast(lltype.Signed, p) @@ -2922,3 +2920,4 @@ ("fail_descr", lltype.Signed), ("jump_target", lltype.Signed), ("gcmap", lltype.Ptr(jitframe.GCMAP))) +AbstractFailDescr._x86_stm_guard_failure = lltype.nullptr(STM_GUARD_FAILURE) From noreply at buildbot.pypy.org Thu Feb 19 09:06:02 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 19 Feb 2015 09:06:02 +0100 (CET) Subject: [pypy-commit] pypy alt_errno: merge default into branch Message-ID: <20150219080602.081901C009F@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: alt_errno Changeset: r75991:c99027ae4f08 Date: 2015-02-19 10:01 +0200 http://bitbucket.org/pypy/pypy/changeset/c99027ae4f08/ Log: merge default into branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -23,3 +23,6 @@ Fix exception being raised by kqueue.control (CPython compatibility) .. branch: gitignore + +.. branch: framestate2 +Refactor rpython.flowspace.framestate.FrameState. diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py --- a/pypy/module/_ssl/test/test_ssl.py +++ b/pypy/module/_ssl/test/test_ssl.py @@ -236,6 +236,9 @@ def test_npn_protocol(self): import socket, _ssl, gc + if not _ssl.HAS_NPN: + skip("NPN requires OpenSSL 1.0.1 or greater") + ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1) ctx._set_npn_protocols(b'\x08http/1.1\x06spdy/2') ss = ctx._wrap_socket(self.s._sock, True, diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py --- a/pypy/objspace/std/intobject.py +++ b/pypy/objspace/std/intobject.py @@ -427,9 +427,11 @@ def descr_bit_length(self, space): val = self.intval + bits = 0 if val < 0: - val = -val - bits = 0 + # warning, "-val" overflows here + val = -((val + 1) >> 1) + bits = 1 while val: bits += 1 val >>= 1 diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py --- a/pypy/objspace/std/test/test_intobject.py +++ b/pypy/objspace/std/test/test_intobject.py @@ -521,11 +521,20 @@ (10, 4), (150, 8), (-1, 1), + (-2, 2), + (-3, 2), + (-4, 3), (-10, 4), (-150, 8), ]: assert val.bit_length() == bits + def test_bit_length_max(self): + import sys + val = -sys.maxint-1 + bits = 32 if val == -2147483648 else 64 + assert val.bit_length() == bits + def test_int_real(self): class A(int): pass b = A(5).real diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py --- a/rpython/flowspace/flowcontext.py +++ b/rpython/flowspace/flowcontext.py @@ -12,8 +12,7 @@ from rpython.flowspace.argument import CallSpec from rpython.flowspace.model import (Constant, Variable, Block, Link, c_last_exception, const, FSException) -from rpython.flowspace.framestate import (FrameState, recursively_unflatten, - recursively_flatten) +from rpython.flowspace.framestate import FrameState from rpython.flowspace.specialcase import (rpython_print_item, rpython_print_newline) from rpython.flowspace.operation import op @@ -278,6 +277,7 @@ "cmp_exc_match", ] + class FlowContext(object): def __init__(self, graph, code): self.graph = graph @@ -307,112 +307,91 @@ The locals are ordered according to self.pycode.signature. """ - self.valuestackdepth = code.co_nlocals - self.locals_stack_w = [None] * (code.co_stacksize + code.co_nlocals) + self.nlocals = code.co_nlocals + self.locals_w = [None] * code.co_nlocals + self.stack = [] + + @property + def stackdepth(self): + return len(self.stack) def pushvalue(self, w_object): - depth = self.valuestackdepth - self.locals_stack_w[depth] = w_object - self.valuestackdepth = depth + 1 + self.stack.append(w_object) def popvalue(self): - depth = self.valuestackdepth - 1 - assert depth >= self.pycode.co_nlocals, "pop from empty value stack" - w_object = self.locals_stack_w[depth] - self.locals_stack_w[depth] = None - self.valuestackdepth = depth - return w_object + return self.stack.pop() def peekvalue(self, index_from_top=0): # NOTE: top of the stack is peekvalue(0). - index = self.valuestackdepth + ~index_from_top - assert index >= self.pycode.co_nlocals, ( - "peek past the bottom of the stack") - return self.locals_stack_w[index] + index = ~index_from_top + return self.stack[index] def settopvalue(self, w_object, index_from_top=0): - index = self.valuestackdepth + ~index_from_top - assert index >= self.pycode.co_nlocals, ( - "settop past the bottom of the stack") - self.locals_stack_w[index] = w_object + index = ~index_from_top + self.stack[index] = w_object def popvalues(self, n): - values_w = [self.popvalue() for i in range(n)] - values_w.reverse() + if n == 0: + return [] + values_w = self.stack[-n:] + del self.stack[-n:] return values_w - def dropvalues(self, n): - finaldepth = self.valuestackdepth - n - for n in range(finaldepth, self.valuestackdepth): - self.locals_stack_w[n] = None - self.valuestackdepth = finaldepth - def dropvaluesuntil(self, finaldepth): - for n in range(finaldepth, self.valuestackdepth): - self.locals_stack_w[n] = None - self.valuestackdepth = finaldepth - - def save_locals_stack(self): - return self.locals_stack_w[:self.valuestackdepth] - - def restore_locals_stack(self, items_w): - self.locals_stack_w[:len(items_w)] = items_w - self.dropvaluesuntil(len(items_w)) + del self.stack[finaldepth:] def getstate(self, next_offset): - # getfastscope() can return real None, for undefined locals - data = self.save_locals_stack() - if self.last_exception is None: - data.append(Constant(None)) - data.append(Constant(None)) - else: - data.append(self.last_exception.w_type) - data.append(self.last_exception.w_value) - recursively_flatten(data) - return FrameState(data, self.blockstack[:], next_offset) + return FrameState(self.locals_w[:], self.stack[:], + self.last_exception, self.blockstack[:], next_offset) def setstate(self, state): """ Reset the context to the given frame state. """ - data = state.mergeable[:] - recursively_unflatten(data) - self.restore_locals_stack(data[:-2]) # Nones == undefined locals - if data[-2] == Constant(None): - assert data[-1] == Constant(None) - self.last_exception = None - else: - self.last_exception = FSException(data[-2], data[-1]) + self.locals_w = state.locals_w[:] + self.stack = state.stack[:] + self.last_exception = state.last_exception self.blockstack = state.blocklist[:] + self._normalize_raise_signals() + + def _normalize_raise_signals(self): + st = self.stack + for i in range(len(st)): + if isinstance(st[i], RaiseImplicit): + st[i] = Raise(st[i].w_exc) def guessbool(self, w_condition): if isinstance(w_condition, Constant): return w_condition.value return self.recorder.guessbool(self, w_condition) - def record(self, spaceop): + def maybe_merge(self): recorder = self.recorder if getattr(recorder, 'final_state', None) is not None: self.mergeblock(recorder.crnt_block, recorder.final_state) raise StopFlowing + + def record(self, spaceop): spaceop.offset = self.last_offset - recorder.append(spaceop) + self.recorder.append(spaceop) def do_op(self, op): + self.maybe_merge() self.record(op) self.guessexception(op.canraise) return op.result - def guessexception(self, exceptions, force=False): + def guessexception(self, exceptions): """ Catch possible exceptions implicitly. """ if not exceptions: return - if not force and not any(isinstance(block, (ExceptBlock, FinallyBlock)) - for block in self.blockstack): - # The implicit exception wouldn't be caught and would later get - # removed, so don't bother creating it. - return - self.recorder.guessexception(self, *exceptions) + # Implicit exceptions are ignored unless they are caught explicitly + if self.has_exc_handler(): + self.recorder.guessexception(self, *exceptions) + + def has_exc_handler(self): + return any(isinstance(block, (ExceptBlock, FinallyBlock)) + for block in self.blockstack) def build_flow(self): graph = self.graph @@ -430,35 +409,8 @@ while True: next_offset = self.handle_bytecode(next_offset) self.recorder.final_state = self.getstate(next_offset) - - except RaiseImplicit as e: - w_exc = e.w_exc - if isinstance(w_exc.w_type, Constant): - exc_cls = w_exc.w_type.value - else: - exc_cls = Exception - msg = "implicit %s shouldn't occur" % exc_cls.__name__ - w_type = Constant(AssertionError) - w_value = Constant(AssertionError(msg)) - link = Link([w_type, w_value], self.graph.exceptblock) - self.recorder.crnt_block.closeblock(link) - - except Raise as e: - w_exc = e.w_exc - if w_exc.w_type == const(ImportError): - msg = 'import statement always raises %s' % e - raise ImportError(msg) - link = Link([w_exc.w_type, w_exc.w_value], self.graph.exceptblock) - self.recorder.crnt_block.closeblock(link) - except StopFlowing: pass - - except Return as exc: - w_result = exc.w_value - link = Link([w_result], self.graph.returnblock) - self.recorder.crnt_block.closeblock(link) - except FlowingError as exc: if exc.ctx is None: exc.ctx = self @@ -476,14 +428,8 @@ if newstate is not None: break else: - newstate = currentstate.copy() - newblock = SpamBlock(newstate) - # unconditionally link the current block to the newblock - outputargs = currentstate.getoutputargs(newstate) - link = Link(outputargs, newblock) - currentblock.closeblock(link) + newblock = self.make_next_block(currentblock, currentstate) candidates.insert(0, newblock) - self.pendingblocks.append(newblock) return if newstate.matches(block.framestate): @@ -493,7 +439,7 @@ newblock = SpamBlock(newstate) varnames = self.pycode.co_varnames - for name, w_value in zip(varnames, newstate.mergeable): + for name, w_value in zip(varnames, newstate.locals_w): if isinstance(w_value, Variable): w_value.rename(name) # unconditionally link the current block to the newblock @@ -513,11 +459,21 @@ candidates.insert(0, newblock) self.pendingblocks.append(newblock) + def make_next_block(self, block, state): + newstate = state.copy() + newblock = SpamBlock(newstate) + # unconditionally link the current block to the newblock + outputargs = state.getoutputargs(newstate) + link = Link(outputargs, newblock) + block.closeblock(link) + self.pendingblocks.append(newblock) + return newblock + # hack for unrolling iterables, don't use this def replace_in_stack(self, oldvalue, newvalue): w_new = Constant(newvalue) - stack_items_w = self.locals_stack_w - for i in range(self.valuestackdepth - 1, self.pycode.co_nlocals - 1, -1): + stack_items_w = self.stack + for i in range(self.stackdepth - 1, - 1, -1): w_v = stack_items_w[i] if isinstance(w_v, Constant): if w_v.value is oldvalue: @@ -541,7 +497,7 @@ if isinstance(signal, block.handles): return block.handle(self, signal) block.cleanupstack(self) - return signal.nomoreblocks() + return signal.nomoreblocks(self) def getlocalvarname(self, index): return self.pycode.co_varnames[index] @@ -870,7 +826,7 @@ op.simple_call(w_exitfunc, w_None, w_None, w_None).eval(self) def LOAD_FAST(self, varindex): - w_value = self.locals_stack_w[varindex] + w_value = self.locals_w[varindex] if w_value is None: raise FlowingError("Local variable referenced before assignment") self.pushvalue(w_value) @@ -915,7 +871,7 @@ def STORE_FAST(self, varindex): w_newvalue = self.popvalue() assert w_newvalue is not None - self.locals_stack_w[varindex] = w_newvalue + self.locals_w[varindex] = w_newvalue if isinstance(w_newvalue, Variable): w_newvalue.rename(self.getlocalvarname(varindex)) @@ -1128,11 +1084,11 @@ op.simple_call(w_append_meth, w_value).eval(self) def DELETE_FAST(self, varindex): - if self.locals_stack_w[varindex] is None: + if self.locals_w[varindex] is None: varname = self.getlocalvarname(varindex) message = "local variable '%s' referenced before assignment" raise UnboundLocalError(message, varname) - self.locals_stack_w[varindex] = None + self.locals_w[varindex] = None def STORE_MAP(self, oparg): w_key = self.popvalue() @@ -1220,25 +1176,32 @@ WHY_CONTINUE, Continue WHY_YIELD not needed """ - def nomoreblocks(self): + def nomoreblocks(self, ctx): raise BytecodeCorruption("misplaced bytecode - should not return") + def __eq__(self, other): + return type(other) is type(self) and other.args == self.args + class Return(FlowSignal): """Signals a 'return' statement. - Argument is the wrapped object to return.""" - + Argument is the wrapped object to return. + """ def __init__(self, w_value): self.w_value = w_value - def nomoreblocks(self): - raise Return(self.w_value) + def nomoreblocks(self, ctx): + w_result = self.w_value + link = Link([w_result], ctx.graph.returnblock) + ctx.recorder.crnt_block.closeblock(link) + raise StopFlowing - def state_unpack_variables(self): + @property + def args(self): return [self.w_value] @staticmethod - def state_pack_variables(w_value): + def rebuild(w_value): return Return(w_value) class Raise(FlowSignal): @@ -1248,28 +1211,48 @@ def __init__(self, w_exc): self.w_exc = w_exc - def nomoreblocks(self): - raise self + def nomoreblocks(self, ctx): + w_exc = self.w_exc + if w_exc.w_type == const(ImportError): + msg = 'import statement always raises %s' % self + raise ImportError(msg) + link = Link([w_exc.w_type, w_exc.w_value], ctx.graph.exceptblock) + ctx.recorder.crnt_block.closeblock(link) + raise StopFlowing - def state_unpack_variables(self): + @property + def args(self): return [self.w_exc.w_type, self.w_exc.w_value] - @staticmethod - def state_pack_variables(w_type, w_value): - return Raise(FSException(w_type, w_value)) + @classmethod + def rebuild(cls, w_type, w_value): + return cls(FSException(w_type, w_value)) class RaiseImplicit(Raise): """Signals an exception raised implicitly""" + def nomoreblocks(self, ctx): + w_exc = self.w_exc + if isinstance(w_exc.w_type, Constant): + exc_cls = w_exc.w_type.value + else: + exc_cls = Exception + msg = "implicit %s shouldn't occur" % exc_cls.__name__ + w_type = Constant(AssertionError) + w_value = Constant(AssertionError(msg)) + link = Link([w_type, w_value], ctx.graph.exceptblock) + ctx.recorder.crnt_block.closeblock(link) + raise StopFlowing class Break(FlowSignal): """Signals a 'break' statement.""" - def state_unpack_variables(self): + @property + def args(self): return [] @staticmethod - def state_pack_variables(): + def rebuild(): return Break.singleton Break.singleton = Break() @@ -1281,11 +1264,12 @@ def __init__(self, jump_to): self.jump_to = jump_to - def state_unpack_variables(self): + @property + def args(self): return [const(self.jump_to)] @staticmethod - def state_pack_variables(w_jump_to): + def rebuild(w_jump_to): return Continue(w_jump_to.value) @@ -1295,21 +1279,21 @@ def __init__(self, ctx, handlerposition): self.handlerposition = handlerposition - self.valuestackdepth = ctx.valuestackdepth + self.stackdepth = ctx.stackdepth def __eq__(self, other): return (self.__class__ is other.__class__ and self.handlerposition == other.handlerposition and - self.valuestackdepth == other.valuestackdepth) + self.stackdepth == other.stackdepth) def __ne__(self, other): return not (self == other) def __hash__(self): - return hash((self.handlerposition, self.valuestackdepth)) + return hash((self.handlerposition, self.stackdepth)) def cleanupstack(self, ctx): - ctx.dropvaluesuntil(self.valuestackdepth) + ctx.dropvaluesuntil(self.stackdepth) def handle(self, ctx, unroller): raise NotImplementedError diff --git a/rpython/flowspace/framestate.py b/rpython/flowspace/framestate.py --- a/rpython/flowspace/framestate.py +++ b/rpython/flowspace/framestate.py @@ -1,21 +1,50 @@ -from rpython.flowspace.model import Variable, Constant +from rpython.flowspace.model import Variable, Constant, FSException from rpython.rlib.unroll import SpecTag +def _copy(v): + from rpython.flowspace.flowcontext import FlowSignal + if isinstance(v, Variable): + return Variable(v) + elif isinstance(v, FlowSignal): + vars = [_copy(var) for var in v.args] + return v.rebuild(*vars) + else: + return v + +def _union(seq1, seq2): + return [union(v1, v2) for v1, v2 in zip(seq1, seq2)] + class FrameState(object): - def __init__(self, mergeable, blocklist, next_offset): - self.mergeable = mergeable + def __init__(self, locals_w, stack, last_exception, blocklist, next_offset): + self.locals_w = locals_w + self.stack = stack + self.last_exception = last_exception self.blocklist = blocklist self.next_offset = next_offset + self._mergeable = None + + @property + def mergeable(self): + if self._mergeable is not None: + return self._mergeable + self._mergeable = data = self.locals_w + self.stack + if self.last_exception is None: + data.append(Constant(None)) + data.append(Constant(None)) + else: + data.append(self.last_exception.w_type) + data.append(self.last_exception.w_value) + recursively_flatten(data) + return data def copy(self): "Make a copy of this state in which all Variables are fresh." - newstate = [] - for w in self.mergeable: - if isinstance(w, Variable): - w = Variable(w) - newstate.append(w) - return FrameState(newstate, self.blocklist, self.next_offset) + exc = self.last_exception + if exc is not None: + exc = FSException(_copy(exc.w_type), _copy(exc.w_value)) + return FrameState(map(_copy, self.locals_w), map(_copy, self.stack), + exc, self.blocklist, self.next_offset) def getvariables(self): return [w for w in self.mergeable if isinstance(w, Variable)] @@ -33,18 +62,31 @@ return False return True + def _exc_args(self): + if self.last_exception is None: + return [Constant(None), Constant(None)] + else: + return [self.last_exception.w_type, + self.last_exception.w_value] + def union(self, other): """Compute a state that is at least as general as both self and other. A state 'a' is more general than a state 'b' if all Variables in 'b' are also Variables in 'a', but 'a' may have more Variables. """ - newstate = [] try: - for w1, w2 in zip(self.mergeable, other.mergeable): - newstate.append(union(w1, w2)) + locals = _union(self.locals_w, other.locals_w) + stack = _union(self.stack, other.stack) + if self.last_exception is None and other.last_exception is None: + exc = None + else: + args1 = self._exc_args() + args2 = other._exc_args() + exc = FSException(union(args1[0], args2[0]), + union(args1[1], args2[1])) except UnionError: return None - return FrameState(newstate, self.blocklist, self.next_offset) + return FrameState(locals, stack, exc, self.blocklist, self.next_offset) def getoutputargs(self, targetstate): "Return the output arguments needed to link self to targetstate." @@ -61,6 +103,7 @@ def union(w1, w2): "Union of two variables or constants." + from rpython.flowspace.flowcontext import FlowSignal if w1 == w2: return w1 if w1 is None or w2 is None: @@ -69,38 +112,21 @@ if isinstance(w1, Variable) or isinstance(w2, Variable): return Variable() # new fresh Variable if isinstance(w1, Constant) and isinstance(w2, Constant): - # FlowSignal represent stack unrollers in the stack. - # They should not be merged because they will be unwrapped. - # This is needed for try:except: and try:finally:, though - # it makes the control flow a bit larger by duplicating the - # handlers. - dont_merge_w1 = w1 in UNPICKLE_TAGS or isinstance(w1.value, SpecTag) - dont_merge_w2 = w2 in UNPICKLE_TAGS or isinstance(w2.value, SpecTag) - if dont_merge_w1 or dont_merge_w2: + if isinstance(w1.value, SpecTag) or isinstance(w2.value, SpecTag): raise UnionError else: return Variable() # generalize different constants + if isinstance(w1, FlowSignal) and isinstance(w2, FlowSignal): + if type(w1) is not type(w2): + raise UnionError + vars = [union(v1, v2) for v1, v2 in zip(w1.args, w2.args)] + return w1.rebuild(*vars) + if isinstance(w1, FlowSignal) or isinstance(w2, FlowSignal): + raise UnionError raise TypeError('union of %r and %r' % (w1.__class__.__name__, w2.__class__.__name__)) -# ____________________________________________________________ -# -# We have to flatten out the state of the frame into a list of -# Variables and Constants. This is done above by collecting the -# locals and the items on the value stack, but the latter may contain -# FlowSignal. We have to handle these specially, because -# some of them hide references to more Variables and Constants. -# The trick is to flatten ("pickle") them into the list so that the -# extra Variables show up directly in the list too. - -class PickleTag: - pass - -PICKLE_TAGS = {} -UNPICKLE_TAGS = {} - - def recursively_flatten(lst): from rpython.flowspace.flowcontext import FlowSignal i = 0 @@ -109,22 +135,4 @@ if not isinstance(unroller, FlowSignal): i += 1 else: - vars = unroller.state_unpack_variables() - key = unroller.__class__, len(vars) - try: - tag = PICKLE_TAGS[key] - except KeyError: - tag = PICKLE_TAGS[key] = Constant(PickleTag()) - UNPICKLE_TAGS[tag] = key - lst[i:i + 1] = [tag] + vars - - -def recursively_unflatten(lst): - for i in xrange(len(lst) - 1, -1, -1): - item = lst[i] - if item in UNPICKLE_TAGS: - unrollerclass, argcount = UNPICKLE_TAGS[item] - arguments = lst[i + 1:i + 1 + argcount] - del lst[i + 1:i + 1 + argcount] - unroller = unrollerclass.state_pack_variables(*arguments) - lst[i] = unroller + lst[i:i + 1] = unroller.args diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py --- a/rpython/flowspace/operation.py +++ b/rpython/flowspace/operation.py @@ -517,7 +517,7 @@ ctx.replace_in_stack(it, next_unroller) return const(v) w_item = ctx.do_op(self) - ctx.guessexception([StopIteration, RuntimeError], force=True) + ctx.recorder.guessexception(ctx, StopIteration, RuntimeError) return w_item class GetAttr(SingleDispatchMixin, HLOperation): diff --git a/rpython/flowspace/pygraph.py b/rpython/flowspace/pygraph.py --- a/rpython/flowspace/pygraph.py +++ b/rpython/flowspace/pygraph.py @@ -11,10 +11,10 @@ def __init__(self, func, code): from rpython.flowspace.flowcontext import SpamBlock - data = [None] * code.co_nlocals + locals = [None] * code.co_nlocals for i in range(code.formalargcount): - data[i] = Variable(code.co_varnames[i]) - state = FrameState(data + [Constant(None), Constant(None)], [], 0) + locals[i] = Variable(code.co_varnames[i]) + state = FrameState(locals, [], None, [], 0) initialblock = SpamBlock(state) super(PyGraph, self).__init__(self._sanitize_funcname(func), initialblock) self.func = func diff --git a/rpython/flowspace/test/test_flowcontext.py b/rpython/flowspace/test/test_flowcontext.py new file mode 100644 --- /dev/null +++ b/rpython/flowspace/test/test_flowcontext.py @@ -0,0 +1,15 @@ +""" Unit tests for flowcontext.py """ +import pytest +from rpython.flowspace.model import Variable, FSException +from rpython.flowspace.flowcontext import ( + Return, Raise, RaiseImplicit, Continue, Break) + + at pytest.mark.parametrize('signal', [ + Return(Variable()), + Raise(FSException(Variable(), Variable())), + RaiseImplicit(FSException(Variable(), Variable())), + Break(), + Continue(42), +]) +def test_signals(signal): + assert signal.rebuild(*signal.args) == signal diff --git a/rpython/flowspace/test/test_framestate.py b/rpython/flowspace/test/test_framestate.py --- a/rpython/flowspace/test/test_framestate.py +++ b/rpython/flowspace/test/test_framestate.py @@ -15,7 +15,7 @@ ctx = FlowContext(graph, code) # hack the frame ctx.setstate(graph.startblock.framestate) - ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Constant(None) + ctx.locals_w[-1] = Constant(None) return ctx def func_simple(x): @@ -31,7 +31,7 @@ def test_neq_hacked_framestate(self): ctx = self.get_context(self.func_simple) fs1 = ctx.getstate(0) - ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable() + ctx.locals_w[-1] = Variable() fs2 = ctx.getstate(0) assert not fs1.matches(fs2) @@ -44,7 +44,7 @@ def test_union_on_hacked_framestates(self): ctx = self.get_context(self.func_simple) fs1 = ctx.getstate(0) - ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable() + ctx.locals_w[-1] = Variable() fs2 = ctx.getstate(0) assert fs1.union(fs2).matches(fs2) # fs2 is more general assert fs2.union(fs1).matches(fs2) # fs2 is more general @@ -52,7 +52,7 @@ def test_restore_frame(self): ctx = self.get_context(self.func_simple) fs1 = ctx.getstate(0) - ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable() + ctx.locals_w[-1] = Variable() ctx.setstate(fs1) assert fs1.matches(ctx.getstate(0)) @@ -71,26 +71,25 @@ def test_getoutputargs(self): ctx = self.get_context(self.func_simple) fs1 = ctx.getstate(0) - ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable() + ctx.locals_w[-1] = Variable() fs2 = ctx.getstate(0) outputargs = fs1.getoutputargs(fs2) # 'x' -> 'x' is a Variable # locals_w[n-1] -> locals_w[n-1] is Constant(None) - assert outputargs == [ctx.locals_stack_w[0], Constant(None)] + assert outputargs == [ctx.locals_w[0], Constant(None)] def test_union_different_constants(self): ctx = self.get_context(self.func_simple) fs1 = ctx.getstate(0) - ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Constant(42) + ctx.locals_w[-1] = Constant(42) fs2 = ctx.getstate(0) fs3 = fs1.union(fs2) ctx.setstate(fs3) - assert isinstance(ctx.locals_stack_w[ctx.pycode.co_nlocals-1], - Variable) # generalized + assert isinstance(ctx.locals_w[-1], Variable) # generalized def test_union_spectag(self): ctx = self.get_context(self.func_simple) fs1 = ctx.getstate(0) - ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Constant(SpecTag()) + ctx.locals_w[-1] = Constant(SpecTag()) fs2 = ctx.getstate(0) assert fs1.union(fs2) is None # UnionError diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -749,18 +749,14 @@ rstack._stack_criticalcode_stop() def handle_async_forcing(self, deadframe): - from rpython.jit.metainterp.resume import (force_from_resumedata, - AlreadyForced) + from rpython.jit.metainterp.resume import force_from_resumedata metainterp_sd = self.metainterp_sd vinfo = self.jitdriver_sd.virtualizable_info ginfo = self.jitdriver_sd.greenfield_info # there is some chance that this is already forced. In this case # the virtualizable would have a token = NULL - try: - all_virtuals = force_from_resumedata(metainterp_sd, self, deadframe, - vinfo, ginfo) - except AlreadyForced: - return + all_virtuals = force_from_resumedata(metainterp_sd, self, deadframe, + vinfo, ginfo) # The virtualizable data was stored on the real virtualizable above. # Handle all_virtuals: keep them for later blackholing from the # future failure of the GUARD_NOT_FORCED diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py --- a/rpython/jit/metainterp/resume.py +++ b/rpython/jit/metainterp/resume.py @@ -19,9 +19,6 @@ # because it needs to support optimize.py which encodes virtuals with # arbitrary cycles and also to compress the information -class AlreadyForced(Exception): - pass - class Snapshot(object): __slots__ = ('prev', 'boxes') diff --git a/rpython/jit/metainterp/test/test_recursive.py b/rpython/jit/metainterp/test/test_recursive.py --- a/rpython/jit/metainterp/test/test_recursive.py +++ b/rpython/jit/metainterp/test/test_recursive.py @@ -774,8 +774,8 @@ def main(codeno): frame = Frame() frame.thing = Thing(0) - portal(codeno, frame) - return frame.thing.val + result = portal(codeno, frame) + return result def portal(codeno, frame): i = 0 @@ -791,7 +791,7 @@ s += subframe.thing.val frame.thing = Thing(nextval + 1) i += 1 - return frame.thing.val + return frame.thing.val + s res = self.meta_interp(main, [0], inline=True) self.check_resops(call=0, cond_call=0) # got removed by optimization diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -59,16 +59,18 @@ rffi.INT, releasegil=True) # release the GIL c_thread_releaselock = llexternal('RPyThreadReleaseLock', [TLOCKP], lltype.Void, - releasegil=True) # release the GIL + _nowrapper=True) # *don't* release the GIL # another set of functions, this time in versions that don't cause the -# GIL to be released. To use to handle the GIL lock itself. +# GIL to be released. Used to be there to handle the GIL lock itself, +# but that was changed (see rgil.py). Now here for performance only. c_thread_acquirelock_NOAUTO = llexternal('RPyThreadAcquireLock', [TLOCKP, rffi.INT], rffi.INT, _nowrapper=True) -c_thread_releaselock_NOAUTO = llexternal('RPyThreadReleaseLock', - [TLOCKP], lltype.Void, - _nowrapper=True) +c_thread_acquirelock_timed_NOAUTO = llexternal('RPyThreadAcquireLockTimed', + [TLOCKP, rffi.LONGLONG, rffi.INT], + rffi.INT, _nowrapper=True) +c_thread_releaselock_NOAUTO = c_thread_releaselock def allocate_lock(): @@ -131,9 +133,16 @@ self._lock = ll_lock def acquire(self, flag): - res = c_thread_acquirelock(self._lock, int(flag)) - res = rffi.cast(lltype.Signed, res) - return bool(res) + if flag: + c_thread_acquirelock(self._lock, 1) + return True + else: + res = c_thread_acquirelock_timed_NOAUTO( + self._lock, + rffi.cast(rffi.LONGLONG, 0), + rffi.cast(rffi.INT, 0)) + res = rffi.cast(lltype.Signed, res) + return bool(res) def acquire_timed(self, timeout): """Timeout is in microseconds. Returns 0 in case of failure, From noreply at buildbot.pypy.org Thu Feb 19 09:06:03 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 19 Feb 2015 09:06:03 +0100 (CET) Subject: [pypy-commit] pypy alt_errno: document branch Message-ID: <20150219080603.375961C009F@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: alt_errno Changeset: r75992:85c4dfba2a78 Date: 2015-02-19 10:03 +0200 http://bitbucket.org/pypy/pypy/changeset/85c4dfba2a78/ Log: document branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -26,3 +26,7 @@ .. branch: framestate2 Refactor rpython.flowspace.framestate.FrameState. + +.. branch alt_errno +Add an alternative location to save LastError, errno around ctypes, +cffi external calls so things like pdb will not overwrite it From noreply at buildbot.pypy.org Thu Feb 19 09:06:04 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 19 Feb 2015 09:06:04 +0100 (CET) Subject: [pypy-commit] pypy alt_errno: close branch to be merged Message-ID: <20150219080604.5C2AF1C009F@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: alt_errno Changeset: r75993:6528b5831517 Date: 2015-02-19 10:04 +0200 http://bitbucket.org/pypy/pypy/changeset/6528b5831517/ Log: close branch to be merged From noreply at buildbot.pypy.org Thu Feb 19 09:06:06 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 19 Feb 2015 09:06:06 +0100 (CET) Subject: [pypy-commit] pypy default: merge alt_errno which provides an alternative location for errno, LastError in calls to cffi, ctypes Message-ID: <20150219080606.2A47A1C009F@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r75994:d0d5493cb5fe Date: 2015-02-19 10:05 +0200 http://bitbucket.org/pypy/pypy/changeset/d0d5493cb5fe/ Log: merge alt_errno which provides an alternative location for errno, LastError in calls to cffi, ctypes diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -26,3 +26,7 @@ .. branch: framestate2 Refactor rpython.flowspace.framestate.FrameState. + +.. branch alt_errno +Add an alternative location to save LastError, errno around ctypes, +cffi external calls so things like pdb will not overwrite it diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py --- a/pypy/module/_cffi_backend/ccallback.py +++ b/pypy/module/_cffi_backend/ccallback.py @@ -210,6 +210,6 @@ space.threadlocals.leave_thread(space) def invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata): - cerrno._errno_after(rffi.RFFI_ERR_ALL) + cerrno._errno_after(rffi.RFFI_ERR_ALL | rffi.RFFI_ALT_ERRNO) _invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata) - cerrno._errno_before(rffi.RFFI_ERR_ALL) + cerrno._errno_before(rffi.RFFI_ERR_ALL | rffi.RFFI_ALT_ERRNO) diff --git a/pypy/module/_cffi_backend/cerrno.py b/pypy/module/_cffi_backend/cerrno.py --- a/pypy/module/_cffi_backend/cerrno.py +++ b/pypy/module/_cffi_backend/cerrno.py @@ -13,18 +13,18 @@ _errno_after = rposix._errno_after def get_errno(space): - return space.wrap(rposix.get_saved_errno()) + return space.wrap(rposix.get_saved_alterrno()) @unwrap_spec(errno=int) def set_errno(space, errno): - rposix.set_saved_errno(errno) + rposix.set_saved_alterrno(errno) # ____________________________________________________________ @unwrap_spec(code=int) def getwinerror(space, code=-1): - from rpython.rlib.rwin32 import GetLastError_saved, FormatError + from rpython.rlib.rwin32 import GetLastError_alt_saved, FormatError if code == -1: - code = GetLastError_saved() + code = GetLastError_alt_saved() message = FormatError(code) return space.newtuple([space.wrap(code), space.wrap(message)]) diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py --- a/pypy/module/_rawffi/interp_rawffi.py +++ b/pypy/module/_rawffi/interp_rawffi.py @@ -608,19 +608,19 @@ return space.wrap(W_CDLL(space, name, cdll)) def get_errno(space): - return space.wrap(rposix.get_saved_errno()) + return space.wrap(rposix.get_saved_alterrno()) def set_errno(space, w_errno): - rposix.set_saved_errno(space.int_w(w_errno)) + rposix.set_saved_alterrno(space.int_w(w_errno)) if sys.platform == 'win32': # see also # https://bitbucket.org/pypy/pypy/issue/1944/ctypes-on-windows-getlasterror def get_last_error(space): - return space.wrap(rwin32.GetLastError_saved()) + return space.wrap(rwin32.GetLastError_alt_saved()) @unwrap_spec(error=int) def set_last_error(space, error): - rwin32.SetLastError_saved(error) + rwin32.SetLastError_alt_saved(error) else: # always have at least a dummy version of these functions # (https://bugs.pypy.org/issue1242) diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py --- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py +++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py @@ -202,7 +202,7 @@ assert loop.match_by_id('cfficall', """ p96 = force_token() setfield_gc(p0, p96, descr=) - f97 = call_release_gil(27, i59, 1.0, 3, descr=) + f97 = call_release_gil(91, i59, 1.0, 3, descr=) guard_not_forced(descr=...) guard_no_exception(descr=...) """, ignore_ops=['guard_not_invalidated']) diff --git a/rpython/jit/backend/arm/callbuilder.py b/rpython/jit/backend/arm/callbuilder.py --- a/rpython/jit/backend/arm/callbuilder.py +++ b/rpython/jit/backend/arm/callbuilder.py @@ -176,11 +176,14 @@ def write_real_errno(self, save_err): if save_err & rffi.RFFI_READSAVED_ERRNO: - # Just before a call, read 'rpy_errno' and write it into the + # Just before a call, read '*_errno' and write it into the # real 'errno'. The r0-r3 registers contain arguments to the # future call; the r5-r7 registers contain various stuff. # We still have r8-r12. - rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu) + if save_err & rffi.RFFI_ALT_ERRNO: + rpy_errno = llerrno.get_alt_errno_offset(self.asm.cpu) + else: + rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu) p_errno = llerrno.get_p_errno_offset(self.asm.cpu) self.mc.LDR_ri(r.r9.value, r.sp.value, self.asm.saved_threadlocal_addr + self.current_sp) @@ -199,10 +202,13 @@ def read_real_errno(self, save_err): if save_err & rffi.RFFI_SAVE_ERRNO: # Just after a call, read the real 'errno' and save a copy of - # it inside our thread-local 'rpy_errno'. Registers r8-r12 + # it inside our thread-local '*_errno'. Registers r8-r12 # are unused here, and registers r2-r3 never contain anything # after the call. - rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu) + if save_err & rffi.RFFI_ALT_ERRNO: + rpy_errno = llerrno.get_alt_errno_offset(self.asm.cpu) + else: + rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu) p_errno = llerrno.get_p_errno_offset(self.asm.cpu) self.mc.LDR_ri(r.r3.value, r.sp.value, self.asm.saved_threadlocal_addr) diff --git a/rpython/jit/backend/llsupport/llerrno.py b/rpython/jit/backend/llsupport/llerrno.py --- a/rpython/jit/backend/llsupport/llerrno.py +++ b/rpython/jit/backend/llsupport/llerrno.py @@ -18,19 +18,41 @@ return 3 * WORD +def get_debug_saved_alterrno(cpu): + return cpu._debug_errno_container[4] + +def set_debug_saved_alterrno(cpu, nerrno): + assert nerrno >= 0 + cpu._debug_errno_container[4] = nerrno + +def get_alt_errno_offset(cpu): + if cpu.translate_support_code: + from rpython.rlib import rthread + return rthread.tlfield_alt_errno.getoffset() + else: + return 4 * WORD + + def get_debug_saved_lasterror(cpu): - return cpu._debug_errno_container[4] + return cpu._debug_errno_container[5] def set_debug_saved_lasterror(cpu, nerrno): assert nerrno >= 0 - cpu._debug_errno_container[4] = nerrno + cpu._debug_errno_container[5] = nerrno def get_rpy_lasterror_offset(cpu): if cpu.translate_support_code: from rpython.rlib import rthread return rthread.tlfield_rpy_lasterror.getoffset() else: - return 4 * WORD + return 5 * WORD + +def get_alt_lasterror_offset(cpu): + if cpu.translate_support_code: + from rpython.rlib import rthread + return rthread.tlfield_alt_lasterror.getoffset() + else: + return 6 * WORD def _fetch_addr_errno(): diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -63,7 +63,7 @@ ad.lendescr, FLAG_FLOAT) self.setup() self._debug_errno_container = lltype.malloc( - rffi.CArray(lltype.Signed), 5, flavor='raw', zero=True, + rffi.CArray(lltype.Signed), 7, flavor='raw', zero=True, track_allocation=False) def getarraydescr_for_frame(self, type): diff --git a/rpython/jit/backend/llsupport/test/zrpy_releasegil_test.py b/rpython/jit/backend/llsupport/test/zrpy_releasegil_test.py --- a/rpython/jit/backend/llsupport/test/zrpy_releasegil_test.py +++ b/rpython/jit/backend/llsupport/test/zrpy_releasegil_test.py @@ -98,6 +98,7 @@ self.run('close_stack') assert 'call_release_gil' in udir.join('TestCompileFramework.log').read() + # XXX this should also test get/set_alterrno ? def define_get_set_errno(self): eci = ExternalCompilationInfo( post_include_bits=[r''' diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -2948,7 +2948,11 @@ calldescr = self.cpu._calldescr_dynamic_for_tests([types.slong]*7, types.slong) # - for saveerr in [rffi.RFFI_ERR_NONE, rffi.RFFI_SAVE_ERRNO]: + for saveerr in [rffi.RFFI_ERR_NONE, + rffi.RFFI_SAVE_ERRNO, + rffi.RFFI_ERR_NONE | rffi.RFFI_ALT_ERRNO, + rffi.RFFI_SAVE_ERRNO | rffi.RFFI_ALT_ERRNO, + ]: faildescr = BasicFailDescr(1) inputargs = [BoxInt() for i in range(7)] i1 = BoxInt() @@ -2965,15 +2969,23 @@ self.cpu.compile_loop(inputargs, ops, looptoken) # llerrno.set_debug_saved_errno(self.cpu, 24) + llerrno.set_debug_saved_alterrno(self.cpu, 25) deadframe = self.cpu.execute_token(looptoken, 9, 8, 7, 6, 5, 4, 3) original_result = self.cpu.get_int_value(deadframe, 0) result = llerrno.get_debug_saved_errno(self.cpu) - print 'saveerr =', saveerr, ': got result =', result + altresult = llerrno.get_debug_saved_alterrno(self.cpu) + print 'saveerr =', saveerr, ': got result =', result, \ + 'altresult =', altresult # - if saveerr == rffi.RFFI_SAVE_ERRNO: - assert result == 42 # from the C code - else: - assert result == 24 # not touched + expected = { + rffi.RFFI_ERR_NONE: (24, 25), + rffi.RFFI_SAVE_ERRNO: (42, 25), + rffi.RFFI_ERR_NONE | rffi.RFFI_ALT_ERRNO: (24, 25), + rffi.RFFI_SAVE_ERRNO | rffi.RFFI_ALT_ERRNO: (24, 42), + } + # expected (24, 25) as originally set, with possibly one + # of the two changed to 42 by the assembler code + assert (result, altresult) == expected[saveerr] assert original_result == 3456789 def test_call_release_gil_readsaved_errno(self): @@ -3007,7 +3019,11 @@ calldescr = self.cpu._calldescr_dynamic_for_tests([types.slong]*7, types.slong) # - for saveerr in [rffi.RFFI_READSAVED_ERRNO, rffi.RFFI_ZERO_ERRNO_BEFORE]: + for saveerr in [rffi.RFFI_READSAVED_ERRNO, + rffi.RFFI_ZERO_ERRNO_BEFORE, + rffi.RFFI_READSAVED_ERRNO | rffi.RFFI_ALT_ERRNO, + rffi.RFFI_ZERO_ERRNO_BEFORE | rffi.RFFI_ALT_ERRNO, + ]: faildescr = BasicFailDescr(1) inputargs = [BoxInt() for i in range(7)] i1 = BoxInt() @@ -3024,12 +3040,17 @@ self.cpu.compile_loop(inputargs, ops, looptoken) # llerrno.set_debug_saved_errno(self.cpu, 24) + llerrno.set_debug_saved_alterrno(self.cpu, 25) deadframe = self.cpu.execute_token(looptoken, 9, 8, 7, 6, 5, 4, 3) result = self.cpu.get_int_value(deadframe, 0) assert llerrno.get_debug_saved_errno(self.cpu) == 24 + assert llerrno.get_debug_saved_alterrno(self.cpu) == 25 # - if saveerr == rffi.RFFI_READSAVED_ERRNO: - assert result == 24 + 345678900 + if saveerr & rffi.RFFI_READSAVED_ERRNO: + if saveerr & rffi.RFFI_ALT_ERRNO: + assert result == 25 + 345678900 + else: + assert result == 24 + 345678900 else: assert result == 0 + 345678900 @@ -3064,7 +3085,10 @@ types.slong) # for saveerr in [rffi.RFFI_SAVE_ERRNO, # but not _LASTERROR - rffi.RFFI_SAVE_LASTERROR]: + rffi.RFFI_SAVE_ERRNO | rffi.RFFI_ALT_ERRNO, + rffi.RFFI_SAVE_LASTERROR, + rffi.RFFI_SAVE_LASTERROR | rffi.RFFI_ALT_ERRNO, + ]: faildescr = BasicFailDescr(1) inputargs = [BoxInt() for i in range(7)] i1 = BoxInt() @@ -3125,7 +3149,9 @@ calldescr = self.cpu._calldescr_dynamic_for_tests([types.slong]*7, types.slong) # - for saveerr in [rffi.RFFI_READSAVED_LASTERROR]: + for saveerr in [rffi.RFFI_READSAVED_LASTERROR, + rffi.RFFI_READSAVED_LASTERROR | rffi.RFFI_ALT_ERRNO, + ]: faildescr = BasicFailDescr(1) inputargs = [BoxInt() for i in range(7)] i1 = BoxInt() @@ -3198,7 +3224,10 @@ calldescr = self.cpu._calldescr_dynamic_for_tests([types.slong]*7, types.slong) # - for saveerr in [rffi.RFFI_ERR_ALL]: + for saveerr in [rffi.RFFI_ERR_ALL, + rffi.RFFI_ERR_ALL | rffi.RFFI_ALT_ERRNO, + ]: + use_alt_errno = saveerr & rffi.RFFI_ALT_ERRNO faildescr = BasicFailDescr(1) inputargs = [BoxInt() for i in range(7)] i1 = BoxInt() @@ -3214,7 +3243,10 @@ looptoken = JitCellToken() self.cpu.compile_loop(inputargs, ops, looptoken) # - llerrno.set_debug_saved_errno(self.cpu, 8) + if use_alt_errno: + llerrno.set_debug_saved_alterrno(self.cpu, 8) + else: + llerrno.set_debug_saved_errno(self.cpu, 8) llerrno.set_debug_saved_lasterror(self.cpu, 9) deadframe = self.cpu.execute_token(looptoken, 1, 2, 3, 4, 5, 6, 7) result = self.cpu.get_int_value(deadframe, 0) diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py --- a/rpython/jit/backend/x86/callbuilder.py +++ b/rpython/jit/backend/x86/callbuilder.py @@ -196,21 +196,27 @@ SetLastError_addr = self.asm.cpu.cast_adr_to_int(adr) assert isinstance(self, CallBuilder32) # Windows 32-bit only # - rpy_lasterror = llerrno.get_rpy_lasterror_offset(self.asm.cpu) + if save_err & rffi.RFFI_ALT_ERRNO: + lasterror = llerrno.get_alt_lasterror_offset(self.asm.cpu) + else: + lasterror = llerrno.get_rpy_lasterror_offset(self.asm.cpu) tlofsreg = self.get_tlofs_reg() # => esi, callee-saved self.save_stack_position() # => edi, callee-saved - mc.PUSH_m((tlofsreg.value, rpy_lasterror)) + mc.PUSH_m((tlofsreg.value, lasterror)) mc.CALL(imm(SetLastError_addr)) # restore the stack position without assuming a particular # calling convention of _SetLastError() self.mc.MOV(esp, self.saved_stack_position_reg) if save_err & rffi.RFFI_READSAVED_ERRNO: - # Just before a call, read 'rpy_errno' and write it into the + # Just before a call, read '*_errno' and write it into the # real 'errno'. Most registers are free here, including the # callee-saved ones, except 'ebx' and except the ones used to # pass the arguments on x86-64. - rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu) + if save_err & rffi.RFFI_ALT_ERRNO: + rpy_errno = llerrno.get_alt_errno_offset(self.asm.cpu) + else: + rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu) p_errno = llerrno.get_p_errno_offset(self.asm.cpu) tlofsreg = self.get_tlofs_reg() # => esi or r12, callee-saved if IS_X86_32: @@ -234,11 +240,14 @@ if save_err & rffi.RFFI_SAVE_ERRNO: # Just after a call, read the real 'errno' and save a copy of - # it inside our thread-local 'rpy_errno'. Most registers are + # it inside our thread-local '*_errno'. Most registers are # free here, including the callee-saved ones, except 'ebx'. # The tlofs register might have been loaded earlier and is # callee-saved, so it does not need to be reloaded. - rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu) + if save_err & rffi.RFFI_ALT_ERRNO: + rpy_errno = llerrno.get_alt_errno_offset(self.asm.cpu) + else: + rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu) p_errno = llerrno.get_p_errno_offset(self.asm.cpu) tlofsreg = self.get_tlofs_reg() # => esi or r12 (possibly reused) mc.MOV_rm(edi.value, (tlofsreg.value, p_errno)) @@ -256,13 +265,16 @@ GetLastError_addr = self.asm.cpu.cast_adr_to_int(adr) assert isinstance(self, CallBuilder32) # Windows 32-bit only # - rpy_lasterror = llerrno.get_rpy_lasterror_offset(self.asm.cpu) + if save_err & rffi.RFFI_ALT_ERRNO: + lasterror = llerrno.get_alt_lasterror_offset(self.asm.cpu) + else: + lasterror = llerrno.get_rpy_lasterror_offset(self.asm.cpu) self.save_result_value(save_edx=True) # save eax/edx/xmm0 self.result_value_saved_early = True mc.CALL(imm(GetLastError_addr)) # tlofsreg = self.get_tlofs_reg() # => esi (possibly reused) - mc.MOV32_mr((tlofsreg.value, rpy_lasterror), eax.value) + mc.MOV32_mr((tlofsreg.value, lasterror), eax.value) def move_real_result_and_call_reacqgil_addr(self, fastgil): from rpython.jit.backend.x86 import rx86 diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -2879,7 +2879,8 @@ box_result = op.result # for now, any call via libffi saves and restores everything # (that is, errno and SetLastError/GetLastError on Windows) - c_saveall = ConstInt(rffi.RFFI_ERR_ALL) + # Note these flags match the ones in clibffi.ll_callback + c_saveall = ConstInt(rffi.RFFI_ERR_ALL | rffi.RFFI_ALT_ERRNO) self.history.record(rop.CALL_RELEASE_GIL, [c_saveall, op.getarg(2)] + arg_boxes, box_result, calldescr) diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py --- a/rpython/rlib/clibffi.py +++ b/rpython/rlib/clibffi.py @@ -331,7 +331,8 @@ c_ffi_call_return_type = lltype.Void c_ffi_call = external('ffi_call', [FFI_CIFP, rffi.VOIDP, rffi.VOIDP, VOIDPP], c_ffi_call_return_type, - save_err=rffi.RFFI_ERR_ALL) + save_err=rffi.RFFI_ERR_ALL | rffi.RFFI_ALT_ERRNO) +# Note: the RFFI_ALT_ERRNO flag matches the one in pyjitpl.direct_libffi_call CALLBACK_TP = rffi.CCallback([FFI_CIFP, rffi.VOIDP, rffi.VOIDPP, rffi.VOIDP], lltype.Void) c_ffi_prep_closure = external('ffi_prep_closure', [FFI_CLOSUREP, FFI_CIFP, @@ -424,9 +425,9 @@ userdata.callback(ll_args, ll_res, userdata) def ll_callback(ffi_cif, ll_res, ll_args, ll_userdata): - rposix._errno_after(rffi.RFFI_ERR_ALL) + rposix._errno_after(rffi.RFFI_ERR_ALL | rffi.RFFI_ALT_ERRNO) _ll_callback(ffi_cif, ll_res, ll_args, ll_userdata) - rposix._errno_before(rffi.RFFI_ERR_ALL) + rposix._errno_before(rffi.RFFI_ERR_ALL | rffi.RFFI_ALT_ERRNO) class StackCheckError(ValueError): diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -116,17 +116,44 @@ from rpython.rlib import rthread rthread.tlfield_rpy_errno.setraw(rffi.cast(INT, errno)) +def get_saved_alterrno(): + """Return the value of the "saved alterrno". + This value is saved after a call to a C function, if it was declared + with the flag llexternal(..., save_err=rffi.RFFI_SAVE_ERRNO | rffl.RFFI_ALT_ERRNO). + Functions without that flag don't change the saved errno. + """ + from rpython.rlib import rthread + return intmask(rthread.tlfield_alt_errno.getraw()) +def set_saved_alterrno(errno): + """Set the value of the saved alterrno. This value will be used to + initialize the real errno just before calling the following C function, + provided it was declared llexternal(..., save_err=RFFI_READSAVED_ERRNO | rffl.RFFI_ALT_ERRNO). + Note also that it is more common to want the real errno to be initially + zero; for that case, use llexternal(..., save_err=RFFI_ZERO_ERRNO_BEFORE) + and then you don't need set_saved_errno(0). + """ + from rpython.rlib import rthread + rthread.tlfield_alt_errno.setraw(rffi.cast(INT, errno)) + + +# These are not posix specific, but where should they move to? @specialize.call_location() def _errno_before(save_err): if save_err & rffi.RFFI_READSAVED_ERRNO: from rpython.rlib import rthread - _set_errno(rthread.tlfield_rpy_errno.getraw()) + if save_err & rffi.RFFI_ALT_ERRNO: + _set_errno(rthread.tlfield_alt_errno.getraw()) + else: + _set_errno(rthread.tlfield_rpy_errno.getraw()) elif save_err & rffi.RFFI_ZERO_ERRNO_BEFORE: _set_errno(rffi.cast(rffi.INT, 0)) if WIN32 and (save_err & rffi.RFFI_READSAVED_LASTERROR): from rpython.rlib import rthread, rwin32 - err = rthread.tlfield_rpy_lasterror.getraw() + if save_err & rffi.RFFI_ALT_ERRNO: + err = rthread.tlfield_alt_lasterror.getraw() + else: + err = rthread.tlfield_rpy_lasterror.getraw() # careful, getraw() overwrites GetLastError. # We must assign it with _SetLastError() as the last # operation, i.e. after the errno handling. @@ -140,14 +167,23 @@ err = rwin32._GetLastError() # careful, setraw() overwrites GetLastError. # We must read it first, before the errno handling. - rthread.tlfield_rpy_lasterror.setraw(err) + if save_err & rffi.RFFI_ALT_ERRNO: + rthread.tlfield_alt_lasterror.setraw(err) + else: + rthread.tlfield_rpy_lasterror.setraw(err) elif save_err & rffi.RFFI_SAVE_WSALASTERROR: from rpython.rlib import rthread, _rsocket_rffi err = _rsocket_rffi._WSAGetLastError() - rthread.tlfield_rpy_lasterror.setraw(err) + if save_err & rffi.RFFI_ALT_ERRNO: + rthread.tlfield_alt_lasterror.setraw(err) + else: + rthread.tlfield_rpy_lasterror.setraw(err) if save_err & rffi.RFFI_SAVE_ERRNO: from rpython.rlib import rthread - rthread.tlfield_rpy_errno.setraw(_get_errno()) + if save_err & rffi.RFFI_ALT_ERRNO: + rthread.tlfield_alt_errno.setraw(_get_errno()) + else: + rthread.tlfield_rpy_errno.setraw(_get_errno()) if os.name == 'nt': diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -379,9 +379,11 @@ tlfield_p_errno = ThreadLocalField(rffi.CArrayPtr(rffi.INT), "p_errno", loop_invariant=True) tlfield_rpy_errno = ThreadLocalField(rffi.INT, "rpy_errno") +tlfield_alt_errno = ThreadLocalField(rffi.INT, "alt_errno") if sys.platform == "win32": from rpython.rlib import rwin32 tlfield_rpy_lasterror = ThreadLocalField(rwin32.DWORD, "rpy_lasterror") + tlfield_alt_lasterror = ThreadLocalField(rwin32.DWORD, "alt_lasterror") def _threadlocalref_seeme(field): "NOT_RPYTHON" diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -145,6 +145,29 @@ from rpython.rlib import rthread rthread.tlfield_rpy_lasterror.setraw(rffi.cast(DWORD, err)) + def GetLastError_alt_saved(): + """Return the value of the "saved alt LastError". + The C-level GetLastError() is saved there after a call to a C + function, if that C function was declared with the flag + llexternal(..., save_err=RFFI_SAVE_LASTERROR | RFFI_ALT_ERRNO). + Functions without that flag don't change the saved LastError. + Alternatively, if the function was declared + RFFI_SAVE_WSALASTERROR | RFFI_ALT_ERRNO, + then the value of the C-level WSAGetLastError() is saved instead + (into the same "saved alt LastError" variable). + """ + from rpython.rlib import rthread + return rffi.cast(lltype.Signed, rthread.tlfield_alt_lasterror.getraw()) + + def SetLastError_alt_saved(err): + """Set the value of the saved alt LastError. This value will be used in + a call to the C-level SetLastError() just before calling the + following C function, provided it was declared + llexternal(..., save_err=RFFI_READSAVED_LASTERROR | RFFI_ALT_ERRNO). + """ + from rpython.rlib import rthread + rthread.tlfield_alt_lasterror.setraw(rffi.cast(DWORD, err)) + # In tests, the first call to _GetLastError() is always wrong, # because error is hidden by operations in ll2ctypes. Call it now. _GetLastError() diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py --- a/rpython/rtyper/lltypesystem/rffi.py +++ b/rpython/rtyper/lltypesystem/rffi.py @@ -70,7 +70,7 @@ RFFI_FULL_LASTERROR = RFFI_SAVE_LASTERROR | RFFI_READSAVED_LASTERROR RFFI_ERR_NONE = 0 RFFI_ERR_ALL = RFFI_FULL_ERRNO | RFFI_FULL_LASTERROR - +RFFI_ALT_ERRNO = 64 # read, save using alt tl destination def llexternal(name, args, result, _callable=None, compilation_info=ExternalCompilationInfo(), sandboxsafe=False, releasegil='auto', From noreply at buildbot.pypy.org Thu Feb 19 09:46:19 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Thu, 19 Feb 2015 09:46:19 +0100 (CET) Subject: [pypy-commit] stmgc default: remove some xfail from test_demo.py Message-ID: <20150219084620.013D01C029A@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1637:72b71c3e59bf Date: 2015-02-19 09:48 +0100 http://bitbucket.org/pypy/stmgc/changeset/72b71c3e59bf/ Log: remove some xfail from test_demo.py diff --git a/c8/test/test_demo.py b/c8/test/test_demo.py --- a/c8/test/test_demo.py +++ b/c8/test/test_demo.py @@ -14,13 +14,11 @@ self._do("../demo/%s > /dev/null" % target) def test_shadowstack(self): - py.test.xfail("no major gc yet") self.make_and_run("debug-test_shadowstack") def test_demo_simple_build(self): self.make_and_run("build-demo_simple") def test_demo_largemalloc_build(self): - py.test.xfail("no largemalloc") self.make_and_run("build-demo_largemalloc") From noreply at buildbot.pypy.org Thu Feb 19 09:49:33 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 19 Feb 2015 09:49:33 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: Fix the slightly broken page Message-ID: <20150219084933.611831C029A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r572:1a826439d598 Date: 2015-02-19 09:49 +0100 http://bitbucket.org/pypy/pypy.org/changeset/1a826439d598/ Log: Fix the slightly broken page diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -114,35 +114,18 @@

    Python2.7 compatible PyPy 2.5.0

    -
    diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -81,12 +81,10 @@ * `ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2, Ubuntu Precise)`__ (see ``[1]`` below) * `Mac OS/X binary (64bit)`__ * `Windows binary (32bit)`__ (you might need the `VS 2008 runtime library -* `Source (tar.bz2)`__ -* `Source (zip)`__ installer vcredist_x86.exe`_.) +* `Source (tar.bz2)`__; `Source (zip)`__. See below for more about the sources. * `All our downloads,`__ including previous versions. We also have a mirror_, but please use only if you have troubles accessing the links above -* See below for the sources. .. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.0-linux.tar.bz2 .. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.0-linux64.tar.bz2 From noreply at buildbot.pypy.org Thu Feb 19 10:16:43 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Thu, 19 Feb 2015 10:16:43 +0100 (CET) Subject: [pypy-commit] stmgc default: test cleanup Message-ID: <20150219091643.04CE21C123E@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: Changeset: r1638:b13bd550b525 Date: 2015-02-19 10:18 +0100 http://bitbucket.org/pypy/stmgc/changeset/b13bd550b525/ Log: test cleanup diff --git a/c8/test/test_basic.py b/c8/test/test_basic.py --- a/c8/test/test_basic.py +++ b/c8/test/test_basic.py @@ -734,38 +734,6 @@ self.abort_transaction() assert self.get_stm_thread_local().last_abort__bytes_in_nursery == 0 - def test_abort_in_segfault_handler(self): - py.test.skip("not doing that anymore") - lp1 = stm_allocate_old(16) - lp2 = stm_allocate_old(16) - - self.start_transaction() - stm_set_char(lp1, 'A') - stm_set_char(lp2, 'B') - self.commit_transaction() - # lp1 = S|N|N|N - # lp2 = S|N|N|N - - self.switch(1) - - self.start_transaction() - assert stm_get_char(lp1) == 'A' - # lp1 = S|P|N|N - # lp2 = S|N|N|N - - self.switch(0) - - self.start_transaction() - stm_set_char(lp1, 'C') - self.commit_transaction() - # lp1 = S|P|N|N - # lp2 = S|N|N|N - - self.switch(1, validate=False) - - # seg1 segfaults, validates, and aborts: - py.test.raises(Conflict, stm_get_char, lp2) - def test_bug(self): lp_char_5 = stm_allocate_old(384) diff --git a/c8/test/test_nursery.py b/c8/test/test_nursery.py --- a/c8/test/test_nursery.py +++ b/c8/test/test_nursery.py @@ -82,7 +82,6 @@ assert young def test_larger_than_limit_for_nursery_die(self): - py.test.xfail() obj_size = lib._STM_FAST_ALLOC + 16 self.start_transaction() @@ -202,7 +201,6 @@ assert lib.stm_can_move(old) == 0 def test_marker_1(self): - py.test.xfail() self.start_transaction() p1 = stm_allocate(600) stm_set_char(p1, 'o') From noreply at buildbot.pypy.org Thu Feb 19 12:09:20 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 19 Feb 2015 12:09:20 +0100 (CET) Subject: [pypy-commit] pypy default: attempt to fix this test Message-ID: <20150219110920.5967A1C009F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75995:3495c123cffb Date: 2015-02-19 12:09 +0100 http://bitbucket.org/pypy/pypy/changeset/3495c123cffb/ Log: attempt to fix this test diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py --- a/pypy/module/pypyjit/test_pypy_c/test_thread.py +++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py @@ -67,21 +67,11 @@ i58 = call_release_gil(0, _, i37, 1, descr=) guard_not_forced(descr=...) guard_no_exception(descr=...) - i59 = int_is_true(i58) - guard_true(i59, descr=...) - i60 = int_sub(i44, 1) - p62 = force_token() - setfield_gc(p0, p62, descr=) - i63 = call_release_gil(0, _, i37, 0, descr=) - guard_not_forced(descr=...) - guard_no_exception(descr=...) - i64 = int_is_true(i63) - guard_false(i64, descr=...) - p65 = force_token() - setfield_gc(p0, p65, descr=) - call_release_gil(0, _, i37, descr=) - guard_not_forced(descr=...) - guard_no_exception(descr=...) + i58 = int_sub(i44, 1) + i59 = call(ConstClass(RPyThreadAcquireLockTimed), i37, _, 0, descr=...) + i60 = int_is_true(i59) + guard_false(i60, descr=...) + call(ConstClass(RPyThreadReleaseLock), i37, descr=) guard_not_invalidated(descr=...) --TICK-- jump(..., descr=...) From noreply at buildbot.pypy.org Thu Feb 19 12:45:22 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 19 Feb 2015 12:45:22 +0100 (CET) Subject: [pypy-commit] pypy default: Group together the two C function calls done during lock.release(). Message-ID: <20150219114522.BD5821C009F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75996:223e8c97aec5 Date: 2015-02-19 12:45 +0100 http://bitbucket.org/pypy/pypy/changeset/223e8c97aec5/ Log: Group together the two C function calls done during lock.release(). diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -58,7 +58,8 @@ [TLOCKP, rffi.LONGLONG, rffi.INT], rffi.INT, releasegil=True) # release the GIL -c_thread_releaselock = llexternal('RPyThreadReleaseLock', [TLOCKP], lltype.Void, +c_thread_releaselock = llexternal('RPyThreadReleaseLock', [TLOCKP], + lltype.Signed, _nowrapper=True) # *don't* release the GIL # another set of functions, this time in versions that don't cause the @@ -152,12 +153,8 @@ return res def release(self): - # Sanity check: the lock must be locked - if self.acquire(False): - c_thread_releaselock(self._lock) - raise error("bad lock") - else: - c_thread_releaselock(self._lock) + if c_thread_releaselock(self._lock) != 0: + raise error("the lock was not previously acquired") def __del__(self): if free_ll_lock is None: # happens when tests are shutting down diff --git a/rpython/translator/c/src/thread_nt.c b/rpython/translator/c/src/thread_nt.c --- a/rpython/translator/c/src/thread_nt.c +++ b/rpython/translator/c/src/thread_nt.c @@ -184,10 +184,12 @@ return RPyThreadAcquireLockTimed(lock, waitflag ? -1 : 0, /*intr_flag=*/0); } -void RPyThreadReleaseLock(struct RPyOpaque_ThreadLock *lock) +long RPyThreadReleaseLock(struct RPyOpaque_ThreadLock *lock) { - if (!LeaveNonRecursiveMutex(lock)) - /* XXX complain? */; + if (LeaveNonRecursiveMutex(lock)) + return 0; /* success */ + else + return -1; /* failure: the lock was not previously acquired */ } /************************************************************/ diff --git a/rpython/translator/c/src/thread_nt.h b/rpython/translator/c/src/thread_nt.h --- a/rpython/translator/c/src/thread_nt.h +++ b/rpython/translator/c/src/thread_nt.h @@ -24,7 +24,7 @@ RPyLockStatus RPyThreadAcquireLockTimed(struct RPyOpaque_ThreadLock *lock, RPY_TIMEOUT_T timeout, int intr_flag); RPY_EXTERN -void RPyThreadReleaseLock(struct RPyOpaque_ThreadLock *lock); +long RPyThreadReleaseLock(struct RPyOpaque_ThreadLock *lock); RPY_EXTERN long RPyThreadGetStackSize(void); RPY_EXTERN diff --git a/rpython/translator/c/src/thread_pthread.c b/rpython/translator/c/src/thread_pthread.c --- a/rpython/translator/c/src/thread_pthread.c +++ b/rpython/translator/c/src/thread_pthread.c @@ -274,13 +274,23 @@ return success; } -void RPyThreadReleaseLock(struct RPyOpaque_ThreadLock *lock) +long RPyThreadReleaseLock(struct RPyOpaque_ThreadLock *lock) { - sem_t *thelock = &lock->sem; - int status, error = 0; + sem_t *thelock = &lock->sem; + int status, error = 0; + int current_value; - status = sem_post(thelock); - CHECK_STATUS("sem_post"); + /* If the current value is > 0, then the lock is not acquired so far. + Oops. */ + sem_getvalue(thelock, ¤t_value); + if (current_value > 0) { + return -1; + } + + status = sem_post(thelock); + CHECK_STATUS("sem_post"); + + return 0; } /************************************************************/ @@ -425,13 +435,18 @@ return success; } -void RPyThreadReleaseLock(struct RPyOpaque_ThreadLock *lock) +long RPyThreadReleaseLock(struct RPyOpaque_ThreadLock *lock) { int status, error = 0; + long result; status = pthread_mutex_lock( &lock->mut ); CHECK_STATUS("pthread_mutex_lock[3]"); + /* If the lock was non-locked, then oops, we return -1 for failure. + Otherwise, we return 0 for success. */ + result = (lock->locked == 0) ? -1 : 0; + lock->locked = 0; status = pthread_mutex_unlock( &lock->mut ); @@ -440,6 +455,8 @@ /* wake up someone (anyone, if any) waiting on the lock */ status = pthread_cond_signal( &lock->lock_released ); CHECK_STATUS("pthread_cond_signal"); + + return result; } /************************************************************/ diff --git a/rpython/translator/c/src/thread_pthread.h b/rpython/translator/c/src/thread_pthread.h --- a/rpython/translator/c/src/thread_pthread.h +++ b/rpython/translator/c/src/thread_pthread.h @@ -71,7 +71,7 @@ RPyLockStatus RPyThreadAcquireLockTimed(struct RPyOpaque_ThreadLock *lock, RPY_TIMEOUT_T timeout, int intr_flag); RPY_EXTERN -void RPyThreadReleaseLock(struct RPyOpaque_ThreadLock *lock); +long RPyThreadReleaseLock(struct RPyOpaque_ThreadLock *lock); RPY_EXTERN long RPyThreadGetStackSize(void); RPY_EXTERN From noreply at buildbot.pypy.org Thu Feb 19 12:50:02 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 19 Feb 2015 12:50:02 +0100 (CET) Subject: [pypy-commit] pypy default: Probable simplification Message-ID: <20150219115002.31AA11C0382@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75997:9a738e595473 Date: 2015-02-19 12:49 +0100 http://bitbucket.org/pypy/pypy/changeset/9a738e595473/ Log: Probable simplification diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py --- a/pypy/module/pypyjit/test_pypy_c/test_thread.py +++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py @@ -68,10 +68,9 @@ guard_not_forced(descr=...) guard_no_exception(descr=...) i58 = int_sub(i44, 1) - i59 = call(ConstClass(RPyThreadAcquireLockTimed), i37, _, 0, descr=...) - i60 = int_is_true(i59) + i59 = call(ConstClass(RPyThreadReleaseLock), i37, descr=) + i60 = int_ne(i59, 0) guard_false(i60, descr=...) - call(ConstClass(RPyThreadReleaseLock), i37, descr=) guard_not_invalidated(descr=...) --TICK-- jump(..., descr=...) From noreply at buildbot.pypy.org Thu Feb 19 16:39:59 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 19 Feb 2015 16:39:59 +0100 (CET) Subject: [pypy-commit] pypy default: fix Message-ID: <20150219153959.DFF461C009F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r75998:09d4c7bbabe4 Date: 2015-02-19 16:39 +0100 http://bitbucket.org/pypy/pypy/changeset/09d4c7bbabe4/ Log: fix diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py --- a/pypy/module/pypyjit/test_pypy_c/test_thread.py +++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py @@ -69,7 +69,7 @@ guard_no_exception(descr=...) i58 = int_sub(i44, 1) i59 = call(ConstClass(RPyThreadReleaseLock), i37, descr=) - i60 = int_ne(i59, 0) + i60 = int_is_true(i59) guard_false(i60, descr=...) guard_not_invalidated(descr=...) --TICK-- From noreply at buildbot.pypy.org Thu Feb 19 19:44:16 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Thu, 19 Feb 2015 19:44:16 +0100 (CET) Subject: [pypy-commit] pypy online-transforms-2: Start work on performing annotation-dependent graph transformations Message-ID: <20150219184416.A17781C11E2@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: online-transforms-2 Changeset: r75999:c1bcfe51647d Date: 2014-10-15 20:46 +0100 http://bitbucket.org/pypy/pypy/changeset/c1bcfe51647d/ Log: Start work on performing annotation-dependent graph transformations during the annotation phase. diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py --- a/rpython/annotator/annrpython.py +++ b/rpython/annotator/annrpython.py @@ -399,12 +399,22 @@ def flowin(self, graph, block): try: - for i, op in enumerate(block.operations): + i = 0 + while i < len(block.operations): + op = block.operations[i] self.bookkeeper.enter((graph, block, i)) try: + new_ops = op.transform(self) + if new_ops is not None: + block.operations[i:i+1] = new_ops + if not new_ops: + continue + new_ops[-1].result = op.result + op = new_ops[0] self.consider_op(op) finally: self.bookkeeper.leave() + i += 1 except BlockedInference as e: if (e.op is block.operations[-1] and diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py --- a/rpython/annotator/unaryop.py +++ b/rpython/annotator/unaryop.py @@ -699,9 +699,6 @@ def next(self): return self._emulate_call('next') - def len(self): - return self._emulate_call('__len__') - def getslice(self, s_start, s_stop): return self._emulate_call('__getslice__', s_start, s_stop) diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py --- a/rpython/flowspace/operation.py +++ b/rpython/flowspace/operation.py @@ -104,6 +104,9 @@ def get_can_only_throw(self, annotator): return None + def transform(self, annotator): + pass + class PureOperation(HLOperation): pure = True @@ -606,6 +609,15 @@ return ArgumentsForTranslation.fromshape(args_s[0].const, list(args_s[1:])) +def transform_len(hlop, annotator): + from rpython.annotator.model import SomeInstance + s_arg = annotator.annotation(hlop.args[0]) + if isinstance(s_arg, SomeInstance): + get_len = op.getattr(hlop.args[0], const('__len__')) + return [get_len, op.simple_call(get_len.result)] + +op.len.transform = transform_len + # Other functions that get directly translated to SpaceOperators func2op[type] = op.type From noreply at buildbot.pypy.org Thu Feb 19 19:44:17 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Thu, 19 Feb 2015 19:44:17 +0100 (CET) Subject: [pypy-commit] pypy online-transforms-2: make it possible to register transformers Message-ID: <20150219184417.E0FB71C11E2@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: online-transforms-2 Changeset: r76000:f787119f18c0 Date: 2014-10-15 21:59 +0100 http://bitbucket.org/pypy/pypy/changeset/f787119f18c0/ Log: make it possible to register transformers diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py --- a/rpython/annotator/unaryop.py +++ b/rpython/annotator/unaryop.py @@ -5,6 +5,7 @@ from __future__ import absolute_import from rpython.flowspace.operation import op +from rpython.flowspace.model import const from rpython.annotator.model import (SomeObject, SomeInteger, SomeBool, SomeString, SomeChar, SomeList, SomeDict, SomeTuple, SomeImpossibleValue, SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeBuiltinMethod, @@ -705,6 +706,12 @@ def setslice(self, s_start, s_stop, s_iterable): return self._emulate_call('__setslice__', s_start, s_stop, s_iterable) + at op.len.register_transform(SomeInstance) +def len_SomeInstance(annotator, v_arg): + get_len = op.getattr(v_arg, const('__len__')) + return [get_len, op.simple_call(get_len.result)] + + class __extend__(SomeBuiltin): def call(self, args, implicit_init=False): args_s, kwds = args.unpack() diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py --- a/rpython/flowspace/operation.py +++ b/rpython/flowspace/operation.py @@ -57,8 +57,10 @@ setattr(op, cls.opname, cls) if cls.dispatch == 1: cls._registry = {} + cls._transform = {} elif cls.dispatch == 2: cls._registry = DoubleDispatchRegistry() + cls._transform = DoubleDispatchRegistry() class HLOperation(SpaceOperation): @@ -104,8 +106,13 @@ def get_can_only_throw(self, annotator): return None + def get_transformer(self, *args_s): + return lambda *args: None + def transform(self, annotator): - pass + args_s = [annotator.annotation(arg) for arg in self.args] + transformer = self.get_transformer(*args_s) + return transformer(annotator, *self.args) class PureOperation(HLOperation): pure = True @@ -188,6 +195,37 @@ except AttributeError: return cls._dispatch(type(s_arg)) + @classmethod + def get_specialization(cls, s_arg, *_ignored): + try: + impl = getattr(s_arg, cls.opname) + + def specialized(annotator, arg, *other_args): + return impl(*[annotator.annotation(x) for x in other_args]) + try: + specialized.can_only_throw = impl.can_only_throw + except AttributeError: + pass + return specialized + except AttributeError: + return cls._dispatch(type(s_arg)) + + @classmethod + def register_transform(cls, Some_cls): + def decorator(func): + cls._transform[Some_cls] = func + return func + return decorator + + @classmethod + def get_transformer(cls, s_arg, *_ignored): + for c in type(s_arg).__mro__: + try: + return cls._transform[c] + except KeyError: + pass + return lambda *args: None + class DoubleDispatchMixin(object): dispatch = 2 @@ -219,6 +257,20 @@ spec = type(self).get_specialization(*args_s) return read_can_only_throw(spec, args_s[0], args_s[1]) + @classmethod + def register_transform(cls, Some1, Some2): + def decorator(func): + cls._transform[Some1, Some2] = func + return func + return decorator + + @classmethod + def get_transformer(cls, s_arg1, s_arg2, *_ignored): + try: + return cls._transform[type(s_arg1), type(s_arg2)] + except KeyError: + return lambda *args: None + def add_operator(name, arity, dispatch=None, pyfunc=None, pure=False, ovf=False): operator_func = getattr(operator, name, None) @@ -609,15 +661,6 @@ return ArgumentsForTranslation.fromshape(args_s[0].const, list(args_s[1:])) -def transform_len(hlop, annotator): - from rpython.annotator.model import SomeInstance - s_arg = annotator.annotation(hlop.args[0]) - if isinstance(s_arg, SomeInstance): - get_len = op.getattr(hlop.args[0], const('__len__')) - return [get_len, op.simple_call(get_len.result)] - -op.len.transform = transform_len - # Other functions that get directly translated to SpaceOperators func2op[type] = op.type From noreply at buildbot.pypy.org Thu Feb 19 19:44:19 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Thu, 19 Feb 2015 19:44:19 +0100 (CET) Subject: [pypy-commit] pypy online-transforms-2: InstanceRepr.rtype_len() is not used any more Message-ID: <20150219184419.0BB5A1C11E2@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: online-transforms-2 Changeset: r76001:db580d4f7b66 Date: 2014-10-15 22:22 +0100 http://bitbucket.org/pypy/pypy/changeset/db580d4f7b66/ Log: InstanceRepr.rtype_len() is not used any more diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py --- a/rpython/rtyper/rclass.py +++ b/rpython/rtyper/rclass.py @@ -851,9 +851,6 @@ def rtype_setslice(self, hop): return self._emulate_call(hop, "__setslice__") - def rtype_len(self, hop): - return self._emulate_call(hop, "__len__") - def ll_str(self, i): # doesn't work for non-gc classes! from rpython.rtyper.lltypesystem.ll_str import ll_int2hex from rpython.rlib.rarithmetic import r_uint From noreply at buildbot.pypy.org Thu Feb 19 19:44:20 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Thu, 19 Feb 2015 19:44:20 +0100 (CET) Subject: [pypy-commit] pypy online-transforms-2: handle __iter__, next, __getslice__, __setslice__ with online transforms Message-ID: <20150219184420.3B41C1C11E2@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: online-transforms-2 Changeset: r76002:246e64867715 Date: 2014-10-15 22:42 +0100 http://bitbucket.org/pypy/pypy/changeset/246e64867715/ Log: handle __iter__, next, __getslice__, __setslice__ with online transforms diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py --- a/rpython/annotator/unaryop.py +++ b/rpython/annotator/unaryop.py @@ -694,23 +694,33 @@ bk.emulate_pbc_call(bk.position_key, s_attr, args_s) return s_attr.call(simple_args(args_s)) - def iter(self): - return self._emulate_call('__iter__') - - def next(self): - return self._emulate_call('next') - - def getslice(self, s_start, s_stop): - return self._emulate_call('__getslice__', s_start, s_stop) - - def setslice(self, s_start, s_stop, s_iterable): - return self._emulate_call('__setslice__', s_start, s_stop, s_iterable) - @op.len.register_transform(SomeInstance) def len_SomeInstance(annotator, v_arg): get_len = op.getattr(v_arg, const('__len__')) return [get_len, op.simple_call(get_len.result)] + at op.iter.register_transform(SomeInstance) +def iter_SomeInstance(annotator, v_arg): + get_iter = op.getattr(v_arg, const('__iter__')) + return [get_iter, op.simple_call(get_iter.result)] + + at op.next.register_transform(SomeInstance) +def next_SomeInstance(annotator, v_arg): + get_next = op.getattr(v_arg, const('next')) + return [get_next, op.simple_call(get_next.result)] + + at op.getslice.register_transform(SomeInstance) +def getslice_SomeInstance(annotator, v_obj, v_start, v_stop): + get_getslice = op.getattr(v_obj, const('__getslice__')) + return [get_getslice, op.simple_call(get_getslice.result, v_start, v_stop)] + + + at op.setslice.register_transform(SomeInstance) +def setslice_SomeInstance(annotator, v_obj, v_start, v_stop, v_iterable): + get_setslice = op.getattr(v_obj, const('__setslice__')) + return [get_setslice, + op.simple_call(get_setslice.result, v_start, v_stop, v_iterable)] + class __extend__(SomeBuiltin): def call(self, args, implicit_init=False): diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py --- a/rpython/rtyper/rclass.py +++ b/rpython/rtyper/rclass.py @@ -839,18 +839,6 @@ hop2.args_s[0] = s_attr return hop2.dispatch() - def rtype_iter(self, hop): - return self._emulate_call(hop, '__iter__') - - def rtype_next(self, hop): - return self._emulate_call(hop, 'next') - - def rtype_getslice(self, hop): - return self._emulate_call(hop, "__getslice__") - - def rtype_setslice(self, hop): - return self._emulate_call(hop, "__setslice__") - def ll_str(self, i): # doesn't work for non-gc classes! from rpython.rtyper.lltypesystem.ll_str import ll_int2hex from rpython.rlib.rarithmetic import r_uint From noreply at buildbot.pypy.org Thu Feb 19 19:44:21 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Thu, 19 Feb 2015 19:44:21 +0100 (CET) Subject: [pypy-commit] pypy online-transforms-2: handle __getitem__, __setitem__ with online transforms Message-ID: <20150219184421.7446D1C11E2@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: online-transforms-2 Changeset: r76003:d2d5da95e062 Date: 2014-10-15 23:28 +0100 http://bitbucket.org/pypy/pypy/changeset/d2d5da95e062/ Log: handle __getitem__, __setitem__ with online transforms diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py --- a/rpython/annotator/binaryop.py +++ b/rpython/annotator/binaryop.py @@ -2,7 +2,6 @@ Binary operations between SomeValues. """ -import operator from rpython.tool.pairtype import pair, pairtype from rpython.annotator.model import ( SomeObject, SomeInteger, SomeBool, s_Bool, SomeString, SomeChar, SomeList, @@ -14,7 +13,7 @@ read_can_only_throw, add_knowntypedata, merge_knowntypedata,) from rpython.annotator.bookkeeper import immutablevalue -from rpython.flowspace.model import Variable, Constant +from rpython.flowspace.model import Variable, Constant, const from rpython.flowspace.operation import op from rpython.rlib import rarithmetic from rpython.annotator.model import AnnotatorError @@ -689,12 +688,16 @@ return super(thistype, pair(ins1, ins2)).improve() -class __extend__(pairtype(SomeInstance, SomeObject)): - def getitem((s_ins, s_idx)): - return s_ins._emulate_call("__getitem__", s_idx) + at op.getitem.register_transform(SomeInstance, SomeObject) +def getitem_SomeInstance(annotator, v_ins, v_idx): + get_getitem = op.getattr(v_ins, const('__getitem__')) + return [get_getitem, op.simple_call(get_getitem.result, v_idx)] - def setitem((s_ins, s_idx), s_value): - return s_ins._emulate_call("__setitem__", s_idx, s_value) + at op.setitem.register_transform(SomeInstance, SomeObject) +def setitem_SomeInstance(annotator, v_ins, v_idx, v_value): + get_setitem = op.getattr(v_ins, const('__setitem__')) + return [get_setitem, + op.simple_call(get_setitem.result, v_idx, v_value)] class __extend__(pairtype(SomeIterator, SomeIterator)): diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py --- a/rpython/rtyper/rclass.py +++ b/rpython/rtyper/rclass.py @@ -1008,14 +1008,6 @@ return hop.gendirectcall(ll_isinstance, v_obj, v_cls) -class __extend__(pairtype(InstanceRepr, Repr)): - def rtype_getitem((r_ins, r_obj), hop): - return r_ins._emulate_call(hop, "__getitem__") - - def rtype_setitem((r_ins, r_obj), hop): - return r_ins._emulate_call(hop, "__setitem__") - - class __extend__(pairtype(InstanceRepr, InstanceRepr)): def convert_from_to((r_ins1, r_ins2), v, llops): # which is a subclass of which? From noreply at buildbot.pypy.org Thu Feb 19 19:44:22 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Thu, 19 Feb 2015 19:44:22 +0100 (CET) Subject: [pypy-commit] pypy online-transforms-2: kill unused _emulate_call() Message-ID: <20150219184422.97D201C11E2@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: online-transforms-2 Changeset: r76004:5fffc446489a Date: 2014-10-15 23:35 +0100 http://bitbucket.org/pypy/pypy/changeset/5fffc446489a/ Log: kill unused _emulate_call() diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py --- a/rpython/annotator/unaryop.py +++ b/rpython/annotator/unaryop.py @@ -687,13 +687,6 @@ if not self.can_be_None: s.const = True - def _emulate_call(self, meth_name, *args_s): - bk = getbookkeeper() - s_attr = self._true_getattr(meth_name) - # record for calltables - bk.emulate_pbc_call(bk.position_key, s_attr, args_s) - return s_attr.call(simple_args(args_s)) - @op.len.register_transform(SomeInstance) def len_SomeInstance(annotator, v_arg): get_len = op.getattr(v_arg, const('__len__')) diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py --- a/rpython/rtyper/rclass.py +++ b/rpython/rtyper/rclass.py @@ -820,25 +820,6 @@ vinst, = hop.inputargs(self) return hop.genop('ptr_nonzero', [vinst], resulttype=Bool) - def _emulate_call(self, hop, meth_name): - vinst = hop.args_v[0] - clsdef = hop.args_s[0].classdef - s_unbound_attr = clsdef.find_attribute(meth_name).getvalue() - s_attr = clsdef.lookup_filter(s_unbound_attr, meth_name, - hop.args_s[0].flags) - # does that even happen? - assert not s_attr.is_constant() - if '__iter__' in self.allinstancefields: - raise Exception("__iter__ on instance disallowed") - r_method = self.rtyper.getrepr(s_attr) - r_method.get_method_from_instance(self, vinst, hop.llops) - hop2 = hop.copy() - hop2.spaceop = op.simple_call(*hop.spaceop.args) - hop2.spaceop.result = hop.spaceop.result - hop2.args_r[0] = r_method - hop2.args_s[0] = s_attr - return hop2.dispatch() - def ll_str(self, i): # doesn't work for non-gc classes! from rpython.rtyper.lltypesystem.ll_str import ll_int2hex from rpython.rlib.rarithmetic import r_uint From noreply at buildbot.pypy.org Fri Feb 20 09:14:51 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 20 Feb 2015 09:14:51 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Change again a bit the API in transaction.py. Now it feels more natural. Message-ID: <20150220081451.9F6721C03C6@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76005:9d4605f70a31 Date: 2015-02-19 23:36 +0100 http://bitbucket.org/pypy/pypy/changeset/9d4605f70a31/ Log: Change again a bit the API in transaction.py. Now it feels more natural. diff --git a/lib_pypy/pypy_test/test_transaction.py b/lib_pypy/pypy_test/test_transaction.py --- a/lib_pypy/pypy_test/test_transaction.py +++ b/lib_pypy/pypy_test/test_transaction.py @@ -8,9 +8,10 @@ def test_simple_random_order(): for x in range(N): lst = [] - with transaction.TransactionQueue(): - for i in range(10): - transaction.add(lst.append, i) + tq = transaction.TransactionQueue() + for i in range(10): + tq.add(lst.append, i) + tq.run() if VERBOSE: print lst assert sorted(lst) == range(10), lst @@ -22,9 +23,10 @@ lst.append(i) i += 1 if i < 10: - transaction.add(do_stuff, i) - with transaction.TransactionQueue(): - transaction.add(do_stuff, 0) + tq.add(do_stuff, i) + tq = transaction.TransactionQueue() + tq.add(do_stuff, 0) + tq.run() if VERBOSE: print lst assert lst == range(10), lst @@ -36,10 +38,11 @@ lsts[i].append(j) j += 1 if j < 10: - transaction.add(do_stuff, i, j) - with transaction.TransactionQueue(): - for i in range(5): - transaction.add(do_stuff, i, 0) + tq.add(do_stuff, i, j) + tq = transaction.TransactionQueue() + for i in range(5): + tq.add(do_stuff, i, 0) + tq.run() if VERBOSE: print lsts assert lsts == (range(10),) * 5, lsts @@ -53,14 +56,15 @@ lsts[i].append(j) j += 1 if j < 5: - transaction.add(do_stuff, i, j) + tq.add(do_stuff, i, j) else: lsts[i].append('foo') raise FooError + tq = transaction.TransactionQueue() + for i in range(10): + tq.add(do_stuff, i, 0) try: - with transaction.TransactionQueue(): - for i in range(10): - transaction.add(do_stuff, i, 0) + tq.run() except FooError: pass else: @@ -78,19 +82,66 @@ def test_number_of_transactions_reported(): - py.test.skip("not reimplemented") - with transaction.TransactionQueue(): - transaction.add(lambda: None) - assert transaction.number_of_transactions_in_last_run() == 1 + tq = transaction.TransactionQueue() + tq.add(lambda: None) + tq.add(lambda: None) + tq.run() + assert tq.number_of_transactions_executed() == 2 + + tq.run() + assert tq.number_of_transactions_executed() == 2 + + tq.add(lambda: None) + tq.run() + assert tq.number_of_transactions_executed() == 3 + + tq.add(lambda: some_name_that_is_not_defined) + try: + tq.run() + except NameError: + pass + else: + raise AssertionError("should have raised NameError") + assert tq.number_of_transactions_executed() == 4 def add_transactions(l): if l: for x in range(l[0]): - transaction.add(add_transactions, l[1:]) + tq.add(add_transactions, l[1:]) - with transaction.TransactionQueue(): - transaction.add(add_transactions, [10, 10, 10]) - assert transaction.number_of_transactions_in_last_run() == 1111 + tq = transaction.TransactionQueue() + tq.add(add_transactions, [10, 10, 10]) + tq.run() + assert tq.number_of_transactions_executed() == 1111 + +def test_unexecuted_transactions_after_exception(): + class FooError(Exception): + pass + class BarError(Exception): + pass + def raiseme(exc): + raise exc + seen = [] + tq = transaction.TransactionQueue() + tq.add(raiseme, FooError) + tq.add(raiseme, BarError) + tq.add(seen.append, 42) + tq.add(seen.append, 42) + try: + tq.run() + except (FooError, BarError), e: + seen_exc = e.__class__ + else: + raise AssertionError("should have raised FooError or BarError") + try: + tq.run() + except (FooError, BarError), e: + assert e.__class__ != seen_exc + else: + raise AssertionError("unexecuted transactions have disappeared") + for i in range(2): + tq.run() + assert seen == [42, 42] def test_stmidset(): diff --git a/lib_pypy/transaction.py b/lib_pypy/transaction.py --- a/lib_pypy/transaction.py +++ b/lib_pypy/transaction.py @@ -108,106 +108,101 @@ pass -def add(f, *args, **kwds): - """Register a new transaction that will be done by 'f(*args, **kwds)'. - Must be called within the transaction in the "with TransactionQueue()" - block, or within a transaction started by this one, directly or - indirectly. - """ - _thread_local.pending.append((f, args, kwds)) +class TransactionQueue(object): + """A queue of pending transactions. - -class TransactionQueue(object): - """Use in 'with TransactionQueue():'. Creates a queue of - transactions. The first transaction in the queue is the content of - the 'with:' block, which is immediately started. - - Any transaction can register new transactions that will be run - after the current one is finished, using the global function add(). + Use the add() method to register new transactions into the queue. + Afterwards, call run() once. While transactions run, it is possible + to add() more transactions, which will run after the current one is + finished. The run() call only returns when the queue is completely + empty. """ - def __init__(self, nb_segments=0): + def __init__(self): + self._deque = collections.deque() + self._pending = self._deque + self._number_transactions_exec = 0 + + def add(self, f, *args, **kwds): + """Register a new transaction to be done by 'f(*args, **kwds)'. + """ + # note: 'self._pending.append' can be two things here: + # * if we are outside run(), it is the regular deque.append method; + # * if we are inside run(), self._pending is a thread._local() + # and then its append attribute is the append method of a + # thread-local list. + self._pending.append((f, args, kwds)) + + def run(self, nb_segments=0): + """Run all transactions, and all transactions started by these + ones, recursively, until the queue is empty. If one transaction + raises, run() re-raises the exception and the unexecuted transaction + are left in the queue. + """ + if is_atomic(): + raise TransactionError( + "TransactionQueue.run() cannot be called in an atomic context") + if not self._pending: + return if nb_segments <= 0: nb_segments = getsegmentlimit() - _thread_pool.ensure_threads(nb_segments) - def __enter__(self): - if hasattr(_thread_local, "pending"): - raise TransactionError( - "recursive invocation of TransactionQueue()") - if is_atomic(): - raise TransactionError( - "invocation of TransactionQueue() from an atomic context") - _thread_local.pending = [] - atomic.__enter__() + assert self._pending is self._deque, "broken state" + try: + self._pending = thread._local() + lock_done_running = thread.allocate_lock() + lock_done_running.acquire() + lock_deque = thread.allocate_lock() + locks = [] + exception = [] + args = (locks, lock_done_running, lock_deque, + exception, nb_segments) + # + for i in range(nb_segments): + thread.start_new_thread(self._thread_runner, args) + # + # The threads run here, and they will release this lock when + # they are all finished. + lock_done_running.acquire() + # + assert len(locks) == nb_segments + for lock in locks: + lock.release() + # + finally: + self._pending = self._deque + # + if exception: + exc_type, exc_value, exc_traceback = exception + raise exc_type, exc_value, exc_traceback - def __exit__(self, exc_type, exc_value, traceback): - atomic.__exit__(exc_type, exc_value, traceback) - pending = _thread_local.pending - del _thread_local.pending - if exc_type is None and len(pending) > 0: - _thread_pool.run(pending) + def number_of_transactions_executed(self): + return self._number_transactions_exec - -# ____________________________________________________________ - - -class _ThreadPool(object): - - def __init__(self): - self.lock_running = thread.allocate_lock() - self.lock_done_running = thread.allocate_lock() - self.lock_done_running.acquire() - self.nb_threads = 0 - self.deque = collections.deque() - self.locks = [] - self.lock_deque = thread.allocate_lock() - self.exception = [] - - def ensure_threads(self, n): - if n > self.nb_threads: - with self.lock_running: - for i in range(self.nb_threads, n): - assert len(self.locks) == self.nb_threads - self.nb_threads += 1 - thread.start_new_thread(self.thread_runner, ()) - # The newly started thread should run immediately into - # the case 'if len(self.locks) == self.nb_threads:' - # and release this lock. Wait until it does. - self.lock_done_running.acquire() - - def run(self, pending): - # For now, can't run multiple threads with each an independent - # TransactionQueue(): they are serialized. - with self.lock_running: - assert self.exception == [] - assert len(self.deque) == 0 - deque = self.deque - with self.lock_deque: - deque.extend(pending) - try: - for i in range(len(pending)): - self.locks.pop().release() - except IndexError: # pop from empty list - pass - # - self.lock_done_running.acquire() - # - if self.exception: - exc_type, exc_value, exc_traceback = self.exception - del self.exception[:] - raise exc_type, exc_value, exc_traceback - - def thread_runner(self): - deque = self.deque + def _thread_runner(self, locks, lock_done_running, lock_deque, + exception, nb_segments): + pending = [] + self._pending.append = pending.append + deque = self._deque lock = thread.allocate_lock() lock.acquire() - pending = [] - _thread_local.pending = pending - lock_deque = self.lock_deque - exception = self.exception + next_transaction = None + count = [0] # - while True: + def _pause_thread(): + self._number_transactions_exec += count[0] + count[0] = 0 + locks.append(lock) + if len(locks) == nb_segments: + lock_done_running.release() + lock_deque.release() + # + # Now wait until our lock is released. + lock.acquire() + return len(locks) == nb_segments + # + while not exception: + assert next_transaction is None # # Look at the deque and try to fetch the next item on the left. # If empty, we add our lock to the 'locks' list. @@ -216,13 +211,8 @@ next_transaction = deque.popleft() lock_deque.release() else: - self.locks.append(lock) - if len(self.locks) == self.nb_threads: - self.lock_done_running.release() - lock_deque.release() - # - # Now wait until our lock is released. - lock.acquire() + if _pause_thread(): + return continue # # Now we have a next_transaction. Run it. @@ -230,13 +220,16 @@ while True: f, args, kwds = next_transaction with atomic: - if len(exception) == 0: - try: - with signals_enabled: - f(*args, **kwds) - except: - exception.extend(sys.exc_info()) - del next_transaction + if exception: + break + next_transaction = None + try: + with signals_enabled: + count[0] += 1 + f(*args, **kwds) + except: + exception.extend(sys.exc_info()) + break # # If no new 'pending' transactions have been added, exit # this loop and go back to fetch more from the deque. @@ -252,19 +245,27 @@ # single item to 'next_transaction', because it looks # like a good idea to preserve some first-in-first-out # approximation.) - with self.lock_deque: + with lock_deque: deque.extend(pending) next_transaction = deque.popleft() try: for i in range(1, len(pending)): - self.locks.pop().release() + locks.pop().release() except IndexError: # pop from empty list pass del pending[:] + # + # We exit here with an exception. Re-add 'next_transaction' + # if it is not None. + lock_deque.acquire() + if next_transaction is not None: + deque.appendleft(next_transaction) + next_transaction = None + while not _pause_thread(): + lock_deque.acquire() -_thread_pool = _ThreadPool() -_thread_local = thread._local() +# ____________________________________________________________ def XXXreport_abort_info(info): From noreply at buildbot.pypy.org Fri Feb 20 09:14:52 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 20 Feb 2015 09:14:52 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Cannot call tq.number_of_transactions_executed() when running. Message-ID: <20150220081452.E55FC1C03C6@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76006:1be37e1a9afe Date: 2015-02-20 09:14 +0100 http://bitbucket.org/pypy/pypy/changeset/1be37e1a9afe/ Log: Cannot call tq.number_of_transactions_executed() when running. diff --git a/lib_pypy/pypy_test/test_transaction.py b/lib_pypy/pypy_test/test_transaction.py --- a/lib_pypy/pypy_test/test_transaction.py +++ b/lib_pypy/pypy_test/test_transaction.py @@ -104,6 +104,14 @@ raise AssertionError("should have raised NameError") assert tq.number_of_transactions_executed() == 4 + tq.add(tq.number_of_transactions_executed) + try: + tq.run() + except transaction.TransactionError: + pass + else: + raise AssertionError("should have raised TransactionError") + def add_transactions(l): if l: for x in range(l[0]): diff --git a/lib_pypy/transaction.py b/lib_pypy/transaction.py --- a/lib_pypy/transaction.py +++ b/lib_pypy/transaction.py @@ -177,7 +177,9 @@ raise exc_type, exc_value, exc_traceback def number_of_transactions_executed(self): - return self._number_transactions_exec + if self._pending is self._deque: + return self._number_transactions_exec + raise TransactionError("TransactionQueue.run() is currently running") def _thread_runner(self, locks, lock_done_running, lock_deque, exception, nb_segments): From noreply at buildbot.pypy.org Fri Feb 20 09:20:59 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 20 Feb 2015 09:20:59 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Use the new API Message-ID: <20150220082059.7B40D1C029E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76007:5de6619fa6de Date: 2015-02-20 09:20 +0100 http://bitbucket.org/pypy/pypy/changeset/5de6619fa6de/ Log: Use the new API diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py --- a/rpython/rtyper/rtyper.py +++ b/rpython/rtyper/rtyper.py @@ -262,9 +262,10 @@ # try a version using the transaction module self.log.event('specializing transactionally %d blocks' % (len(pending),)) - with transaction.TransactionQueue(): - for block in pending: - transaction.add(self.specialize_block, block) + tq = transaction.TransactionQueue() + for block in pending: + tq.add(self.specialize_block, block) + tq.run() self.log.event('left transactional mode') blockcount += len(pending) self.already_seen.update(dict.fromkeys(pending, True)) From noreply at buildbot.pypy.org Fri Feb 20 09:33:28 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 20 Feb 2015 09:33:28 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: start working on a talk Message-ID: <20150220083328.C83071C11AA@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: extradoc Changeset: r5505:2dc819062ba3 Date: 2015-02-20 10:33 +0200 http://bitbucket.org/pypy/extradoc/changeset/2dc819062ba3/ Log: start working on a talk diff --git a/talk/divio-zurich-2015/talk.rst b/talk/divio-zurich-2015/talk.rst new file mode 100644 --- /dev/null +++ b/talk/divio-zurich-2015/talk.rst @@ -0,0 +1,78 @@ +==== +PyPy +==== + +Who am I? +--------- + +* Maciej Fijalkowski + +* PyPy core developer + +* owner of baroquesoftware.com + +What is PyPy? +------------- + +* a fast, compliant Python interpreter + +* comes with a just in time compiler + +* covers Python 2.7 and beta 3.2/3.3 + +Compatibility status +-------------------- + +* pure-python code should just run + +* C-extensions sometimes work sometimes don't + +* library support is large and growing + +* numeric is more complicated + +Compatibility - C extensions +---------------------------- + +* CPython C API - compatibility layer (slow, potentially problematic) + +* Cython uses that, does not work + +* ctypes sort of works, but don't use ctypes + +* our solutions is cffi + +cffi +---- + +XXX slide about cffi and demo + +Speed status +------------ + +xxx graph + +Speed status (2) +---------------- + +* more mudded picture :-) + +* a lot of programs are faster (20% to 100x) + +* some programs are slower + +* warmup time is a key factor + +Memory consumption +------------------ + +* objects are smaller than CPython + +Can I run PyPy? +--------------- + +Xxxx + +Case study - magnetic +--------------------- + From noreply at buildbot.pypy.org Fri Feb 20 09:58:49 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 20 Feb 2015 09:58:49 +0100 (CET) Subject: [pypy-commit] pypy default: Kill an unused function Message-ID: <20150220085849.7E5F11C11F1@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76008:29e1d6afb3c7 Date: 2015-02-20 09:58 +0100 http://bitbucket.org/pypy/pypy/changeset/29e1d6afb3c7/ Log: Kill an unused function diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py --- a/pypy/module/time/interp_time.py +++ b/pypy/module/time/interp_time.py @@ -168,7 +168,6 @@ if cConfig.has_gettimeofday: c_gettimeofday = external('gettimeofday', [rffi.VOIDP, rffi.VOIDP], rffi.INT) TM_P = lltype.Ptr(tm) -c_clock = external('clock', [rffi.TIME_TP], clock_t) c_time = external('time', [rffi.TIME_TP], rffi.TIME_T) c_ctime = external('ctime', [rffi.TIME_TP], rffi.CCHARP) c_gmtime = external('gmtime', [rffi.TIME_TP], TM_P, From noreply at buildbot.pypy.org Fri Feb 20 10:42:32 2015 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 20 Feb 2015 10:42:32 +0100 (CET) Subject: [pypy-commit] pypy default: whoops Message-ID: <20150220094232.8C95F1C03C6@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76009:623fdd43ae53 Date: 2015-02-20 11:06 +0200 http://bitbucket.org/pypy/pypy/changeset/623fdd43ae53/ Log: whoops diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -27,6 +27,6 @@ .. branch: framestate2 Refactor rpython.flowspace.framestate.FrameState. -.. branch alt_errno +.. branch: alt_errno Add an alternative location to save LastError, errno around ctypes, cffi external calls so things like pdb will not overwrite it From noreply at buildbot.pypy.org Fri Feb 20 11:00:09 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 20 Feb 2015 11:00:09 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: work on talk Message-ID: <20150220100009.12DD11C11F1@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: extradoc Changeset: r5506:ff4a28b22fc0 Date: 2015-02-20 12:00 +0200 http://bitbucket.org/pypy/extradoc/changeset/ff4a28b22fc0/ Log: work on talk diff --git a/talk/divio-zurich-2015/talk.rst b/talk/divio-zurich-2015/talk.rst --- a/talk/divio-zurich-2015/talk.rst +++ b/talk/divio-zurich-2015/talk.rst @@ -45,12 +45,28 @@ cffi ---- -XXX slide about cffi and demo +xxxx image + +cffi (2) +-------- + +* demo + +numpy and numerics +------------------ + +* big subset of numpy reimplemented + +* same speed as numpy, much faster with python code + +* aim at compatibility + +* pymetabiosis (scipy, matplotlib) Speed status ------------ -xxx graph +* speed.pypy.org Speed status (2) ---------------- @@ -68,11 +84,65 @@ * objects are smaller than CPython +* interpreter size is large (~60M) + +* additional assembler consumes memory + +* plus bookkeeping info + +* measure! + Can I run PyPy? --------------- -Xxxx +* mostly, yes! -Case study - magnetic +* look at C extensions + +* try, measure, benchmark + +Case study - magnetic.com +------------------------- + +* magnetic.com - online search retargetting + +* 2 weeks of work, including benchmarking and analysis + +* swapped ujson for json + +* rewrote bindings for protobuf + +* ~2x speedup after warmup, no additional noticable memory + +Future +------ + +* work on warmup + +* STM work + +* more numpy compatibility + +Crowdfunding +------------ + +* moderate success in crowdfunding + +* numpy, STM, py3k + +* donate + +Commercial consulting --------------------- +* baroquesoftware.com + +* measure, analyze, port to pypy + +Q&A +--- + +* http://pypy.org + +* http://baroquesoftware.com + From noreply at buildbot.pypy.org Fri Feb 20 11:05:12 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 20 Feb 2015 11:05:12 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: typos Message-ID: <20150220100512.9BBD81C12D1@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r5507:e3d32ff5312a Date: 2015-02-20 11:05 +0100 http://bitbucket.org/pypy/extradoc/changeset/e3d32ff5312a/ Log: typos diff --git a/talk/divio-zurich-2015/abstract.rst b/talk/divio-zurich-2015/abstract.rst --- a/talk/divio-zurich-2015/abstract.rst +++ b/talk/divio-zurich-2015/abstract.rst @@ -1,14 +1,14 @@ PyPy is a fast and compliant alternative implementation for the Python -programming language. In this talk I would like to present PyPy, it's goals, -it's history and the current status of running real world Python code. +programming language. In this talk I would like to present PyPy, its goals, +its history and the current status of running real world Python code. The talk will cover some architectural details like the implementation -of our Just in Time compiler as well a bit more esoteric features like Software -Transactional Memory, which promises to provide a GIL less PyPy. It'll also +of our Just-in-Time compiler as well a bit more esoteric features like Software +Transactional Memory, which promises to provide a GIL-less PyPy. It'll also contain a fair bit about benchmarking and how to port/move your code to PyPy based on a case study. Maciej Fijalkowski is a long time PyPy core developer, running a consulting business around PyPy, baroquesoftware.com. He has been involved in various -pieces of PyPy, including garbage collector, just in time compiler, +pieces of PyPy, including garbage collector, just-in-time compiler, assembler generation etc. diff --git a/talk/divio-zurich-2015/talk.rst b/talk/divio-zurich-2015/talk.rst --- a/talk/divio-zurich-2015/talk.rst +++ b/talk/divio-zurich-2015/talk.rst @@ -16,7 +16,7 @@ * a fast, compliant Python interpreter -* comes with a just in time compiler +* comes with a just-in-time compiler * covers Python 2.7 and beta 3.2/3.3 From noreply at buildbot.pypy.org Fri Feb 20 11:28:05 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 20 Feb 2015 11:28:05 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: some adjustments+ pdf Message-ID: <20150220102805.2847B1C1519@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: extradoc Changeset: r5508:49f9ae982668 Date: 2015-02-20 12:28 +0200 http://bitbucket.org/pypy/extradoc/changeset/49f9ae982668/ Log: some adjustments+ pdf diff --git a/talk/divio-zurich-2015/talk.pdf b/talk/divio-zurich-2015/talk.pdf new file mode 100644 index 0000000000000000000000000000000000000000..0c3b8cd22df26daeb6ab339388a81401c2295adf GIT binary patch [cut] diff --git a/talk/divio-zurich-2015/talk.rst b/talk/divio-zurich-2015/talk.rst --- a/talk/divio-zurich-2015/talk.rst +++ b/talk/divio-zurich-2015/talk.rst @@ -1,3 +1,5 @@ +.. include:: ../default/beamerdefs.txt + ==== PyPy ==== @@ -45,7 +47,9 @@ cffi ---- -xxxx image +.. image:: standards.png + :scale: 50% + :align: center cffi (2) -------- @@ -68,6 +72,8 @@ * speed.pypy.org +* basic demo + Speed status (2) ---------------- @@ -123,6 +129,8 @@ * more numpy compatibility +* tools for performance analysis + Crowdfunding ------------ From noreply at buildbot.pypy.org Fri Feb 20 11:35:34 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 20 Feb 2015 11:35:34 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: Move 'pypy_have_debug_prints' to the ExcData instance, which Message-ID: <20150220103534.ED68F1C1519@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76010:a5feb989591f Date: 2015-02-20 11:34 +0100 http://bitbucket.org/pypy/pypy/changeset/a5feb989591f/ Log: Move 'pypy_have_debug_prints' to the ExcData instance, which is also thread-local, but which is cleared on transaction aborts. diff --git a/rpython/translator/c/src/debug_print.c b/rpython/translator/c/src/debug_print.c --- a/rpython/translator/c/src/debug_print.c +++ b/rpython/translator/c/src/debug_print.c @@ -12,10 +12,10 @@ #include #endif #include "common_header.h" +#include "structdef.h" #include "src/profiling.h" #include "src/debug_print.h" -__thread_if_stm long pypy_have_debug_prints = -1; FILE *pypy_debug_file = NULL; /* XXX make it thread-local too? */ static unsigned char debug_ready = 0; static unsigned char debug_profile = 0; diff --git a/rpython/translator/c/src/debug_print.h b/rpython/translator/c/src/debug_print.h --- a/rpython/translator/c/src/debug_print.h +++ b/rpython/translator/c/src/debug_print.h @@ -23,6 +23,14 @@ Note that 'fname' can be '-' to send the logging data to stderr. */ +/* We stick pypy_have_debug_prints as a field of pypy_g_ExcData, + where it will be cleared in case of STM transaction aborts. + XXX XXX this will set to zero the bits that were at one before + the transaction started; the log will be truncated sometimes. +*/ +RPY_EXTERN struct pypy_ExcData0 pypy_g_ExcData; +#define pypy_have_debug_prints pypy_g_ExcData.ed_have_debug_prints + /* macros used by the generated code */ #define PYPY_HAVE_DEBUG_PRINTS (pypy_have_debug_prints & 1 ? \ (pypy_debug_ensure_opened(), 1) : 0) @@ -49,7 +57,6 @@ #define __thread_if_stm /* nothing */ #endif -RPY_EXTERN __thread_if_stm long pypy_have_debug_prints; RPY_EXTERN __thread_if_stm char pypy_debug_threadid[]; RPY_EXPORTED FILE *pypy_debug_file; diff --git a/rpython/translator/exceptiontransform.py b/rpython/translator/exceptiontransform.py --- a/rpython/translator/exceptiontransform.py +++ b/rpython/translator/exceptiontransform.py @@ -449,12 +449,14 @@ EXCDATA = lltype.Struct('ExcData', ('exc_type', self.lltype_of_exception_type), ('exc_value', self.lltype_of_exception_value), + ('have_debug_prints', lltype.Signed), hints={'stm_thread_local': True, 'stm_dont_track_raw_accesses':True, 'is_excdata': True}) self.EXCDATA = EXCDATA exc_data = lltype.malloc(EXCDATA, immortal=True) + exc_data.have_debug_prints = -1 null_type = lltype.nullptr(self.lltype_of_exception_type.TO) null_value = lltype.nullptr(self.lltype_of_exception_value.TO) From noreply at buildbot.pypy.org Fri Feb 20 12:19:53 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 20 Feb 2015 12:19:53 +0100 (CET) Subject: [pypy-commit] pypy default: doc precision from the stm branch Message-ID: <20150220111953.140C41C009F@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76011:acab03818b23 Date: 2015-02-20 12:19 +0100 http://bitbucket.org/pypy/pypy/changeset/acab03818b23/ Log: doc precision from the stm branch diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py --- a/rpython/jit/backend/x86/callbuilder.py +++ b/rpython/jit/backend/x86/callbuilder.py @@ -326,8 +326,8 @@ # in 'ebx'), and if not, we fall back to 'reacqgil_addr'. mc.J_il8(rx86.Conditions['NE'], 0) jne_location = mc.get_relative_pos() - # here, ecx is zero (so rpy_fastgil was in 'released' state - # before the XCHG, but the XCHG acquired it by writing 1) + # here, ecx (=old_value) is zero (so rpy_fastgil was in 'released' + # state before the XCHG, but the XCHG acquired it by writing 1) rst = gcrootmap.get_root_stack_top_addr() mc = self.mc mc.CMP(ebx, heap(rst)) From noreply at buildbot.pypy.org Fri Feb 20 12:55:30 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 20 Feb 2015 12:55:30 +0100 (CET) Subject: [pypy-commit] pypy stmgc-c7: A quick way to avoid the many forks (thanks fijal) Message-ID: <20150220115530.76AEB1C1307@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stmgc-c7 Changeset: r76012:80f3a98777c0 Date: 2015-02-20 12:54 +0100 http://bitbucket.org/pypy/pypy/changeset/80f3a98777c0/ Log: A quick way to avoid the many forks (thanks fijal) diff --git a/rpython/tool/runsubprocess.py b/rpython/tool/runsubprocess.py --- a/rpython/tool/runsubprocess.py +++ b/rpython/tool/runsubprocess.py @@ -65,7 +65,9 @@ def spawn_subprocess(): global _child - _child = Popen([sys.executable, _source], bufsize=0, + # For STM, it doesn't make sense to run another STM subprocess. + # Better just start cpython. + _child = Popen(['/usr/bin/python', _source], bufsize=0, stdin=PIPE, stdout=PIPE, close_fds=True) spawn_subprocess() From noreply at buildbot.pypy.org Fri Feb 20 14:26:04 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 20 Feb 2015 14:26:04 +0100 (CET) Subject: [pypy-commit] pypy default: Add a test from the stm branch Message-ID: <20150220132604.0CAFC1C11AA@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r76013:f90ce9affa58 Date: 2015-02-20 13:14 +0100 http://bitbucket.org/pypy/pypy/changeset/f90ce9affa58/ Log: Add a test from the stm branch diff --git a/rpython/translator/c/test/test_lltyped.py b/rpython/translator/c/test/test_lltyped.py --- a/rpython/translator/c/test/test_lltyped.py +++ b/rpython/translator/c/test/test_lltyped.py @@ -957,6 +957,16 @@ fn = self.getcompiled(f, [int]) assert fn(0) == 9 + def test_call_null_funcptr(self): + fnptr = nullptr(FuncType([], Void)) + def f(n): + if n > 10: + fnptr() # never reached, or so we hope + return n + + fn = self.getcompiled(f, [int]) + assert fn(6) == 6 + def test_likely_unlikely(self): from rpython.rlib.objectmodel import likely, unlikely From noreply at buildbot.pypy.org Fri Feb 20 14:54:30 2015 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 20 Feb 2015 14:54:30 +0100 (CET) Subject: [pypy-commit] pypy default: try to debug test that fails only on full test runs Message-ID: <20150220135430.E941C1C11F1@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r76014:3a472fd0f59f Date: 2015-02-20 15:55 +0200 http://bitbucket.org/pypy/pypy/changeset/3a472fd0f59f/ Log: try to debug test that fails only on full test runs diff --git a/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c b/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c --- a/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c +++ b/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c @@ -272,7 +272,11 @@ { double x, sum=0.0, dx=(b-a)/(double)nstep; for(x=a+0.5*dx; (b-x)*(x-a)>0.0; x+=dx) + { + double y = f(x); + printf("f(x)=%.1f\n", y); sum += f(x); + } return sum/(double)nstep; } diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py b/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py --- a/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py +++ b/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py @@ -138,6 +138,7 @@ integrate.restype = c_double def func(x): + print 'calculating x**2 of',x return x**2 result = integrate(0.0, 1.0, CALLBACK(func), 10) From noreply at buildbot.pypy.org Fri Feb 20 16:55:37 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 20 Feb 2015 16:55:37 +0100 (CET) Subject: [pypy-commit] pypy vmprof: kill the code handling for now Message-ID: <20150220155537.373CF1C03C6@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76015:5bb5dbd1a60b Date: 2015-02-20 17:54 +0200 http://bitbucket.org/pypy/pypy/changeset/5bb5dbd1a60b/ Log: kill the code handling for now diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -138,7 +138,6 @@ self.fileno = -1 def enable(self, space, fileno, period): - xxx if self.is_enabled: raise oefmt(space.w_ValueError, "_vmprof already enabled") self.fileno = fileno @@ -148,11 +147,6 @@ if we_are_translated(): pypy_vmprof_init() self.ever_enabled = True - for weakcode in space.all_code_objs.get_all_handles(): - code = weakcode() - if code: - self.register_code(space, code) - space.set_code_callback(vmprof_register_code) if we_are_translated(): # does not work untranslated res = vmprof_enable(fileno, period, 0, From noreply at buildbot.pypy.org Fri Feb 20 17:13:25 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 20 Feb 2015 17:13:25 +0100 (CET) Subject: [pypy-commit] pypy vmprof: kill this case too Message-ID: <20150220161325.90C751C009F@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: vmprof Changeset: r76016:2eeb5641a842 Date: 2015-02-20 18:12 +0200 http://bitbucket.org/pypy/pypy/changeset/2eeb5641a842/ Log: kill this case too diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -192,7 +192,6 @@ res = vmprof_disable() else: res = 0 - space.set_code_callback(None) if res == -1: raise wrap_oserror(space, OSError(rposix.get_saved_errno(), "_vmprof.disable")) From noreply at buildbot.pypy.org Fri Feb 20 17:38:30 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 20 Feb 2015 17:38:30 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: pdf, final Message-ID: <20150220163830.AE2C01C009F@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: extradoc Changeset: r5509:6ff6d6615185 Date: 2015-02-20 18:37 +0200 http://bitbucket.org/pypy/extradoc/changeset/6ff6d6615185/ Log: pdf, final diff --git a/talk/divio-zurich-2015/talk.pdf b/talk/divio-zurich-2015/talk.pdf index 0c3b8cd22df26daeb6ab339388a81401c2295adf..11de45ab15bed1f8da788386b611fef8e2ea07a0 GIT binary patch [cut] From noreply at buildbot.pypy.org Fri Feb 20 20:08:16 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 20 Feb 2015 20:08:16 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: fix doc Message-ID: <20150220190816.508A11C029E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r573:2ef8358bc3bc Date: 2015-02-20 20:08 +0100 http://bitbucket.org/pypy/pypy.org/changeset/2ef8358bc3bc/ Log: fix doc diff --git a/features.html b/features.html --- a/features.html +++ b/features.html @@ -141,7 +141,7 @@

    To run the sandboxed process, you need pypy-sandbox. You also need to get the full sources (step 1 only). Run:

    -cd pypy/pypy/translator/sandbox
+cd pypy/sandbox
 pypy_interact.py path/to/pypy-sandbox

    You get a fully sandboxed interpreter, in its own filesystem hierarchy diff --git a/source/features.txt b/source/features.txt --- a/source/features.txt +++ b/source/features.txt @@ -109,7 +109,7 @@ To run the sandboxed process, you need `pypy-sandbox`_. You also need to get the `full sources`_ (step 1 only). Run:: - cd pypy/pypy/translator/sandbox + cd pypy/sandbox pypy_interact.py path/to/pypy-sandbox You get a fully sandboxed interpreter, in its own filesystem hierarchy From noreply at buildbot.pypy.org Fri Feb 20 21:33:46 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 20 Feb 2015 21:33:46 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: clarify this step Message-ID: <20150220203346.4F3B41C0499@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r574:9c01b24c6425 Date: 2015-02-20 21:34 +0100 http://bitbucket.org/pypy/pypy.org/changeset/9c01b24c6425/ Log: clarify this step diff --git a/features.html b/features.html --- a/features.html +++ b/features.html @@ -138,8 +138,10 @@ with an external process handling the policy.

    Please be aware that it is a prototype only. It needs work to become more complete, and you are welcome to help.

    -

    To run the sandboxed process, you need pypy-sandbox. You also need to -get the full sources (step 1 only). Run:

    +

    To run the sandboxed process, you need to get the full sources and +build pypy-sandbox from it (see Building from source). These +instructions give you a pypy-c that you should rename to +pypy-sandbox to avoid future confusion. Then run:

     cd pypy/sandbox
 pypy_interact.py path/to/pypy-sandbox
diff --git a/source/features.txt b/source/features.txt
--- a/source/features.txt
+++ b/source/features.txt
@@ -106,8 +106,10 @@
 Please be aware that it is a **prototype** only.  It needs work to become
 more complete, and you are welcome to help.
 
-To run the sandboxed process, you need `pypy-sandbox`_.  You also need to
-get the `full sources`_ (step 1 only).  Run::
+To run the sandboxed process, you need to get the full sources and
+build ``pypy-sandbox`` from it (see `Building from source`_).  These
+instructions give you a ``pypy-c`` that you should rename to
+``pypy-sandbox`` to avoid future confusion.  Then run::
 
    cd pypy/sandbox
    pypy_interact.py path/to/pypy-sandbox
@@ -127,8 +129,7 @@
 To read more about its features, try ``pypy_interact.py --help`` or go to
 `our documentation site`_.
 
-.. _`pypy-sandbox`: download.html#sandboxed-version
-.. _`full sources`: download.html#translate
+.. _`Building from source`: download.html#building-from-source
 .. _`our documentation site`: http://pypy.readthedocs.org/en/latest/sandbox.html
 
 

From noreply at buildbot.pypy.org  Sat Feb 21 09:34:16 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Feb 2015 09:34:16 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: more fixes
Message-ID: <20150221083416.66E421C0129@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r76019:d2efcb5e05aa
Date: 2015-02-20 19:23 +0100
http://bitbucket.org/pypy/pypy/changeset/d2efcb5e05aa/

Log:	more fixes

diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -820,9 +820,6 @@
     if bk.thread_local_fields:
         print >> f, '\tRPython_ThreadLocals_ProgramInit();'
 
-    for line in database.gcpolicy.gc_startup_code():
-        print >> f,"\t" + line
-
     # put float infinities in global constants, we should not have so many of them for now to make
     # a table+loop preferable
     for dest, value in database.late_initializations:
diff --git a/rpython/translator/c/src/mem.c b/rpython/translator/c/src/mem.c
--- a/rpython/translator/c/src/mem.c
+++ b/rpython/translator/c/src/mem.c
@@ -6,7 +6,6 @@
 
 #ifdef RPY_STM
 # include "src/mem.h"
-# include "src/allocator.h"
 # ifdef RPY_ASSERT
 int try_pypy_debug_alloc_stop(void *);
 # else
@@ -21,7 +20,7 @@
        free the object in the normal way.  Finally we increment the
        free counter to keep it in sync. */
     try_pypy_debug_alloc_stop(ptr);
-    PyObject_Free(ptr);
+    free(ptr);
     COUNT_FREE;
 }
 void _pypy_stm_op_free(void *ptr)

From noreply at buildbot.pypy.org  Sat Feb 21 09:34:15 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Feb 2015 09:34:15 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: fixes
Message-ID: <20150221083415.324D31C0129@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r76018:79bb7fe37c41
Date: 2015-02-20 19:16 +0100
http://bitbucket.org/pypy/pypy/changeset/79bb7fe37c41/

Log:	fixes

diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -276,6 +276,13 @@
 # KEEP THE REFERENCE ALIVE, THE GC DOES NOT FOLLOW THEM SO FAR!
 # We use _make_sure_does_not_move() to make sure the pointer will not move.
 
+def _field2structptr(FIELDTYPE, cache={}):
+    if FIELDTYPE not in cache:
+        cache[FIELDTYPE] = lltype.Ptr(lltype.Struct(
+            'pypythread%d' % len(cache), ('c_value', FIELDTYPE),
+            hints={'stm_dont_track_raw_accesses': True}))
+    return cache[FIELDTYPE]
+
 
 class ThreadLocalField(object):
     def __init__(self, FIELDTYPE, fieldname, loop_invariant=False):
@@ -290,6 +297,9 @@
         offset.loop_invariant = loop_invariant
         self.offset = offset
 
+        # for STM only
+        PSTRUCTTYPE = _field2structptr(FIELDTYPE)
+
         def getraw():
             if we_are_translated():
                 _threadlocalref_seeme(self)
@@ -302,7 +312,11 @@
             if we_are_translated():
                 _threadlocalref_seeme(self)
                 addr = llop.threadlocalref_addr(llmemory.Address)
-                return llop.raw_load(FIELDTYPE, addr, offset)
+                if rgc.stm_is_enabled():
+                    p = llmemory.cast_adr_to_ptr(addr + offset, PSTRUCTTYPE)
+                    return p.c_value
+                else:
+                    return llop.raw_load(FIELDTYPE, addr, offset)
             else:
                 return getattr(self.local, 'rawvalue', zero)
 
@@ -311,7 +325,11 @@
             if we_are_translated():
                 _threadlocalref_seeme(self)
                 addr = llop.threadlocalref_addr(llmemory.Address)
-                llop.raw_store(lltype.Void, addr, offset, value)
+                if rgc.stm_is_enabled():
+                    p = llmemory.cast_adr_to_ptr(addr + offset, PSTRUCTTYPE)
+                    p.c_value = value
+                else:
+                    llop.raw_store(lltype.Void, addr, offset, value)
             else:
                 self.local.rawvalue = value
 
diff --git a/rpython/translator/stm/inevitable.py b/rpython/translator/stm/inevitable.py
--- a/rpython/translator/stm/inevitable.py
+++ b/rpython/translator/stm/inevitable.py
@@ -21,7 +21,7 @@
     'weakref_create', 'weakref_deref',
     'jit_assembler_call', 'gc_writebarrier',
     'shrink_array',
-    'threadlocalref_get', 'threadlocalref_set',
+    'threadlocalref_addr', 'threadlocalref_get',
     ])
 ALWAYS_ALLOW_OPERATIONS |= set(lloperation.enum_tryfold_ops())
 
diff --git a/rpython/translator/stm/test/test_inevitable.py b/rpython/translator/stm/test/test_inevitable.py
--- a/rpython/translator/stm/test/test_inevitable.py
+++ b/rpython/translator/stm/test/test_inevitable.py
@@ -287,14 +287,19 @@
         assert res is None
 
     def test_threadlocal(self):
-        from rpython.rlib.rthread import ThreadLocalReference
-        opaque_id = lltype.opaqueptr(ThreadLocalReference.OPAQUEID, "foobar")
-        X = lltype.GcStruct('X', ('foo', lltype.Signed))
+        from rpython.rlib.rthread import ThreadLocalField
+        from rpython.rlib.rthread import _threadlocalref_seeme
+        from rpython.rlib.rthread import _field2structptr
+        foobar = ThreadLocalField(lltype.Signed, 'foobar')
+        offset = foobar.offset
+        PSTRUCTTYPE = _field2structptr(lltype.Signed)
         def f1():
-            x = lltype.malloc(X)
-            llop.threadlocalref_set(lltype.Void, opaque_id, x)
-            y = llop.threadlocalref_get(lltype.Ptr(X), opaque_id)
-            return x == y
+            addr = llop.threadlocalref_addr(llmemory.Address)
+            # ...The rest of this test does not run on the llinterp so far...
+            #p = llmemory.cast_adr_to_ptr(addr + offset, PSTRUCTTYPE)
+            #p.c_value = 42
+            #x = llop.threadlocalref_get(lltype.Signed, offset)
+            #assert x == 42
 
         res = self.interpret_inevitable(f1, [])
         assert res is None

From noreply at buildbot.pypy.org  Sat Feb 21 09:34:14 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Feb 2015 09:34:14 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: hg merge default (not tested so far)
Message-ID: <20150221083414.0270B1C0129@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r76017:3335ebd30ce5
Date: 2015-02-20 13:14 +0100
http://bitbucket.org/pypy/pypy/changeset/3335ebd30ce5/

Log:	hg merge default (not tested so far)

diff too long, truncating to 2000 out of 38264 lines

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -28,7 +28,7 @@
     DEALINGS IN THE SOFTWARE.
 
 
-PyPy Copyright holders 2003-2014
+PyPy Copyright holders 2003-2015
 ----------------------------------- 
 
 Except when otherwise stated (look for LICENSE files or information at
@@ -42,19 +42,19 @@
   Amaury Forgeot d'Arc
   Samuele Pedroni
   Alex Gaynor
+  Brian Kearns
+  Matti Picus
+  Philip Jenvey
   Michael Hudson
   David Schneider
-  Matti Picus
-  Brian Kearns
-  Philip Jenvey
   Holger Krekel
   Christian Tismer
   Hakan Ardo
   Benjamin Peterson
   Manuel Jacob
+  Ronan Lamy
   Anders Chrigstrom
   Eric van Riet Paap
-  Ronan Lamy
   Wim Lavrijsen
   Richard Emslie
   Alexander Schremmer
@@ -68,9 +68,9 @@
   Camillo Bruni
   Laura Creighton
   Toon Verwaest
+  Romain Guillebert
   Leonardo Santagada
   Seo Sanghyeon
-  Romain Guillebert
   Justin Peel
   Ronny Pfannschmidt
   David Edelsohn
@@ -91,15 +91,16 @@
   Michal Bendowski
   Jan de Mooij
   stian
+  Tyler Wade
   Michael Foord
   Stephan Diehl
-  Tyler Wade
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
   Patrick Maupin
   Bob Ippolito
   Bruno Gola
+  David Malcolm
   Jean-Paul Calderone
   Timo Paulssen
   Squeaky
@@ -108,18 +109,19 @@
   Marius Gedminas
   Martin Matusiak
   Konstantin Lopuhin
+  Wenzhu Man
   John Witulski
-  Wenzhu Man
+  Laurence Tratt
+  Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
   Mark Pearse
   Simon Cross
-  Ivan Sichmann Freitas
   Andreas Stührk
+  Stefano Rivera
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Stefano Rivera
   Paweł Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -129,7 +131,6 @@
   tav
   Taavi Burns
   Georg Brandl
-  Laurence Tratt
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -141,13 +142,12 @@
   Jeremy Thurgood
   Rami Chowdhury
   Tobias Pape
-  David Malcolm
   Eugene Oden
   Henry Mason
   Vasily Kuznetsov
   Preston Timmons
+  David Ripton
   Jeff Terrace
-  David Ripton
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
@@ -166,13 +166,16 @@
   Gintautas Miliauskas
   Michael Twomey
   Lucian Branescu Mihaila
+  Yichao Yu
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
   Karl Bartel
+  Wouter van Heyst
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  anatoly techtonik
   Stuart Williams
   Jasper Schulz
   Christian Hudon
@@ -182,12 +185,11 @@
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
-  anatoly techtonik
   Neil Shepperd
+  Stanislaw Halik
   Mikael Schönenberg
   Elmo M?ntynen
   Jonathan David Riehl
-  Stanislaw Halik
   Anders Qvist
   Corbin Simpson
   Chirag Jadwani
@@ -196,10 +198,13 @@
   Vincent Legoll
   Alan McIntyre
   Alexander Sedov
+  Attila Gobi
   Christopher Pope
   Christian Tismer 
   Marc Abramowitz
   Dan Stromberg
+  Arjun Naik
+  Valentina Mukhamedzhanova
   Stefano Parmesan
   Alexis Daboville
   Jens-Uwe Mager
@@ -213,8 +218,6 @@
   Sylvain Thenault
   Nathan Taylor
   Vladimir Kryachko
-  Arjun Naik
-  Attila Gobi
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -222,22 +225,23 @@
   Ryan Gonzalez
   Ian Foote
   Kristjan Valur Jonsson
+  David Lievens
   Neil Blakey-Milner
   Lutz Paelike
   Lucio Torre
   Lars Wassermann
-  Valentina Mukhamedzhanova
   Henrik Vendelbo
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
   Sergey Kishchenko
-  Yichao Yu
   Ignas Mikalajunas
   Christoph Gerum
   Martin Blais
   Lene Wagner
   Tomo Cocoa
+  Toni Mattis
+  Lucas Stadler
   roberto at goyle
   Yury V. Zaytsev
   Anna Katrina Dominguez
@@ -265,23 +269,30 @@
   Stephan Busemann
   Rafał Gałczyński
   Christian Muirhead
+  Berker Peksag
   James Lan
   shoma hosaka
-  Daniel Neuh?user
-  Matthew Miller
+  Daniel Neuhäuser
+  Ben Mather
+  halgari
+  Boglarka Vezer
+  Chris Pressey
   Buck Golemon
   Konrad Delong
   Dinu Gherman
   Chris Lambacher
   coolbutuseless at gmail.com
+  Jim Baker
   Rodrigo Araújo
-  Jim Baker
+  Nikolaos-Digenis Karagiannis
   James Robert
   Armin Ronacher
   Brett Cannon
+  Donald Stufft
   yrttyr
   aliceinwire
   OlivierBlanvillain
+  Dan Sanders
   Zooko Wilcox-O Hearn
   Tomer Chachamu
   Christopher Groskopf
@@ -295,6 +306,7 @@
   Markus Unterwaditzer
   Even Wiik Thomassen
   jbs
+  squeaky
   soareschen
   Kurt Griffiths
   Mike Bayer
@@ -306,6 +318,7 @@
   Anna Ravencroft
   Dan Crosta
   Julien Phalip
+  Roman Podoliaka
   Dan Loewenherz
 
   Heinrich-Heine University, Germany 
diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -17,6 +17,10 @@
 except ImportError:
     assert '__pypy__' not in _sys.builtin_module_names
     newdict = lambda _ : {}
+try:
+    from __pypy__ import reversed_dict
+except ImportError:
+    reversed_dict = lambda d: reversed(d.keys())
 
 try:
     from thread import get_ident as _get_ident
@@ -29,142 +33,35 @@
 ################################################################################
 
 class OrderedDict(dict):
-    'Dictionary that remembers insertion order'
-    # An inherited dict maps keys to values.
-    # The inherited dict provides __getitem__, __len__, __contains__, and get.
-    # The remaining methods are order-aware.
-    # Big-O running times for all methods are the same as regular dictionaries.
+    '''Dictionary that remembers insertion order.
 
-    # The internal self.__map dict maps keys to links in a doubly linked list.
-    # The circular doubly linked list starts and ends with a sentinel element.
-    # The sentinel element never gets deleted (this simplifies the algorithm).
-    # Each link is stored as a list of length three:  [PREV, NEXT, KEY].
+    In PyPy all dicts are ordered anyway.  This is mostly useful as a
+    placeholder to mean "this dict must be ordered even on CPython".
 
-    def __init__(self, *args, **kwds):
-        '''Initialize an ordered dictionary.  The signature is the same as
-        regular dictionaries, but keyword arguments are not recommended because
-        their insertion order is arbitrary.
-
-        '''
-        if len(args) > 1:
-            raise TypeError('expected at most 1 arguments, got %d' % len(args))
-        try:
-            self.__root
-        except AttributeError:
-            self.__root = root = []                     # sentinel node
-            root[:] = [root, root, None]
-            self.__map = {}
-        self.__update(*args, **kwds)
-
-    def __setitem__(self, key, value, dict_setitem=dict.__setitem__):
-        'od.__setitem__(i, y) <==> od[i]=y'
-        # Setting a new item creates a new link at the end of the linked list,
-        # and the inherited dictionary is updated with the new key/value pair.
-        if key not in self:
-            root = self.__root
-            last = root[0]
-            last[1] = root[0] = self.__map[key] = [last, root, key]
-        return dict_setitem(self, key, value)
-
-    def __delitem__(self, key, dict_delitem=dict.__delitem__):
-        'od.__delitem__(y) <==> del od[y]'
-        # Deleting an existing item uses self.__map to find the link which gets
-        # removed by updating the links in the predecessor and successor nodes.
-        dict_delitem(self, key)
-        link_prev, link_next, _ = self.__map.pop(key)
-        link_prev[1] = link_next                        # update link_prev[NEXT]
-        link_next[0] = link_prev                        # update link_next[PREV]
-
-    def __iter__(self):
-        'od.__iter__() <==> iter(od)'
-        # Traverse the linked list in order.
-        root = self.__root
-        curr = root[1]                                  # start at the first node
-        while curr is not root:
-            yield curr[2]                               # yield the curr[KEY]
-            curr = curr[1]                              # move to next node
+    Known difference: iterating over an OrderedDict which is being
+    concurrently modified raises RuntimeError in PyPy.  In CPython
+    instead we get some behavior that appears reasonable in some
+    cases but is nonsensical in other cases.  This is officially
+    forbidden by the CPython docs, so we forbid it explicitly for now.
+    '''
 
     def __reversed__(self):
-        'od.__reversed__() <==> reversed(od)'
-        # Traverse the linked list in reverse order.
-        root = self.__root
-        curr = root[0]                                  # start at the last node
-        while curr is not root:
-            yield curr[2]                               # yield the curr[KEY]
-            curr = curr[0]                              # move to previous node
-
-    def clear(self):
-        'od.clear() -> None.  Remove all items from od.'
-        root = self.__root
-        root[:] = [root, root, None]
-        self.__map.clear()
-        dict.clear(self)
-
-    # -- the following methods do not depend on the internal structure --
-
-    def keys(self):
-        'od.keys() -> list of keys in od'
-        return list(self)
-
-    def values(self):
-        'od.values() -> list of values in od'
-        return [self[key] for key in self]
-
-    def items(self):
-        'od.items() -> list of (key, value) pairs in od'
-        return [(key, self[key]) for key in self]
-
-    def iterkeys(self):
-        'od.iterkeys() -> an iterator over the keys in od'
-        return iter(self)
-
-    def itervalues(self):
-        'od.itervalues -> an iterator over the values in od'
-        for k in self:
-            yield self[k]
-
-    def iteritems(self):
-        'od.iteritems -> an iterator over the (key, value) pairs in od'
-        for k in self:
-            yield (k, self[k])
-
-    update = MutableMapping.update
-
-    __update = update # let subclasses override update without breaking __init__
-
-    __marker = object()
-
-    def pop(self, key, default=__marker):
-        '''od.pop(k[,d]) -> v, remove specified key and return the corresponding
-        value.  If key is not found, d is returned if given, otherwise KeyError
-        is raised.
-
-        '''
-        if key in self:
-            result = self[key]
-            del self[key]
-            return result
-        if default is self.__marker:
-            raise KeyError(key)
-        return default
-
-    def setdefault(self, key, default=None):
-        'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
-        if key in self:
-            return self[key]
-        self[key] = default
-        return default
+        return reversed_dict(self)
 
     def popitem(self, last=True):
         '''od.popitem() -> (k, v), return and remove a (key, value) pair.
         Pairs are returned in LIFO order if last is true or FIFO order if false.
 
         '''
-        if not self:
-            raise KeyError('dictionary is empty')
-        key = next(reversed(self) if last else iter(self))
-        value = self.pop(key)
-        return key, value
+        if last:
+            return dict.popitem(self)
+        else:
+            it = dict.__iter__(self)
+            try:
+                k = it.next()
+            except StopIteration:
+                raise KeyError('dictionary is empty')
+            return (k, self.pop(k))
 
     def __repr__(self, _repr_running={}):
         'od.__repr__() <==> repr(od)'
@@ -183,8 +80,6 @@
         'Return state information for pickling'
         items = [[k, self[k]] for k in self]
         inst_dict = vars(self).copy()
-        for k in vars(OrderedDict()):
-            inst_dict.pop(k, None)
         if inst_dict:
             return (self.__class__, (items,), inst_dict)
         return self.__class__, (items,)
@@ -193,17 +88,6 @@
         'od.copy() -> a shallow copy of od'
         return self.__class__(self)
 
-    @classmethod
-    def fromkeys(cls, iterable, value=None):
-        '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S.
-        If not specified, the value defaults to None.
-
-        '''
-        self = cls()
-        for key in iterable:
-            self[key] = value
-        return self
-
     def __eq__(self, other):
         '''od.__eq__(y) <==> od==y.  Comparison to another OD is order-sensitive
         while comparison to a regular mapping is order-insensitive.
diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py
@@ -2,7 +2,6 @@
 import array
 import gc
 import unittest
-from ctypes.test import xfail
 
 class X(Structure):
     _fields_ = [("c_int", c_int)]
@@ -11,7 +10,6 @@
         self._init_called = True
 
 class Test(unittest.TestCase):
-    @xfail
     def test_fom_buffer(self):
         a = array.array("i", range(16))
         x = (c_int * 16).from_buffer(a)
@@ -34,10 +32,9 @@
         del a; gc.collect(); gc.collect(); gc.collect()
         self.assertEqual(x[:], expected)
 
-        self.assertRaises(TypeError,
+        self.assertRaises((TypeError, ValueError),
                           (c_char * 16).from_buffer, "a" * 16)
 
-    @xfail
     def test_fom_buffer_with_offset(self):
         a = array.array("i", range(16))
         x = (c_int * 15).from_buffer(a, sizeof(c_int))
@@ -46,7 +43,6 @@
         self.assertRaises(ValueError, lambda: (c_int * 16).from_buffer(a, sizeof(c_int)))
         self.assertRaises(ValueError, lambda: (c_int * 1).from_buffer(a, 16 * sizeof(c_int)))
 
-    @xfail
     def test_from_buffer_copy(self):
         a = array.array("i", range(16))
         x = (c_int * 16).from_buffer_copy(a)
@@ -71,7 +67,6 @@
         x = (c_char * 16).from_buffer_copy("a" * 16)
         self.assertEqual(x[:], "a" * 16)
 
-    @xfail
     def test_fom_buffer_copy_with_offset(self):
         a = array.array("i", range(16))
         x = (c_int * 15).from_buffer_copy(a, sizeof(c_int))
diff --git a/lib-python/2.7/distutils/unixccompiler.py b/lib-python/2.7/distutils/unixccompiler.py
--- a/lib-python/2.7/distutils/unixccompiler.py
+++ b/lib-python/2.7/distutils/unixccompiler.py
@@ -58,7 +58,7 @@
     executables = {'preprocessor' : None,
                    'compiler'     : ["cc"],
                    'compiler_so'  : ["cc"],
-                   'compiler_cxx' : ["cc"],
+                   'compiler_cxx' : ["c++"],  # pypy: changed, 'cc' is bogus
                    'linker_so'    : ["cc", "-shared"],
                    'linker_exe'   : ["cc"],
                    'archiver'     : ["ar", "-cr"],
diff --git a/lib-python/2.7/sqlite3/test/dbapi.py b/lib-python/2.7/sqlite3/test/dbapi.py
--- a/lib-python/2.7/sqlite3/test/dbapi.py
+++ b/lib-python/2.7/sqlite3/test/dbapi.py
@@ -478,6 +478,29 @@
         except TypeError:
             pass
 
+    def CheckCurDescription(self):
+        self.cu.execute("select * from test")
+
+        actual = self.cu.description
+        expected = [
+            ('id', None, None, None, None, None, None),
+            ('name', None, None, None, None, None, None),
+            ('income', None, None, None, None, None, None),
+        ]
+        self.assertEqual(expected, actual)
+
+    def CheckCurDescriptionVoidStatement(self):
+        self.cu.execute("insert into test(name) values (?)", ("foo",))
+        self.assertIsNone(self.cu.description)
+
+    def CheckCurDescriptionWithoutStatement(self):
+        cu = self.cx.cursor()
+        try:
+            self.assertIsNone(cu.description)
+        finally:
+            cu.close()
+
+
 @unittest.skipUnless(threading, 'This test requires threading.')
 class ThreadTests(unittest.TestCase):
     def setUp(self):
diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -1589,7 +1589,7 @@
                   'copyfile' in caller.f_globals):
         dest_dir = sys.pypy_resolvedirof(target_executable)
         src_dir = sys.pypy_resolvedirof(sys.executable)
-        for libname in ['libpypy-c.so']:
+        for libname in ['libpypy-c.so', 'libpypy-c.dylib']:
             dest_library = os.path.join(dest_dir, libname)
             src_library = os.path.join(src_dir, libname)
             if os.path.exists(src_library):
diff --git a/lib-python/2.7/test/test_collections.py b/lib-python/2.7/test/test_collections.py
--- a/lib-python/2.7/test/test_collections.py
+++ b/lib-python/2.7/test/test_collections.py
@@ -578,7 +578,12 @@
             def __repr__(self):
                 return "MySet(%s)" % repr(list(self))
         s = MySet([5,43,2,1])
-        self.assertEqual(s.pop(), 1)
+        # changed from CPython 2.7: it was "s.pop() == 1" but I see
+        # nothing that guarantees a particular order here.  In the
+        # 'all_ordered_dicts' branch of PyPy (or with OrderedDict
+        # instead of sets), it consistently returns 5, but this test
+        # should not rely on this or any other order.
+        self.assert_(s.pop() in [5,43,2,1])
 
     def test_issue8750(self):
         empty = WithSet()
@@ -1010,8 +1015,9 @@
                                           c=3, e=5).items()), pairs)                # mixed input
 
         # make sure no positional args conflict with possible kwdargs
-        self.assertEqual(inspect.getargspec(OrderedDict.__dict__['__init__']).args,
-                         ['self'])
+        if '__init__' in OrderedDict.__dict__:   # absent in PyPy
+            self.assertEqual(inspect.getargspec(OrderedDict.__dict__['__init__']).args,
+                             ['self'])
 
         # Make sure that direct calls to __init__ do not clear previous contents
         d = OrderedDict([('a', 1), ('b', 2), ('c', 3), ('d', 44), ('e', 55)])
@@ -1108,6 +1114,16 @@
             od.popitem()
         self.assertEqual(len(od), 0)
 
+    def test_popitem_first(self):
+        pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
+        shuffle(pairs)
+        od = OrderedDict(pairs)
+        while pairs:
+            self.assertEqual(od.popitem(last=False), pairs.pop(0))
+        with self.assertRaises(KeyError):
+            od.popitem(last=False)
+        self.assertEqual(len(od), 0)
+
     def test_pop(self):
         pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
         shuffle(pairs)
@@ -1179,7 +1195,11 @@
         od = OrderedDict(pairs)
         # yaml.dump(od) -->
         # '!!python/object/apply:__main__.OrderedDict\n- - [a, 1]\n  - [b, 2]\n'
-        self.assertTrue(all(type(pair)==list for pair in od.__reduce__()[1]))
+
+        # PyPy bug fix: added [0] at the end of this line, because the
+        # test is really about the 2-tuples that need to be 2-lists
+        # inside the list of 6 of them
+        self.assertTrue(all(type(pair)==list for pair in od.__reduce__()[1][0]))
 
     def test_reduce_not_too_fat(self):
         # do not save instance dictionary if not needed
@@ -1189,6 +1209,16 @@
         od.x = 10
         self.assertEqual(len(od.__reduce__()), 3)
 
+    def test_reduce_exact_output(self):
+        # PyPy: test that __reduce__() produces the exact same answer as
+        # CPython does, even though in the 'all_ordered_dicts' branch we
+        # have to emulate it.
+        pairs = [['c', 1], ['b', 2], ['d', 4]]
+        od = OrderedDict(pairs)
+        self.assertEqual(od.__reduce__(), (OrderedDict, (pairs,)))
+        od.x = 10
+        self.assertEqual(od.__reduce__(), (OrderedDict, (pairs,), {'x': 10}))
+
     def test_repr(self):
         od = OrderedDict([('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)])
         self.assertEqual(repr(od),
diff --git a/lib-python/2.7/test/test_xml_etree.py b/lib-python/2.7/test/test_xml_etree.py
--- a/lib-python/2.7/test/test_xml_etree.py
+++ b/lib-python/2.7/test/test_xml_etree.py
@@ -225,9 +225,9 @@
     >>> element.remove(subelement)
     >>> serialize(element) # 5
     ''
-    >>> element.remove(subelement)
+    >>> element.remove(subelement)    # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    ValueError: list.remove(x): x not in list
+    ValueError: list.remove(...
     >>> serialize(element) # 6
     ''
     >>> element[0:0] = [subelement, subelement, subelement]
diff --git a/lib-python/stdlib-upgrade.txt b/lib-python/stdlib-upgrade.txt
--- a/lib-python/stdlib-upgrade.txt
+++ b/lib-python/stdlib-upgrade.txt
@@ -7,7 +7,7 @@
 
 1. check out the branch vendor/stdlib
 2. upgrade the files there
-3. update stdlib-versions.txt with the output of hg -id from the cpython repo
+3. update stdlib-version.txt with the output of hg -id from the cpython repo
 4. commit
 5. update to default/py3k
 6. create a integration branch for the new stdlib
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -83,6 +83,37 @@
     def in_dll(self, dll, name):
         return self.from_address(dll._handle.getaddressindll(name))
 
+    def from_buffer(self, obj, offset=0):
+        size = self._sizeofinstances()
+        buf = buffer(obj, offset, size)
+        if len(buf) < size:
+            raise ValueError(
+                "Buffer size too small (%d instead of at least %d bytes)"
+                % (len(buf) + offset, size + offset))
+        raw_addr = buf._pypy_raw_address()
+        result = self.from_address(raw_addr)
+        result._ensure_objects()['ffffffff'] = obj
+        return result
+
+    def from_buffer_copy(self, obj, offset=0):
+        size = self._sizeofinstances()
+        buf = buffer(obj, offset, size)
+        if len(buf) < size:
+            raise ValueError(
+                "Buffer size too small (%d instead of at least %d bytes)"
+                % (len(buf) + offset, size + offset))
+        result = self()
+        dest = result._buffer.buffer
+        try:
+            raw_addr = buf._pypy_raw_address()
+        except ValueError:
+            _rawffi.rawstring2charp(dest, buf)
+        else:
+            from ctypes import memmove
+            memmove(dest, raw_addr, size)
+        return result
+
+
 class CArgObject(object):
     """ simple wrapper around buffer, just for the case of freeing
     it afterwards
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -1,3 +1,4 @@
+import sys
 import _rawffi
 from _ctypes.basics import _CData, _CDataMeta, keepalive_key,\
      store_reference, ensure_objects, CArgObject
@@ -178,6 +179,8 @@
         instance = StructOrUnion.__new__(self)
         if isinstance(address, _rawffi.StructureInstance):
             address = address.buffer
+        # fix the address: turn it into as unsigned, in case it is negative
+        address = address & (sys.maxint * 2 + 1)
         instance.__dict__['_buffer'] = self._ffistruct.fromaddress(address)
         return instance
 
diff --git a/lib_pypy/_functools.py b/lib_pypy/_functools.py
--- a/lib_pypy/_functools.py
+++ b/lib_pypy/_functools.py
@@ -9,7 +9,10 @@
     of the given arguments and keywords.
     """
 
-    def __init__(self, func, *args, **keywords):
+    def __init__(self, *args, **keywords):
+        if not args:
+            raise TypeError('__init__() takes at least 2 arguments (1 given)')
+        func, args = args[0], args[1:]
         if not callable(func):
             raise TypeError("the first argument must be callable")
         self._func = func
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -1175,8 +1175,9 @@
         try:
             return self.__description
         except AttributeError:
-            self.__description = self.__statement._get_description()
-            return self.__description
+            if self.__statement:
+                self.__description = self.__statement._get_description()
+                return self.__description
     description = property(__get_description)
 
     def __get_lastrowid(self):
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -6,3 +6,8 @@
 
 __version__ = "0.8.6"
 __version_info__ = (0, 8, 6)
+
+# The verifier module file names are based on the CRC32 of a string that
+# contains the following version number.  It may be older than __version__
+# if nothing is clearly incompatible.
+__version_verifier_modules__ = "0.8.6"
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -69,6 +69,7 @@
         self._function_caches = []
         self._libraries = []
         self._cdefsources = []
+        self._windows_unicode = None
         if hasattr(backend, 'set_ffi'):
             backend.set_ffi(self)
         for name in backend.__dict__:
@@ -77,6 +78,7 @@
         #
         with self._lock:
             self.BVoidP = self._get_cached_btype(model.voidp_type)
+            self.BCharA = self._get_cached_btype(model.char_array_type)
         if isinstance(backend, types.ModuleType):
             # _cffi_backend: attach these constants to the class
             if not hasattr(FFI, 'NULL'):
@@ -189,13 +191,16 @@
             cdecl = self._typeof(cdecl)
         return self._backend.alignof(cdecl)
 
-    def offsetof(self, cdecl, fieldname):
+    def offsetof(self, cdecl, *fields_or_indexes):
         """Return the offset of the named field inside the given
-        structure, which must be given as a C type name.
+        structure or array, which must be given as a C type name.  
+        You can give several field names in case of nested structures.
+        You can also give numeric values which correspond to array
+        items, in case of an array type.
         """
         if isinstance(cdecl, basestring):
             cdecl = self._typeof(cdecl)
-        return self._backend.typeoffsetof(cdecl, fieldname)[1]
+        return self._typeoffsetof(cdecl, *fields_or_indexes)[1]
 
     def new(self, cdecl, init=None):
         """Allocate an instance according to the specified C type and
@@ -264,6 +269,16 @@
         """
         return self._backend.buffer(cdata, size)
 
+    def from_buffer(self, python_buffer):
+        """Return a  that points to the data of the
+        given Python object, which must support the buffer interface.
+        Note that this is not meant to be used on the built-in types str,
+        unicode, or bytearray (you can build 'char[]' arrays explicitly)
+        but only on objects containing large quantities of raw data
+        in some other format, like 'array.array' or numpy arrays.
+        """
+        return self._backend.from_buffer(self.BCharA, python_buffer)
+
     def callback(self, cdecl, python_callable=None, error=None):
         """Return a callback object or a decorator making such a
         callback object.  'cdecl' must name a C function pointer type.
@@ -335,9 +350,23 @@
         which requires binary compatibility in the signatures.
         """
         from .verifier import Verifier, _caller_dir_pycache
+        #
+        # If set_unicode(True) was called, insert the UNICODE and
+        # _UNICODE macro declarations
+        if self._windows_unicode:
+            self._apply_windows_unicode(kwargs)
+        #
+        # Set the tmpdir here, and not in Verifier.__init__: it picks
+        # up the caller's directory, which we want to be the caller of
+        # ffi.verify(), as opposed to the caller of Veritier().
         tmpdir = tmpdir or _caller_dir_pycache()
+        #
+        # Make a Verifier() and use it to load the library.
         self.verifier = Verifier(self, source, tmpdir, **kwargs)
         lib = self.verifier.load_library()
+        #
+        # Save the loaded library for keep-alive purposes, even
+        # if the caller doesn't keep it alive itself (it should).
         self._libraries.append(lib)
         return lib
 
@@ -356,15 +385,29 @@
         with self._lock:
             return model.pointer_cache(self, ctype)
 
-    def addressof(self, cdata, field=None):
+    def addressof(self, cdata, *fields_or_indexes):
         """Return the address of a .
-        If 'field' is specified, return the address of this field.
+        If 'fields_or_indexes' are given, returns the address of that
+        field or array item in the structure or array, recursively in
+        case of nested structures.
         """
         ctype = self._backend.typeof(cdata)
-        ctype, offset = self._backend.typeoffsetof(ctype, field)
+        if fields_or_indexes:
+            ctype, offset = self._typeoffsetof(ctype, *fields_or_indexes)
+        else:
+            if ctype.kind == "pointer":
+                raise TypeError("addressof(pointer)")
+            offset = 0
         ctypeptr = self._pointer_to(ctype)
         return self._backend.rawaddressof(ctypeptr, cdata, offset)
 
+    def _typeoffsetof(self, ctype, field_or_index, *fields_or_indexes):
+        ctype, offset = self._backend.typeoffsetof(ctype, field_or_index)
+        for field1 in fields_or_indexes:
+            ctype, offset1 = self._backend.typeoffsetof(ctype, field1, 1)
+            offset += offset1
+        return ctype, offset
+
     def include(self, ffi_to_include):
         """Includes the typedefs, structs, unions and enums defined
         in another FFI instance.  Usage is similar to a #include in C,
@@ -387,6 +430,44 @@
     def from_handle(self, x):
         return self._backend.from_handle(x)
 
+    def set_unicode(self, enabled_flag):
+        """Windows: if 'enabled_flag' is True, enable the UNICODE and
+        _UNICODE defines in C, and declare the types like TCHAR and LPTCSTR
+        to be (pointers to) wchar_t.  If 'enabled_flag' is False,
+        declare these types to be (pointers to) plain 8-bit characters.
+        This is mostly for backward compatibility; you usually want True.
+        """
+        if self._windows_unicode is not None:
+            raise ValueError("set_unicode() can only be called once")
+        enabled_flag = bool(enabled_flag)
+        if enabled_flag:
+            self.cdef("typedef wchar_t TBYTE;"
+                      "typedef wchar_t TCHAR;"
+                      "typedef const wchar_t *LPCTSTR;"
+                      "typedef const wchar_t *PCTSTR;"
+                      "typedef wchar_t *LPTSTR;"
+                      "typedef wchar_t *PTSTR;"
+                      "typedef TBYTE *PTBYTE;"
+                      "typedef TCHAR *PTCHAR;")
+        else:
+            self.cdef("typedef char TBYTE;"
+                      "typedef char TCHAR;"
+                      "typedef const char *LPCTSTR;"
+                      "typedef const char *PCTSTR;"
+                      "typedef char *LPTSTR;"
+                      "typedef char *PTSTR;"
+                      "typedef TBYTE *PTBYTE;"
+                      "typedef TCHAR *PTCHAR;")
+        self._windows_unicode = enabled_flag
+
+    def _apply_windows_unicode(self, kwds):
+        defmacros = kwds.get('define_macros', ())
+        if not isinstance(defmacros, (list, tuple)):
+            raise TypeError("'define_macros' must be a list or tuple")
+        defmacros = list(defmacros) + [('UNICODE', '1'),
+                                       ('_UNICODE', '1')]
+        kwds['define_macros'] = defmacros
+
 
 def _load_backend_lib(backend, name, flags):
     if name is None:
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -169,6 +169,7 @@
 class CTypesGenericPtr(CTypesData):
     __slots__ = ['_address', '_as_ctype_ptr']
     _automatic_casts = False
+    kind = "pointer"
 
     @classmethod
     def _newp(cls, init):
@@ -370,10 +371,12 @@
                                 (CTypesPrimitive, type(source).__name__))
             return source
         #
+        kind1 = kind
         class CTypesPrimitive(CTypesGenericPrimitive):
             __slots__ = ['_value']
             _ctype = ctype
             _reftypename = '%s &' % name
+            kind = kind1
 
             def __init__(self, value):
                 self._value = value
@@ -703,12 +706,13 @@
         class struct_or_union(base_ctypes_class):
             pass
         struct_or_union.__name__ = '%s_%s' % (kind, name)
+        kind1 = kind
         #
         class CTypesStructOrUnion(CTypesBaseStructOrUnion):
             __slots__ = ['_blob']
             _ctype = struct_or_union
             _reftypename = '%s &' % (name,)
-            _kind = kind
+            _kind = kind = kind1
         #
         CTypesStructOrUnion._fix_class()
         return CTypesStructOrUnion
@@ -994,27 +998,42 @@
     def getcname(self, BType, replace_with):
         return BType._get_c_name(replace_with)
 
-    def typeoffsetof(self, BType, fieldname):
-        if fieldname is not None and issubclass(BType, CTypesGenericPtr):
-            BType = BType._BItem
-        if not issubclass(BType, CTypesBaseStructOrUnion):
-            raise TypeError("expected a struct or union ctype")
-        if fieldname is None:
-            return (BType, 0)
-        else:
+    def typeoffsetof(self, BType, fieldname, num=0):
+        if isinstance(fieldname, str):
+            if num == 0 and issubclass(BType, CTypesGenericPtr):
+                BType = BType._BItem
+            if not issubclass(BType, CTypesBaseStructOrUnion):
+                raise TypeError("expected a struct or union ctype")
             BField = BType._bfield_types[fieldname]
             if BField is Ellipsis:
                 raise TypeError("not supported for bitfields")
             return (BField, BType._offsetof(fieldname))
+        elif isinstance(fieldname, (int, long)):
+            if issubclass(BType, CTypesGenericArray):
+                BType = BType._CTPtr
+            if not issubclass(BType, CTypesGenericPtr):
+                raise TypeError("expected an array or ptr ctype")
+            BItem = BType._BItem
+            offset = BItem._get_size() * fieldname
+            if offset > sys.maxsize:
+                raise OverflowError
+            return (BItem, offset)
+        else:
+            raise TypeError(type(fieldname))
 
-    def rawaddressof(self, BTypePtr, cdata, offset):
+    def rawaddressof(self, BTypePtr, cdata, offset=None):
         if isinstance(cdata, CTypesBaseStructOrUnion):
             ptr = ctypes.pointer(type(cdata)._to_ctypes(cdata))
         elif isinstance(cdata, CTypesGenericPtr):
+            if offset is None or not issubclass(type(cdata)._BItem,
+                                                CTypesBaseStructOrUnion):
+                raise TypeError("unexpected cdata type")
+            ptr = type(cdata)._to_ctypes(cdata)
+        elif isinstance(cdata, CTypesGenericArray):
             ptr = type(cdata)._to_ctypes(cdata)
         else:
             raise TypeError("expected a ")
-        if offset != 0:
+        if offset:
             ptr = ctypes.cast(
                 ctypes.c_void_p(
                     ctypes.cast(ptr, ctypes.c_void_p).value + offset),
diff --git a/lib_pypy/cffi/commontypes.py b/lib_pypy/cffi/commontypes.py
--- a/lib_pypy/cffi/commontypes.py
+++ b/lib_pypy/cffi/commontypes.py
@@ -29,6 +29,9 @@
                 result = model.PointerType(resolve_common_type(result[:-2]))
         elif result in model.PrimitiveType.ALL_PRIMITIVE_TYPES:
             result = model.PrimitiveType(result)
+        elif result == 'set-unicode-needed':
+            raise api.FFIError("The Windows type %r is only available after "
+                               "you call ffi.set_unicode()" % (commontype,))
         else:
             if commontype == result:
                 raise api.FFIError("Unsupported type: %r.  Please file a bug "
@@ -86,8 +89,6 @@
         "ULONGLONG": "unsigned long long",
         "WCHAR": "wchar_t",
         "SHORT": "short",
-        "TBYTE": "WCHAR",
-        "TCHAR": "WCHAR",
         "UCHAR": "unsigned char",
         "UINT": "unsigned int",
         "UINT8": "unsigned char",
@@ -157,14 +158,12 @@
 
         "LPCVOID": model.const_voidp_type,
         "LPCWSTR": "const WCHAR *",
-        "LPCTSTR": "LPCWSTR",
         "LPDWORD": "DWORD *",
         "LPHANDLE": "HANDLE *",
         "LPINT": "int *",
         "LPLONG": "long *",
         "LPSTR": "CHAR *",
         "LPWSTR": "WCHAR *",
-        "LPTSTR": "LPWSTR",
         "LPVOID": model.voidp_type,
         "LPWORD": "WORD *",
         "LRESULT": "LONG_PTR",
@@ -173,7 +172,6 @@
         "PBYTE": "BYTE *",
         "PCHAR": "CHAR *",
         "PCSTR": "const CHAR *",
-        "PCTSTR": "LPCWSTR",
         "PCWSTR": "const WCHAR *",
         "PDWORD": "DWORD *",
         "PDWORDLONG": "DWORDLONG *",
@@ -200,9 +198,6 @@
         "PSIZE_T": "SIZE_T *",
         "PSSIZE_T": "SSIZE_T *",
         "PSTR": "CHAR *",
-        "PTBYTE": "TBYTE *",
-        "PTCHAR": "TCHAR *",
-        "PTSTR": "LPWSTR",
         "PUCHAR": "UCHAR *",
         "PUHALF_PTR": "UHALF_PTR *",
         "PUINT": "UINT *",
@@ -240,6 +235,15 @@
         "USN": "LONGLONG",
         "VOID": model.void_type,
         "WPARAM": "UINT_PTR",
+
+        "TBYTE": "set-unicode-needed",
+        "TCHAR": "set-unicode-needed",
+        "LPCTSTR": "set-unicode-needed",
+        "PCTSTR": "set-unicode-needed",
+        "LPTSTR": "set-unicode-needed",
+        "PTSTR": "set-unicode-needed",
+        "PTBYTE": "set-unicode-needed",
+        "PTCHAR": "set-unicode-needed",
         })
     return result
 
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -1,4 +1,3 @@
-
 from . import api, model
 from .commontypes import COMMON_TYPES, resolve_common_type
 try:
@@ -209,6 +208,8 @@
 
     def _add_constants(self, key, val):
         if key in self._int_constants:
+            if self._int_constants[key] == val:
+                return     # ignore identical double declarations
             raise api.FFIError(
                 "multiple declarations of constant: %s" % (key,))
         self._int_constants[key] = val
@@ -228,12 +229,18 @@
 
                 pyvalue = int(int_str, 0)
                 self._add_constants(key, pyvalue)
+                self._declare('macro ' + key, pyvalue)
             elif value == '...':
                 self._declare('macro ' + key, value)
             else:
-                raise api.CDefError('only supports the syntax "#define '
-                                    '%s ..." (literally) or "#define '
-                                    '%s 0x1FF" for now' % (key, key))
+                raise api.CDefError(
+                    'only supports one of the following syntax:\n'
+                    '  #define %s ...     (literally dot-dot-dot)\n'
+                    '  #define %s NUMBER  (with NUMBER an integer'
+                                    ' constant, decimal/hex/octal)\n'
+                    'got:\n'
+                    '  #define %s %s'
+                    % (key, key, key, value))
 
     def _parse_decl(self, decl):
         node = decl.type
@@ -460,6 +467,8 @@
             elif kind == 'union':
                 tp = model.UnionType(explicit_name, None, None, None)
             elif kind == 'enum':
+                if explicit_name == '__dotdotdot__':
+                    raise CDefError("Enums cannot be declared with ...")
                 tp = self._build_enum_type(explicit_name, type.values)
             else:
                 raise AssertionError("kind = %r" % (kind,))
@@ -532,9 +541,24 @@
 
     def _parse_constant(self, exprnode, partial_length_ok=False):
         # for now, limited to expressions that are an immediate number
-        # or negative number
+        # or positive/negative number
         if isinstance(exprnode, pycparser.c_ast.Constant):
-            return int(exprnode.value, 0)
+            s = exprnode.value
+            if s.startswith('0'):
+                if s.startswith('0x') or s.startswith('0X'):
+                    return int(s, 16)
+                return int(s, 8)
+            elif '1' <= s[0] <= '9':
+                return int(s, 10)
+            elif s[0] == "'" and s[-1] == "'" and (
+                    len(s) == 3 or (len(s) == 4 and s[1] == "\\")):
+                return ord(s[-2])
+            else:
+                raise api.CDefError("invalid constant %r" % (s,))
+        #
+        if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and
+                exprnode.op == '+'):
+            return self._parse_constant(exprnode.expr)
         #
         if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and
                 exprnode.op == '-'):
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -11,6 +11,9 @@
     """
 
 
+LIST_OF_FILE_NAMES = ['sources', 'include_dirs', 'library_dirs',
+                      'extra_objects', 'depends']
+
 def get_extension(srcfilename, modname, sources=(), **kwds):
     from distutils.core import Extension
     allsources = [srcfilename]
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -235,6 +235,8 @@
         BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist)
         return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length)
 
+char_array_type = ArrayType(PrimitiveType('char'), None)
+
 
 class StructOrUnionOrEnum(BaseTypeByIdentity):
     _attrs_ = ('name',)
@@ -478,7 +480,7 @@
     try:
         res = getattr(ffi._backend, funcname)(*args)
     except NotImplementedError as e:
-        raise NotImplementedError("%r: %s" % (srctype, e))
+        raise NotImplementedError("%s: %r: %s" % (funcname, srctype, e))
     # note that setdefault() on WeakValueDictionary is not atomic
     # and contains a rare bug (http://bugs.python.org/issue19542);
     # we have to use a lock and do it ourselves
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -65,7 +65,7 @@
         # The following two 'chained_list_constants' items contains
         # the head of these two chained lists, as a string that gives the
         # call to do, if any.
-        self._chained_list_constants = ['0', '0']
+        self._chained_list_constants = ['((void)lib,0)', '((void)lib,0)']
         #
         prnt = self._prnt
         # first paste some standard set of lines that are mostly '#define'
@@ -138,15 +138,22 @@
         prnt()
         prnt('#endif')
 
-    def load_library(self):
+    def load_library(self, flags=None):
         # XXX review all usages of 'self' here!
         # import it as a new extension module
+        if hasattr(sys, "getdlopenflags"):
+            previous_flags = sys.getdlopenflags()
         try:
+            if hasattr(sys, "setdlopenflags") and flags is not None:
+                sys.setdlopenflags(flags)
             module = imp.load_dynamic(self.verifier.get_module_name(),
                                       self.verifier.modulefilename)
         except ImportError as e:
             error = "importing %r: %s" % (self.verifier.modulefilename, e)
             raise ffiplatform.VerificationError(error)
+        finally:
+            if hasattr(sys, "setdlopenflags"):
+                sys.setdlopenflags(previous_flags)
         #
         # call loading_cpy_struct() to get the struct layout inferred by
         # the C compiler
@@ -228,7 +235,8 @@
                 converter = '_cffi_to_c_int'
                 extraarg = ', %s' % tp.name
             else:
-                converter = '_cffi_to_c_%s' % (tp.name.replace(' ', '_'),)
+                converter = '(%s)_cffi_to_c_%s' % (tp.get_c_name(''),
+                                                   tp.name.replace(' ', '_'))
             errvalue = '-1'
         #
         elif isinstance(tp, model.PointerType):
@@ -267,8 +275,8 @@
         self._prnt('  if (datasize != 0) {')
         self._prnt('    if (datasize < 0)')
         self._prnt('      %s;' % errcode)
-        self._prnt('    %s = alloca(datasize);' % (tovar,))
-        self._prnt('    memset((void *)%s, 0, datasize);' % (tovar,))
+        self._prnt('    %s = alloca((size_t)datasize);' % (tovar,))
+        self._prnt('    memset((void *)%s, 0, (size_t)datasize);' % (tovar,))
         self._prnt('    if (_cffi_convert_array_from_object('
                    '(char *)%s, _cffi_type(%d), %s) < 0)' % (
             tovar, self._gettypenum(tp), fromvar))
@@ -336,7 +344,7 @@
         prnt = self._prnt
         numargs = len(tp.args)
         if numargs == 0:
-            argname = 'no_arg'
+            argname = 'noarg'
         elif numargs == 1:
             argname = 'arg0'
         else:
@@ -386,6 +394,9 @@
         prnt('  Py_END_ALLOW_THREADS')
         prnt()
         #
+        prnt('  (void)self; /* unused */')
+        if numargs == 0:
+            prnt('  (void)noarg; /* unused */')
         if result_code:
             prnt('  return %s;' %
                  self._convert_expr_from_c(tp.result, 'result', 'result type'))
@@ -452,6 +463,7 @@
         prnt('static void %s(%s *p)' % (checkfuncname, cname))
         prnt('{')
         prnt('  /* only to generate compile-time warnings or errors */')
+        prnt('  (void)p;')
         for fname, ftype, fbitsize in tp.enumfields():
             if (isinstance(ftype, model.PrimitiveType)
                 and ftype.is_integer_type()) or fbitsize >= 0:
@@ -482,6 +494,8 @@
                 prnt('    sizeof(((%s *)0)->%s),' % (cname, fname))
         prnt('    -1')
         prnt('  };')
+        prnt('  (void)self; /* unused */')
+        prnt('  (void)noarg; /* unused */')
         prnt('  return _cffi_get_struct_layout(nums);')
         prnt('  /* the next line is not executed, but compiled */')
         prnt('  %s(0);' % (checkfuncname,))
@@ -578,7 +592,8 @@
     # constants, likely declared with '#define'
 
     def _generate_cpy_const(self, is_int, name, tp=None, category='const',
-                            vartp=None, delayed=True, size_too=False):
+                            vartp=None, delayed=True, size_too=False,
+                            check_value=None):
         prnt = self._prnt
         funcname = '_cffi_%s_%s' % (category, name)
         prnt('static int %s(PyObject *lib)' % funcname)
@@ -590,6 +605,9 @@
         else:
             assert category == 'const'
         #
+        if check_value is not None:
+            self._check_int_constant_value(name, check_value)
+        #
         if not is_int:
             if category == 'var':
                 realexpr = '&' + name
@@ -637,6 +655,27 @@
     # ----------
     # enums
 
+    def _check_int_constant_value(self, name, value, err_prefix=''):
+        prnt = self._prnt
+        if value <= 0:
+            prnt('  if ((%s) > 0 || (long)(%s) != %dL) {' % (
+                name, name, value))
+        else:
+            prnt('  if ((%s) <= 0 || (unsigned long)(%s) != %dUL) {' % (
+                name, name, value))
+        prnt('    char buf[64];')
+        prnt('    if ((%s) <= 0)' % name)
+        prnt('        snprintf(buf, 63, "%%ld", (long)(%s));' % name)
+        prnt('    else')
+        prnt('        snprintf(buf, 63, "%%lu", (unsigned long)(%s));' %
+             name)
+        prnt('    PyErr_Format(_cffi_VerificationError,')
+        prnt('                 "%s%s has the real value %s, not %s",')
+        prnt('                 "%s", "%s", buf, "%d");' % (
+            err_prefix, name, value))
+        prnt('    return -1;')
+        prnt('  }')
+
     def _enum_funcname(self, prefix, name):
         # "$enum_$1" => "___D_enum____D_1"
         name = name.replace('$', '___D_')
@@ -653,25 +692,8 @@
         prnt('static int %s(PyObject *lib)' % funcname)
         prnt('{')
         for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
-            if enumvalue < 0:
-                prnt('  if ((%s) >= 0 || (long)(%s) != %dL) {' % (
-                    enumerator, enumerator, enumvalue))
-            else:
-                prnt('  if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % (
-                    enumerator, enumerator, enumvalue))
-            prnt('    char buf[64];')
-            prnt('    if ((%s) < 0)' % enumerator)
-            prnt('        snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator)
-            prnt('    else')
-            prnt('        snprintf(buf, 63, "%%lu", (unsigned long)(%s));' %
-                 enumerator)
-            prnt('    PyErr_Format(_cffi_VerificationError,')
-            prnt('                 "enum %s: %s has the real value %s, '
-                 'not %s",')
-            prnt('                 "%s", "%s", buf, "%d");' % (
-                name, enumerator, enumvalue))
-            prnt('    return -1;')
-            prnt('  }')
+            self._check_int_constant_value(enumerator, enumvalue,
+                                           "enum %s: " % name)
         prnt('  return %s;' % self._chained_list_constants[True])
         self._chained_list_constants[True] = funcname + '(lib)'
         prnt('}')
@@ -695,8 +717,11 @@
     # macros: for now only for integers
 
     def _generate_cpy_macro_decl(self, tp, name):
-        assert tp == '...'
-        self._generate_cpy_const(True, name)
+        if tp == '...':
+            check_value = None
+        else:
+            check_value = tp     # an integer
+        self._generate_cpy_const(True, name, check_value=check_value)
 
     _generate_cpy_macro_collecttype = _generate_nothing
     _generate_cpy_macro_method = _generate_nothing
@@ -783,6 +808,24 @@
    typedef unsigned __int16 uint16_t;
    typedef unsigned __int32 uint32_t;
    typedef unsigned __int64 uint64_t;
+   typedef __int8 int_least8_t;
+   typedef __int16 int_least16_t;
+   typedef __int32 int_least32_t;
+   typedef __int64 int_least64_t;
+   typedef unsigned __int8 uint_least8_t;
+   typedef unsigned __int16 uint_least16_t;
+   typedef unsigned __int32 uint_least32_t;
+   typedef unsigned __int64 uint_least64_t;
+   typedef __int8 int_fast8_t;
+   typedef __int16 int_fast16_t;
+   typedef __int32 int_fast32_t;
+   typedef __int64 int_fast64_t;
+   typedef unsigned __int8 uint_fast8_t;
+   typedef unsigned __int16 uint_fast16_t;
+   typedef unsigned __int32 uint_fast32_t;
+   typedef unsigned __int64 uint_fast64_t;
+   typedef __int64 intmax_t;
+   typedef unsigned __int64 uintmax_t;
 # else
 #  include 
 # endif
@@ -828,12 +871,15 @@
             PyLong_FromLongLong((long long)(x)))
 
 #define _cffi_from_c_int(x, type)                                        \
-    (((type)-1) > 0 ?   /* unsigned */                                   \
-        (sizeof(type) < sizeof(long) ? PyInt_FromLong(x) :               \
-         sizeof(type) == sizeof(long) ? PyLong_FromUnsignedLong(x) :     \
-                                        PyLong_FromUnsignedLongLong(x))  \
-      : (sizeof(type) <= sizeof(long) ? PyInt_FromLong(x) :              \
-                                        PyLong_FromLongLong(x)))
+    (((type)-1) > 0 ? /* unsigned */                                     \
+        (sizeof(type) < sizeof(long) ?                                   \
+            PyInt_FromLong((long)x) :                                    \
+         sizeof(type) == sizeof(long) ?                                  \
+            PyLong_FromUnsignedLong((unsigned long)x) :                  \
+            PyLong_FromUnsignedLongLong((unsigned long long)x)) :        \
+        (sizeof(type) <= sizeof(long) ?                                  \
+            PyInt_FromLong((long)x) :                                    \
+            PyLong_FromLongLong((long long)x)))
 
 #define _cffi_to_c_int(o, type)                                          \
     (sizeof(type) == 1 ? (((type)-1) > 0 ? (type)_cffi_to_c_u8(o)        \
@@ -844,7 +890,7 @@
                                          : (type)_cffi_to_c_i32(o)) :    \
      sizeof(type) == 8 ? (((type)-1) > 0 ? (type)_cffi_to_c_u64(o)       \
                                          : (type)_cffi_to_c_i64(o)) :    \
-     (Py_FatalError("unsupported size for type " #type), 0))
+     (Py_FatalError("unsupported size for type " #type), (type)0))
 
 #define _cffi_to_c_i8                                                    \
                  ((int(*)(PyObject *))_cffi_exports[1])
@@ -907,6 +953,7 @@
 {
     PyObject *library;
     int was_alive = (_cffi_types != NULL);
+    (void)self; /* unused */
     if (!PyArg_ParseTuple(args, "OOO", &_cffi_types, &_cffi_VerificationError,
                                        &library))
         return NULL;
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -58,12 +58,12 @@
             modname = self.verifier.get_module_name()
             prnt("void %s%s(void) { }\n" % (prefix, modname))
 
-    def load_library(self):
+    def load_library(self, flags=0):
         # import it with the CFFI backend
         backend = self.ffi._backend
         # needs to make a path that contains '/', on Posix
         filename = os.path.join(os.curdir, self.verifier.modulefilename)
-        module = backend.load_library(filename)
+        module = backend.load_library(filename, flags)
         #
         # call loading_gen_struct() to get the struct layout inferred by
         # the C compiler
@@ -235,6 +235,7 @@
         prnt('static void %s(%s *p)' % (checkfuncname, cname))
         prnt('{')
         prnt('  /* only to generate compile-time warnings or errors */')
+        prnt('  (void)p;')
         for fname, ftype, fbitsize in tp.enumfields():
             if (isinstance(ftype, model.PrimitiveType)
                 and ftype.is_integer_type()) or fbitsize >= 0:
@@ -354,11 +355,20 @@
     # ----------
     # constants, likely declared with '#define'
 
-    def _generate_gen_const(self, is_int, name, tp=None, category='const'):
+    def _generate_gen_const(self, is_int, name, tp=None, category='const',
+                            check_value=None):
         prnt = self._prnt
         funcname = '_cffi_%s_%s' % (category, name)
         self.export_symbols.append(funcname)
-        if is_int:
+        if check_value is not None:
+            assert is_int
+            assert category == 'const'
+            prnt('int %s(char *out_error)' % funcname)
+            prnt('{')
+            self._check_int_constant_value(name, check_value)
+            prnt('  return 0;')
+            prnt('}')
+        elif is_int:
             assert category == 'const'
             prnt('int %s(long long *out_value)' % funcname)
             prnt('{')
@@ -367,6 +377,7 @@
             prnt('}')
         else:
             assert tp is not None
+            assert check_value is None
             prnt(tp.get_c_name(' %s(void)' % funcname, name),)
             prnt('{')
             if category == 'var':
@@ -383,9 +394,13 @@
 
     _loading_gen_constant = _loaded_noop
 
-    def _load_constant(self, is_int, tp, name, module):
+    def _load_constant(self, is_int, tp, name, module, check_value=None):
         funcname = '_cffi_const_%s' % name
-        if is_int:
+        if check_value is not None:
+            assert is_int
+            self._load_known_int_constant(module, funcname)
+            value = check_value
+        elif is_int:
             BType = self.ffi._typeof_locked("long long*")[0]
             BFunc = self.ffi._typeof_locked("int(*)(long long*)")[0]
             function = module.load_function(BFunc, funcname)
@@ -396,6 +411,7 @@
                 BLongLong = self.ffi._typeof_locked("long long")[0]
                 value += (1 << (8*self.ffi.sizeof(BLongLong)))
         else:
+            assert check_value is None
             BFunc = self.ffi._typeof_locked(tp.get_c_name('(*)(void)', name))[0]
             function = module.load_function(BFunc, funcname)
             value = function()
@@ -410,6 +426,36 @@
     # ----------
     # enums
 
+    def _check_int_constant_value(self, name, value):
+        prnt = self._prnt
+        if value <= 0:
+            prnt('  if ((%s) > 0 || (long)(%s) != %dL) {' % (
+                name, name, value))
+        else:
+            prnt('  if ((%s) <= 0 || (unsigned long)(%s) != %dUL) {' % (
+                name, name, value))
+        prnt('    char buf[64];')
+        prnt('    if ((%s) <= 0)' % name)
+        prnt('        sprintf(buf, "%%ld", (long)(%s));' % name)
+        prnt('    else')
+        prnt('        sprintf(buf, "%%lu", (unsigned long)(%s));' %
+             name)
+        prnt('    sprintf(out_error, "%s has the real value %s, not %s",')
+        prnt('            "%s", buf, "%d");' % (name[:100], value))
+        prnt('    return -1;')
+        prnt('  }')
+
+    def _load_known_int_constant(self, module, funcname):
+        BType = self.ffi._typeof_locked("char[]")[0]
+        BFunc = self.ffi._typeof_locked("int(*)(char*)")[0]
+        function = module.load_function(BFunc, funcname)
+        p = self.ffi.new(BType, 256)
+        if function(p) < 0:
+            error = self.ffi.string(p)
+            if sys.version_info >= (3,):
+                error = str(error, 'utf-8')
+            raise ffiplatform.VerificationError(error)
+
     def _enum_funcname(self, prefix, name):
         # "$enum_$1" => "___D_enum____D_1"
         name = name.replace('$', '___D_')
@@ -427,24 +473,7 @@
         prnt('int %s(char *out_error)' % funcname)
         prnt('{')
         for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
-            if enumvalue < 0:
-                prnt('  if ((%s) >= 0 || (long)(%s) != %dL) {' % (
-                    enumerator, enumerator, enumvalue))
-            else:
-                prnt('  if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % (
-                    enumerator, enumerator, enumvalue))
-            prnt('    char buf[64];')
-            prnt('    if ((%s) < 0)' % enumerator)
-            prnt('        sprintf(buf, "%%ld", (long)(%s));' % enumerator)
-            prnt('    else')
-            prnt('        sprintf(buf, "%%lu", (unsigned long)(%s));' %
-                 enumerator)
-            prnt('    sprintf(out_error,'
-                             ' "%s has the real value %s, not %s",')
-            prnt('            "%s", buf, "%d");' % (
-                enumerator[:100], enumvalue))
-            prnt('    return -1;')
-            prnt('  }')
+            self._check_int_constant_value(enumerator, enumvalue)
         prnt('  return 0;')
         prnt('}')
         prnt()
@@ -456,16 +485,8 @@
             tp.enumvalues = tuple(enumvalues)
             tp.partial_resolved = True
         else:
-            BType = self.ffi._typeof_locked("char[]")[0]
-            BFunc = self.ffi._typeof_locked("int(*)(char*)")[0]
             funcname = self._enum_funcname(prefix, name)
-            function = module.load_function(BFunc, funcname)
-            p = self.ffi.new(BType, 256)
-            if function(p) < 0:
-                error = self.ffi.string(p)
-                if sys.version_info >= (3,):
-                    error = str(error, 'utf-8')
-                raise ffiplatform.VerificationError(error)
+            self._load_known_int_constant(module, funcname)
 
     def _loaded_gen_enum(self, tp, name, module, library):
         for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
@@ -476,13 +497,21 @@
     # macros: for now only for integers
 
     def _generate_gen_macro_decl(self, tp, name):
-        assert tp == '...'
-        self._generate_gen_const(True, name)
+        if tp == '...':
+            check_value = None
+        else:
+            check_value = tp     # an integer
+        self._generate_gen_const(True, name, check_value=check_value)
 
     _loading_gen_macro = _loaded_noop
 
     def _loaded_gen_macro(self, tp, name, module, library):
-        value = self._load_constant(True, tp, name, module)
+        if tp == '...':
+            check_value = None
+        else:
+            check_value = tp     # an integer
+        value = self._load_constant(True, tp, name, module,
+                                    check_value=check_value)
         setattr(library, name, value)
         type(library)._cffi_dir.append(name)
 
@@ -565,6 +594,24 @@
    typedef unsigned __int16 uint16_t;
    typedef unsigned __int32 uint32_t;
    typedef unsigned __int64 uint64_t;
+   typedef __int8 int_least8_t;
+   typedef __int16 int_least16_t;
+   typedef __int32 int_least32_t;
+   typedef __int64 int_least64_t;
+   typedef unsigned __int8 uint_least8_t;
+   typedef unsigned __int16 uint_least16_t;
+   typedef unsigned __int32 uint_least32_t;
+   typedef unsigned __int64 uint_least64_t;
+   typedef __int8 int_fast8_t;
+   typedef __int16 int_fast16_t;
+   typedef __int32 int_fast32_t;
+   typedef __int64 int_fast64_t;
+   typedef unsigned __int8 uint_fast8_t;
+   typedef unsigned __int16 uint_fast16_t;
+   typedef unsigned __int32 uint_fast32_t;
+   typedef unsigned __int64 uint_fast64_t;
+   typedef __int64 intmax_t;
+   typedef unsigned __int64 uintmax_t;
 # else
 #  include 
 # endif
diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py
--- a/lib_pypy/cffi/verifier.py
+++ b/lib_pypy/cffi/verifier.py
@@ -1,12 +1,23 @@
-import sys, os, binascii, imp, shutil
-from . import __version__
+import sys, os, binascii, shutil
+from . import __version_verifier_modules__
 from . import ffiplatform
 
+if sys.version_info >= (3, 3):
+    import importlib.machinery
+    def _extension_suffixes():
+        return importlib.machinery.EXTENSION_SUFFIXES[:]
+else:
+    import imp
+    def _extension_suffixes():
+        return [suffix for suffix, _, type in imp.get_suffixes()
+                if type == imp.C_EXTENSION]
+
 
 class Verifier(object):
 
     def __init__(self, ffi, preamble, tmpdir=None, modulename=None,
-                 ext_package=None, tag='', force_generic_engine=False, **kwds):
+                 ext_package=None, tag='', force_generic_engine=False,
+                 source_extension='.c', flags=None, relative_to=None, **kwds):
         self.ffi = ffi
         self.preamble = preamble
         if not modulename:
@@ -14,14 +25,15 @@
         vengine_class = _locate_engine_class(ffi, force_generic_engine)
         self._vengine = vengine_class(self)
         self._vengine.patch_extension_kwds(kwds)
-        self.kwds = kwds
+        self.flags = flags
+        self.kwds = self.make_relative_to(kwds, relative_to)
         #
         if modulename:
             if tag:
                 raise TypeError("can't specify both 'modulename' and 'tag'")
         else:
-            key = '\x00'.join([sys.version[:3], __version__, preamble,
-                               flattened_kwds] +
+            key = '\x00'.join([sys.version[:3], __version_verifier_modules__,
+                               preamble, flattened_kwds] +
                               ffi._cdefsources)
             if sys.version_info >= (3,):
                 key = key.encode('utf-8')
@@ -33,7 +45,7 @@
                                               k1, k2)
         suffix = _get_so_suffixes()[0]
         self.tmpdir = tmpdir or _caller_dir_pycache()
-        self.sourcefilename = os.path.join(self.tmpdir, modulename + '.c')
+        self.sourcefilename = os.path.join(self.tmpdir, modulename + source_extension)
         self.modulefilename = os.path.join(self.tmpdir, modulename + suffix)
         self.ext_package = ext_package
         self._has_source = False
@@ -97,6 +109,20 @@
     def generates_python_module(self):
         return self._vengine._gen_python_module
 
+    def make_relative_to(self, kwds, relative_to):
+        if relative_to and os.path.dirname(relative_to):
+            dirname = os.path.dirname(relative_to)
+            kwds = kwds.copy()
+            for key in ffiplatform.LIST_OF_FILE_NAMES:
+                if key in kwds:
+                    lst = kwds[key]
+                    if not isinstance(lst, (list, tuple)):
+                        raise TypeError("keyword '%s' should be a list or tuple"
+                                        % (key,))
+                    lst = [os.path.join(dirname, fn) for fn in lst]
+                    kwds[key] = lst
+        return kwds
+
     # ----------
 
     def _locate_module(self):
@@ -148,7 +174,10 @@
 
     def _load_library(self):
         assert self._has_module
-        return self._vengine.load_library()
+        if self.flags is not None:
+            return self._vengine.load_library(self.flags)
+        else:
+            return self._vengine.load_library()
 
 # ____________________________________________________________
 
@@ -181,6 +210,9 @@
 def _caller_dir_pycache():
     if _TMPDIR:
         return _TMPDIR
+    result = os.environ.get('CFFI_TMPDIR')
+    if result:
+        return result
     filename = sys._getframe(2).f_code.co_filename
     return os.path.abspath(os.path.join(os.path.dirname(filename),
                            '__pycache__'))
@@ -222,11 +254,7 @@
             pass
 
 def _get_so_suffixes():
-    suffixes = []
-    for suffix, mode, type in imp.get_suffixes():
-        if type == imp.C_EXTENSION:
-            suffixes.append(suffix)
-
+    suffixes = _extension_suffixes()
     if not suffixes:
         # bah, no C_EXTENSION available.  Occurs on pypy without cpyext
         if sys.platform == 'win32':
diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
--- a/lib_pypy/greenlet.egg-info
+++ b/lib_pypy/greenlet.egg-info
@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: greenlet
-Version: 0.4.0
+Version: 0.4.5
 Summary: Lightweight in-process concurrent programming
 Home-page: https://github.com/python-greenlet/greenlet
 Author: Ralf Schmitt (for CPython), PyPy team
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -1,7 +1,7 @@
 import sys
 import _continuation
 
-__version__ = "0.4.0"
+__version__ = "0.4.5"
 
 # ____________________________________________________________
 # Exceptions
diff --git a/lib_pypy/readline.py b/lib_pypy/readline.py
--- a/lib_pypy/readline.py
+++ b/lib_pypy/readline.py
@@ -6,4 +6,11 @@
 are only stubs at the moment.
 """
 
-from pyrepl.readline import *
+try:
+    from pyrepl.readline import *
+except ImportError:
+    import sys
+    if sys.platform == 'win32':
+        raise ImportError("the 'readline' module is not available on Windows"
+                          " (on either PyPy or CPython)")
+    raise
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -47,6 +47,11 @@
 
 Install build-time dependencies
 -------------------------------
+(**Note**: for some hints on how to translate the Python interpreter under
+Windows, see the `windows document`_)
+
+.. _`windows document`: windows.html
+
 
 To build PyPy on Unix using the C translation backend, you need at least a C
 compiler and ``make`` installed. Further, some optional modules have additional
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -65,9 +65,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '2.4'
+version = '2.5'
 # The full version, including alpha/beta/rc tags.
-release = '2.4.0'
+release = '2.5.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -12,19 +12,19 @@
   Amaury Forgeot d'Arc
   Samuele Pedroni
   Alex Gaynor
+  Brian Kearns
+  Matti Picus
+  Philip Jenvey
   Michael Hudson
   David Schneider
-  Matti Picus
-  Brian Kearns
-  Philip Jenvey
   Holger Krekel
   Christian Tismer
   Hakan Ardo
   Benjamin Peterson
   Manuel Jacob
+  Ronan Lamy
   Anders Chrigstrom
   Eric van Riet Paap
-  Ronan Lamy
   Wim Lavrijsen
   Richard Emslie
   Alexander Schremmer
@@ -38,9 +38,9 @@
   Camillo Bruni
   Laura Creighton
   Toon Verwaest
+  Romain Guillebert
   Leonardo Santagada
   Seo Sanghyeon
-  Romain Guillebert
   Justin Peel
   Ronny Pfannschmidt
   David Edelsohn
@@ -61,15 +61,16 @@
   Michal Bendowski
   Jan de Mooij
   stian
+  Tyler Wade
   Michael Foord
   Stephan Diehl
-  Tyler Wade
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
   Patrick Maupin
   Bob Ippolito
   Bruno Gola
+  David Malcolm
   Jean-Paul Calderone
   Timo Paulssen
   Squeaky
@@ -78,18 +79,19 @@
   Marius Gedminas
   Martin Matusiak
   Konstantin Lopuhin
+  Wenzhu Man
   John Witulski
-  Wenzhu Man
+  Laurence Tratt
+  Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
   Mark Pearse
   Simon Cross
-  Ivan Sichmann Freitas
   Andreas Stührk
+  Stefano Rivera
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Stefano Rivera
   Paweł Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -99,7 +101,6 @@
   tav
   Taavi Burns
   Georg Brandl
-  Laurence Tratt
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -111,13 +112,12 @@
   Jeremy Thurgood
   Rami Chowdhury
   Tobias Pape
-  David Malcolm
   Eugene Oden
   Henry Mason
   Vasily Kuznetsov
   Preston Timmons
+  David Ripton
   Jeff Terrace
-  David Ripton
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
@@ -136,13 +136,16 @@
   Gintautas Miliauskas
   Michael Twomey
   Lucian Branescu Mihaila
+  Yichao Yu
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
   Karl Bartel
+  Wouter van Heyst
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  anatoly techtonik
   Stuart Williams
   Jasper Schulz
   Christian Hudon
@@ -152,12 +155,11 @@
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
-  anatoly techtonik
   Neil Shepperd
+  Stanislaw Halik
   Mikael Schönenberg
   Elmo M?ntynen
   Jonathan David Riehl
-  Stanislaw Halik
   Anders Qvist
   Corbin Simpson
   Chirag Jadwani
@@ -166,10 +168,13 @@
   Vincent Legoll
   Alan McIntyre
   Alexander Sedov
+  Attila Gobi
   Christopher Pope
   Christian Tismer 
   Marc Abramowitz
   Dan Stromberg
+  Arjun Naik
+  Valentina Mukhamedzhanova
   Stefano Parmesan
   Alexis Daboville
   Jens-Uwe Mager
@@ -183,8 +188,6 @@
   Sylvain Thenault
   Nathan Taylor
   Vladimir Kryachko
-  Arjun Naik
-  Attila Gobi
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -192,22 +195,23 @@
   Ryan Gonzalez
   Ian Foote
   Kristjan Valur Jonsson
+  David Lievens
   Neil Blakey-Milner
   Lutz Paelike
   Lucio Torre
   Lars Wassermann
-  Valentina Mukhamedzhanova
   Henrik Vendelbo
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
   Sergey Kishchenko
-  Yichao Yu
   Ignas Mikalajunas
   Christoph Gerum
   Martin Blais
   Lene Wagner
   Tomo Cocoa

From noreply at buildbot.pypy.org  Sat Feb 21 09:34:17 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Feb 2015 09:34:17 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: translation fixes
Message-ID: <20150221083417.9C8C71C0129@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r76020:30fb754cc44a
Date: 2015-02-20 19:48 +0100
http://bitbucket.org/pypy/pypy/changeset/30fb754cc44a/

Log:	translation fixes

diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py
--- a/rpython/jit/backend/x86/callbuilder.py
+++ b/rpython/jit/backend/x86/callbuilder.py
@@ -189,6 +189,7 @@
         """This occurs just before emit_raw_call().
         """
         mc = self.mc
+        SEGMENT_NO = self.asm.SEGMENT_NO
 
         if handle_lasterror and (save_err & rffi.RFFI_READSAVED_LASTERROR):
             # must call SetLastError().  There are no registers to save
@@ -202,7 +203,7 @@
             rpy_lasterror = llerrno.get_rpy_lasterror_offset(self.asm.cpu)
             tlofsreg = self.get_tlofs_reg()    # => esi, callee-saved
             self.save_stack_position()         # => edi, callee-saved
-            mc.PUSH_m((tlofsreg.value, rpy_lasterror))
+            mc.PUSH_m((SEGMENT_NO, tlofsreg.value, rpy_lasterror))
             mc.CALL(imm(SetLastError_addr))
             # restore the stack position without assuming a particular
             # calling convention of _SetLastError()
@@ -220,20 +221,21 @@
                 tmpreg = edx
             else:
                 tmpreg = r11     # edx is used for 3rd argument
-            mc.MOV_rm(tmpreg.value, (tlofsreg.value, p_errno))
-            mc.MOV32_rm(eax.value, (tlofsreg.value, rpy_errno))
-            mc.MOV32_mr((tmpreg.value, 0), eax.value)
+            mc.MOV_rm(tmpreg.value, (SEGMENT_NO, tlofsreg.value, p_errno))
+            mc.MOV32_rm(eax.value, (SEGMENT_NO, tlofsreg.value, rpy_errno))
+            mc.MOV32_mr((SEGMENT_NO, tmpreg.value, 0), eax.value)
         elif save_err & rffi.RFFI_ZERO_ERRNO_BEFORE:
             # Same, but write zero.
             p_errno = llerrno.get_p_errno_offset(self.asm.cpu)
             tlofsreg = self.get_tlofs_reg()    # => esi or r12, callee-saved
-            mc.MOV_rm(eax.value, (tlofsreg.value, p_errno))
-            mc.MOV32_mi((eax.value, 0), 0)
+            mc.MOV_rm(eax.value, (SEGMENT_NO, tlofsreg.value, p_errno))
+            mc.MOV32_mi((SEGMENT_NO, eax.value, 0), 0)
 
     def read_real_errno(self, save_err):
         """This occurs after emit_raw_call() and after restore_stack_pointer().
         """
         mc = self.mc
+        SEGMENT_NO = self.asm.SEGMENT_NO
 
         if save_err & rffi.RFFI_SAVE_ERRNO:
             # Just after a call, read the real 'errno' and save a copy of
@@ -244,9 +246,9 @@
             rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu)
             p_errno = llerrno.get_p_errno_offset(self.asm.cpu)
             tlofsreg = self.get_tlofs_reg()   # => esi or r12 (possibly reused)
-            mc.MOV_rm(edi.value, (tlofsreg.value, p_errno))
-            mc.MOV32_rm(edi.value, (edi.value, 0))
-            mc.MOV32_mr((tlofsreg.value, rpy_errno), edi.value)
+            mc.MOV_rm(edi.value, (SEGMENT_NO, tlofsreg.value, p_errno))
+            mc.MOV32_rm(edi.value, (SEGMENT_NO, edi.value, 0))
+            mc.MOV32_mr((SEGMENT_NO, tlofsreg.value, rpy_errno), edi.value)
 
         if handle_lasterror and (save_err & (rffi.RFFI_SAVE_LASTERROR |
                                              rffi.RFFI_SAVE_WSALASTERROR)):
@@ -265,7 +267,7 @@
             mc.CALL(imm(GetLastError_addr))
             #
             tlofsreg = self.get_tlofs_reg()    # => esi (possibly reused)
-            mc.MOV32_mr((tlofsreg.value, rpy_lasterror), eax.value)
+            mc.MOV32_mr((SEGMENT_NO, tlofsreg.value, rpy_lasterror), eax.value)
 
     def move_real_result_and_call_reacqgil_addr(self, fastgil):
         from rpython.jit.backend.x86 import rx86
@@ -688,9 +690,9 @@
         from rpython.rlib import rstm
         # after any CALL_RELEASE_GIL, invoke the
         # pypy_stm_start_if_not_atomic() function
-        self.save_result_value_reacq()
+        self.save_result_value(True)
         self.mc.CALL(imm(rstm.adr_pypy_stm_start_if_not_atomic))
-        self.restore_result_value_reacq()
+        self.restore_result_value(True)
 
 
 if IS_X86_32:
diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -106,7 +106,7 @@
 
     def guard_already_patched(self, faildescr):
         # only needed for STM so far
-        return faildescr._x86_adr_jump_offset == 0
+        return faildescr.adr_jump_offset == 0
 
         
     def clear_latest_values(self, count):
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -491,8 +491,9 @@
 
 class ResumeGuardDescr(ResumeDescr):
     _attrs_ = ('rd_numb', 'rd_count', 'rd_consts', 'rd_virtuals',
-               'rd_frame_info_list', 'rd_pendingfields', 'status')
-    
+               'rd_frame_info_list', 'rd_pendingfields', 'status',
+               'stm_location_int', 'stm_location_ref')   # stm only
+
     rd_numb = lltype.nullptr(NUMBERING)
     rd_count = 0
     rd_consts = None
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -154,7 +154,8 @@
     index = -1
     final_descr = False
 
-    _attrs_ = ('adr_jump_offset', 'rd_locs', 'rd_loop_token')
+    _attrs_ = ('adr_jump_offset', 'rd_locs', 'rd_loop_token',
+               '_x86_stm_guard_failure')    # only with stm
 
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
         raise NotImplementedError

From noreply at buildbot.pypy.org  Sat Feb 21 09:34:18 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Feb 2015 09:34:18 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: The new rgc.ll_arrayclear() needs a
 stm fall-back version too
Message-ID: <20150221083418.CE8BA1C0129@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r76021:5c42be959acd
Date: 2015-02-20 19:56 +0100
http://bitbucket.org/pypy/pypy/changeset/5c42be959acd/

Log:	The new rgc.ll_arrayclear() needs a stm fall-back version too

diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -374,6 +374,14 @@
 
     length = len(p)
     ARRAY = lltype.typeOf(p).TO
+    if stm_is_enabled():
+        # do the clearing element by element
+        from rpython.rtyper.lltypesystem import rffi
+        i = 0
+        while i < length:
+            p[i] = rffi.cast(ARRAY.OF, 0)
+            i += 1
+        return
     offset = llmemory.itemoffsetof(ARRAY, 0)
     dest_addr = llmemory.cast_ptr_to_adr(p) + offset
     llmemory.raw_memclear(dest_addr, llmemory.sizeof(ARRAY.OF) * length)
diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py
--- a/rpython/translator/stm/test/test_ztranslated.py
+++ b/rpython/translator/stm/test/test_ztranslated.py
@@ -636,3 +636,19 @@
         t, cbuilder = self.compile(main)
         data = cbuilder.cmdexec('')
         assert 'ok!\n' in data
+
+    def test_ll_arrayclear(self):
+        A = lltype.GcArray(rffi.SHORT)
+        def main(argv):
+            p = lltype.malloc(A, 11)
+            for i in range(11):
+                p[i] = rffi.cast(rffi.SHORT, -4242)
+            rgc.ll_arrayclear(p)
+            for i in range(11):
+                assert rffi.cast(lltype.Signed, p[i]) == 0
+            print "ok!"
+            return 0
+
+        t, cbuilder = self.compile(main)
+        data = cbuilder.cmdexec('')
+        assert 'ok!\n' in data

From noreply at buildbot.pypy.org  Sat Feb 21 09:34:19 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Feb 2015 09:34:19 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: stm fix
Message-ID: <20150221083419.EAC1E1C0129@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r76022:19cb65723b8c
Date: 2015-02-20 20:13 +0100
http://bitbucket.org/pypy/pypy/changeset/19cb65723b8c/

Log:	stm fix

diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -678,9 +678,13 @@
     OP_CAST_PTR_TO_ADR = OP_CAST_POINTER
 
     def OP_LENGTH_OF_SIMPLE_GCARRAY_FROM_OPAQUE(self, op):
-        return ('%s = *(long *)(((char *)%s) + sizeof(struct pypy_header0));'
+        header0 = 'struct pypy_header0'
+        if hasattr(self.db.gcpolicy, 'headertype'):
+            header0 = self.db.gcpolicy.headertype('')
+        return ('%s = *(long *)(((char *)%s) + sizeof(%s));'
                 '  /* length_of_simple_gcarray_from_opaque */'
-            % (self.expr(op.result), self.expr(op.args[0])))
+            % (self.expr(op.result), self.expr(op.args[0]),
+               header0))
 
     def OP_CAST_INT_TO_PTR(self, op):
         TYPE = self.lltypemap(op.result)

From noreply at buildbot.pypy.org  Sat Feb 21 09:34:21 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Feb 2015 09:34:21 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: really fix it
Message-ID: <20150221083421.1632E1C0129@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r76023:1dba5a90fd28
Date: 2015-02-20 21:36 +0100
http://bitbucket.org/pypy/pypy/changeset/1dba5a90fd28/

Log:	really fix it

diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -678,13 +678,15 @@
     OP_CAST_PTR_TO_ADR = OP_CAST_POINTER
 
     def OP_LENGTH_OF_SIMPLE_GCARRAY_FROM_OPAQUE(self, op):
-        header0 = 'struct pypy_header0'
-        if hasattr(self.db.gcpolicy, 'headertype'):
-            header0 = self.db.gcpolicy.headertype('')
-        return ('%s = *(long *)(((char *)%s) + sizeof(%s));'
+        if self.db.with_stm:
+            header0 = 'rpyobj_t'
+            long0 = 'TLPREFIX long'
+        else:
+            header0 = 'struct pypy_header0'
+            long0 = 'long'
+        return ('%s = *(%s *)(((%s *)%s) + 1);'
                 '  /* length_of_simple_gcarray_from_opaque */'
-            % (self.expr(op.result), self.expr(op.args[0]),
-               header0))
+            % (self.expr(op.result), long0, header0, self.expr(op.args[0])))
 
     def OP_CAST_INT_TO_PTR(self, op):
         TYPE = self.lltypemap(op.result)

From noreply at buildbot.pypy.org  Sat Feb 21 09:34:22 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Feb 2015 09:34:22 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: translation fix
Message-ID: <20150221083422.33E871C0129@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r76024:5e43d3d7fb16
Date: 2015-02-20 22:25 +0100
http://bitbucket.org/pypy/pypy/changeset/5e43d3d7fb16/

Log:	translation fix

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -2815,7 +2815,8 @@
         assert self.cpu.translate_support_code
         assert isinstance(resloc, RegLoc)
         self.mc.MOV_rs(resloc.value, THREADLOCAL_OFS)
-        self.load_from_mem(resloc, addr_add_const(resloc, offset),
+        self.load_from_mem(resloc,
+                           addr_add_const(self.SEGMENT_NO, resloc, offset),
                            imm(size), imm(sign))
 
     def genop_discard_zero_array(self, op, arglocs):

From noreply at buildbot.pypy.org  Sat Feb 21 09:34:23 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Feb 2015 09:34:23 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: merge heads
Message-ID: <20150221083423.5791C1C0129@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r76025:9cb90693e784
Date: 2015-02-20 23:15 +0100
http://bitbucket.org/pypy/pypy/changeset/9cb90693e784/

Log:	merge heads

diff --git a/rpython/tool/runsubprocess.py b/rpython/tool/runsubprocess.py
--- a/rpython/tool/runsubprocess.py
+++ b/rpython/tool/runsubprocess.py
@@ -65,7 +65,9 @@
 
     def spawn_subprocess():
         global _child
-        _child = Popen([sys.executable, _source], bufsize=0,
+        # For STM, it doesn't make sense to run another STM subprocess.
+        # Better just start cpython.
+        _child = Popen(['/usr/bin/python', _source], bufsize=0,
                        stdin=PIPE, stdout=PIPE, close_fds=True)
     spawn_subprocess()
 
diff --git a/rpython/translator/c/src/debug_print.c b/rpython/translator/c/src/debug_print.c
--- a/rpython/translator/c/src/debug_print.c
+++ b/rpython/translator/c/src/debug_print.c
@@ -12,10 +12,10 @@
 #include 
 #endif
 #include "common_header.h"
+#include "structdef.h"
 #include "src/profiling.h"
 #include "src/debug_print.h"
 
-__thread_if_stm long pypy_have_debug_prints = -1;
 FILE *pypy_debug_file = NULL;   /* XXX make it thread-local too? */
 static unsigned char debug_ready = 0;
 static unsigned char debug_profile = 0;
diff --git a/rpython/translator/c/src/debug_print.h b/rpython/translator/c/src/debug_print.h
--- a/rpython/translator/c/src/debug_print.h
+++ b/rpython/translator/c/src/debug_print.h
@@ -23,6 +23,14 @@
    Note that 'fname' can be '-' to send the logging data to stderr.
 */
 
+/* We stick pypy_have_debug_prints as a field of pypy_g_ExcData,
+   where it will be cleared in case of STM transaction aborts.
+   XXX XXX this will set to zero the bits that were at one before
+   the transaction started; the log will be truncated sometimes.
+*/
+RPY_EXTERN struct pypy_ExcData0 pypy_g_ExcData;
+#define pypy_have_debug_prints    pypy_g_ExcData.ed_have_debug_prints
+
 /* macros used by the generated code */
 #define PYPY_HAVE_DEBUG_PRINTS    (pypy_have_debug_prints & 1 ? \
                                    (pypy_debug_ensure_opened(), 1) : 0)
@@ -49,7 +57,6 @@
 #define __thread_if_stm  /* nothing */
 #endif
 
-RPY_EXTERN __thread_if_stm long pypy_have_debug_prints;
 RPY_EXTERN __thread_if_stm char pypy_debug_threadid[];
 RPY_EXPORTED FILE *pypy_debug_file;
 
diff --git a/rpython/translator/exceptiontransform.py b/rpython/translator/exceptiontransform.py
--- a/rpython/translator/exceptiontransform.py
+++ b/rpython/translator/exceptiontransform.py
@@ -453,12 +453,14 @@
         EXCDATA = lltype.Struct('ExcData',
             ('exc_type',  self.lltype_of_exception_type),
             ('exc_value', self.lltype_of_exception_value),
+            ('have_debug_prints', lltype.Signed),
             hints={'stm_thread_local': True,
                    'stm_dont_track_raw_accesses':True,
                    'is_excdata': True})
         self.EXCDATA = EXCDATA
 
         exc_data = lltype.malloc(EXCDATA, immortal=True)
+        exc_data.have_debug_prints = -1
         null_type = lltype.nullptr(self.lltype_of_exception_type.TO)
         null_value = lltype.nullptr(self.lltype_of_exception_value.TO)
 

From noreply at buildbot.pypy.org  Sat Feb 21 09:34:24 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Feb 2015 09:34:24 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Two tests.  One fails and one passes
Message-ID: <20150221083424.7AC231C0129@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r76026:670bf8f99260
Date: 2015-02-21 09:33 +0100
http://bitbucket.org/pypy/pypy/changeset/670bf8f99260/

Log:	Two tests. One fails and one passes

diff --git a/pypy/module/pypystm/test_pypy_c/test_no_conflict.py b/pypy/module/pypystm/test_pypy_c/test_no_conflict.py
--- a/pypy/module/pypystm/test_pypy_c/test_no_conflict.py
+++ b/pypy/module/pypystm/test_pypy_c/test_no_conflict.py
@@ -29,6 +29,27 @@
         #
         self.check_almost_no_conflict(f)
 
+    def test_stmdict_populate(self):
+        def f():
+            import pypystm, random
+            d = pypystm.stmdict()     # shared
+            def g(r):
+                value = r.randrange(0, 1000000000)
+                d[value] = None
+            run_in_threads(g, arg_class=random.Random)
+        #
+        self.check_almost_no_conflict(f)
+
+    def test_threadlocal(self):
+        def f():
+            import thread
+            tls = thread._local()
+            def g():
+                tls.value = getattr(tls, 'value', 0) + 1
+            run_in_threads(g)
+        #
+        self.check_almost_no_conflict(f)
+
     def test_weakrefs(self):
         py.test.skip("next issue")
         def f():

From noreply at buildbot.pypy.org  Sat Feb 21 12:55:36 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Feb 2015 12:55:36 +0100 (CET)
Subject: [pypy-commit] stmgc default: Add an assert that fails
Message-ID: <20150221115536.239CB1C026F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1639:e121fe82bf2c
Date: 2015-02-21 12:26 +0100
http://bitbucket.org/pypy/stmgc/changeset/e121fe82bf2c/

Log:	Add an assert that fails

diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c
--- a/c7/stm/gcpage.c
+++ b/c7/stm/gcpage.c
@@ -139,6 +139,9 @@
         char *dest = get_segment_base(j) + nobj;
         memcpy(dest, initial_data, size_rounded_up);
         ((struct object_s *)dest)->stm_flags = GCFLAG_WRITE_BARRIER;
+        if (j) {
+            assert(!was_read_remote(get_segment_base(j), (object_t *)nobj));
+        }
     }
 
     release_privatization_lock();

From noreply at buildbot.pypy.org  Sat Feb 21 12:55:37 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Feb 2015 12:55:37 +0100 (CET)
Subject: [pypy-commit] stmgc default: Fix
Message-ID: <20150221115537.5A1AB1C026F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1640:78281a990907
Date: 2015-02-21 12:49 +0100
http://bitbucket.org/pypy/stmgc/changeset/78281a990907/

Log:	Fix

diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c
--- a/c7/stm/gcpage.c
+++ b/c7/stm/gcpage.c
@@ -134,14 +134,19 @@
 
     char *p = allocate_outside_nursery_large(size_rounded_up);
     uintptr_t nobj = p - stm_object_pages;
+    dprintf(("allocate_preexisting: %p\n", (object_t *)nobj));
     long j;
     for (j = 0; j <= NB_SEGMENTS; j++) {
         char *dest = get_segment_base(j) + nobj;
         memcpy(dest, initial_data, size_rounded_up);
         ((struct object_s *)dest)->stm_flags = GCFLAG_WRITE_BARRIER;
-        if (j) {
+#ifdef STM_TESTS
+        /* can't really enable this check outside tests, because there is
+           a change that the transaction_state changes in parallel */
+        if (j && get_priv_segment(j)->transaction_state != TS_NONE) {
             assert(!was_read_remote(get_segment_base(j), (object_t *)nobj));
         }
+#endif
     }
 
     release_privatization_lock();
diff --git a/c7/stm/hashtable.c b/c7/stm/hashtable.c
--- a/c7/stm/hashtable.c
+++ b/c7/stm/hashtable.c
@@ -107,8 +107,12 @@
 
 static bool _stm_was_read_by_anybody(object_t *obj)
 {
+    /* can only be safely called during major GC, when all other threads
+       are suspended */
     long i;
     for (i = 1; i <= NB_SEGMENTS; i++) {
+        if (get_priv_segment(i)->transaction_state == TS_NONE)
+            continue;
         if (was_read_remote(get_segment_base(i), obj))
             return true;
     }
diff --git a/c7/test/test_hashtable.py b/c7/test/test_hashtable.py
--- a/c7/test/test_hashtable.py
+++ b/c7/test/test_hashtable.py
@@ -105,10 +105,12 @@
         self.start_transaction()
         h = self.pop_root()
         stm_set_char(lp2, 'B')
+        self.switch(1)
+        self.start_transaction()
+        self.switch(0)
         htset(h, 9991234, lp2, tl0)
         #
         self.switch(1)
-        self.start_transaction()
         lp1b = htget(h, 1234)
         assert lp1b != ffi.NULL
         assert stm_get_char(lp1b) == 'A'
@@ -303,6 +305,36 @@
         self.switch(0)
         stm_major_collect()       # to get rid of the hashtable object
 
+    def test_grow_without_conflict(self):
+        self.start_transaction()
+        h = self.allocate_hashtable()
+        self.push_root(h)
+        self.commit_transaction()
+        h = self.pop_root()
+        self.push_root(h)
+        #
+        STEPS = 50
+        for i in range(STEPS):
+            self.switch(1)
+            self.start_transaction()
+            tl0 = self.tls[self.current_thread]
+            htset(h, i + STEPS, stm_allocate(32), tl0)
+            #
+            self.switch(0)
+            self.start_transaction()
+            tl0 = self.tls[self.current_thread]
+            htset(h, i, stm_allocate(24), tl0)
+            #
+            self.switch(1)
+            self.commit_transaction()
+            #
+            self.switch(0)
+            self.commit_transaction()
+        #
+        self.pop_root()
+        self.start_transaction()
+        stm_major_collect()       # to get rid of the hashtable object
+
 
 class TestRandomHashtable(BaseTestHashtable):
 

From noreply at buildbot.pypy.org  Sat Feb 21 15:00:20 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 21 Feb 2015 15:00:20 +0100 (CET)
Subject: [pypy-commit] pypy rewrite-unrolling: implement importing of values
	for input args
Message-ID: <20150221140020.C35F91C0041@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: rewrite-unrolling
Changeset: r76027:d23a1eaee646
Date: 2015-02-21 12:32 +0200
http://bitbucket.org/pypy/pypy/changeset/d23a1eaee646/

Log:	implement importing of values for input args

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -67,11 +67,6 @@
         self._tag = (self._tag & (~0x3)) | level
 
     def import_from(self, other, optimizer):
-        if self.getlevel() == LEVEL_CONSTANT:
-            assert other.getlevel() == LEVEL_CONSTANT
-            assert other.box.same_constant(self.box)
-            return
-        assert self.getlevel() <= LEVEL_NONNULL
         if other.getlevel() == LEVEL_CONSTANT:
             self.make_constant(other.get_key_box())
         elif other.getlevel() == LEVEL_KNOWNCLASS:
@@ -256,14 +251,13 @@
 
     def import_from(self, other, optimizer):
         OptValue.import_from(self, other, optimizer)
-        if self.getlevel() != LEVEL_CONSTANT:
-            if other.getlenbound():
-                if self.lenbound:
-                    assert other.getlenbound().mode == self.lenbound.mode
-                    assert other.getlenbound().descr == self.lenbound.descr
-                    self.lenbound.bound.intersect(other.getlenbound().bound)
-                else:
-                    self.lenbound = other.getlenbound().clone()
+        if other.getlenbound():
+            if self.lenbound:
+                assert other.getlenbound().mode == self.lenbound.mode
+                assert other.getlenbound().descr == self.lenbound.descr
+                self.lenbound.bound.intersect(other.getlenbound().bound)
+            else:
+                self.lenbound = other.getlenbound().clone()
 
     def make_guards(self, box):
         guards = []
@@ -360,9 +354,8 @@
 
     def import_from(self, other, optimizer):
         OptValue.import_from(self, other, optimizer)
-        if self.getlevel() != LEVEL_CONSTANT:
-            if other.getintbound() is not None: # VRawBufferValue
-                self.intbound.intersect(other.getintbound())
+        if other.getintbound() is not None: # VRawBufferValue
+            self.intbound.intersect(other.getintbound())
 
     def make_guards(self, box):
         guards = []
@@ -606,7 +599,6 @@
                 value = self.values[box] = IntOptValue(box)
             else:
                 value = self.values[box] = OptValue(box)
-        self.ensure_imported(value)
         return value
 
     def get_box_replacement(self, box):
@@ -616,16 +608,12 @@
             return box
         return v.get_key_box()
 
-    def ensure_imported(self, value):
-        pass
-
     @specialize.argtype(0)
     def get_constant_box(self, box):
         if isinstance(box, Const):
             return box
         try:
             value = self.values[box]
-            self.ensure_imported(value)
         except KeyError:
             return None
         if value.is_constant():
@@ -738,7 +726,6 @@
             except KeyError:
                 pass
             else:
-                self.ensure_imported(value)
                 newbox = value.force_box(self)
                 if arg is not newbox:
                     if not changed:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
@@ -1,27 +1,32 @@
 
 from rpython.jit.tool.oparser import parse
 from rpython.jit.metainterp.optimizeopt import optimize_trace
-from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer
+from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer,\
+     PtrOptValue, LEVEL_KNOWNCLASS
 from rpython.jit.metainterp.optimizeopt.unroll import Unroller
 from rpython.jit.metainterp.optimizeopt.test.test_util import BaseTest,\
-     FakeMetaInterpStaticData
+     FakeMetaInterpStaticData, LLtypeMixin
 from rpython.jit.metainterp.optimizeopt.pure import OptPure
 from rpython.jit.metainterp.resoperation import rop, ResOperation
 from rpython.jit.metainterp.history import ConstInt, BoxInt
 from rpython.jit.backend.llgraph import runner
+from rpython.jit.codewriter.heaptracker import adr2int
 
-class TestUnrollDirect(BaseTest):
+class TestUnrollDirect(BaseTest, LLtypeMixin):
     cpu = runner.LLGraphCPU(None)
     enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll"
     metainterp_sd = FakeMetaInterpStaticData(cpu)
 
+    def parse(self, loop):
+        return BaseTest.parse(self, loop, postprocess=self.postprocess)
+
     def optimize(self, loop, expected=None, unroller=None):
         if isinstance(loop, str):
-            loop = parse(loop)
+            loop = self.parse(loop)
         state = optimize_trace(self.metainterp_sd, None, loop, self.enable_opts,
                                unroller=unroller)
         if expected is not None:
-            expected = parse(expected)
+            expected = self.parse(expected)
             self.assert_equal(loop, expected)
         return state
     
@@ -65,3 +70,25 @@
         jump(i0, i1)
         """
         self.optimize(loop, expected, unroller=unroller)
+
+    def test_inherit_known_class(self):
+        loop = self.parse("""
+        [p0]
+        label(p0)
+        guard_class(p0, ConstClass(node_vtable)) []
+        jump(p0)
+        """)
+        expected = """
+        [p0]
+        label(p0)
+        jump(p0)
+        """
+        p0 = loop.inputargs[0]
+        unroller = Unroller()
+        unroller.optimizer = Optimizer(self.metainterp_sd, None, None, [])
+        cls = adr2int(self.node_vtable_adr)
+        unroller.optimizer.values = {
+            p0: PtrOptValue(p0, known_class=ConstInt(cls),
+                            level=LEVEL_KNOWNCLASS),
+        }
+        self.optimize(loop, expected, unroller=unroller)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -361,7 +361,7 @@
 class BaseTest(object):
 
     def parse(self, s, boxkinds=None, want_fail_descr=True, postprocess=None):
-        self.oparse = OpParser(s, self.cpu, self.namespace, 'lltype',
+        self.oparse = OpParser(s, self.cpu, self.namespace.copy(), 'lltype',
                                boxkinds,
                                None, False, postprocess)
         return self.oparse.parse()
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -25,6 +25,7 @@
 
     def force_box(self, optforce):
         if self.box is None:
+            # XXX add myself to the short preamble
             self.box = self.keybox
             optforce.optimizer.reuse_pure_result(self.box)
         return self.box
@@ -57,6 +58,15 @@
                     new_optpure.pure(op.getopnum(), op.getarglist(),
                                      op.result, pure_value)
                     self.optimizer.pure_reverse(op)
+        for box in self.optimizer.loop.operations[0].getarglist():
+            try:
+                # XXX do the same thing for pure opt value
+                other = old_optimizer.values[box]
+                self.optimizer.getvalue(box).import_from(other,
+                                                         self.optimizer)
+            except KeyError:
+                pass
+        
     #         for opargs, value in old_optpure.pure_operations.items():
     #             if not value.is_virtual():
     #                 pure_value = OptPureValue(self, value.box)

From noreply at buildbot.pypy.org  Sat Feb 21 16:31:22 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 21 Feb 2015 16:31:22 +0100 (CET)
Subject: [pypy-commit] benchmarks default: function call warmup benchmark
Message-ID: <20150221153122.607021C0135@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r304:945b94561e5c
Date: 2015-02-21 17:31 +0200
http://bitbucket.org/pypy/benchmarks/changeset/945b94561e5c/

Log:	function call warmup benchmark

diff --git a/warmup/function_call.py b/warmup/function_call.py
new file mode 100644
--- /dev/null
+++ b/warmup/function_call.py
@@ -0,0 +1,39 @@
+
+import time
+l = []
+
+for i in range(100):
+    print i
+    t0 = time.time()
+    exec """
+def j(a, b, c):
+    pass
+
+def k(a, b, c):
+    j(a, b, c)
+    j(a, b, c)
+    j(a, b, c)
+    j(a, b, c)
+    j(a, b, c)
+    j(a, b, c)
+
+def g(a, b, c):
+    k(a, b + 1, c + 2)
+    k(a, b + 1, c + 2)
+    k(a, b + 1, c + 2)
+    k(a, b + 1, c + 2)
+    k(a, b + 1, c + 2)
+
+def f(i):
+    g(i, i + 1, i + 2)
+    g(i, i + 1, i + 2)
+    g(i, i + 1, i + 2)
+    g(i, i + 1, i + 2)
+    g(i, i + 1, i + 2)
+    g(i, i + 1, i + 2)
+for i in range(1000):
+    f(i)
+"""
+    l.append(time.time() - t0)
+
+print l

From noreply at buildbot.pypy.org  Sat Feb 21 17:03:23 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 21 Feb 2015 17:03:23 +0100 (CET)
Subject: [pypy-commit] benchmarks default: add graph alloc removal bench
Message-ID: <20150221160323.AC1731C0471@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r305:7a35d603a9fd
Date: 2015-02-21 18:03 +0200
http://bitbucket.org/pypy/benchmarks/changeset/7a35d603a9fd/

Log:	add graph alloc removal bench

diff --git a/warmup/pypy-graph-alloc-removal.py b/warmup/pypy-graph-alloc-removal.py
new file mode 100644
--- /dev/null
+++ b/warmup/pypy-graph-alloc-removal.py
@@ -0,0 +1,43 @@
+from rpython.translator.interactive import Translation
+from rpython.translator.translator import graphof
+from rpython.translator.backendopt.malloc import LLTypeMallocRemover
+from rpython.flowspace.model import copygraph
+
+import time
+
+class X(object):
+    pass
+
+def f(n):
+    x = X()
+    x.attr = n
+    y = X()
+    y.attr = 6 if n > 100 else 5
+    return x.attr + y.attr
+
+
+t = Translation(f, [int])
+t.annotate()
+t.rtype()
+
+graph = graphof(t.context, f)
+
+def main(graph, repeat):
+    start = time.time()
+    l = []
+    for i in range(repeat):
+        print i
+        l.append(f(graph))
+    print l
+    end = time.time()
+    print end - start
+
+def f(graph):
+    start = time.time()
+    for k in range(10):
+        g = copygraph(graph)
+        remover = LLTypeMallocRemover()
+        remover.remove_mallocs_once(g)
+    return time.time() - start
+
+main(graph, 500)

From noreply at buildbot.pypy.org  Sat Feb 21 17:15:57 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Feb 2015 17:15:57 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Improve the test, still failing
Message-ID: <20150221161557.D067A1C009F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r76028:c0860a9ad1db
Date: 2015-02-21 09:50 +0100
http://bitbucket.org/pypy/pypy/changeset/c0860a9ad1db/

Log:	Improve the test, still failing

diff --git a/pypy/module/pypystm/test_pypy_c/test_no_conflict.py b/pypy/module/pypystm/test_pypy_c/test_no_conflict.py
--- a/pypy/module/pypystm/test_pypy_c/test_no_conflict.py
+++ b/pypy/module/pypystm/test_pypy_c/test_no_conflict.py
@@ -29,14 +29,31 @@
         #
         self.check_almost_no_conflict(f)
 
+    def test_hashtable_populate(self):
+        def f():
+            import pypystm
+            class TL(object):
+                step = 0
+            d = pypystm.hashtable()     # shared
+            def g(n, tl):
+                value = n + tl.step
+                tl.step += 10
+                d[value] = tl
+            run_in_threads(g, arg_thread_num=True, arg_class=TL)
+        #
+        self.check_almost_no_conflict(f)
+
     def test_stmdict_populate(self):
         def f():
-            import pypystm, random
+            import pypystm
+            class TL(object):
+                step = 0
             d = pypystm.stmdict()     # shared
-            def g(r):
-                value = r.randrange(0, 1000000000)
-                d[value] = None
-            run_in_threads(g, arg_class=random.Random)
+            def g(n, tl):
+                value = n + tl.step
+                tl.step += 10
+                d[value] = tl
+            run_in_threads(g, arg_thread_num=True, arg_class=TL)
         #
         self.check_almost_no_conflict(f)
 

From noreply at buildbot.pypy.org  Sat Feb 21 17:15:59 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Feb 2015 17:15:59 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: import stmgc/78281a990907
Message-ID: <20150221161559.1508F1C009F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r76029:59ea05c4e7c5
Date: 2015-02-21 12:53 +0100
http://bitbucket.org/pypy/pypy/changeset/59ea05c4e7c5/

Log:	import stmgc/78281a990907

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-6c6a18d9e312
+78281a990907
diff --git a/rpython/translator/stm/src_stm/stm/gcpage.c b/rpython/translator/stm/src_stm/stm/gcpage.c
--- a/rpython/translator/stm/src_stm/stm/gcpage.c
+++ b/rpython/translator/stm/src_stm/stm/gcpage.c
@@ -135,11 +135,19 @@
 
     char *p = allocate_outside_nursery_large(size_rounded_up);
     uintptr_t nobj = p - stm_object_pages;
+    dprintf(("allocate_preexisting: %p\n", (object_t *)nobj));
     long j;
     for (j = 0; j <= NB_SEGMENTS; j++) {
         char *dest = get_segment_base(j) + nobj;
         memcpy(dest, initial_data, size_rounded_up);
         ((struct object_s *)dest)->stm_flags = GCFLAG_WRITE_BARRIER;
+#ifdef STM_TESTS
+        /* can't really enable this check outside tests, because there is
+           a change that the transaction_state changes in parallel */
+        if (j && get_priv_segment(j)->transaction_state != TS_NONE) {
+            assert(!was_read_remote(get_segment_base(j), (object_t *)nobj));
+        }
+#endif
     }
 
     release_privatization_lock();
diff --git a/rpython/translator/stm/src_stm/stm/hashtable.c b/rpython/translator/stm/src_stm/stm/hashtable.c
--- a/rpython/translator/stm/src_stm/stm/hashtable.c
+++ b/rpython/translator/stm/src_stm/stm/hashtable.c
@@ -108,8 +108,12 @@
 
 static bool _stm_was_read_by_anybody(object_t *obj)
 {
+    /* can only be safely called during major GC, when all other threads
+       are suspended */
     long i;
     for (i = 1; i <= NB_SEGMENTS; i++) {
+        if (get_priv_segment(i)->transaction_state == TS_NONE)
+            continue;
         if (was_read_remote(get_segment_base(i), obj))
             return true;
     }

From noreply at buildbot.pypy.org  Sat Feb 21 17:16:00 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Feb 2015 17:16:00 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Fix test_lloperation
Message-ID: <20150221161600.3631D1C009F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r76030:f08715bea48a
Date: 2015-02-21 17:15 +0100
http://bitbucket.org/pypy/pypy/changeset/f08715bea48a/

Log:	Fix test_lloperation

diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -960,34 +960,51 @@
 
     def _stm_not_implemented(self, *args):
         raise NotImplementedError
-    op_stm_initialize = _stm_not_implemented
-    op_stm_finalize = _stm_not_implemented
-    op_stm_perform_transaction = _stm_not_implemented
-    op_stm_commit_transaction = _stm_not_implemented
-    op_stm_begin_inevitable_transaction = _stm_not_implemented
-    op_stm_barrier = _stm_not_implemented
     op_stm_push_root = _stm_not_implemented
     op_stm_pop_root_into = _stm_not_implemented
-    op_stm_get_adr_of_nursery_current = _stm_not_implemented
-    op_stm_get_adr_of_nursery_nextlimit = _stm_not_implemented
-    op_stm_get_adr_of_active = _stm_not_implemented
-    op_stm_get_adr_of_read_barrier_cache = _stm_not_implemented
-    op_stm_get_adr_of_private_rev_num = _stm_not_implemented
+    op_stm_get_root_stack_top = _stm_not_implemented
+    op_stm_start_if_not_atomic = _stm_not_implemented
+    op_stm_commit_if_not_atomic = _stm_not_implemented
     op_stm_enter_callback_call = _stm_not_implemented
     op_stm_leave_callback_call = _stm_not_implemented
     op_stm_get_atomic = _stm_not_implemented
     op_stm_is_inevitable = _stm_not_implemented
-    op_stm_change_atomic = _stm_not_implemented
     op_stm_set_transaction_length = _stm_not_implemented
-    op_stm_hash = _stm_not_implemented
     op_stm_id = _stm_not_implemented
-    op_stm_allocate = _stm_not_implemented
-    op_stm_weakref_allocate = _stm_not_implemented
-    op_stm_allocate_nonmovable_int_adr = _stm_not_implemented
-    op_stm_minor_collect = _stm_not_implemented
-    op_stm_major_collect = _stm_not_implemented
     op_stm_abort_and_retry = _stm_not_implemented
     op_stm_become_inevitable = _stm_not_implemented
+    op_stm_stop_all_other_threads = _stm_not_implemented
+    op_stm_resume_all_other_threads = _stm_not_implemented
+    op_stm_set_into_obj = _stm_not_implemented
+    op_stm_addr_get_tid = _stm_not_implemented
+    op_stm_allocate_tid = _stm_not_implemented
+    op_stm_allocate_weakref = _stm_not_implemented
+    op_stm_allocate_f_light = _stm_not_implemented
+    op_stm_allocate_finalizer = _stm_not_implemented
+    op_stm_allocate_nonmovable = _stm_not_implemented
+    op_stm_allocate_preexisting = _stm_not_implemented
+    op_stm_malloc_nonmovable = _stm_not_implemented
+    op_stm_can_move = _stm_not_implemented
+    op_stm_read = _stm_not_implemented
+    op_stm_write = _stm_not_implemented
+    op_stm_hashtable_create = _stm_not_implemented
+    op_stm_hashtable_lookup = _stm_not_implemented
+    op_stm_hashtable_read = _stm_not_implemented
+    op_stm_hashtable_write = _stm_not_implemented
+    op_stm_hashtable_write_entry = _stm_not_implemented
+    op_stm_hashtable_tracefn = _stm_not_implemented
+    op_stm_hashtable_length_upper_bound = _stm_not_implemented
+    op_stm_hashtable_list = _stm_not_implemented
+    op_stm_hashtable_free = _stm_not_implemented
+    op_stm_register_thread_local = _stm_not_implemented
+    op_stm_unregister_thread_local = _stm_not_implemented
+    op_stm_really_force_cast_ptr = _stm_not_implemented
+    op_stm_identityhash = _stm_not_implemented
+    op_stm_expand_marker = _stm_not_implemented
+    op_stm_setup_expand_marker_for_pypy = _stm_not_implemented
+    op_stm_increment_atomic = _stm_not_implemented
+    op_stm_decrement_atomic = _stm_not_implemented
+    op_stm_collect = _stm_not_implemented
 
     def op_stm_should_break_transaction(self, keep):
         return False
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -425,13 +425,11 @@
     'stm_allocate_preexisting':LLOp(sideeffects=False, canmallocgc=True),
     'stm_allocate_nonmovable':LLOp(sideeffects=False, canmallocgc=True),
     'stm_malloc_nonmovable':  LLOp(sideeffects=False, canmallocgc=True),
-    'stm_get_from_obj':       LLOp(sideeffects=False),
-    'stm_get_from_obj_const': LLOp(canfold=True),
     'stm_set_into_obj':       LLOp(),
     'stm_collect':            LLOp(canmallocgc=True),
     'stm_id':                 LLOp(sideeffects=False),
-    'stm_identityhash':       LLOp(canfold=True),
-    'stm_addr_get_tid':       LLOp(canfold=True),
+    'stm_identityhash':       LLOp(sideeffects=False),
+    'stm_addr_get_tid':       LLOp(sideeffects=False),
     'stm_get_root_stack_top': LLOp(sideeffects=False),
     'stm_become_inevitable':  LLOp(canmallocgc=True),
     'stm_push_root':          LLOp(),
diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -126,17 +126,6 @@
                 (result, arg_size, arg_size) +
             '((rpyobj_t *)%s)->tid = %s;' % (result, arg_type_id))
 
-def stm_get_from_obj(funcgen, op):
-    assert op.args[0].concretetype == llmemory.GCREF
-    arg_obj = funcgen.expr(op.args[0])
-    arg_ofs = funcgen.expr(op.args[1])
-    result  = funcgen.expr(op.result)
-    resulttype = cdecl(funcgen.lltypename(op.result), '')
-    return '%s = *(TLPREFIX %s *)(%s + %s);' % (
-        result, resulttype, arg_obj, arg_ofs)
-
-stm_get_from_obj_const = stm_get_from_obj
-
 def stm_set_into_obj(funcgen, op):
     assert op.args[0].concretetype == llmemory.GCREF
     arg_obj = funcgen.expr(op.args[0])

From noreply at buildbot.pypy.org  Sat Feb 21 18:32:40 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Feb 2015 18:32:40 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: fix
Message-ID: <20150221173240.EAB7C1C0382@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r76031:9b0a5a4d6ba2
Date: 2015-02-21 17:50 +0100
http://bitbucket.org/pypy/pypy/changeset/9b0a5a4d6ba2/

Log:	fix

diff --git a/rpython/translator/c/src/debug_print.h b/rpython/translator/c/src/debug_print.h
--- a/rpython/translator/c/src/debug_print.h
+++ b/rpython/translator/c/src/debug_print.h
@@ -28,7 +28,13 @@
    XXX XXX this will set to zero the bits that were at one before
    the transaction started; the log will be truncated sometimes.
 */
-RPY_EXTERN struct pypy_ExcData0 pypy_g_ExcData;
+#ifdef RPY_STM
+#define __thread_if_stm  __thread
+#else
+#define __thread_if_stm  /* nothing */
+#endif
+
+RPY_EXTERN __thread struct pypy_ExcData0 pypy_g_ExcData;
 #define pypy_have_debug_prints    pypy_g_ExcData.ed_have_debug_prints
 
 /* macros used by the generated code */
@@ -51,12 +57,6 @@
 RPY_EXTERN long pypy_debug_offset(void);
 RPY_EXTERN void pypy_debug_forked(long original_offset);
 
-#ifdef RPY_STM
-#define __thread_if_stm  __thread
-#else
-#define __thread_if_stm  /* nothing */
-#endif
-
 RPY_EXTERN __thread_if_stm char pypy_debug_threadid[];
 RPY_EXPORTED FILE *pypy_debug_file;
 

From noreply at buildbot.pypy.org  Sat Feb 21 18:32:42 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Feb 2015 18:32:42 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: merge heads
Message-ID: <20150221173242.1B2231C0382@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r76032:47e4c4e0ffbd
Date: 2015-02-21 17:51 +0100
http://bitbucket.org/pypy/pypy/changeset/47e4c4e0ffbd/

Log:	merge heads

diff --git a/rpython/translator/c/src/debug_print.h b/rpython/translator/c/src/debug_print.h
--- a/rpython/translator/c/src/debug_print.h
+++ b/rpython/translator/c/src/debug_print.h
@@ -28,7 +28,13 @@
    XXX XXX this will set to zero the bits that were at one before
    the transaction started; the log will be truncated sometimes.
 */
-RPY_EXTERN struct pypy_ExcData0 pypy_g_ExcData;
+#ifdef RPY_STM
+#define __thread_if_stm  __thread
+#else
+#define __thread_if_stm  /* nothing */
+#endif
+
+RPY_EXTERN __thread struct pypy_ExcData0 pypy_g_ExcData;
 #define pypy_have_debug_prints    pypy_g_ExcData.ed_have_debug_prints
 
 /* macros used by the generated code */
@@ -51,12 +57,6 @@
 RPY_EXTERN long pypy_debug_offset(void);
 RPY_EXTERN void pypy_debug_forked(long original_offset);
 
-#ifdef RPY_STM
-#define __thread_if_stm  __thread
-#else
-#define __thread_if_stm  /* nothing */
-#endif
-
 RPY_EXTERN __thread_if_stm char pypy_debug_threadid[];
 RPY_EXPORTED FILE *pypy_debug_file;
 

From noreply at buildbot.pypy.org  Sat Feb 21 18:32:43 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Feb 2015 18:32:43 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Classify all operations into one of
	the groups.
Message-ID: <20150221173243.367E21C0382@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r76033:c1c83613b370
Date: 2015-02-21 18:18 +0100
http://bitbucket.org/pypy/pypy/changeset/c1c83613b370/

Log:	Classify all operations into one of the groups.

diff --git a/rpython/translator/stm/inevitable.py b/rpython/translator/stm/inevitable.py
--- a/rpython/translator/stm/inevitable.py
+++ b/rpython/translator/stm/inevitable.py
@@ -7,9 +7,13 @@
 
 ALWAYS_ALLOW_OPERATIONS = set([
     'force_cast', 'keepalive', 'cast_ptr_to_adr',
-    'cast_adr_to_int',
-    'debug_print', 'debug_assert',
+    'cast_adr_to_int', 'cast_int_to_ptr',
+    'cast_ptr_to_weakrefptr', 'cast_weakrefptr_to_ptr',
+    'debug_print', 'debug_assert', 'debug_flush', 'debug_offset',
     'debug_start', 'debug_stop', 'have_debug_prints',
+    'debug_catch_exception', 'debug_nonnull_pointer',
+    'debug_record_traceback', 'debug_start_traceback',
+    'debug_reraise_traceback',
     'cast_opaque_ptr', 'hint',
     'stack_current', 'gc_stack_bottom', 'cast_ptr_to_int',
     'jit_force_virtual', 'jit_force_virtualizable',
@@ -22,6 +26,13 @@
     'jit_assembler_call', 'gc_writebarrier',
     'shrink_array',
     'threadlocalref_addr', 'threadlocalref_get',
+    'gc_get_rpy_memory_usage', 'gc_get_rpy_referents',
+    'gc_get_rpy_roots', 'gc_get_rpy_type_index', 'gc_get_type_info_group',
+    'gc_heap_stats', 'gc_is_rpy_instance', 'gc_typeids_list',
+    'gc_typeids_z', 'nop',
+    'length_of_simple_gcarray_from_opaque',
+    'll_read_timestamp',
+    'jit_conditional_call',
     ])
 ALWAYS_ALLOW_OPERATIONS |= set(lloperation.enum_tryfold_ops())
 
@@ -29,6 +40,7 @@
     if opname.startswith('stm_'):
         ALWAYS_ALLOW_OPERATIONS.add(opname)
 
+CALLS   = set(['direct_call', 'indirect_call'])
 GETTERS = set(['getfield', 'getarrayitem', 'getinteriorfield', 'raw_load'])
 SETTERS = set(['setfield', 'setarrayitem', 'setinteriorfield', 'raw_store'])
 MALLOCS = set(['malloc', 'malloc_varsize',
@@ -37,8 +49,46 @@
                'do_malloc_fixedsize_clear', 'do_malloc_varsize_clear'])
 FREES   = set(['free', 'raw_free'])
 
-for opname in ALWAYS_ALLOW_OPERATIONS | GETTERS | SETTERS | MALLOCS | FREES:
-    getattr(lloperation.llop, opname)   # the opname must exist!
+# These operations should not appear at all in an stm build at the
+# point this file is invoked (before gctransform)
+INCOMPATIBLE_OPS = set([
+    'bare_raw_store', 'bare_setarrayitem', 'bare_setfield',
+    'bare_setinteriorfield',
+    'boehm_disappearing_link', 'boehm_malloc', 'boehm_malloc_atomic',
+    'boehm_register_finalizer',
+    'check_and_clear_exc', 'check_no_more_arg', 'check_self_nonzero',
+    'debug_stm_flush_barrier', 'debug_view',
+    'decode_arg', 'decode_arg_def',
+    'gc_adr_of_nursery_free', 'gc_adr_of_nursery_top',
+    'gc_adr_of_root_stack_base', 'gc_asmgcroot_static',
+    'gc_call_rtti_destructor', 'gc_deallocate',
+    'gc_detach_callback_pieces', 'gc_fetch_exception',
+    'gc_set_max_heap_size',
+    'gc_forget_current_state', 'gc_free',
+    'gc_gcflag_extra', 'gc_obtain_free_space',
+    'gc_reattach_callback_pieces', 'gc_reload_possibly_moved',
+    'gc_restore_exception', 'gc_restore_state_from',
+    'gc_save_current_state_away',
+    'gc_shadowstackref_context', 'gc_shadowstackref_new',
+    'gc_thread_after_fork', 'gc_thread_before_fork',
+    'gc_start_fresh_new_state', 'gc_thread_run',
+    'gc_writebarrier_before_copy',
+    'get_exc_value_addr', 'get_exception_addr',
+    'get_write_barrier_failing_case',
+    'get_write_barrier_from_array_failing_case',
+    'getslice', 'instrument_count',
+    'jit_ffi_save_result',
+    'raw_malloc_usage',
+    'raw_memclear', 'raw_memcopy', 'raw_memmove', 'raw_memset',
+    'stack_malloc', 'track_alloc_start', 'track_alloc_stop',
+    'zero_gc_pointers_inside',
+    ])
+
+# These operations always turn the transaction inevitable.
+TURN_INEVITABLE_OPS = set([
+    'debug_fatalerror', 'debug_llinterpcall', 'debug_print_traceback',
+    'gc_dump_rpy_heap', 'gc_thread_start', 'gc_thread_die',
+    ])
 
 # ____________________________________________________________
 
@@ -86,6 +136,7 @@
     # Always-allowed operations never cause a 'turn inevitable'
     if op.opname in ALWAYS_ALLOW_OPERATIONS:
         return False
+    assert op.opname not in INCOMPATIBLE_OPS
     #
     # Getters and setters
     if op.opname in GETTERS:
@@ -104,7 +155,7 @@
         return False
     #
     # Function calls
-    if op.opname == 'direct_call' or op.opname == 'indirect_call':
+    if op.opname in CALLS:
         return should_turn_inevitable_call(op)
     #
     # Entirely unsupported operations cause a 'turn inevitable'
diff --git a/rpython/translator/stm/test/test_inevitable.py b/rpython/translator/stm/test/test_inevitable.py
--- a/rpython/translator/stm/test/test_inevitable.py
+++ b/rpython/translator/stm/test/test_inevitable.py
@@ -1,12 +1,30 @@
-from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, lloperation
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.llinterp import LLFrame
 from rpython.rtyper.test import test_llinterp
 from rpython.rtyper.test.test_llinterp import get_interpreter, clear_tcache
 from rpython.translator.stm.inevitable import insert_turn_inevitable
+from rpython.translator.stm import inevitable
 from rpython.conftest import option
 
 
+KNOWN_OPERATIONS = (inevitable.ALWAYS_ALLOW_OPERATIONS |
+                    inevitable.CALLS |
+                    inevitable.GETTERS | inevitable.SETTERS |
+                    inevitable.MALLOCS | inevitable.FREES |
+                    inevitable.INCOMPATIBLE_OPS |
+                    inevitable.TURN_INEVITABLE_OPS)
+
+def test_defined_operations():
+    for opname in KNOWN_OPERATIONS:
+        getattr(llop, opname)   # the opname must exist!
+
+def test_no_missing_operation():
+    ALL_OPERATIONS = set(lloperation.LL_OPERATIONS)
+    MISSING_OPERATIONS = ALL_OPERATIONS - KNOWN_OPERATIONS
+    assert not sorted(MISSING_OPERATIONS)
+
+
 class LLSTMInevFrame(LLFrame):
     def op_stm_become_inevitable(self, info):
         assert info is not None

From noreply at buildbot.pypy.org  Sat Feb 21 18:32:44 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Feb 2015 18:32:44 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Fix test
Message-ID: <20150221173244.56D5D1C0382@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r76034:58200123718c
Date: 2015-02-21 18:32 +0100
http://bitbucket.org/pypy/pypy/changeset/58200123718c/

Log:	Fix test

diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py
--- a/rpython/translator/stm/test/test_ztranslated.py
+++ b/rpython/translator/stm/test/test_ztranslated.py
@@ -1,6 +1,6 @@
 import py
 from rpython.rlib import rstm, rgc, objectmodel
-from rpython.rlib.debug import debug_print
+from rpython.rlib.debug import debug_start, debug_print, debug_stop
 from rpython.rlib.rarithmetic import intmask
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rtyper.lltypesystem.lloperation import llop
@@ -280,7 +280,9 @@
             if retry_counter < 1000:
                 if (retry_counter & 3) == 0:
                     lltype.free(x, flavor='raw')
+                debug_start('foo')
                 debug_print(rffi.cast(lltype.Signed, x))
+                debug_stop('foo')
                 rstm.abort_and_retry()
             lltype.free(x, flavor='raw')
             return 0
@@ -295,10 +297,14 @@
             return 0
 
         t, cbuilder = self.compile(main)
-        data, dataerr = cbuilder.cmdexec('', err=True)
-        lines = dataerr.split('\n')
-        assert len(lines) > 1000
-        addresses = map(int, lines[:1000])
+        data, dataerr = cbuilder.cmdexec('', err=True, env={'PYPYLOG': 'foo:-'})
+        lines = dataerr.splitlines()
+        assert len(lines) > 3000
+        addresses = []
+        for i in range(0, 3000, 3):
+            assert lines[i].endswith('{foo')
+            addresses.append(int(lines[i+1].split()[-1]))
+            assert lines[i+2].endswith('foo}')
         assert len(addresses) == 1000
         assert len(set(addresses)) < 500    # should ideally just be a few
         import re

From noreply at buildbot.pypy.org  Sat Feb 21 18:39:38 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 21 Feb 2015 18:39:38 +0100 (CET)
Subject: [pypy-commit] pypy nonquadratic-heapcache: close to be merged branch
Message-ID: <20150221173938.49C731C009F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: nonquadratic-heapcache
Changeset: r76035:da6eac6ebb1b
Date: 2015-02-21 19:38 +0200
http://bitbucket.org/pypy/pypy/changeset/da6eac6ebb1b/

Log:	close to be merged branch


From noreply at buildbot.pypy.org  Sat Feb 21 18:39:39 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 21 Feb 2015 18:39:39 +0100 (CET)
Subject: [pypy-commit] pypy default: (cfbolz) Improve the complexity of
 heapcache so it's no longer quadratic
Message-ID: <20150221173939.8C2EC1C009F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r76036:7cf2438f9023
Date: 2015-02-21 19:39 +0200
http://bitbucket.org/pypy/pypy/changeset/7cf2438f9023/

Log:	(cfbolz) Improve the complexity of heapcache so it's no longer
	quadratic in the number of identical of virtuals

diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -1,61 +1,111 @@
 from rpython.jit.metainterp.history import ConstInt
 from rpython.jit.metainterp.resoperation import rop
 
+class HeapCacheValue(object):
+    def __init__(self, box):
+        self.box = box
+        self.likely_virtual = False
+        self.reset_keep_likely_virtual()
+
+    def reset_keep_likely_virtual(self):
+        self.known_class = False
+        # did we see the allocation during tracing?
+        self.seen_allocation = False
+        self.is_unescaped = False
+        self.nonstandard_virtualizable = False
+        self.length = None
+        self.dependencies = None
+
+    def __repr__(self):
+        return 'HeapCacheValue(%s)' % (self.box, )
+
+
+class CacheEntry(object):
+    def __init__(self):
+        # both are {from_value: to_value} dicts
+        # the first is for boxes where we did not see the allocation, the
+        # second for anything else. the reason that distinction makes sense is
+        # because if we saw the allocation, we know it cannot alias with
+        # anything else where we saw the allocation.
+        self.cache_anything = {}
+        self.cache_seen_allocation = {}
+
+    def _clear_cache_on_write(self, seen_allocation_of_target):
+        if not seen_allocation_of_target:
+            self.cache_seen_allocation.clear()
+        self.cache_anything.clear()
+
+    def _getdict(self, value):
+        if value.seen_allocation:
+            return self.cache_seen_allocation
+        else:
+            return self.cache_anything
+
+    def do_write_with_aliasing(self, value, fieldvalue):
+        self._clear_cache_on_write(value.seen_allocation)
+        self._getdict(value)[value] = fieldvalue
+
+    def read(self, value):
+        return self._getdict(value).get(value, None)
+
+    def read_now_known(self, value, fieldvalue):
+        self._getdict(value)[value] = fieldvalue
+
+    def invalidate_unescaped(self):
+        self._invalidate_unescaped(self.cache_anything)
+        self._invalidate_unescaped(self.cache_seen_allocation)
+
+    def _invalidate_unescaped(self, d):
+        for value in d.keys():
+            if not value.is_unescaped:
+                del d[value]
 
 class HeapCache(object):
     def __init__(self):
         self.reset()
 
-    def reset(self, reset_virtuals=True, trace_branch=True):
-        # contains boxes where the class is already known
-        self.known_class_boxes = {}
+    def reset(self):
+        # maps boxes to values
+        self.values = {}
         # store the boxes that contain newly allocated objects, this maps the
         # boxes to a bool, the bool indicates whether or not the object has
         # escaped the trace or not (True means the box never escaped, False
         # means it did escape), its presences in the mapping shows that it was
         # allocated inside the trace
-        if trace_branch:
-            self.new_boxes = {}
-        else:
-            for box in self.new_boxes:
-                self.new_boxes[box] = False
-        if reset_virtuals:
-            self.likely_virtuals = {}      # only for jit.isvirtual()
+        #if trace_branch:
+            #self.new_boxes = {}
+        #    pass
+        #else:
+            #for box in self.new_boxes:
+            #    self.new_boxes[box] = False
+        #    pass
+        #if reset_virtuals:
+        #    self.likely_virtuals = {}      # only for jit.isvirtual()
         # Tracks which boxes should be marked as escaped when the key box
         # escapes.
-        self.dependencies = {}
-        # contains frame boxes that are not virtualizables
-        if trace_branch:
-            self.nonstandard_virtualizables = {}
+        #self.dependencies = {}
 
         # heap cache
-        # maps descrs to {from_box, to_box} dicts
+        # maps descrs to CacheEntry
         self.heap_cache = {}
         # heap array cache
-        # maps descrs to {index: {from_box: to_box}} dicts
+        # maps descrs to {index: {from_value: to_value}} dicts
         self.heap_array_cache = {}
-        # cache the length of arrays
-        self.length_cache = {}
 
-        # replace_box is called surprisingly often, therefore it's not efficient
-        # to go over all the dicts and fix them.
-        # instead, these two dicts are kept, and a replace_box adds an entry to
-        # each of them.
-        # every time one of the dicts heap_cache, heap_array_cache, length_cache
-        # is accessed, suitable indirections need to be performed
+    def reset_keep_likely_virtuals(self):
+        for value in self.values.itervalues():
+            value.reset_keep_likely_virtual()
+        self.heap_cache = {}
+        self.heap_array_cache = {}
 
-        # this looks all very subtle, but in practice the patterns of
-        # replacements should not be that complex. Usually a box is replaced by
-        # a const, once. Also, if something goes wrong, the effect is that less
-        # caching than possible is done, which is not a huge problem.
-        self.input_indirections = {}
-        self.output_indirections = {}
+    def getvalue(self, box):
+        value = self.values.get(box, None)
+        if not value:
+            value = self.values[box] = HeapCacheValue(box)
+        return value
 
-    def _input_indirection(self, box):
-        return self.input_indirections.get(box, box)
-
-    def _output_indirection(self, box):
-        return self.output_indirections.get(box, box)
+    def getvalues(self, boxes):
+        return [self.getvalue(box) for box in boxes]
 
     def invalidate_caches(self, opnum, descr, argboxes):
         self.mark_escaped(opnum, descr, argboxes)
@@ -64,18 +114,22 @@
     def mark_escaped(self, opnum, descr, argboxes):
         if opnum == rop.SETFIELD_GC:
             assert len(argboxes) == 2
-            box, valuebox = argboxes
-            if self.is_unescaped(box) and self.is_unescaped(valuebox):
-                self.dependencies.setdefault(box, []).append(valuebox)
+            value, fieldvalue = self.getvalues(argboxes)
+            if value.is_unescaped and fieldvalue.is_unescaped:
+                if value.dependencies is None:
+                    value.dependencies = []
+                value.dependencies.append(fieldvalue)
             else:
-                self._escape(valuebox)
+                self._escape(fieldvalue)
         elif opnum == rop.SETARRAYITEM_GC:
             assert len(argboxes) == 3
-            box, indexbox, valuebox = argboxes
-            if self.is_unescaped(box) and self.is_unescaped(valuebox):
-                self.dependencies.setdefault(box, []).append(valuebox)
+            value, indexvalue, fieldvalue = self.getvalues(argboxes)
+            if value.is_unescaped and fieldvalue.is_unescaped:
+                if value.dependencies is None:
+                    value.dependencies = []
+                value.dependencies.append(fieldvalue)
             else:
-                self._escape(valuebox)
+                self._escape(fieldvalue)
         elif (opnum == rop.CALL and
               descr.get_extra_info().oopspecindex == descr.get_extra_info().OS_ARRAYCOPY and
               isinstance(argboxes[3], ConstInt) and
@@ -84,6 +138,7 @@
               len(descr.get_extra_info().write_descrs_arrays) == 1):
             # ARRAYCOPY with constant starts and constant length doesn't escape
             # its argument
+            # XXX really?
             pass
         # GETFIELD_GC, MARK_OPAQUE_PTR, PTR_EQ, and PTR_NE don't escape their
         # arguments
@@ -95,25 +150,20 @@
               opnum != rop.INSTANCE_PTR_EQ and
               opnum != rop.INSTANCE_PTR_NE):
             for box in argboxes:
-                self._escape(box)
+                self._escape_box(box)
 
-    def _escape(self, box):
-        try:
-            unescaped = self.new_boxes[box]
-        except KeyError:
-            pass
-        else:
-            if unescaped:
-                self.new_boxes[box] = False
-        try:
-            del self.likely_virtuals[box]
-        except KeyError:
-            pass
-        try:
-            deps = self.dependencies.pop(box)
-        except KeyError:
-            pass
-        else:
+    def _escape_box(self, box):
+        value = self.values.get(box, None)
+        if not value:
+            return
+        self._escape(value)
+
+    def _escape(self, value):
+        value.is_unescaped = False
+        value.likely_virtual = False
+        deps = value.dependencies
+        value.dependencies = None
+        if deps is not None:
             for dep in deps:
                 self._escape(dep)
 
@@ -146,181 +196,198 @@
             # A special case for ll_arraycopy, because it is so common, and its
             # effects are so well defined.
             elif effectinfo.oopspecindex == effectinfo.OS_ARRAYCOPY:
-                if (
-                    isinstance(argboxes[3], ConstInt) and
-                    isinstance(argboxes[4], ConstInt) and
-                    isinstance(argboxes[5], ConstInt) and
-                    len(effectinfo.write_descrs_arrays) == 1
-                ):
-                    descr = effectinfo.write_descrs_arrays[0]
-                    cache = self.heap_array_cache.get(descr, None)
-                    srcstart = argboxes[3].getint()
-                    dststart = argboxes[4].getint()
-                    length = argboxes[5].getint()
-                    for i in xrange(length):
-                        value = self.getarrayitem(
-                            argboxes[1],
-                            ConstInt(srcstart + i),
-                            descr,
-                        )
-                        if value is not None:
-                            self.setarrayitem(
-                                argboxes[2],
-                                ConstInt(dststart + i),
-                                value,
-                                descr,
-                            )
-                        elif cache is not None:
-                            try:
-                                idx_cache = cache[dststart + i]
-                            except KeyError:
-                                pass
-                            else:
-                                if argboxes[2] in self.new_boxes:
-                                    for frombox in idx_cache.keys():
-                                        if not self.is_unescaped(frombox):
-                                            del idx_cache[frombox]
-                                else:
-                                    idx_cache.clear()
-                    return
-                elif (
-                    argboxes[2] in self.new_boxes and
-                    len(effectinfo.write_descrs_arrays) == 1
-                ):
-                    # Fish the descr out of the effectinfo
-                    cache = self.heap_array_cache.get(effectinfo.write_descrs_arrays[0], None)
-                    if cache is not None:
-                        for idx, cache in cache.iteritems():
-                            for frombox in cache.keys():
-                                if not self.is_unescaped(frombox):
-                                    del cache[frombox]
-                    return
+                self._clear_caches_arraycopy(opnum, descr, argboxes, effectinfo)
+                return
             else:
-                # Only invalidate things that are either escaped or arguments
-                for descr, boxes in self.heap_cache.iteritems():
-                    for box in boxes.keys():
-                        if not self.is_unescaped(box) or box in argboxes:
-                            del boxes[box]
+                # first escape arguments:
+                for argbox in argboxes:
+                    self._escape_box(argbox)
+
+                # Only invalidate things that are escaped
+                # XXX can do better, only do it for the descrs in the effectinfo
+                for descr, cache in self.heap_cache.iteritems():
+                    cache.invalidate_unescaped()
                 for descr, indices in self.heap_array_cache.iteritems():
-                    for boxes in indices.itervalues():
-                        for box in boxes.keys():
-                            if not self.is_unescaped(box) or box in argboxes:
-                                del boxes[box]
+                    for cache in indices.itervalues():
+                        cache.invalidate_unescaped()
                 return
 
         # XXX not completely sure, but I *think* it is needed to reset() the
         # state at least in the 'CALL_*' operations that release the GIL.  We
         # tried to do only the kind of resetting done by the two loops just
         # above, but hit an assertion in "pypy test_multiprocessing.py".
-        self.reset(reset_virtuals=False, trace_branch=False)
+        self.reset_keep_likely_virtuals()
+
+    def _clear_caches_arraycopy(self, opnum, desrc, argboxes, effectinfo):
+        seen_allocation_of_target = self.getvalue(argboxes[2]).seen_allocation
+        if (
+            isinstance(argboxes[3], ConstInt) and
+            isinstance(argboxes[4], ConstInt) and
+            isinstance(argboxes[5], ConstInt) and
+            len(effectinfo.write_descrs_arrays) == 1
+        ):
+            descr = effectinfo.write_descrs_arrays[0]
+            cache = self.heap_array_cache.get(descr, None)
+            srcstart = argboxes[3].getint()
+            dststart = argboxes[4].getint()
+            length = argboxes[5].getint()
+            for i in xrange(length):
+                value = self.getarrayitem(
+                    argboxes[1],
+                    ConstInt(srcstart + i),
+                    descr,
+                )
+                if value is not None:
+                    self.setarrayitem(
+                        argboxes[2],
+                        ConstInt(dststart + i),
+                        value,
+                        descr,
+                    )
+                elif cache is not None:
+                    try:
+                        idx_cache = cache[dststart + i]
+                    except KeyError:
+                        pass
+                    else:
+                        idx_cache._clear_cache_on_write(seen_allocation_of_target)
+            return
+        elif (
+            len(effectinfo.write_descrs_arrays) == 1
+        ):
+            # Fish the descr out of the effectinfo
+            cache = self.heap_array_cache.get(effectinfo.write_descrs_arrays[0], None)
+            if cache is not None:
+                for idx, cache in cache.iteritems():
+                    cache._clear_cache_on_write(seen_allocation_of_target)
+            return
+        self.reset_keep_likely_virtuals()
 
     def is_class_known(self, box):
-        return box in self.known_class_boxes
+        value = self.values.get(box, None)
+        if value:
+            return value.known_class
+        return False
 
     def class_now_known(self, box):
-        self.known_class_boxes[box] = None
+        self.getvalue(box).known_class = True
 
     def is_nonstandard_virtualizable(self, box):
-        return box in self.nonstandard_virtualizables
+        value = self.values.get(box, None)
+        if value:
+            return value.nonstandard_virtualizable
+        return False
 
     def nonstandard_virtualizables_now_known(self, box):
-        self.nonstandard_virtualizables[box] = None
+        self.getvalue(box).nonstandard_virtualizable = True
 
     def is_unescaped(self, box):
-        return self.new_boxes.get(box, False)
+        value = self.values.get(box, None)
+        if value:
+            return value.is_unescaped
+        return False
 
     def is_likely_virtual(self, box):
-        return box in self.likely_virtuals
+        value = self.values.get(box, None)
+        if value:
+            return value.likely_virtual
+        return False
 
     def new(self, box):
-        self.new_boxes[box] = True
-        self.likely_virtuals[box] = None
+        value = self.getvalue(box)
+        value.is_unescaped = True
+        value.likely_virtual = True
+        value.seen_allocation = True
 
     def new_array(self, box, lengthbox):
         self.new(box)
         self.arraylen_now_known(box, lengthbox)
 
     def getfield(self, box, descr):
-        box = self._input_indirection(box)
-        d = self.heap_cache.get(descr, None)
-        if d:
-            tobox = d.get(box, None)
-            return self._output_indirection(tobox)
+        value = self.values.get(box, None)
+        if value:
+            cache = self.heap_cache.get(descr, None)
+            if cache:
+                tovalue = cache.read(value)
+                if tovalue:
+                    return tovalue.box
         return None
 
     def getfield_now_known(self, box, descr, fieldbox):
-        box = self._input_indirection(box)
-        fieldbox = self._input_indirection(fieldbox)
-        self.heap_cache.setdefault(descr, {})[box] = fieldbox
+        value = self.getvalue(box)
+        fieldvalue = self.getvalue(fieldbox)
+        cache = self.heap_cache.get(descr, None)
+        if cache is None:
+            cache = self.heap_cache[descr] = CacheEntry()
+        cache.read_now_known(value, fieldvalue)
 
     def setfield(self, box, fieldbox, descr):
-        d = self.heap_cache.get(descr, None)
-        new_d = self._do_write_with_aliasing(d, box, fieldbox)
-        self.heap_cache[descr] = new_d
-
-    def _do_write_with_aliasing(self, d, box, fieldbox):
-        box = self._input_indirection(box)
-        fieldbox = self._input_indirection(fieldbox)
-        # slightly subtle logic here
-        # a write to an arbitrary box, all other boxes can alias this one
-        if not d or box not in self.new_boxes:
-            # therefore we throw away the cache
-            return {box: fieldbox}
-        # the object we are writing to is freshly allocated
-        # only remove some boxes from the cache
-        new_d = {}
-        for frombox, tobox in d.iteritems():
-            # the other box is *also* freshly allocated
-            # therefore frombox and box *must* contain different objects
-            # thus we can keep it in the cache
-            if frombox in self.new_boxes:
-                new_d[frombox] = tobox
-        new_d[box] = fieldbox
-        return new_d
+        cache = self.heap_cache.get(descr, None)
+        if cache is None:
+            cache = self.heap_cache[descr] = CacheEntry()
+        value = self.getvalue(box)
+        fieldvalue = self.getvalue(fieldbox)
+        cache.do_write_with_aliasing(value, fieldvalue)
 
     def getarrayitem(self, box, indexbox, descr):
         if not isinstance(indexbox, ConstInt):
-            return
-        box = self._input_indirection(box)
+            return None
+        value = self.values.get(box, None)
+        if value is None:
+            return None
         index = indexbox.getint()
         cache = self.heap_array_cache.get(descr, None)
         if cache:
             indexcache = cache.get(index, None)
             if indexcache is not None:
-                return self._output_indirection(indexcache.get(box, None))
+                resvalue = indexcache.read(value)
+                if resvalue:
+                    return resvalue.box
+        return None
 
-    def getarrayitem_now_known(self, box, indexbox, valuebox, descr):
+    def _get_or_make_array_cache_entry(self, indexbox, descr):
         if not isinstance(indexbox, ConstInt):
-            return
-        box = self._input_indirection(box)
-        valuebox = self._input_indirection(valuebox)
+            return None
         index = indexbox.getint()
         cache = self.heap_array_cache.setdefault(descr, {})
         indexcache = cache.get(index, None)
-        if indexcache is not None:
-            indexcache[box] = valuebox
-        else:
-            cache[index] = {box: valuebox}
+        if indexcache is None:
+            cache[index] = indexcache = CacheEntry()
+        return indexcache
 
-    def setarrayitem(self, box, indexbox, valuebox, descr):
+
+    def getarrayitem_now_known(self, box, indexbox, fieldbox, descr):
+        value = self.getvalue(box)
+        fieldvalue = self.getvalue(fieldbox)
+        indexcache = self._get_or_make_array_cache_entry(indexbox, descr)
+        if indexcache:
+            indexcache.read_now_known(value, fieldvalue)
+
+    def setarrayitem(self, box, indexbox, fieldbox, descr):
         if not isinstance(indexbox, ConstInt):
             cache = self.heap_array_cache.get(descr, None)
             if cache is not None:
                 cache.clear()
             return
-        index = indexbox.getint()
-        cache = self.heap_array_cache.setdefault(descr, {})
-        indexcache = cache.get(index, None)
-        cache[index] = self._do_write_with_aliasing(indexcache, box, valuebox)
+        value = self.getvalue(box)
+        fieldvalue = self.getvalue(fieldbox)
+        indexcache = self._get_or_make_array_cache_entry(indexbox, descr)
+        if indexcache:
+            indexcache.do_write_with_aliasing(value, fieldvalue)
 
     def arraylen(self, box):
-        box = self._input_indirection(box)
-        return self._output_indirection(self.length_cache.get(box, None))
+        value = self.values.get(box, None)
+        if value and value.length:
+            return value.length.box
+        return None
 
     def arraylen_now_known(self, box, lengthbox):
-        box = self._input_indirection(box)
-        self.length_cache[box] = self._input_indirection(lengthbox)
+        value = self.getvalue(box)
+        value.length = self.getvalue(lengthbox)
 
     def replace_box(self, oldbox, newbox):
-        self.input_indirections[self._output_indirection(newbox)] = self._input_indirection(oldbox)
-        self.output_indirections[self._input_indirection(oldbox)] = self._output_indirection(newbox)
+        value = self.values.get(oldbox, None)
+        if value is None:
+            return
+        value.box = newbox
+        self.values[newbox] = value
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2193,7 +2193,8 @@
                 duplicates[box] = None
 
     def reached_loop_header(self, greenboxes, redboxes):
-        self.heapcache.reset(reset_virtuals=False)
+        self.heapcache.reset() #reset_virtuals=False)
+        #self.heapcache.reset_keep_likely_virtuals()
 
         duplicates = {}
         self.remove_consts_and_duplicates(redboxes, len(redboxes),
diff --git a/rpython/jit/metainterp/test/test_heapcache.py b/rpython/jit/metainterp/test/test_heapcache.py
--- a/rpython/jit/metainterp/test/test_heapcache.py
+++ b/rpython/jit/metainterp/test/test_heapcache.py
@@ -29,24 +29,31 @@
 
     OS_ARRAYCOPY = 0
 
-    def __init__(self, extraeffect, oopspecindex, write_descrs_arrays):
+    def __init__(self, extraeffect, oopspecindex, write_descrs_fields, write_descrs_arrays):
         self.extraeffect = extraeffect
         self.oopspecindex = oopspecindex
+        self.write_descrs_fields = write_descrs_fields
         self.write_descrs_arrays = write_descrs_arrays
 
+    def has_random_effects(self):
+        return self.extraeffect == self.EF_RANDOM_EFFECTS
 
 class FakeCallDescr(object):
-    def __init__(self, extraeffect, oopspecindex=None, write_descrs_arrays=[]):
+    def __init__(self, extraeffect, oopspecindex=None, write_descrs_fields=[], write_descrs_arrays=[]):
         self.extraeffect = extraeffect
         self.oopspecindex = oopspecindex
+        self.write_descrs_fields = write_descrs_fields
         self.write_descrs_arrays = write_descrs_arrays
 
     def get_extra_info(self):
         return FakeEffectinfo(
             self.extraeffect, self.oopspecindex,
-            write_descrs_arrays=self.write_descrs_arrays
+            write_descrs_fields=self.write_descrs_fields,
+            write_descrs_arrays=self.write_descrs_arrays,
         )
 
+arraycopydescr1 = FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1])
+
 
 class TestHeapCache(object):
     def test_known_class_box(self):
@@ -369,13 +376,13 @@
         # Just need the destination box for this call
         h.invalidate_caches(
             rop.CALL,
-            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
+            arraycopydescr1,
             [None, box5, box2, index1, index1, index1]
         )
         assert h.getarrayitem(box1, index1, descr1) is box2
         h.invalidate_caches(
             rop.CALL,
-            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
+            arraycopydescr1,
             [None, box5, box3, index1, index1, index1]
         )
         assert h.getarrayitem(box1, index1, descr1) is box2
@@ -384,29 +391,40 @@
         assert h.getarrayitem(box4, index1, descr1) is box2
         h.invalidate_caches(
             rop.CALL,
-            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
+            arraycopydescr1,
             [None, box3, box5, index1, index1, index2]
         )
         assert h.getarrayitem(box4, index1, descr1) is None
 
     def test_ll_arraycopy_differing_descrs(self):
         h = HeapCache()
-        h.setarrayitem(box1, index1, box2, descr1)
-        assert h.getarrayitem(box1, index1, descr1) is box2
+        h.setarrayitem(box1, index1, box2, descr2)
+        assert h.getarrayitem(box1, index1, descr2) is box2
         h.new_array(box2, lengthbox2)
         h.invalidate_caches(
             rop.CALL,
-            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr2]),
+            arraycopydescr1,
             [None, box3, box2, index1, index1, index2]
         )
-        assert h.getarrayitem(box1, index1, descr1) is box2
+        assert h.getarrayitem(box1, index1, descr2) is box2
+
+    def test_ll_arraycopy_differing_descrs_nonconst_index(self):
+        h = HeapCache()
+        h.setarrayitem(box1, index1, box2, descr2)
+        assert h.getarrayitem(box1, index1, descr2) is box2
+        h.invalidate_caches(
+            rop.CALL,
+            arraycopydescr1,
+            [None, box3, box2, index1, index1, BoxInt()]
+        )
+        assert h.getarrayitem(box1, index1, descr2) is box2
 
     def test_ll_arraycopy_result_propogated(self):
         h = HeapCache()
         h.setarrayitem(box1, index1, box2, descr1)
         h.invalidate_caches(
             rop.CALL,
-            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
+            arraycopydescr1,
             [None, box1, box3, index1, index1, index2]
         )
         assert h.getarrayitem(box3, index1, descr1) is box2
@@ -417,7 +435,7 @@
         h.setarrayitem(box3, index1, box4, descr1)
         h.invalidate_caches(
             rop.CALL,
-            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
+            arraycopydescr1,
             [None, box2, box1, index1, index1, index2]
         )
 
@@ -427,14 +445,14 @@
         h.new_array(box2, lengthbox2)
         h.invalidate_caches(
             rop.CALL,
-            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
+            arraycopydescr1,
             [None, box2, box1, index1, index1, index2]
         )
         assert h.is_unescaped(box1)
         assert h.is_unescaped(box2)
         h.invalidate_caches(
             rop.CALL,
-            FakeCallDescr(FakeEffectinfo.EF_CANNOT_RAISE, FakeEffectinfo.OS_ARRAYCOPY, write_descrs_arrays=[descr1]),
+            arraycopydescr1,
             [None, box2, box1, index1, index1, BoxInt()]
         )
         assert not h.is_unescaped(box1)
@@ -594,9 +612,9 @@
         h.new(box1)
         assert h.is_unescaped(box1)
         assert h.is_likely_virtual(box1)
-        h.reset(reset_virtuals=False)
+        h.reset_keep_likely_virtuals()
         assert not h.is_unescaped(box1)
         assert h.is_likely_virtual(box1)
-        h._escape(box1)
+        h._escape_box(box1)
         assert not h.is_unescaped(box1)
         assert not h.is_likely_virtual(box1)

From noreply at buildbot.pypy.org  Sat Feb 21 18:50:04 2015
From: noreply at buildbot.pypy.org (xando)
Date: Sat, 21 Feb 2015 18:50:04 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: (xando,
 mjacob) Don't allow integer like objects with bytes constructor,
 although regular integers should be allowed
Message-ID: <20150221175004.C46A81C0382@cobra.cs.uni-duesseldorf.de>

Author: Sebastian Pawlu? 
Branch: py3.3
Changeset: r76037:3f49d90dbd9e
Date: 2015-02-21 18:45 +0100
http://bitbucket.org/pypy/pypy/changeset/3f49d90dbd9e/

Log:	(xando, mjacob) Don't allow integer like objects with bytes
	constructor, although regular integers should be allowed

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -685,7 +685,7 @@
         return []
     # Is it an int?
     try:
-        count = space.int_w(w_source)
+        count = space.int_w(w_source, allow_conversion=False)
     except OperationError, e:
         if not e.match(space, space.w_TypeError):
             raise
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -823,6 +823,13 @@
             assert bytes.maketrans(bb, bb)
             assert bytearray.maketrans(bb, bb)
 
+    def test_constructor_dont_convert_int(self):
+        class A(object):
+            def __int__(self):
+                return 42
+        raises(TypeError, bytes, A())
+
+
 class AppTestPrebuilt(AppTestBytesObject):
     spaceconfig = {"objspace.std.withprebuiltchar": True}
 

From noreply at buildbot.pypy.org  Sat Feb 21 23:54:09 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Feb 2015 23:54:09 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: print the command-line executed
Message-ID: <20150221225409.46E671C0499@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r76038:4adc4dd995c6
Date: 2015-02-21 23:41 +0100
http://bitbucket.org/pypy/pypy/changeset/4adc4dd995c6/

Log:	print the command-line executed

diff --git a/pypy/module/pypystm/test_pypy_c/support.py b/pypy/module/pypystm/test_pypy_c/support.py
--- a/pypy/module/pypystm/test_pypy_c/support.py
+++ b/pypy/module/pypystm/test_pypy_c/support.py
@@ -109,6 +109,7 @@
         if not import_site:
             cmdline.append('-S')
         cmdline.append(str(self.filepath))
+        print '*', ' '.join(cmdline)
         env = os.environ.copy()
         env['PYPYSTM'] = str(self.stmfile)
         env['PYPYLOG'] = str(self.logfile)

From noreply at buildbot.pypy.org  Sat Feb 21 23:54:10 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Feb 2015 23:54:10 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Finish the classification of all
 operations that are seen when translating pypy-stm
Message-ID: <20150221225410.91D291C0499@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r76039:ef25158220b4
Date: 2015-02-21 23:54 +0100
http://bitbucket.org/pypy/pypy/changeset/ef25158220b4/

Log:	Finish the classification of all operations that are seen when
	translating pypy-stm

diff --git a/rpython/translator/stm/inevitable.py b/rpython/translator/stm/inevitable.py
--- a/rpython/translator/stm/inevitable.py
+++ b/rpython/translator/stm/inevitable.py
@@ -18,7 +18,7 @@
     'stack_current', 'gc_stack_bottom', 'cast_ptr_to_int',
     'jit_force_virtual', 'jit_force_virtualizable',
     'jit_force_quasi_immutable', 'jit_marker', 'jit_is_virtual',
-    'jit_record_known_class',
+    'jit_record_known_class', 'jit_ffi_save_result',
     'gc_identityhash', 'gc_id', 'gc_can_move', 'gc__collect',
     'gc_adr_of_root_stack_top', 'gc_add_memory_pressure',
     'gc_pin', 'gc_unpin', 'gc__is_pinned',
@@ -33,6 +33,12 @@
     'length_of_simple_gcarray_from_opaque',
     'll_read_timestamp',
     'jit_conditional_call',
+    'get_exc_value_addr', 'get_exception_addr',
+    'get_write_barrier_failing_case',
+    'get_write_barrier_from_array_failing_case',
+    'gc_set_max_heap_size', 'gc_gcflag_extra',
+    'raw_malloc_usage',
+    'track_alloc_start', 'track_alloc_stop',
     ])
 ALWAYS_ALLOW_OPERATIONS |= set(lloperation.enum_tryfold_ops())
 
@@ -63,24 +69,16 @@
     'gc_adr_of_root_stack_base', 'gc_asmgcroot_static',
     'gc_call_rtti_destructor', 'gc_deallocate',
     'gc_detach_callback_pieces', 'gc_fetch_exception',
-    'gc_set_max_heap_size',
     'gc_forget_current_state', 'gc_free',
-    'gc_gcflag_extra', 'gc_obtain_free_space',
+    'gc_obtain_free_space',
     'gc_reattach_callback_pieces', 'gc_reload_possibly_moved',
     'gc_restore_exception', 'gc_restore_state_from',
     'gc_save_current_state_away',
     'gc_shadowstackref_context', 'gc_shadowstackref_new',
-    'gc_thread_after_fork', 'gc_thread_before_fork',
     'gc_start_fresh_new_state', 'gc_thread_run',
     'gc_writebarrier_before_copy',
-    'get_exc_value_addr', 'get_exception_addr',
-    'get_write_barrier_failing_case',
-    'get_write_barrier_from_array_failing_case',
     'getslice', 'instrument_count',
-    'jit_ffi_save_result',
-    'raw_malloc_usage',
-    'raw_memclear', 'raw_memcopy', 'raw_memmove', 'raw_memset',
-    'stack_malloc', 'track_alloc_start', 'track_alloc_stop',
+    'stack_malloc',
     'zero_gc_pointers_inside',
     ])
 
@@ -88,6 +86,8 @@
 TURN_INEVITABLE_OPS = set([
     'debug_fatalerror', 'debug_llinterpcall', 'debug_print_traceback',
     'gc_dump_rpy_heap', 'gc_thread_start', 'gc_thread_die',
+    'raw_memclear', 'raw_memcopy', 'raw_memmove', 'raw_memset',
+    'gc_thread_after_fork', 'gc_thread_before_fork',
     ])
 
 # ____________________________________________________________
@@ -136,7 +136,7 @@
     # Always-allowed operations never cause a 'turn inevitable'
     if op.opname in ALWAYS_ALLOW_OPERATIONS:
         return False
-    assert op.opname not in INCOMPATIBLE_OPS
+    assert op.opname not in INCOMPATIBLE_OPS, repr(op)
     #
     # Getters and setters
     if op.opname in GETTERS:
diff --git a/rpython/translator/stm/test/test_inevitable.py b/rpython/translator/stm/test/test_inevitable.py
--- a/rpython/translator/stm/test/test_inevitable.py
+++ b/rpython/translator/stm/test/test_inevitable.py
@@ -8,17 +8,27 @@
 from rpython.conftest import option
 
 
-KNOWN_OPERATIONS = (inevitable.ALWAYS_ALLOW_OPERATIONS |
-                    inevitable.CALLS |
-                    inevitable.GETTERS | inevitable.SETTERS |
-                    inevitable.MALLOCS | inevitable.FREES |
-                    inevitable.INCOMPATIBLE_OPS |
-                    inevitable.TURN_INEVITABLE_OPS)
+CATEGORIES = [inevitable.ALWAYS_ALLOW_OPERATIONS,
+              inevitable.CALLS,
+              inevitable.GETTERS, inevitable.SETTERS,
+              inevitable.MALLOCS, inevitable.FREES,
+              inevitable.INCOMPATIBLE_OPS,
+              inevitable.TURN_INEVITABLE_OPS]
+
+KNOWN_OPERATIONS = set()
+for _cat in CATEGORIES:
+    KNOWN_OPERATIONS |= _cat
 
 def test_defined_operations():
     for opname in KNOWN_OPERATIONS:
         getattr(llop, opname)   # the opname must exist!
 
+def test_no_duplicate_operations():
+    for i in range(len(CATEGORIES)):
+        for j in range(i):
+            common = (CATEGORIES[i] & CATEGORIES[j])
+            assert not common
+
 def test_no_missing_operation():
     ALL_OPERATIONS = set(lloperation.LL_OPERATIONS)
     MISSING_OPERATIONS = ALL_OPERATIONS - KNOWN_OPERATIONS

From noreply at buildbot.pypy.org  Sun Feb 22 12:21:07 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Feb 2015 12:21:07 +0100 (CET)
Subject: [pypy-commit] pypy default: (xoraxax,
	arigo)  Fix two broken behaviors of zipimport:
Message-ID: <20150222112107.D56001C14C9@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76040:5ada2fb6796f
Date: 2015-02-22 12:21 +0100
http://bitbucket.org/pypy/pypy/changeset/5ada2fb6796f/

Log:	(xoraxax, arigo) Fix two broken behaviors of zipimport:

	* random exceptions produced by importing the modules were eaten
	and (sometimes) re-raised as ZipImportErrors with the same
	message, which does not make sense (added test_import_exception)

	* calling import_py_file() from import_pyc_file() would try to
	parse the pyc bytes as Python code!

diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -199,8 +199,7 @@
         magic = importing._get_long(buf[:4])
         timestamp = importing._get_long(buf[4:8])
         if not self.can_use_pyc(space, filename, magic, timestamp):
-            return self.import_py_file(space, modname, filename[:-1], buf,
-                                       pkgpath)
+            return None
         buf = buf[8:] # XXX ugly copy, should use sequential read instead
         w_mod = w(Module(space, w(modname)))
         real_name = self.filename + os.path.sep + self.corr_zname(filename)
@@ -249,7 +248,6 @@
     def load_module(self, space, fullname):
         w = space.wrap
         filename = self.make_filename(fullname)
-        last_exc = None
         for compiled, is_package, ext in ENUMERATE_EXTS:
             fname = filename + ext
             try:
@@ -268,19 +266,18 @@
                     pkgpath = None
                 try:
                     if compiled:
-                        return self.import_pyc_file(space, fullname, fname,
-                                                    buf, pkgpath)
+                        w_result = self.import_pyc_file(space, fullname, fname,
+                                                        buf, pkgpath)
+                        if w_result is not None:
+                            return w_result
                     else:
                         return self.import_py_file(space, fullname, fname,
                                                    buf, pkgpath)
-                except OperationError, e:
-                    last_exc = e
+                except:
                     w_mods = space.sys.get('modules')
-                space.call_method(w_mods, 'pop', w(fullname), space.w_None)
-        if last_exc:
-            raise OperationError(get_error(space), last_exc.get_w_value(space))
-        # should never happen I think
-        return space.w_None
+                    space.call_method(w_mods, 'pop', w(fullname), space.w_None)
+                    raise
+        raise oefmt(get_error(space), "can't find module '%s'", fullname)
 
     @unwrap_spec(filename=str)
     def get_data(self, space, filename):
diff --git a/pypy/module/zipimport/test/test_zipimport.py b/pypy/module/zipimport/test/test_zipimport.py
--- a/pypy/module/zipimport/test/test_zipimport.py
+++ b/pypy/module/zipimport/test/test_zipimport.py
@@ -195,7 +195,8 @@
         m0 ^= 0x04
         test_pyc = chr(m0) + self.test_pyc[1:]
         self.writefile("uu.pyc", test_pyc)
-        raises(ImportError, "__import__('uu', globals(), locals(), [])")
+        raises(zipimport.ZipImportError,
+               "__import__('uu', globals(), locals(), [])")
         assert 'uu' not in sys.modules
 
     def test_force_py(self):
@@ -360,6 +361,11 @@
         co_filename = code.co_filename
         assert co_filename == expected
 
+    def test_import_exception(self):
+        self.writefile('x1test.py', '1/0')
+        self.writefile('x1test/__init__.py', 'raise ValueError')
+        raises(ValueError, __import__, 'x1test', None, None, [])
+
 
 if os.sep != '/':
     class AppTestNativePathSep(AppTestZipimport):

From noreply at buildbot.pypy.org  Sun Feb 22 12:59:02 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Feb 2015 12:59:02 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: (xoraxax,
 arigo)  Fix two broken behaviors of zipimport:
Message-ID: <20150222115902.146441C02ED@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r76041:752077ecf930
Date: 2015-02-22 12:21 +0100
http://bitbucket.org/pypy/pypy/changeset/752077ecf930/

Log:	(xoraxax, arigo) Fix two broken behaviors of zipimport:

	* random exceptions produced by importing the modules were eaten
	and (sometimes) re-raised as ZipImportErrors with the same
	message, which does not make sense (added test_import_exception)

	* calling import_py_file() from import_pyc_file() would try to
	parse the pyc bytes as Python code! (transplanted from
	5ada2fb6796fbc176e7ea49c513fd1edf1ddf709)

diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -199,8 +199,7 @@
         magic = importing._get_long(buf[:4])
         timestamp = importing._get_long(buf[4:8])
         if not self.can_use_pyc(space, filename, magic, timestamp):
-            return self.import_py_file(space, modname, filename[:-1], buf,
-                                       pkgpath)
+            return None
         buf = buf[8:] # XXX ugly copy, should use sequential read instead
         w_mod = w(Module(space, w(modname)))
         real_name = self.filename + os.path.sep + self.corr_zname(filename)
@@ -249,7 +248,6 @@
     def load_module(self, space, fullname):
         w = space.wrap
         filename = self.make_filename(fullname)
-        last_exc = None
         for compiled, is_package, ext in ENUMERATE_EXTS:
             fname = filename + ext
             try:
@@ -268,19 +266,18 @@
                     pkgpath = None
                 try:
                     if compiled:
-                        return self.import_pyc_file(space, fullname, fname,
-                                                    buf, pkgpath)
+                        w_result = self.import_pyc_file(space, fullname, fname,
+                                                        buf, pkgpath)
+                        if w_result is not None:
+                            return w_result
                     else:
                         return self.import_py_file(space, fullname, fname,
                                                    buf, pkgpath)
-                except OperationError, e:
-                    last_exc = e
+                except:
                     w_mods = space.sys.get('modules')
-                space.call_method(w_mods, 'pop', w(fullname), space.w_None)
-        if last_exc:
-            raise OperationError(get_error(space), last_exc.get_w_value(space))
-        # should never happen I think
-        return space.w_None
+                    space.call_method(w_mods, 'pop', w(fullname), space.w_None)
+                    raise
+        raise oefmt(get_error(space), "can't find module '%s'", fullname)
 
     @unwrap_spec(filename=str)
     def get_data(self, space, filename):
diff --git a/pypy/module/zipimport/test/test_zipimport.py b/pypy/module/zipimport/test/test_zipimport.py
--- a/pypy/module/zipimport/test/test_zipimport.py
+++ b/pypy/module/zipimport/test/test_zipimport.py
@@ -195,7 +195,8 @@
         m0 ^= 0x04
         test_pyc = chr(m0) + self.test_pyc[1:]
         self.writefile("uu.pyc", test_pyc)
-        raises(ImportError, "__import__('uu', globals(), locals(), [])")
+        raises(zipimport.ZipImportError,
+               "__import__('uu', globals(), locals(), [])")
         assert 'uu' not in sys.modules
 
     def test_force_py(self):
@@ -360,6 +361,11 @@
         co_filename = code.co_filename
         assert co_filename == expected
 
+    def test_import_exception(self):
+        self.writefile('x1test.py', '1/0')
+        self.writefile('x1test/__init__.py', 'raise ValueError')
+        raises(ValueError, __import__, 'x1test', None, None, [])
+
 
 if os.sep != '/':
     class AppTestNativePathSep(AppTestZipimport):

From noreply at buildbot.pypy.org  Sun Feb 22 12:59:03 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Feb 2015 12:59:03 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Fix platform.python_implementation()
	for the stm branch
Message-ID: <20150222115903.4186D1C02ED@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r76042:b99508208181
Date: 2015-02-22 12:59 +0100
http://bitbucket.org/pypy/pypy/changeset/b99508208181/

Log:	Fix platform.python_implementation() for the stm branch

diff --git a/lib-python/2.7/platform.py b/lib-python/2.7/platform.py
--- a/lib-python/2.7/platform.py
+++ b/lib-python/2.7/platform.py
@@ -1382,7 +1382,7 @@
 _pypy_sys_version_parser = re.compile(
     r'([\w.+]+)\s*'
     '\(#?([^,]+),\s*([\w ]+),\s*([\w :]+)\)\s*'
-    '\[PyPy [^\]]+\]?')
+    '\[PyPy[^\]]+\]?')   # this also covers 'PyPy-STM x.y'
 
 _sys_version_cache = {}
 

From noreply at buildbot.pypy.org  Sun Feb 22 14:42:09 2015
From: noreply at buildbot.pypy.org (Alexander Schremmer)
Date: Sun, 22 Feb 2015 14:42:09 +0100 (CET)
Subject: [pypy-commit] pypy default: (fox, xoraxax)  ansi_mandelbrot: Cleanup,
 add new "slides", add comments.
Message-ID: <20150222134209.29AA31C009D@cobra.cs.uni-duesseldorf.de>

Author: Alexander Schremmer 
Branch: 
Changeset: r76043:5cac60cf129a
Date: 2015-02-22 14:42 +0100
http://bitbucket.org/pypy/pypy/changeset/5cac60cf129a/

Log:	(fox, xoraxax) ansi_mandelbrot: Cleanup, add new "slides", add
	comments.

diff --git a/rpython/tool/ansi_mandelbrot.py b/rpython/tool/ansi_mandelbrot.py
--- a/rpython/tool/ansi_mandelbrot.py
+++ b/rpython/tool/ansi_mandelbrot.py
@@ -21,22 +21,15 @@
 else:
     palette = [39, 34, 35, 36, 31, 33, 32, 37]
 
-colour_range = None # used for debugging
-
-
-def print_pixel(colour, value_range, invert=1):
-    global colour_range
-    chars = [".", ".", "+", "*", "%", "#"]
-    idx = lambda chars: (colour+1) * (len(chars) - 1) / value_range
-    if invert:
-        idx = lambda chars, idx=idx:len(chars) - 1 - idx(chars)
-    char = chars[idx(chars)]
-    ansi_colour = palette[idx(palette)]
-    ansi_print(char, ansi_colour, newline=False, flush=True)
-    #if colour_range is None:
-    #    colour_range = [colour, colour]
-    #else:
-    #    colour_range = [min(colour_range[0], colour), max(colour_range[1], colour)]
+# used for debugging/finding new coordinates
+# How to:
+#   1. Set DEBUG to True
+#   2. Add a new coordinate to coordinates with a high distance and high max colour (e.g. 300)
+#   3. Run, pick an interesting coordinate from the shown list and replace the newly added
+#      coordinate by it.
+#   4. Rerun to see the max colour, insert this max colour where you put the high max colour.
+#   5. Set DEBUG to False
+DEBUG = False
 
 
 class Mandelbrot:
@@ -58,9 +51,6 @@
         ymin = self.ypos - self.yscale * self.y / 2
         self.x_range = [xmin + self.xscale * ix for ix in range(self.x)]
         self.y_range = [ymin + self.yscale * iy for iy in range(self.y)]
-        
-        #print "x", self.x_range[0], self.x_range[-1]
-        #print "y", self.y_range[0], self.y_range[-1]
 
     def reset(self, cnt):
         self.reset_lines = cnt
@@ -100,8 +90,11 @@
 
 class Driver(object):
     zoom_locations = [
-        # x, y, "distance", range
+        # x, y, "distance", max color range
         (0.37865401, 0.669227668, 0.04, 111),
+        (-1.2693, -0.4145, 0.2, 105),
+        (-1.2693, -0.4145, 0.05, 97),
+        (-1.2642, -0.4185, 0.01, 95),
         (-1.15, -0.28, 0.9, 94),
         (-1.15, -0.28, 0.3, 58),
         (-1.15, -0.28, 0.05, 26),
@@ -109,8 +102,10 @@
     def __init__(self, **kwargs):
         self.kwargs = kwargs
         self.zoom_location = -1
-        self.colour_range = 256
+        self.max_colour = 256
+        self.colour_range = None
         self.invert = True
+        self.interesting_coordinates = []
         self.init()
 
     def init(self):
@@ -123,14 +118,6 @@
         """ Resets to the beginning of the line and drops cnt lines internally. """
         self.mandelbrot.reset(cnt)
 
-    def catchup(self):
-        """ Fills the current line. """
-        x = 0
-        while x != self.width - 1:
-            x, y, c = self.gen.next()
-            print_pixel(c, self.colour_range, self.invert)
-        print >>sys.stderr
-
     def restart(self):
         """ Restarts the current generator. """
         print >>sys.stderr
@@ -146,36 +133,54 @@
                 if width != self.width:
                     self.init()
         except StopIteration:
+            if DEBUG and self.interesting_coordinates:
+                print >>sys.stderr, "Interesting coordinates:", self.interesting_coordinates
+                self.interesting_coordinates = []
             kwargs = self.kwargs
             self.zoom_location += 1
             self.zoom_location %= len(self.zoom_locations)
             loc = self.zoom_locations[self.zoom_location]
             kwargs.update({"x_pos": loc[0], "y_pos": loc[1], "distance": loc[2]})
-            self.colour_range = loc[3]
-            #global colour_range
-            #print colour_range, loc[2]
-            #colour_range = None
-            return self.restart()
-        print_pixel(c, self.colour_range, self.invert)
+            self.max_colour = loc[3]
+            if DEBUG:
+                # Only used for debugging new locations:
+                print "Colour range", self.colour_range
+            self.colour_range = None
+            self.restart()
+            return
+        if self.print_pixel(c, self.invert):
+            self.interesting_coordinates.append(dict(x=(x, self.mandelbrot.x_range[x]),
+                                                     y=(y, self.mandelbrot.y_range[y])))
         if x == self.width - 1:
             print >>sys.stderr
 
+    def print_pixel(self, colour, invert=1):
+        chars = [".", ".", "+", "*", "%", "#"]
+        idx = lambda chars: (colour+1) * (len(chars) - 1) / self.max_colour
+        if invert:
+            idx = lambda chars, idx=idx:len(chars) - 1 - idx(chars)
+        char = chars[idx(chars)]
+        ansi_colour = palette[idx(palette)]
+        ansi_print(char, ansi_colour, newline=False, flush=True)
+        if DEBUG:
+            if self.colour_range is None:
+                self.colour_range = [colour, colour]
+            else:
+                old_colour_range = self.colour_range
+                self.colour_range = [min(self.colour_range[0], colour), max(self.colour_range[1], colour)]
+                if old_colour_range[0] - colour > 3 or colour - old_colour_range[1] > 3:
+                    return True
+
 
 if __name__ == '__main__':
     import random
     from time import sleep
 
     d = Driver()
-    for x in xrange(15000):
-        sleep(random.random() / 300)
+    while True:
+        sleep(random.random() / 800)
         d.dot()
         if 0 and random.random() < 0.01:
-            d.catchup()
-            print "WARNING! " * 3
-            d.reset(1)
-        #    print "R",
-        if 0 and random.random() < 0.01:
             string = "WARNING! " * 3
             d.jump(len(string))
             print string,
-

From noreply at buildbot.pypy.org  Sun Feb 22 15:17:00 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Feb 2015 15:17:00 +0100 (CET)
Subject: [pypy-commit] pypy default: Fix for older versions of OpenSSL: use
 the official 'STACK_OF(TYPE)' as
Message-ID: <20150222141700.EFE7B1C1356@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76044:df36b95933f9
Date: 2015-02-22 15:16 +0100
http://bitbucket.org/pypy/pypy/changeset/df36b95933f9/

Log:	Fix for older versions of OpenSSL: use the official 'STACK_OF(TYPE)'
	as a type name

diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -56,9 +56,9 @@
 ASN1_OBJECT = rffi.COpaquePtr('ASN1_OBJECT')
 X509_NAME = rffi.COpaquePtr('X509_NAME')
 X509_VERIFY_PARAM = rffi.COpaquePtr('X509_VERIFY_PARAM')
-stack_st_X509_OBJECT = rffi.COpaquePtr('struct stack_st_X509_OBJECT')
+stack_st_X509_OBJECT = rffi.COpaquePtr('STACK_OF(X509_OBJECT)')
 DIST_POINT = rffi.COpaquePtr('DIST_POINT')
-stack_st_DIST_POINT = rffi.COpaquePtr('struct stack_st_DIST_POINT')
+stack_st_DIST_POINT = rffi.COpaquePtr('STACK_OF(X509_OBJECT)')
 DH = rffi.COpaquePtr('DH')
 EC_KEY = rffi.COpaquePtr('EC_KEY')
 AUTHORITY_INFO_ACCESS = rffi.COpaquePtr('AUTHORITY_INFO_ACCESS')

From noreply at buildbot.pypy.org  Sun Feb 22 16:17:09 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Feb 2015 16:17:09 +0100 (CET)
Subject: [pypy-commit] pypy default: Force alignment (to 16 bytes) of any
 call to materialize().
Message-ID: <20150222151709.F01211C02BD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76045:0d1ee380a33e
Date: 2015-02-22 16:17 +0100
http://bitbucket.org/pypy/pypy/changeset/0d1ee380a33e/

Log:	Force alignment (to 16 bytes) of any call to materialize(). Kill
	stuff in the ARM backend.

diff --git a/rpython/jit/backend/arm/arch.py b/rpython/jit/backend/arm/arch.py
--- a/rpython/jit/backend/arm/arch.py
+++ b/rpython/jit/backend/arm/arch.py
@@ -1,4 +1,3 @@
-FUNC_ALIGN = 8
 WORD = 4
 DOUBLE_WORD = 8
 
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -4,7 +4,7 @@
 
 from rpython.jit.backend.arm import conditions as c, registers as r
 from rpython.jit.backend.arm import shift
-from rpython.jit.backend.arm.arch import (WORD, DOUBLE_WORD, FUNC_ALIGN,
+from rpython.jit.backend.arm.arch import (WORD, DOUBLE_WORD,
     JITFRAME_FIXED_SIZE)
 from rpython.jit.backend.arm.codebuilder import InstrBuilder, OverwritingBuilder
 from rpython.jit.backend.arm.locations import imm, StackLocation, get_fp_offset
@@ -484,10 +484,6 @@
         self.mc.BL(target)
         return startpos
 
-    def align(self):
-        while(self.mc.currpos() % FUNC_ALIGN != 0):
-            self.mc.writechar(chr(0))
-
     def gen_func_epilog(self, mc=None, cond=c.AL):
         gcrootmap = self.cpu.gc_ll_descr.gcrootmap
         if mc is None:
@@ -557,7 +553,7 @@
         debug_stop('jit-backend-ops')
 
     def _call_header(self):
-        self.align()
+        assert self.currpos() == 0
         self.gen_func_prolog()
 
     def _call_header_with_stack_check(self):
diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py
--- a/rpython/jit/backend/arm/codebuilder.py
+++ b/rpython/jit/backend/arm/codebuilder.py
@@ -1,7 +1,7 @@
 from rpython.jit.backend.arm import conditions as cond
 from rpython.jit.backend.arm import registers as reg
 from rpython.jit.backend.arm import support
-from rpython.jit.backend.arm.arch import (WORD, FUNC_ALIGN, PC_OFFSET)
+from rpython.jit.backend.arm.arch import WORD, PC_OFFSET
 from rpython.jit.backend.arm.instruction_builder import define_instructions
 from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin
 from rpython.rlib.objectmodel import we_are_translated
@@ -29,14 +29,9 @@
 
 
 class AbstractARMBuilder(object):
-
     def __init__(self, arch_version=7):
         self.arch_version = arch_version
 
-    def align(self):
-        while(self.currpos() % FUNC_ALIGN != 0):
-            self.writechar(chr(0))
-
     def NOP(self):
         self.MOV_rr(0, 0)
 
@@ -467,21 +462,6 @@
                 f.write(data[i])
             f.close()
 
-    # XXX remove and setup aligning in llsupport
-    def materialize(self, asmmemmgr, allblocks, gcrootmap=None):
-        size = self.get_relative_pos() + WORD
-        malloced = asmmemmgr.malloc(size, size + 7)
-        allblocks.append(malloced)
-        rawstart = malloced[0]
-        while(rawstart % FUNC_ALIGN != 0):
-            rawstart += 1
-        self.copy_to_raw_memory(rawstart)
-        if self.gcroot_markers is not None:
-            assert gcrootmap is not None
-            for pos, mark in self.gcroot_markers:
-                gcrootmap.put(rawstart + pos, mark)
-        return rawstart
-
     def clear_cache(self, addr):
         if we_are_translated():
             startaddr = rffi.cast(llmemory.Address, addr)
diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py
--- a/rpython/jit/backend/llsupport/asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/asmmemmgr.py
@@ -208,6 +208,8 @@
                    ('data', lltype.FixedSizeArray(lltype.Char, SUBBLOCK_SIZE)))
     SUBBLOCK_PTR.TO.become(SUBBLOCK)
 
+    ALIGN_MATERIALIZE = 16
+
     gcroot_markers = None
 
     def __init__(self, translated=None):
@@ -303,9 +305,12 @@
 
     def materialize(self, asmmemmgr, allblocks, gcrootmap=None):
         size = self.get_relative_pos()
+        align = self.ALIGN_MATERIALIZE
+        size += align - 1
         malloced = asmmemmgr.malloc(size, size)
         allblocks.append(malloced)
         rawstart = malloced[0]
+        rawstart = (rawstart + align - 1) & (-align)
         self.copy_to_raw_memory(rawstart)
         if self.gcroot_markers is not None:
             assert gcrootmap is not None

From noreply at buildbot.pypy.org  Sun Feb 22 17:28:13 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Sun, 22 Feb 2015 17:28:13 +0100 (CET)
Subject: [pypy-commit] pypy default: fix test: the cond_call=0 was actually
 a long-standing bug that the new
Message-ID: <20150222162813.A68BF1C02BD@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: 
Changeset: r76046:3930b6579b7e
Date: 2015-02-22 17:26 +0100
http://bitbucket.org/pypy/pypy/changeset/3930b6579b7e/

Log:	fix test: the cond_call=0 was actually a long-standing bug that the
	new heapcache fixes

diff --git a/rpython/jit/metainterp/test/test_recursive.py b/rpython/jit/metainterp/test/test_recursive.py
--- a/rpython/jit/metainterp/test/test_recursive.py
+++ b/rpython/jit/metainterp/test/test_recursive.py
@@ -794,7 +794,7 @@
             return frame.thing.val + s
 
         res = self.meta_interp(main, [0], inline=True)
-        self.check_resops(call=0, cond_call=0) # got removed by optimization
+        self.check_resops(call=0, cond_call=2)
         assert res == main(0)
 
     def test_directly_call_assembler_virtualizable_reset_token(self):

From noreply at buildbot.pypy.org  Sun Feb 22 17:28:14 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Sun, 22 Feb 2015 17:28:14 +0100 (CET)
Subject: [pypy-commit] pypy default: whatsnew entry
Message-ID: <20150222162814.D79281C02BD@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: 
Changeset: r76047:3356a2ba8490
Date: 2015-02-22 17:27 +0100
http://bitbucket.org/pypy/pypy/changeset/3356a2ba8490/

Log:	whatsnew entry

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -30,3 +30,7 @@
 .. branch: alt_errno
 Add an alternative location to save LastError, errno around ctypes,
 cffi external calls so things like pdb will not overwrite it
+
+.. branch: nonquadratic-heapcache
+Speed up the warmup times of the JIT by removing a quadratic algorithm in the
+heapcache.

From noreply at buildbot.pypy.org  Sun Feb 22 17:48:01 2015
From: noreply at buildbot.pypy.org (vvladymyrov)
Date: Sun, 22 Feb 2015 17:48:01 +0100 (CET)
Subject: [pypy-commit] pypy default: improve _contains_ and _getitem_
 performance almost 2x times using trick I've seen in py-lmdb to avoid key
 alloc/dup:
Message-ID: <20150222164801.5D2C31C0271@cobra.cs.uni-duesseldorf.de>

Author: Volodymyr Vladymyrov 
Branch: 
Changeset: r76048:0531bc9adbf4
Date: 2015-02-22 01:33 +0000
http://bitbucket.org/pypy/pypy/changeset/0531bc9adbf4/

Log:	improve _contains_ and _getitem_ performance almost 2x times using
	trick I've seen in py-lmdb to avoid key alloc/dup: see
	https://github.com/dw/py-lmdb/blob/master/lmdb/cffi.py#L228-L230

diff --git a/lib_pypy/gdbm.py b/lib_pypy/gdbm.py
--- a/lib_pypy/gdbm.py
+++ b/lib_pypy/gdbm.py
@@ -20,9 +20,11 @@
 } datum;
 
 datum gdbm_fetch(void*, datum);
+datum pygdbm_fetch(void*, char*, int);
 int gdbm_delete(void*, datum);
 int gdbm_store(void*, datum, datum, int);
 int gdbm_exists(void*, datum);
+int pygdbm_exists(void*, char*, int);
 
 int gdbm_reorganize(void*);
 
@@ -37,19 +39,29 @@
 ''')
 
 try:
+    verify_code = '''
+    #include "gdbm.h"
+
+    static datum pygdbm_fetch(GDBM_FILE gdbm_file, char *dptr, int dsize) {
+        datum key = {dptr, dsize};
+        return gdbm_fetch(gdbm_file, key);
+    }
+
+    static int pygdbm_exists(GDBM_FILE gdbm_file, char *dptr, int dsize) {
+        datum key = {dptr, dsize};
+        return gdbm_exists(gdbm_file, key);
+    }
+    
+    '''
     if sys.platform.startswith('freebsd'):
         import os.path
         _localbase = os.environ.get('LOCALBASE', '/usr/local')
-        lib = ffi.verify('''
-        #include "gdbm.h"
-        ''', libraries=['gdbm'],
+        lib = ffi.verify(verify_code, libraries=['gdbm'],
              include_dirs=[os.path.join(_localbase, 'include')],
              library_dirs=[os.path.join(_localbase, 'lib')]
         )
     else:
-        lib = ffi.verify('''
-        #include "gdbm.h"
-        ''', libraries=['gdbm'])
+        lib = ffi.verify(verify_code, libraries=['gdbm'])
 except cffi.VerificationError as e:
     # distutils does not preserve the actual message,
     # but the verification is simple enough that the
@@ -59,6 +71,13 @@
 class error(Exception):
     pass
 
+def _checkstr(key):
+    if isinstance(key, unicode):
+        key = key.encode("ascii")
+    if not isinstance(key, str):
+        raise TypeError("gdbm mappings have string indices only")
+    return key
+
 def _fromstr(key):
     if isinstance(key, unicode):
         key = key.encode("ascii")
@@ -107,12 +126,14 @@
 
     def __contains__(self, key):
         self._check_closed()
-        return lib.gdbm_exists(self.ll_dbm, _fromstr(key))
+        key = _checkstr(key)
+        return lib.pygdbm_exists(self.ll_dbm, key, len(key)
     has_key = __contains__
 
     def __getitem__(self, key):
         self._check_closed()
-        drec = lib.gdbm_fetch(self.ll_dbm, _fromstr(key))
+        key = _checkstr(key)        
+        drec = lib.pygdbm_fetch(self.ll_dbm, key, len(key))
         if not drec.dptr:
             raise KeyError(key)
         res = str(ffi.buffer(drec.dptr, drec.dsize))

From noreply at buildbot.pypy.org  Sun Feb 22 17:48:02 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 22 Feb 2015 17:48:02 +0100 (CET)
Subject: [pypy-commit] pypy default: I'm merging this PR anyway,
 but if you need more improvements, pop in on #pypy on IRC
Message-ID: <20150222164802.990EA1C0271@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r76049:4aecd4091baf
Date: 2015-02-22 18:48 +0200
http://bitbucket.org/pypy/pypy/changeset/4aecd4091baf/

Log:	I'm merging this PR anyway, but if you need more improvements, pop
	in on #pypy on IRC

	Merged in vvladymyrov/pypy (pull request #307)

	improve _contains_ and _getitem_ performance almost 2x times using
	trick I've seen in py-lmdb to avoid key alloc/dup:

diff --git a/lib_pypy/gdbm.py b/lib_pypy/gdbm.py
--- a/lib_pypy/gdbm.py
+++ b/lib_pypy/gdbm.py
@@ -20,9 +20,11 @@
 } datum;
 
 datum gdbm_fetch(void*, datum);
+datum pygdbm_fetch(void*, char*, int);
 int gdbm_delete(void*, datum);
 int gdbm_store(void*, datum, datum, int);
 int gdbm_exists(void*, datum);
+int pygdbm_exists(void*, char*, int);
 
 int gdbm_reorganize(void*);
 
@@ -37,19 +39,29 @@
 ''')
 
 try:
+    verify_code = '''
+    #include "gdbm.h"
+
+    static datum pygdbm_fetch(GDBM_FILE gdbm_file, char *dptr, int dsize) {
+        datum key = {dptr, dsize};
+        return gdbm_fetch(gdbm_file, key);
+    }
+
+    static int pygdbm_exists(GDBM_FILE gdbm_file, char *dptr, int dsize) {
+        datum key = {dptr, dsize};
+        return gdbm_exists(gdbm_file, key);
+    }
+    
+    '''
     if sys.platform.startswith('freebsd'):
         import os.path
         _localbase = os.environ.get('LOCALBASE', '/usr/local')
-        lib = ffi.verify('''
-        #include "gdbm.h"
-        ''', libraries=['gdbm'],
+        lib = ffi.verify(verify_code, libraries=['gdbm'],
              include_dirs=[os.path.join(_localbase, 'include')],
              library_dirs=[os.path.join(_localbase, 'lib')]
         )
     else:
-        lib = ffi.verify('''
-        #include "gdbm.h"
-        ''', libraries=['gdbm'])
+        lib = ffi.verify(verify_code, libraries=['gdbm'])
 except cffi.VerificationError as e:
     # distutils does not preserve the actual message,
     # but the verification is simple enough that the
@@ -59,6 +71,13 @@
 class error(Exception):
     pass
 
+def _checkstr(key):
+    if isinstance(key, unicode):
+        key = key.encode("ascii")
+    if not isinstance(key, str):
+        raise TypeError("gdbm mappings have string indices only")
+    return key
+
 def _fromstr(key):
     if isinstance(key, unicode):
         key = key.encode("ascii")
@@ -107,12 +126,14 @@
 
     def __contains__(self, key):
         self._check_closed()
-        return lib.gdbm_exists(self.ll_dbm, _fromstr(key))
+        key = _checkstr(key)
+        return lib.pygdbm_exists(self.ll_dbm, key, len(key)
     has_key = __contains__
 
     def __getitem__(self, key):
         self._check_closed()
-        drec = lib.gdbm_fetch(self.ll_dbm, _fromstr(key))
+        key = _checkstr(key)        
+        drec = lib.pygdbm_fetch(self.ll_dbm, key, len(key))
         if not drec.dptr:
             raise KeyError(key)
         res = str(ffi.buffer(drec.dptr, drec.dsize))

From noreply at buildbot.pypy.org  Sun Feb 22 17:50:26 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Feb 2015 17:50:26 +0100 (CET)
Subject: [pypy-commit] stmgc default: (remi, xoraxax, me around) Bug fix
Message-ID: <20150222165026.DA69B1C03C6@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1641:b26fe28d6f2b
Date: 2015-02-22 17:50 +0100
http://bitbucket.org/pypy/stmgc/changeset/b26fe28d6f2b/

Log:	(remi, xoraxax, me around) Bug fix

diff --git a/c7/stm/finalizer.c b/c7/stm/finalizer.c
--- a/c7/stm/finalizer.c
+++ b/c7/stm/finalizer.c
@@ -40,6 +40,8 @@
             *STM_PSEGMENT->finalizers->running_next = (uintptr_t)-1;
         }
         if (frm < list_count(src)) {
+            if (g_finalizers.run_finalizers == NULL)
+                g_finalizers.run_finalizers = list_create();
             g_finalizers.run_finalizers = list_extend(
                 g_finalizers.run_finalizers,
                 src, frm);

From noreply at buildbot.pypy.org  Sun Feb 22 17:51:18 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Feb 2015 17:51:18 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: import stmgc/b26fe28d6f2b
Message-ID: <20150222165118.244EA1C03C6@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r76050:21ed62f741b8
Date: 2015-02-22 17:51 +0100
http://bitbucket.org/pypy/pypy/changeset/21ed62f741b8/

Log:	import stmgc/b26fe28d6f2b

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-78281a990907
+b26fe28d6f2b
diff --git a/rpython/translator/stm/src_stm/stm/finalizer.c b/rpython/translator/stm/src_stm/stm/finalizer.c
--- a/rpython/translator/stm/src_stm/stm/finalizer.c
+++ b/rpython/translator/stm/src_stm/stm/finalizer.c
@@ -41,6 +41,8 @@
             *STM_PSEGMENT->finalizers->running_next = (uintptr_t)-1;
         }
         if (frm < list_count(src)) {
+            if (g_finalizers.run_finalizers == NULL)
+                g_finalizers.run_finalizers = list_create();
             g_finalizers.run_finalizers = list_extend(
                 g_finalizers.run_finalizers,
                 src, frm);

From noreply at buildbot.pypy.org  Sun Feb 22 18:22:57 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Feb 2015 18:22:57 +0100 (CET)
Subject: [pypy-commit] pypy default: (fijal, remi, arigo)
Message-ID: <20150222172257.C519C1C0041@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76051:b3a08c810002
Date: 2015-02-22 17:23 +0100
http://bitbucket.org/pypy/pypy/changeset/b3a08c810002/

Log:	(fijal, remi, arigo)

	Meh, sometimes we get an extra "mov" and sometimes we don't,
	depending on whether some pointers fit into 2GB.

diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -47,6 +47,7 @@
 
     add_loop_instructions = ['overload for a specific cpu']
     bridge_loop_instructions = ['overload for a specific cpu']
+    bridge_loop_instructions_alternative = None   # or another possible answer
 
     def execute_operation(self, opname, valueboxes, result_type, descr=None):
         inputargs, operations = self._get_single_operation_list(opname,
@@ -4284,7 +4285,9 @@
         # XXX we have to check the precise assembler, otherwise
         # we don't quite know if borders are correct
 
-        def checkops(mc, ops):
+        def checkops(mc, ops, alt_ops=None):
+            if len(mc) != len(ops) and alt_ops is not None:
+                ops = alt_ops
             assert len(mc) == len(ops)
             for i in range(len(mc)):
                 if ops[i] == '*':
@@ -4299,7 +4302,8 @@
             data = ctypes.string_at(bridge_info.asmaddr, bridge_info.asmlen)
             mc = list(machine_code_dump(data, bridge_info.asmaddr, cpuname))
             lines = [line for line in mc if line.count('\t') >= 2]
-            checkops(lines, self.bridge_loop_instructions)
+            checkops(lines, self.bridge_loop_instructions,
+                            self.bridge_loop_instructions_alternative)
         except ObjdumpNotFound:
             py.test.skip("requires (g)objdump")
 
diff --git a/rpython/jit/backend/x86/test/test_runner.py b/rpython/jit/backend/x86/test/test_runner.py
--- a/rpython/jit/backend/x86/test/test_runner.py
+++ b/rpython/jit/backend/x86/test/test_runner.py
@@ -34,8 +34,10 @@
     if WORD == 4:
         bridge_loop_instructions = ['cmp', 'jge', 'mov', 'mov', 'call', 'jmp']
     else:
-        bridge_loop_instructions = ['cmp', 'jge', 'mov', 'mov', 'mov', 'mov',
-                                    'call', 'mov', 'jmp']
+        bridge_loop_instructions = [
+            'cmp', 'jge', 'mov', 'mov', 'mov', 'mov', 'call', 'mov', 'jmp']
+        bridge_loop_instructions_alternative = [
+            'cmp', 'jge', 'mov', 'mov', 'mov', 'call', 'mov', 'jmp']
 
     def get_cpu(self):
         cpu = CPU(rtyper=None, stats=FakeStats())

From noreply at buildbot.pypy.org  Sun Feb 22 18:22:58 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Feb 2015 18:22:58 +0100 (CET)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20150222172258.F0BDD1C0041@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76052:ce0ca1b30216
Date: 2015-02-22 17:23 +0100
http://bitbucket.org/pypy/pypy/changeset/ce0ca1b30216/

Log:	merge heads

diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -199,8 +199,7 @@
         magic = importing._get_long(buf[:4])
         timestamp = importing._get_long(buf[4:8])
         if not self.can_use_pyc(space, filename, magic, timestamp):
-            return self.import_py_file(space, modname, filename[:-1], buf,
-                                       pkgpath)
+            return None
         buf = buf[8:] # XXX ugly copy, should use sequential read instead
         w_mod = w(Module(space, w(modname)))
         real_name = self.filename + os.path.sep + self.corr_zname(filename)
@@ -249,7 +248,6 @@
     def load_module(self, space, fullname):
         w = space.wrap
         filename = self.make_filename(fullname)
-        last_exc = None
         for compiled, is_package, ext in ENUMERATE_EXTS:
             fname = filename + ext
             try:
@@ -268,19 +266,18 @@
                     pkgpath = None
                 try:
                     if compiled:
-                        return self.import_pyc_file(space, fullname, fname,
-                                                    buf, pkgpath)
+                        w_result = self.import_pyc_file(space, fullname, fname,
+                                                        buf, pkgpath)
+                        if w_result is not None:
+                            return w_result
                     else:
                         return self.import_py_file(space, fullname, fname,
                                                    buf, pkgpath)
-                except OperationError, e:
-                    last_exc = e
+                except:
                     w_mods = space.sys.get('modules')
-                space.call_method(w_mods, 'pop', w(fullname), space.w_None)
-        if last_exc:
-            raise OperationError(get_error(space), last_exc.get_w_value(space))
-        # should never happen I think
-        return space.w_None
+                    space.call_method(w_mods, 'pop', w(fullname), space.w_None)
+                    raise
+        raise oefmt(get_error(space), "can't find module '%s'", fullname)
 
     @unwrap_spec(filename=str)
     def get_data(self, space, filename):
diff --git a/pypy/module/zipimport/test/test_zipimport.py b/pypy/module/zipimport/test/test_zipimport.py
--- a/pypy/module/zipimport/test/test_zipimport.py
+++ b/pypy/module/zipimport/test/test_zipimport.py
@@ -195,7 +195,8 @@
         m0 ^= 0x04
         test_pyc = chr(m0) + self.test_pyc[1:]
         self.writefile("uu.pyc", test_pyc)
-        raises(ImportError, "__import__('uu', globals(), locals(), [])")
+        raises(zipimport.ZipImportError,
+               "__import__('uu', globals(), locals(), [])")
         assert 'uu' not in sys.modules
 
     def test_force_py(self):
@@ -360,6 +361,11 @@
         co_filename = code.co_filename
         assert co_filename == expected
 
+    def test_import_exception(self):
+        self.writefile('x1test.py', '1/0')
+        self.writefile('x1test/__init__.py', 'raise ValueError')
+        raises(ValueError, __import__, 'x1test', None, None, [])
+
 
 if os.sep != '/':
     class AppTestNativePathSep(AppTestZipimport):
diff --git a/rpython/jit/backend/arm/arch.py b/rpython/jit/backend/arm/arch.py
--- a/rpython/jit/backend/arm/arch.py
+++ b/rpython/jit/backend/arm/arch.py
@@ -1,4 +1,3 @@
-FUNC_ALIGN = 8
 WORD = 4
 DOUBLE_WORD = 8
 
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -4,7 +4,7 @@
 
 from rpython.jit.backend.arm import conditions as c, registers as r
 from rpython.jit.backend.arm import shift
-from rpython.jit.backend.arm.arch import (WORD, DOUBLE_WORD, FUNC_ALIGN,
+from rpython.jit.backend.arm.arch import (WORD, DOUBLE_WORD,
     JITFRAME_FIXED_SIZE)
 from rpython.jit.backend.arm.codebuilder import InstrBuilder, OverwritingBuilder
 from rpython.jit.backend.arm.locations import imm, StackLocation, get_fp_offset
@@ -484,10 +484,6 @@
         self.mc.BL(target)
         return startpos
 
-    def align(self):
-        while(self.mc.currpos() % FUNC_ALIGN != 0):
-            self.mc.writechar(chr(0))
-
     def gen_func_epilog(self, mc=None, cond=c.AL):
         gcrootmap = self.cpu.gc_ll_descr.gcrootmap
         if mc is None:
@@ -557,7 +553,7 @@
         debug_stop('jit-backend-ops')
 
     def _call_header(self):
-        self.align()
+        assert self.currpos() == 0
         self.gen_func_prolog()
 
     def _call_header_with_stack_check(self):
diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py
--- a/rpython/jit/backend/arm/codebuilder.py
+++ b/rpython/jit/backend/arm/codebuilder.py
@@ -1,7 +1,7 @@
 from rpython.jit.backend.arm import conditions as cond
 from rpython.jit.backend.arm import registers as reg
 from rpython.jit.backend.arm import support
-from rpython.jit.backend.arm.arch import (WORD, FUNC_ALIGN, PC_OFFSET)
+from rpython.jit.backend.arm.arch import WORD, PC_OFFSET
 from rpython.jit.backend.arm.instruction_builder import define_instructions
 from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin
 from rpython.rlib.objectmodel import we_are_translated
@@ -29,14 +29,9 @@
 
 
 class AbstractARMBuilder(object):
-
     def __init__(self, arch_version=7):
         self.arch_version = arch_version
 
-    def align(self):
-        while(self.currpos() % FUNC_ALIGN != 0):
-            self.writechar(chr(0))
-
     def NOP(self):
         self.MOV_rr(0, 0)
 
@@ -467,21 +462,6 @@
                 f.write(data[i])
             f.close()
 
-    # XXX remove and setup aligning in llsupport
-    def materialize(self, asmmemmgr, allblocks, gcrootmap=None):
-        size = self.get_relative_pos() + WORD
-        malloced = asmmemmgr.malloc(size, size + 7)
-        allblocks.append(malloced)
-        rawstart = malloced[0]
-        while(rawstart % FUNC_ALIGN != 0):
-            rawstart += 1
-        self.copy_to_raw_memory(rawstart)
-        if self.gcroot_markers is not None:
-            assert gcrootmap is not None
-            for pos, mark in self.gcroot_markers:
-                gcrootmap.put(rawstart + pos, mark)
-        return rawstart
-
     def clear_cache(self, addr):
         if we_are_translated():
             startaddr = rffi.cast(llmemory.Address, addr)
diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py
--- a/rpython/jit/backend/llsupport/asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/asmmemmgr.py
@@ -208,6 +208,8 @@
                    ('data', lltype.FixedSizeArray(lltype.Char, SUBBLOCK_SIZE)))
     SUBBLOCK_PTR.TO.become(SUBBLOCK)
 
+    ALIGN_MATERIALIZE = 16
+
     gcroot_markers = None
 
     def __init__(self, translated=None):
@@ -303,9 +305,12 @@
 
     def materialize(self, asmmemmgr, allblocks, gcrootmap=None):
         size = self.get_relative_pos()
+        align = self.ALIGN_MATERIALIZE
+        size += align - 1
         malloced = asmmemmgr.malloc(size, size)
         allblocks.append(malloced)
         rawstart = malloced[0]
+        rawstart = (rawstart + align - 1) & (-align)
         self.copy_to_raw_memory(rawstart)
         if self.gcroot_markers is not None:
             assert gcrootmap is not None
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -56,9 +56,9 @@
 ASN1_OBJECT = rffi.COpaquePtr('ASN1_OBJECT')
 X509_NAME = rffi.COpaquePtr('X509_NAME')
 X509_VERIFY_PARAM = rffi.COpaquePtr('X509_VERIFY_PARAM')
-stack_st_X509_OBJECT = rffi.COpaquePtr('struct stack_st_X509_OBJECT')
+stack_st_X509_OBJECT = rffi.COpaquePtr('STACK_OF(X509_OBJECT)')
 DIST_POINT = rffi.COpaquePtr('DIST_POINT')
-stack_st_DIST_POINT = rffi.COpaquePtr('struct stack_st_DIST_POINT')
+stack_st_DIST_POINT = rffi.COpaquePtr('STACK_OF(X509_OBJECT)')
 DH = rffi.COpaquePtr('DH')
 EC_KEY = rffi.COpaquePtr('EC_KEY')
 AUTHORITY_INFO_ACCESS = rffi.COpaquePtr('AUTHORITY_INFO_ACCESS')
diff --git a/rpython/tool/ansi_mandelbrot.py b/rpython/tool/ansi_mandelbrot.py
--- a/rpython/tool/ansi_mandelbrot.py
+++ b/rpython/tool/ansi_mandelbrot.py
@@ -21,22 +21,15 @@
 else:
     palette = [39, 34, 35, 36, 31, 33, 32, 37]
 
-colour_range = None # used for debugging
-
-
-def print_pixel(colour, value_range, invert=1):
-    global colour_range
-    chars = [".", ".", "+", "*", "%", "#"]
-    idx = lambda chars: (colour+1) * (len(chars) - 1) / value_range
-    if invert:
-        idx = lambda chars, idx=idx:len(chars) - 1 - idx(chars)
-    char = chars[idx(chars)]
-    ansi_colour = palette[idx(palette)]
-    ansi_print(char, ansi_colour, newline=False, flush=True)
-    #if colour_range is None:
-    #    colour_range = [colour, colour]
-    #else:
-    #    colour_range = [min(colour_range[0], colour), max(colour_range[1], colour)]
+# used for debugging/finding new coordinates
+# How to:
+#   1. Set DEBUG to True
+#   2. Add a new coordinate to coordinates with a high distance and high max colour (e.g. 300)
+#   3. Run, pick an interesting coordinate from the shown list and replace the newly added
+#      coordinate by it.
+#   4. Rerun to see the max colour, insert this max colour where you put the high max colour.
+#   5. Set DEBUG to False
+DEBUG = False
 
 
 class Mandelbrot:
@@ -58,9 +51,6 @@
         ymin = self.ypos - self.yscale * self.y / 2
         self.x_range = [xmin + self.xscale * ix for ix in range(self.x)]
         self.y_range = [ymin + self.yscale * iy for iy in range(self.y)]
-        
-        #print "x", self.x_range[0], self.x_range[-1]
-        #print "y", self.y_range[0], self.y_range[-1]
 
     def reset(self, cnt):
         self.reset_lines = cnt
@@ -100,8 +90,11 @@
 
 class Driver(object):
     zoom_locations = [
-        # x, y, "distance", range
+        # x, y, "distance", max color range
         (0.37865401, 0.669227668, 0.04, 111),
+        (-1.2693, -0.4145, 0.2, 105),
+        (-1.2693, -0.4145, 0.05, 97),
+        (-1.2642, -0.4185, 0.01, 95),
         (-1.15, -0.28, 0.9, 94),
         (-1.15, -0.28, 0.3, 58),
         (-1.15, -0.28, 0.05, 26),
@@ -109,8 +102,10 @@
     def __init__(self, **kwargs):
         self.kwargs = kwargs
         self.zoom_location = -1
-        self.colour_range = 256
+        self.max_colour = 256
+        self.colour_range = None
         self.invert = True
+        self.interesting_coordinates = []
         self.init()
 
     def init(self):
@@ -123,14 +118,6 @@
         """ Resets to the beginning of the line and drops cnt lines internally. """
         self.mandelbrot.reset(cnt)
 
-    def catchup(self):
-        """ Fills the current line. """
-        x = 0
-        while x != self.width - 1:
-            x, y, c = self.gen.next()
-            print_pixel(c, self.colour_range, self.invert)
-        print >>sys.stderr
-
     def restart(self):
         """ Restarts the current generator. """
         print >>sys.stderr
@@ -146,36 +133,54 @@
                 if width != self.width:
                     self.init()
         except StopIteration:
+            if DEBUG and self.interesting_coordinates:
+                print >>sys.stderr, "Interesting coordinates:", self.interesting_coordinates
+                self.interesting_coordinates = []
             kwargs = self.kwargs
             self.zoom_location += 1
             self.zoom_location %= len(self.zoom_locations)
             loc = self.zoom_locations[self.zoom_location]
             kwargs.update({"x_pos": loc[0], "y_pos": loc[1], "distance": loc[2]})
-            self.colour_range = loc[3]
-            #global colour_range
-            #print colour_range, loc[2]
-            #colour_range = None
-            return self.restart()
-        print_pixel(c, self.colour_range, self.invert)
+            self.max_colour = loc[3]
+            if DEBUG:
+                # Only used for debugging new locations:
+                print "Colour range", self.colour_range
+            self.colour_range = None
+            self.restart()
+            return
+        if self.print_pixel(c, self.invert):
+            self.interesting_coordinates.append(dict(x=(x, self.mandelbrot.x_range[x]),
+                                                     y=(y, self.mandelbrot.y_range[y])))
         if x == self.width - 1:
             print >>sys.stderr
 
+    def print_pixel(self, colour, invert=1):
+        chars = [".", ".", "+", "*", "%", "#"]
+        idx = lambda chars: (colour+1) * (len(chars) - 1) / self.max_colour
+        if invert:
+            idx = lambda chars, idx=idx:len(chars) - 1 - idx(chars)
+        char = chars[idx(chars)]
+        ansi_colour = palette[idx(palette)]
+        ansi_print(char, ansi_colour, newline=False, flush=True)
+        if DEBUG:
+            if self.colour_range is None:
+                self.colour_range = [colour, colour]
+            else:
+                old_colour_range = self.colour_range
+                self.colour_range = [min(self.colour_range[0], colour), max(self.colour_range[1], colour)]
+                if old_colour_range[0] - colour > 3 or colour - old_colour_range[1] > 3:
+                    return True
+
 
 if __name__ == '__main__':
     import random
     from time import sleep
 
     d = Driver()
-    for x in xrange(15000):
-        sleep(random.random() / 300)
+    while True:
+        sleep(random.random() / 800)
         d.dot()
         if 0 and random.random() < 0.01:
-            d.catchup()
-            print "WARNING! " * 3
-            d.reset(1)
-        #    print "R",
-        if 0 and random.random() < 0.01:
             string = "WARNING! " * 3
             d.jump(len(string))
             print string,
-

From noreply at buildbot.pypy.org  Sun Feb 22 18:23:00 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Feb 2015 18:23:00 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Obscure: after inserting an extra line
 at the start of the file, 
Message-ID: <20150222172300.309681C0041@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r76053:b92017c00165
Date: 2015-02-22 18:22 +0100
http://bitbucket.org/pypy/pypy/changeset/b92017c00165/

Log:	Obscure: after inserting an extra line at the start of the file,
	kill the first empty line to restore the correct line numbers

diff --git a/rpython/translator/stm/import_stmgc.py b/rpython/translator/stm/import_stmgc.py
--- a/rpython/translator/stm/import_stmgc.py
+++ b/rpython/translator/stm/import_stmgc.py
@@ -11,7 +11,11 @@
 
 def mangle(lines):
     yield "/* Imported by rpython/translator/stm/import_stmgc.py */\n"
+    kill_first_empty_line = True
     for line in lines:
+        if kill_first_empty_line and line.strip() == '':
+            kill_first_empty_line = False
+            continue
         yield line
 
 def main(stmgc_dir):
diff --git a/rpython/translator/stm/src_stm/stm/atomic.h b/rpython/translator/stm/src_stm/stm/atomic.h
--- a/rpython/translator/stm/src_stm/stm/atomic.h
+++ b/rpython/translator/stm/src_stm/stm/atomic.h
@@ -1,7 +1,6 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
 #ifndef _STM_ATOMIC_H
 #define _STM_ATOMIC_H
-
 /* spin_loop() corresponds to the PAUSE instruction on x86.  On
    other architectures, we generate no instruction (but still need
    the compiler barrier); if on another architecture you find the
diff --git a/rpython/translator/stm/src_stm/stm/contention.c b/rpython/translator/stm/src_stm/stm/contention.c
--- a/rpython/translator/stm/src_stm/stm/contention.c
+++ b/rpython/translator/stm/src_stm/stm/contention.c
@@ -3,7 +3,6 @@
 # error "must be compiled via stmgc.c"
 #endif
 
-
 /* Here are the possible kinds of contention:
 
    STM_CONTENTION_WRITE_WRITE
diff --git a/rpython/translator/stm/src_stm/stm/contention.h b/rpython/translator/stm/src_stm/stm/contention.h
--- a/rpython/translator/stm/src_stm/stm/contention.h
+++ b/rpython/translator/stm/src_stm/stm/contention.h
@@ -1,5 +1,4 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
-
 static void write_write_contention_management(uintptr_t lock_idx,
                                               object_t *obj);
 static bool write_read_contention_management(uint8_t other_segment_num,
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -3,7 +3,6 @@
 # error "must be compiled via stmgc.c"
 #endif
 
-
 static void teardown_core(void)
 {
     memset(write_locks, 0, sizeof(write_locks));
diff --git a/rpython/translator/stm/src_stm/stm/core.h b/rpython/translator/stm/src_stm/stm/core.h
--- a/rpython/translator/stm/src_stm/stm/core.h
+++ b/rpython/translator/stm/src_stm/stm/core.h
@@ -1,6 +1,5 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
 #define _STM_CORE_H_
-
 #include 
 #include 
 #include 
diff --git a/rpython/translator/stm/src_stm/stm/extra.c b/rpython/translator/stm/src_stm/stm/extra.c
--- a/rpython/translator/stm/src_stm/stm/extra.c
+++ b/rpython/translator/stm/src_stm/stm/extra.c
@@ -3,7 +3,6 @@
 # error "must be compiled via stmgc.c"
 #endif
 
-
 static long register_callbacks(stm_thread_local_t *tl,
                                void *key, void callback(void *), long index)
 {
diff --git a/rpython/translator/stm/src_stm/stm/extra.h b/rpython/translator/stm/src_stm/stm/extra.h
--- a/rpython/translator/stm/src_stm/stm/extra.h
+++ b/rpython/translator/stm/src_stm/stm/extra.h
@@ -1,4 +1,3 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
-
 static void invoke_and_clear_user_callbacks(long index);
 /* 0 = for commit, 1 = for abort */
diff --git a/rpython/translator/stm/src_stm/stm/finalizer.c b/rpython/translator/stm/src_stm/stm/finalizer.c
--- a/rpython/translator/stm/src_stm/stm/finalizer.c
+++ b/rpython/translator/stm/src_stm/stm/finalizer.c
@@ -1,6 +1,5 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
 
-
 /* callbacks */
 void (*stmcb_light_finalizer)(object_t *);
 void (*stmcb_finalizer)(object_t *);
diff --git a/rpython/translator/stm/src_stm/stm/finalizer.h b/rpython/translator/stm/src_stm/stm/finalizer.h
--- a/rpython/translator/stm/src_stm/stm/finalizer.h
+++ b/rpython/translator/stm/src_stm/stm/finalizer.h
@@ -1,5 +1,4 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
-
 struct finalizers_s {
     struct list_s *objects_with_finalizers;
     uintptr_t count_non_young;
diff --git a/rpython/translator/stm/src_stm/stm/forksupport.c b/rpython/translator/stm/src_stm/stm/forksupport.c
--- a/rpython/translator/stm/src_stm/stm/forksupport.c
+++ b/rpython/translator/stm/src_stm/stm/forksupport.c
@@ -3,7 +3,6 @@
 # error "must be compiled via stmgc.c"
 #endif
 
-
 /* XXX this is currently not doing copy-on-write, but simply forces a
    copy of all pages as soon as fork() is called. */
 
diff --git a/rpython/translator/stm/src_stm/stm/fprintcolor.c b/rpython/translator/stm/src_stm/stm/fprintcolor.c
--- a/rpython/translator/stm/src_stm/stm/fprintcolor.c
+++ b/rpython/translator/stm/src_stm/stm/fprintcolor.c
@@ -3,7 +3,6 @@
 #ifdef STM_DEBUGPRINT
 /* ------------------------------------------------------------ */
 
-
 static int threadcolor_printf(const char *format, ...)
 {
     char buffer[2048];
diff --git a/rpython/translator/stm/src_stm/stm/fprintcolor.h b/rpython/translator/stm/src_stm/stm/fprintcolor.h
--- a/rpython/translator/stm/src_stm/stm/fprintcolor.h
+++ b/rpython/translator/stm/src_stm/stm/fprintcolor.h
@@ -3,7 +3,6 @@
 #ifdef STM_DEBUGPRINT
 /* ------------------------------------------------------------ */
 
-
 #include 
 
 
diff --git a/rpython/translator/stm/src_stm/stm/gcpage.c b/rpython/translator/stm/src_stm/stm/gcpage.c
--- a/rpython/translator/stm/src_stm/stm/gcpage.c
+++ b/rpython/translator/stm/src_stm/stm/gcpage.c
@@ -3,7 +3,6 @@
 # error "must be compiled via stmgc.c"
 #endif
 
-
 static struct list_s *testing_prebuilt_objs = NULL;
 static struct tree_s *tree_prebuilt_objs = NULL;     /* XXX refactor */
 
diff --git a/rpython/translator/stm/src_stm/stm/gcpage.h b/rpython/translator/stm/src_stm/stm/gcpage.h
--- a/rpython/translator/stm/src_stm/stm/gcpage.h
+++ b/rpython/translator/stm/src_stm/stm/gcpage.h
@@ -1,5 +1,4 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
-
 /* Outside the nursery, we are taking from the highest addresses
    complete pages, one at a time, which uniformly contain objects of
    size "8 * N" for some N in range(2, GC_N_SMALL_REQUESTS).  We are
diff --git a/rpython/translator/stm/src_stm/stm/hash_id.c b/rpython/translator/stm/src_stm/stm/hash_id.c
--- a/rpython/translator/stm/src_stm/stm/hash_id.c
+++ b/rpython/translator/stm/src_stm/stm/hash_id.c
@@ -3,7 +3,6 @@
 # error "must be compiled via stmgc.c"
 #endif
 
-
 static long mangle_hash(long i)
 {
     /* To hash pointers in dictionaries.  Assumes that i shows some
diff --git a/rpython/translator/stm/src_stm/stm/hashtable.c b/rpython/translator/stm/src_stm/stm/hashtable.c
--- a/rpython/translator/stm/src_stm/stm/hashtable.c
+++ b/rpython/translator/stm/src_stm/stm/hashtable.c
@@ -2,7 +2,6 @@
 /*
 Design of stmgc's "hashtable" objects
 =====================================
-
 A "hashtable" is theoretically a lazily-filled array of objects of
 length 2**64.  Initially it is full of NULLs.  It's obviously
 implemented as a dictionary in which NULL objects are not needed.
diff --git a/rpython/translator/stm/src_stm/stm/largemalloc.c b/rpython/translator/stm/src_stm/stm/largemalloc.c
--- a/rpython/translator/stm/src_stm/stm/largemalloc.c
+++ b/rpython/translator/stm/src_stm/stm/largemalloc.c
@@ -2,7 +2,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
 #endif
-
 /* This contains a lot of inspiration from malloc() in the GNU C Library.
    More precisely, this is (a subset of) the part that handles large
    blocks, which in our case means at least 288 bytes.  It is actually
diff --git a/rpython/translator/stm/src_stm/stm/largemalloc.h b/rpython/translator/stm/src_stm/stm/largemalloc.h
--- a/rpython/translator/stm/src_stm/stm/largemalloc.h
+++ b/rpython/translator/stm/src_stm/stm/largemalloc.h
@@ -1,5 +1,4 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
-
 /* all addresses passed to this interface should be "char *" pointers
    in the segment 0. */
 void _stm_largemalloc_init_arena(char *data_start, size_t data_size);
diff --git a/rpython/translator/stm/src_stm/stm/list.c b/rpython/translator/stm/src_stm/stm/list.c
--- a/rpython/translator/stm/src_stm/stm/list.c
+++ b/rpython/translator/stm/src_stm/stm/list.c
@@ -3,7 +3,6 @@
 # error "must be compiled via stmgc.c"
 #endif
 
-
 #define LIST_SETSIZE(n)    (sizeof(struct list_s) + LIST_ITEMSSIZE(n))
 #define LIST_ITEMSSIZE(n)  ((n) * sizeof(uintptr_t))
 #define LIST_OVERCNT(n)    (33 + ((((n) / 2) * 3) | 1))
diff --git a/rpython/translator/stm/src_stm/stm/list.h b/rpython/translator/stm/src_stm/stm/list.h
--- a/rpython/translator/stm/src_stm/stm/list.h
+++ b/rpython/translator/stm/src_stm/stm/list.h
@@ -1,7 +1,6 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
 #include 
 #include 
-
 /************************************************************/
 
 struct list_s {
diff --git a/rpython/translator/stm/src_stm/stm/marker.c b/rpython/translator/stm/src_stm/stm/marker.c
--- a/rpython/translator/stm/src_stm/stm/marker.c
+++ b/rpython/translator/stm/src_stm/stm/marker.c
@@ -3,7 +3,6 @@
 # error "must be compiled via stmgc.c"
 #endif
 
-
 static void marker_fetch(stm_loc_marker_t *out_marker)
 {
     /* Fetch the current marker from the 'out_marker->tl's shadow stack,
diff --git a/rpython/translator/stm/src_stm/stm/marker.h b/rpython/translator/stm/src_stm/stm/marker.h
--- a/rpython/translator/stm/src_stm/stm/marker.h
+++ b/rpython/translator/stm/src_stm/stm/marker.h
@@ -1,5 +1,4 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
-
 static void _timing_record_write(void);
 static void _timing_fetch_inev(void);
 static void _timing_contention(enum stm_event_e kind,
diff --git a/rpython/translator/stm/src_stm/stm/misc.c b/rpython/translator/stm/src_stm/stm/misc.c
--- a/rpython/translator/stm/src_stm/stm/misc.c
+++ b/rpython/translator/stm/src_stm/stm/misc.c
@@ -3,7 +3,6 @@
 # error "must be compiled via stmgc.c"
 #endif
 
-
 char *_stm_real_address(object_t *o)
 {
     if (o == NULL)
diff --git a/rpython/translator/stm/src_stm/stm/nursery.c b/rpython/translator/stm/src_stm/stm/nursery.c
--- a/rpython/translator/stm/src_stm/stm/nursery.c
+++ b/rpython/translator/stm/src_stm/stm/nursery.c
@@ -2,7 +2,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
 #endif
-
 /************************************************************/
 
 /* xxx later: divide the nursery into sections, and zero them
diff --git a/rpython/translator/stm/src_stm/stm/nursery.h b/rpython/translator/stm/src_stm/stm/nursery.h
--- a/rpython/translator/stm/src_stm/stm/nursery.h
+++ b/rpython/translator/stm/src_stm/stm/nursery.h
@@ -1,5 +1,4 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
-
 /* 'nursery_end' is either NURSERY_END or one of NSE_SIGxxx */
 #define NSE_SIGABORT        1
 #define NSE_SIGPAUSE        2
diff --git a/rpython/translator/stm/src_stm/stm/pagecopy.c b/rpython/translator/stm/src_stm/stm/pagecopy.c
--- a/rpython/translator/stm/src_stm/stm/pagecopy.c
+++ b/rpython/translator/stm/src_stm/stm/pagecopy.c
@@ -1,5 +1,4 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
-
 #define PAGECOPY_128(dest, src)                                         \
         asm volatile("movdqa (%0), %%xmm0\n"                            \
                      "movdqa 16(%0), %%xmm1\n"                          \
diff --git a/rpython/translator/stm/src_stm/stm/pagecopy.h b/rpython/translator/stm/src_stm/stm/pagecopy.h
--- a/rpython/translator/stm/src_stm/stm/pagecopy.h
+++ b/rpython/translator/stm/src_stm/stm/pagecopy.h
@@ -1,3 +1,2 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
-
 static void pagecopy(void *dest, const void *src);      // 4096 bytes
diff --git a/rpython/translator/stm/src_stm/stm/pages.c b/rpython/translator/stm/src_stm/stm/pages.c
--- a/rpython/translator/stm/src_stm/stm/pages.c
+++ b/rpython/translator/stm/src_stm/stm/pages.c
@@ -3,7 +3,6 @@
 # error "must be compiled via stmgc.c"
 #endif
 
-
 /************************************************************/
 
 struct {
diff --git a/rpython/translator/stm/src_stm/stm/pages.h b/rpython/translator/stm/src_stm/stm/pages.h
--- a/rpython/translator/stm/src_stm/stm/pages.h
+++ b/rpython/translator/stm/src_stm/stm/pages.h
@@ -1,5 +1,4 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
-
 /* This handles pages of objects outside the nursery.  Every page
    has a "shared copy" and zero or more "private copies".
 
diff --git a/rpython/translator/stm/src_stm/stm/prebuilt.c b/rpython/translator/stm/src_stm/stm/prebuilt.c
--- a/rpython/translator/stm/src_stm/stm/prebuilt.c
+++ b/rpython/translator/stm/src_stm/stm/prebuilt.c
@@ -3,7 +3,6 @@
 # error "must be compiled via stmgc.c"
 #endif
 
-
 static struct list_s *prebuilt_objects_to_trace;
 static struct tree_s *tree_prebuilt_objs;  /* XXX from gcpage.c */
 
diff --git a/rpython/translator/stm/src_stm/stm/prof.c b/rpython/translator/stm/src_stm/stm/prof.c
--- a/rpython/translator/stm/src_stm/stm/prof.c
+++ b/rpython/translator/stm/src_stm/stm/prof.c
@@ -1,7 +1,6 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
 #include 
 
-
 static FILE *profiling_file;
 static char *profiling_basefn = NULL;
 static int (*profiling_expand_marker)(stm_loc_marker_t *, char *, int);
diff --git a/rpython/translator/stm/src_stm/stm/rewind_setjmp.c b/rpython/translator/stm/src_stm/stm/rewind_setjmp.c
--- a/rpython/translator/stm/src_stm/stm/rewind_setjmp.c
+++ b/rpython/translator/stm/src_stm/stm/rewind_setjmp.c
@@ -5,7 +5,6 @@
 #include 
 #include 
 
-
 struct _rewind_jmp_moved_s {
     struct _rewind_jmp_moved_s *next;
     size_t stack_size;
diff --git a/rpython/translator/stm/src_stm/stm/rewind_setjmp.h b/rpython/translator/stm/src_stm/stm/rewind_setjmp.h
--- a/rpython/translator/stm/src_stm/stm/rewind_setjmp.h
+++ b/rpython/translator/stm/src_stm/stm/rewind_setjmp.h
@@ -2,7 +2,6 @@
 #ifndef _REWIND_SETJMP_H_
 #define _REWIND_SETJMP_H_
 
-
 #include 
 
 /************************************************************
diff --git a/rpython/translator/stm/src_stm/stm/setup.c b/rpython/translator/stm/src_stm/stm/setup.c
--- a/rpython/translator/stm/src_stm/stm/setup.c
+++ b/rpython/translator/stm/src_stm/stm/setup.c
@@ -3,7 +3,6 @@
 # error "must be compiled via stmgc.c"
 #endif
 
-
 #ifdef USE_REMAP_FILE_PAGES
 static char *setup_mmap(char *reason, int *ignored)
 {
diff --git a/rpython/translator/stm/src_stm/stm/setup.h b/rpython/translator/stm/src_stm/stm/setup.h
--- a/rpython/translator/stm/src_stm/stm/setup.h
+++ b/rpython/translator/stm/src_stm/stm/setup.h
@@ -1,5 +1,4 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
-
 static char *setup_mmap(char *reason, int *map_fd);
 static void close_fd_mmap(int map_fd);
 static void setup_protection_settings(void);
diff --git a/rpython/translator/stm/src_stm/stm/sync.c b/rpython/translator/stm/src_stm/stm/sync.c
--- a/rpython/translator/stm/src_stm/stm/sync.c
+++ b/rpython/translator/stm/src_stm/stm/sync.c
@@ -2,7 +2,6 @@
 #include 
 #include 
 #include 
-
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
 #endif
diff --git a/rpython/translator/stm/src_stm/stm/sync.h b/rpython/translator/stm/src_stm/stm/sync.h
--- a/rpython/translator/stm/src_stm/stm/sync.h
+++ b/rpython/translator/stm/src_stm/stm/sync.h
@@ -1,6 +1,5 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
 
-
 static void setup_sync(void);
 static void teardown_sync(void);
 
diff --git a/rpython/translator/stm/src_stm/stm/weakref.c b/rpython/translator/stm/src_stm/stm/weakref.c
--- a/rpython/translator/stm/src_stm/stm/weakref.c
+++ b/rpython/translator/stm/src_stm/stm/weakref.c
@@ -2,7 +2,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
 #endif
-
 #define WEAKREF_PTR(wr, sz)  ((object_t * TLPREFIX *)(((stm_char *)(wr)) + (sz) - sizeof(void*)))
 
 object_t *stm_allocate_weakref(ssize_t size_rounded_up)
diff --git a/rpython/translator/stm/src_stm/stm/weakref.h b/rpython/translator/stm/src_stm/stm/weakref.h
--- a/rpython/translator/stm/src_stm/stm/weakref.h
+++ b/rpython/translator/stm/src_stm/stm/weakref.h
@@ -1,7 +1,6 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
 #ifndef _SRCSTM_WEAKREF_H
 #define _SRCSTM_WEAKREF_H
-
 object_t *stm_allocate_weakref(ssize_t size_rounded_up);
 static void stm_move_young_weakrefs(void);
 static void stm_visit_old_weakrefs(void);
diff --git a/rpython/translator/stm/src_stm/stmgc.c b/rpython/translator/stm/src_stm/stmgc.c
--- a/rpython/translator/stm/src_stm/stmgc.c
+++ b/rpython/translator/stm/src_stm/stmgc.c
@@ -17,7 +17,6 @@
 #include "stm/weakref.h"
 #include "stm/marker.h"
 #include "stm/finalizer.h"
-
 #include "stm/misc.c"
 #include "stm/list.c"
 #include "stm/pagecopy.c"
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -2,7 +2,6 @@
 #ifndef _STMGC_H
 #define _STMGC_H
 
-
 /* ==================== INTERNAL ==================== */
 
 /* See "API" below. */

From noreply at buildbot.pypy.org  Sun Feb 22 18:23:01 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Feb 2015 18:23:01 +0100 (CET)
Subject: [pypy-commit] pypy default: Painfully insert the same NOPs as gcc
 does before labels to align them
Message-ID: <20150222172301.558D61C0041@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76054:b6b0af53a51d
Date: 2015-02-22 18:22 +0100
http://bitbucket.org/pypy/pypy/changeset/b6b0af53a51d/

Log:	Painfully insert the same NOPs as gcc does before labels to align
	them to 16 bytes.

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -339,14 +339,21 @@
             self.possibly_free_var(arg)
 
     def flush_loop(self):
+        # Force the code to be aligned to a multiple of 16.  Also,
         # rare case: if the loop is too short, or if we are just after
-        # a GUARD_NOT_INVALIDATED, pad with NOPs.  Important!  This must
-        # be called to ensure that there are enough bytes produced,
-        # because GUARD_NOT_INVALIDATED or redirect_call_assembler()
-        # will maybe overwrite them.
+        # a GUARD_NOT_INVALIDATED, we need to make sure we insert enough
+        # NOPs.  This is important to ensure that there are enough bytes
+        # produced, because GUARD_NOT_INVALIDATED or
+        # redirect_call_assembler() will maybe overwrite them.  (In that
+        # rare case we don't worry too much about alignment.)
         mc = self.assembler.mc
-        while mc.get_relative_pos() < self.min_bytes_before_label:
-            mc.NOP()
+        current_pos = mc.get_relative_pos()
+        target_pos = (current_pos + 15) & ~15
+        target_pos = max(target_pos, self.min_bytes_before_label)
+        insert_nops = target_pos - current_pos
+        assert 0 <= insert_nops <= 15
+        for c in mc.MULTIBYTE_NOPs[insert_nops]:
+            mc.writechar(c)
 
     def loc(self, v):
         if v is None: # xxx kludgy
diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -671,11 +671,39 @@
 def invert_condition(cond_num):
     return cond_num ^ 1
 
+
 class X86_32_CodeBuilder(AbstractX86CodeBuilder):
     WORD = 4
 
     PMOVMSKB_rx = xmminsn('\x66', rex_nw, '\x0F\xD7', register(1, 8), register(2), '\xC0')
 
+    # multibyte nops, from 0 to 15 bytes
+    MULTIBYTE_NOPs = [
+        '',
+        '\x90',                          # nop
+        '\x66\x90',                      # xchg ax, ax
+        '\x8d\x76\x00',                  # lea    0x0(%esi),%esi
+        '\x8d\x74\x26\x00',              # lea    0x0(%esi,%eiz,1),%esi
+        '\x90\x8d\x74\x26\x00',          # nop; lea 0x0(%esi,%eiz,1),%esi
+        '\x8d\xb6\x00\x00\x00\x00',      # lea    0x0(%esi),%esi
+        '\x8d\xb4\x26\x00\x00\x00\x00',  # lea    0x0(%esi,%eiz,1),%esi
+        ('\x90'                          # nop
+         '\x8d\xb4\x26\x00\x00\x00\x00'),#   lea    0x0(%esi,%eiz,1),%esi
+        ('\x89\xf6'                      # mov    %esi,%esi
+         '\x8d\xbc\x27\x00\x00\x00\x00'),#   lea    0x0(%edi,%eiz,1),%edi
+        ('\x8d\x76\x00'                  # lea    0x0(%esi),%esi
+         '\x8d\xbc\x27\x00\x00\x00\x00'),#   lea    0x0(%edi,%eiz,1),%edi
+        ('\x8d\x74\x26\x00'              # lea    0x0(%esi,%eiz,1),%esi
+         '\x8d\xbc\x27\x00\x00\x00\x00'),#   lea    0x0(%edi,%eiz,1),%edi
+        ('\x8d\xb6\x00\x00\x00\x00'      # lea    0x0(%esi),%esi
+         '\x8d\xbf\x00\x00\x00\x00'),    #   lea    0x0(%edi),%edi
+        ('\x8d\xb6\x00\x00\x00\x00'      # lea    0x0(%esi),%esi
+         '\x8d\xbc\x27\x00\x00\x00\x00'),#   lea    0x0(%edi,%eiz,1),%edi
+        ('\x8d\xb4\x26\x00\x00\x00\x00'  # lea    0x0(%esi,%eiz,1),%esi
+         '\x8d\xbc\x27\x00\x00\x00\x00'),#   lea    0x0(%edi,%eiz,1),%edi
+        ('\xeb\x0d' + '\x90' * 13)]      # jmp +x0d; a bunch of nops
+
+
 class X86_64_CodeBuilder(AbstractX86CodeBuilder):
     WORD = 8
 
@@ -706,6 +734,24 @@
         else:
             self.MOV_ri64(reg, immed)
 
+    # multibyte nops, from 0 to 15 bytes
+    MULTIBYTE_NOPs = ([
+        '',
+        '\x90',                          # nop
+        '\x66\x90',                      # xchg ax, ax
+        '\x0f\x1f\x00',                  # nopl   (%rax)
+        '\x0f\x1f\x40\x00',              # nopl   0x0(%rax)
+        '\x0f\x1f\x44\x00\x00',          # nopl   0x0(%rax,%rax,1)
+        '\x66\x0f\x1f\x44\x00\x00',      # nopw   0x0(%rax,%rax,1)
+        '\x0f\x1f\x80\x00\x00\x00\x00',  # nopl   0x0(%rax)
+        ('\x0f\x1f\x84\x00\x00\x00\x00'  # nopl   0x0(%rax,%rax,1)
+         '\x00'),
+        ('\x66\x0f\x1f\x84\x00\x00\x00'  # nopw   0x0(%rax,%rax,1)
+         '\x00\x00')] +
+        ['\x66' * _i + '\x2e\x0f\x1f'    # nopw   %cs:0x0(%rax,%rax,1)
+         '\x84\x00\x00\x00\x00\x00' for _i in range(1, 7)])
+
+
 def define_modrm_modes(insnname_template, before_modrm, after_modrm=[], regtype='GPR'):
     def add_insn(code, *modrm):
         args = before_modrm + list(modrm)
diff --git a/rpython/jit/backend/x86/test/test_runner.py b/rpython/jit/backend/x86/test/test_runner.py
--- a/rpython/jit/backend/x86/test/test_runner.py
+++ b/rpython/jit/backend/x86/test/test_runner.py
@@ -30,14 +30,21 @@
     # for the individual tests see
     # ====> ../../test/runner_test.py
 
-    add_loop_instructions = ['mov', 'add', 'test', 'je', 'jmp']
+    add_loop_instructions = ['mov',
+                             'nop',    # for the label
+                             'add', 'test', 'je', 'jmp',
+                             'data32',   # padding
+                             ]
     if WORD == 4:
-        bridge_loop_instructions = ['cmp', 'jge', 'mov', 'mov', 'call', 'jmp']
+        bridge_loop_instructions = ['cmp', 'jge', 'mov', 'mov', 'call', 'jmp'
+                                    'nop']   # padding
     else:
         bridge_loop_instructions = [
-            'cmp', 'jge', 'mov', 'mov', 'mov', 'mov', 'call', 'mov', 'jmp']
+            'cmp', 'jge', 'mov', 'mov', 'mov', 'mov', 'call', 'mov', 'jmp',
+            'nop']      # padding
         bridge_loop_instructions_alternative = [
-            'cmp', 'jge', 'mov', 'mov', 'mov', 'call', 'mov', 'jmp']
+            'cmp', 'jge', 'mov', 'mov', 'mov', 'call', 'mov', 'jmp',
+            'nop']      # padding
 
     def get_cpu(self):
         cpu = CPU(rtyper=None, stats=FakeStats())
diff --git a/rpython/jit/backend/x86/test/test_rx86.py b/rpython/jit/backend/x86/test/test_rx86.py
--- a/rpython/jit/backend/x86/test/test_rx86.py
+++ b/rpython/jit/backend/x86/test/test_rx86.py
@@ -229,3 +229,9 @@
     s = CodeBuilder64()
     s.MOVSD_xj(xmm2, 0x01234567)
     assert s.getvalue() == '\xF2\x0F\x10\x14\x25\x67\x45\x23\x01'
+
+def test_multibyte_nops():
+    for cls in [X86_64_CodeBuilder, X86_32_CodeBuilder]:
+        assert len(cls.MULTIBYTE_NOPs) == 16
+        for i in range(16):
+            assert len(cls.MULTIBYTE_NOPs[i]) == i

From noreply at buildbot.pypy.org  Sun Feb 22 18:23:02 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Feb 2015 18:23:02 +0100 (CET)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20150222172302.7E1641C0041@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76055:6419bd45b4a3
Date: 2015-02-22 18:22 +0100
http://bitbucket.org/pypy/pypy/changeset/6419bd45b4a3/

Log:	merge heads

diff --git a/lib_pypy/gdbm.py b/lib_pypy/gdbm.py
--- a/lib_pypy/gdbm.py
+++ b/lib_pypy/gdbm.py
@@ -20,9 +20,11 @@
 } datum;
 
 datum gdbm_fetch(void*, datum);
+datum pygdbm_fetch(void*, char*, int);
 int gdbm_delete(void*, datum);
 int gdbm_store(void*, datum, datum, int);
 int gdbm_exists(void*, datum);
+int pygdbm_exists(void*, char*, int);
 
 int gdbm_reorganize(void*);
 
@@ -37,19 +39,29 @@
 ''')
 
 try:
+    verify_code = '''
+    #include "gdbm.h"
+
+    static datum pygdbm_fetch(GDBM_FILE gdbm_file, char *dptr, int dsize) {
+        datum key = {dptr, dsize};
+        return gdbm_fetch(gdbm_file, key);
+    }
+
+    static int pygdbm_exists(GDBM_FILE gdbm_file, char *dptr, int dsize) {
+        datum key = {dptr, dsize};
+        return gdbm_exists(gdbm_file, key);
+    }
+    
+    '''
     if sys.platform.startswith('freebsd'):
         import os.path
         _localbase = os.environ.get('LOCALBASE', '/usr/local')
-        lib = ffi.verify('''
-        #include "gdbm.h"
-        ''', libraries=['gdbm'],
+        lib = ffi.verify(verify_code, libraries=['gdbm'],
              include_dirs=[os.path.join(_localbase, 'include')],
              library_dirs=[os.path.join(_localbase, 'lib')]
         )
     else:
-        lib = ffi.verify('''
-        #include "gdbm.h"
-        ''', libraries=['gdbm'])
+        lib = ffi.verify(verify_code, libraries=['gdbm'])
 except cffi.VerificationError as e:
     # distutils does not preserve the actual message,
     # but the verification is simple enough that the
@@ -59,6 +71,13 @@
 class error(Exception):
     pass
 
+def _checkstr(key):
+    if isinstance(key, unicode):
+        key = key.encode("ascii")
+    if not isinstance(key, str):
+        raise TypeError("gdbm mappings have string indices only")
+    return key
+
 def _fromstr(key):
     if isinstance(key, unicode):
         key = key.encode("ascii")
@@ -107,12 +126,14 @@
 
     def __contains__(self, key):
         self._check_closed()
-        return lib.gdbm_exists(self.ll_dbm, _fromstr(key))
+        key = _checkstr(key)
+        return lib.pygdbm_exists(self.ll_dbm, key, len(key)
     has_key = __contains__
 
     def __getitem__(self, key):
         self._check_closed()
-        drec = lib.gdbm_fetch(self.ll_dbm, _fromstr(key))
+        key = _checkstr(key)        
+        drec = lib.pygdbm_fetch(self.ll_dbm, key, len(key))
         if not drec.dptr:
             raise KeyError(key)
         res = str(ffi.buffer(drec.dptr, drec.dsize))
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -30,3 +30,7 @@
 .. branch: alt_errno
 Add an alternative location to save LastError, errno around ctypes,
 cffi external calls so things like pdb will not overwrite it
+
+.. branch: nonquadratic-heapcache
+Speed up the warmup times of the JIT by removing a quadratic algorithm in the
+heapcache.
diff --git a/rpython/jit/metainterp/test/test_recursive.py b/rpython/jit/metainterp/test/test_recursive.py
--- a/rpython/jit/metainterp/test/test_recursive.py
+++ b/rpython/jit/metainterp/test/test_recursive.py
@@ -794,7 +794,7 @@
             return frame.thing.val + s
 
         res = self.meta_interp(main, [0], inline=True)
-        self.check_resops(call=0, cond_call=0) # got removed by optimization
+        self.check_resops(call=0, cond_call=2)
         assert res == main(0)
 
     def test_directly_call_assembler_virtualizable_reset_token(self):

From noreply at buildbot.pypy.org  Sun Feb 22 18:25:46 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Sun, 22 Feb 2015 18:25:46 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Remove unneeded code in gendfa -- the
 generated file doesn't change.
Message-ID: <20150222172546.E67711C009D@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76056:0644f2470b91
Date: 2015-02-22 18:25 +0100
http://bitbucket.org/pypy/pypy/changeset/0644f2470b91/

Log:	Remove unneeded code in gendfa -- the generated file doesn't change.

diff --git a/pypy/interpreter/pyparser/gendfa.py b/pypy/interpreter/pyparser/gendfa.py
--- a/pypy/interpreter/pyparser/gendfa.py
+++ b/pypy/interpreter/pyparser/gendfa.py
@@ -254,20 +254,10 @@
                               any(states, notGroupStr(states, '"\\')))),
                     chainStr(states, '"""'))
     double3DFA = NonGreedyDFA(*nfaToDfa(states, *double3))
-    map = {"'" : singleDFA,
-           '"' : doubleDFA,
-           "r" : None,
-           "R" : None,
-           "u" : None,
-           "U" : None,
-           "b" : None,
-           "B" : None}
-    for uniPrefix in ("", "u", "U", "b", "B", ):
-        for rawPrefix in ("", "r", "R"):
-            prefix = uniPrefix + rawPrefix
-            map[prefix + "'''"] = single3DFA
-            map[prefix + '"""'] = double3DFA
-    return map
+    return {"'" : singleDFA,
+            '"' : doubleDFA,
+            "'''": single3DFA,
+            '"""': double3DFA}
 
 # ______________________________________________________________________
 

From noreply at buildbot.pypy.org  Sun Feb 22 18:26:04 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Feb 2015 18:26:04 +0100 (CET)
Subject: [pypy-commit] pypy default: Fix on 32-bit
Message-ID: <20150222172604.242E61C009D@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76057:097db6dff79b
Date: 2015-02-22 18:26 +0100
http://bitbucket.org/pypy/pypy/changeset/097db6dff79b/

Log:	Fix on 32-bit

diff --git a/rpython/jit/backend/x86/test/test_runner.py b/rpython/jit/backend/x86/test/test_runner.py
--- a/rpython/jit/backend/x86/test/test_runner.py
+++ b/rpython/jit/backend/x86/test/test_runner.py
@@ -30,15 +30,18 @@
     # for the individual tests see
     # ====> ../../test/runner_test.py
 
-    add_loop_instructions = ['mov',
-                             'nop',    # for the label
-                             'add', 'test', 'je', 'jmp',
-                             'data32',   # padding
-                             ]
     if WORD == 4:
-        bridge_loop_instructions = ['cmp', 'jge', 'mov', 'mov', 'call', 'jmp'
-                                    'nop']   # padding
+        add_loop_instructions = ['mov',
+                                 'lea',    # a nop, for the label
+                                 'add', 'test', 'je', 'jmp',
+                                 'nop']    # padding
+        bridge_loop_instructions = ['cmp', 'jge', 'mov', 'mov', 'call', 'jmp',
+                                    'lea', 'lea']   # padding
     else:
+        add_loop_instructions = ['mov',
+                                 'nop',    # for the label
+                                 'add', 'test', 'je', 'jmp',
+                                 'data32']   # padding
         bridge_loop_instructions = [
             'cmp', 'jge', 'mov', 'mov', 'mov', 'mov', 'call', 'mov', 'jmp',
             'nop']      # padding

From noreply at buildbot.pypy.org  Sun Feb 22 18:45:48 2015
From: noreply at buildbot.pypy.org (xando)
Date: Sun, 22 Feb 2015 18:45:48 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: (xando,
	mjacob) implementing new string literals syntax
Message-ID: <20150222174548.C40521C0271@cobra.cs.uni-duesseldorf.de>

Author: Sebastian Pawlu? 
Branch: py3.3
Changeset: r76058:527e933dbf5a
Date: 2015-02-22 18:45 +0100
http://bitbucket.org/pypy/pypy/changeset/527e933dbf5a/

Log:	(xando, mjacob) implementing new string literals syntax

diff --git a/pypy/interpreter/pyparser/dfa_generated.py b/pypy/interpreter/pyparser/dfa_generated.py
--- a/pypy/interpreter/pyparser/dfa_generated.py
+++ b/pypy/interpreter/pyparser/dfa_generated.py
@@ -5,39 +5,39 @@
 
 from pypy.interpreter.pyparser import automata
 accepts = [True, True, True, True, True, True, True, True,
-           True, True, False, True, True, True, True, True,
-           False, False, False, True, False, False, False,
-           True, False, True, False, True, False, False,
-           True, False, False, True, False, False, True,
-           True, True, False, False, True, False, False,
-           False, True]
+           True, True, True, False, True, True, True, True,
+           True, False, False, False, True, False, False,
+           False, True, False, True, False, True, False,
+           False, True, False, False, True, False, False,
+           True, True, True, False, False, True, False,
+           False, False, True]
 states = [
     # 0
-    {'\t': 0, '\n': 14, '\x0c': 0,
-     '\r': 15, ' ': 0, '!': 10, '"': 17,
-     '#': 19, '%': 13, '&': 13, "'": 16,
-     '(': 14, ')': 14, '*': 7, '+': 13,
-     ',': 14, '-': 11, '.': 6, '/': 12,
-     '0': 4, '1': 5, '2': 5, '3': 5,
-     '4': 5, '5': 5, '6': 5, '7': 5,
-     '8': 5, '9': 5, ':': 14, ';': 14,
-     '<': 9, '=': 13, '>': 8, '@': 14,
+    {'\t': 0, '\n': 15, '\x0c': 0,
+     '\r': 16, ' ': 0, '!': 11, '"': 18,
+     '#': 20, '%': 14, '&': 14, "'": 17,
+     '(': 15, ')': 15, '*': 8, '+': 14,
+     ',': 15, '-': 12, '.': 7, '/': 13,
+     '0': 5, '1': 6, '2': 6, '3': 6,
+     '4': 6, '5': 6, '6': 6, '7': 6,
+     '8': 6, '9': 6, ':': 15, ';': 15,
+     '<': 10, '=': 14, '>': 9, '@': 15,
      'A': 1, 'B': 2, 'C': 1, 'D': 1,
      'E': 1, 'F': 1, 'G': 1, 'H': 1,
      'I': 1, 'J': 1, 'K': 1, 'L': 1,
      'M': 1, 'N': 1, 'O': 1, 'P': 1,
      'Q': 1, 'R': 3, 'S': 1, 'T': 1,
-     'U': 3, 'V': 1, 'W': 1, 'X': 1,
-     'Y': 1, 'Z': 1, '[': 14, '\\': 18,
-     ']': 14, '^': 13, '_': 1, '`': 14,
+     'U': 4, 'V': 1, 'W': 1, 'X': 1,
+     'Y': 1, 'Z': 1, '[': 15, '\\': 19,
+     ']': 15, '^': 14, '_': 1, '`': 15,
      'a': 1, 'b': 2, 'c': 1, 'd': 1,
      'e': 1, 'f': 1, 'g': 1, 'h': 1,
      'i': 1, 'j': 1, 'k': 1, 'l': 1,
      'm': 1, 'n': 1, 'o': 1, 'p': 1,
      'q': 1, 'r': 3, 's': 1, 't': 1,
-     'u': 3, 'v': 1, 'w': 1, 'x': 1,
-     'y': 1, 'z': 1, '{': 14, '|': 13,
-     '}': 14, '~': 14, '\x80': 1},
+     'u': 4, 'v': 1, 'w': 1, 'x': 1,
+     'y': 1, 'z': 1, '{': 15, '|': 14,
+     '}': 15, '~': 15, '\x80': 1},
     # 1
     {'0': 1, '1': 1, '2': 1, '3': 1,
      '4': 1, '5': 1, '6': 1, '7': 1,
@@ -56,25 +56,43 @@
      't': 1, 'u': 1, 'v': 1, 'w': 1,
      'x': 1, 'y': 1, 'z': 1, '\x80': 1},
     # 2
-    {'"': 17, "'": 16, '0': 1, '1': 1,
+    {'"': 18, "'": 17, '0': 1, '1': 1,
      '2': 1, '3': 1, '4': 1, '5': 1,
      '6': 1, '7': 1, '8': 1, '9': 1,
      'A': 1, 'B': 1, 'C': 1, 'D': 1,
      'E': 1, 'F': 1, 'G': 1, 'H': 1,
      'I': 1, 'J': 1, 'K': 1, 'L': 1,
      'M': 1, 'N': 1, 'O': 1, 'P': 1,
-     'Q': 1, 'R': 3, 'S': 1, 'T': 1,
+     'Q': 1, 'R': 4, 'S': 1, 'T': 1,
      'U': 1, 'V': 1, 'W': 1, 'X': 1,
      'Y': 1, 'Z': 1, '_': 1, 'a': 1,
      'b': 1, 'c': 1, 'd': 1, 'e': 1,
      'f': 1, 'g': 1, 'h': 1, 'i': 1,
      'j': 1, 'k': 1, 'l': 1, 'm': 1,
      'n': 1, 'o': 1, 'p': 1, 'q': 1,
-     'r': 3, 's': 1, 't': 1, 'u': 1,
+     'r': 4, 's': 1, 't': 1, 'u': 1,
      'v': 1, 'w': 1, 'x': 1, 'y': 1,
      'z': 1, '\x80': 1},
     # 3
-    {'"': 17, "'": 16, '0': 1, '1': 1,
+    {'"': 18, "'": 17, '0': 1, '1': 1,
+     '2': 1, '3': 1, '4': 1, '5': 1,
+     '6': 1, '7': 1, '8': 1, '9': 1,
+     'A': 1, 'B': 4, 'C': 1, 'D': 1,
+     'E': 1, 'F': 1, 'G': 1, 'H': 1,
+     'I': 1, 'J': 1, 'K': 1, 'L': 1,
+     'M': 1, 'N': 1, 'O': 1, 'P': 1,
+     'Q': 1, 'R': 1, 'S': 1, 'T': 1,
+     'U': 1, 'V': 1, 'W': 1, 'X': 1,
+     'Y': 1, 'Z': 1, '_': 1, 'a': 1,
+     'b': 4, 'c': 1, 'd': 1, 'e': 1,
+     'f': 1, 'g': 1, 'h': 1, 'i': 1,
+     'j': 1, 'k': 1, 'l': 1, 'm': 1,
+     'n': 1, 'o': 1, 'p': 1, 'q': 1,
+     'r': 1, 's': 1, 't': 1, 'u': 1,
+     'v': 1, 'w': 1, 'x': 1, 'y': 1,
+     'z': 1, '\x80': 1},
+    # 4
+    {'"': 18, "'": 17, '0': 1, '1': 1,
      '2': 1, '3': 1, '4': 1, '5': 1,
      '6': 1, '7': 1, '8': 1, '9': 1,
      'A': 1, 'B': 1, 'C': 1, 'D': 1,
@@ -91,142 +109,142 @@
      'r': 1, 's': 1, 't': 1, 'u': 1,
      'v': 1, 'w': 1, 'x': 1, 'y': 1,
      'z': 1, '\x80': 1},
-    # 4
-    {'.': 25, '0': 23, '1': 24, '2': 24,
-     '3': 24, '4': 24, '5': 24, '6': 24,
-     '7': 24, '8': 24, '9': 24, 'B': 22,
-     'E': 26, 'J': 14, 'O': 21, 'X': 20,
-     'b': 22, 'e': 26, 'j': 14, 'o': 21,
-     'x': 20},
     # 5
-    {'.': 25, '0': 5, '1': 5, '2': 5,
-     '3': 5, '4': 5, '5': 5, '6': 5,
-     '7': 5, '8': 5, '9': 5, 'E': 26,
-     'J': 14, 'e': 26, 'j': 14},
+    {'.': 26, '0': 24, '1': 25, '2': 25,
+     '3': 25, '4': 25, '5': 25, '6': 25,
+     '7': 25, '8': 25, '9': 25, 'B': 23,
+     'E': 27, 'J': 15, 'O': 22, 'X': 21,
+     'b': 23, 'e': 27, 'j': 15, 'o': 22,
+     'x': 21},
     # 6
-    {'.': 28, '0': 27, '1': 27, '2': 27,
-     '3': 27, '4': 27, '5': 27, '6': 27,
-     '7': 27, '8': 27, '9': 27},
+    {'.': 26, '0': 6, '1': 6, '2': 6,
+     '3': 6, '4': 6, '5': 6, '6': 6,
+     '7': 6, '8': 6, '9': 6, 'E': 27,
+     'J': 15, 'e': 27, 'j': 15},
     # 7
-    {'*': 13, '=': 14},
+    {'.': 29, '0': 28, '1': 28, '2': 28,
+     '3': 28, '4': 28, '5': 28, '6': 28,
+     '7': 28, '8': 28, '9': 28},
     # 8
-    {'=': 14, '>': 13},
+    {'*': 14, '=': 15},
     # 9
-    {'<': 13, '=': 14, '>': 14},
+    {'=': 15, '>': 14},
     # 10
-    {'=': 14},
+    {'<': 14, '=': 15, '>': 15},
     # 11
-    {'=': 14, '>': 14},
+    {'=': 15},
     # 12
-    {'/': 13, '=': 14},
+    {'=': 15, '>': 15},
     # 13
-    {'=': 14},
+    {'/': 14, '=': 15},
     # 14
+    {'=': 15},
+    # 15
     {},
-    # 15
-    {'\n': 14},
     # 16
-    {automata.DEFAULT: 32, '\n': 29,
-     '\r': 29, "'": 30, '\\': 31},
+    {'\n': 15},
     # 17
-    {automata.DEFAULT: 35, '\n': 29,
-     '\r': 29, '"': 33, '\\': 34},
+    {automata.DEFAULT: 33, '\n': 30,
+     '\r': 30, "'": 31, '\\': 32},
     # 18
-    {'\n': 14, '\r': 15},
+    {automata.DEFAULT: 36, '\n': 30,
+     '\r': 30, '"': 34, '\\': 35},
     # 19
-    {automata.DEFAULT: 19, '\n': 29, '\r': 29},
+    {'\n': 15, '\r': 16},
     # 20
-    {'0': 36, '1': 36, '2': 36, '3': 36,
-     '4': 36, '5': 36, '6': 36, '7': 36,
-     '8': 36, '9': 36, 'A': 36, 'B': 36,
-     'C': 36, 'D': 36, 'E': 36, 'F': 36,
-     'a': 36, 'b': 36, 'c': 36, 'd': 36,
-     'e': 36, 'f': 36},
+    {automata.DEFAULT: 20, '\n': 30, '\r': 30},
     # 21
     {'0': 37, '1': 37, '2': 37, '3': 37,
-     '4': 37, '5': 37, '6': 37, '7': 37},
+     '4': 37, '5': 37, '6': 37, '7': 37,
+     '8': 37, '9': 37, 'A': 37, 'B': 37,
+     'C': 37, 'D': 37, 'E': 37, 'F': 37,
+     'a': 37, 'b': 37, 'c': 37, 'd': 37,
+     'e': 37, 'f': 37},
     # 22
-    {'0': 38, '1': 38},
+    {'0': 38, '1': 38, '2': 38, '3': 38,
+     '4': 38, '5': 38, '6': 38, '7': 38},
     # 23
-    {'.': 25, '0': 23, '1': 24, '2': 24,
-     '3': 24, '4': 24, '5': 24, '6': 24,
-     '7': 24, '8': 24, '9': 24, 'E': 26,
-     'J': 14, 'e': 26, 'j': 14},
+    {'0': 39, '1': 39},
     # 24
-    {'.': 25, '0': 24, '1': 24, '2': 24,
-     '3': 24, '4': 24, '5': 24, '6': 24,
-     '7': 24, '8': 24, '9': 24, 'E': 26,
-     'J': 14, 'e': 26, 'j': 14},
+    {'.': 26, '0': 24, '1': 25, '2': 25,
+     '3': 25, '4': 25, '5': 25, '6': 25,
+     '7': 25, '8': 25, '9': 25, 'E': 27,
+     'J': 15, 'e': 27, 'j': 15},
     # 25
-    {'0': 25, '1': 25, '2': 25, '3': 25,
-     '4': 25, '5': 25, '6': 25, '7': 25,
-     '8': 25, '9': 25, 'E': 39, 'J': 14,
-     'e': 39, 'j': 14},
+    {'.': 26, '0': 25, '1': 25, '2': 25,
+     '3': 25, '4': 25, '5': 25, '6': 25,
+     '7': 25, '8': 25, '9': 25, 'E': 27,
+     'J': 15, 'e': 27, 'j': 15},
     # 26
-    {'+': 40, '-': 40, '0': 41, '1': 41,
-     '2': 41, '3': 41, '4': 41, '5': 41,
-     '6': 41, '7': 41, '8': 41, '9': 41},
+    {'0': 26, '1': 26, '2': 26, '3': 26,
+     '4': 26, '5': 26, '6': 26, '7': 26,
+     '8': 26, '9': 26, 'E': 40, 'J': 15,
+     'e': 40, 'j': 15},
     # 27
-    {'0': 27, '1': 27, '2': 27, '3': 27,
-     '4': 27, '5': 27, '6': 27, '7': 27,
-     '8': 27, '9': 27, 'E': 39, 'J': 14,
-     'e': 39, 'j': 14},
+    {'+': 41, '-': 41, '0': 42, '1': 42,
+     '2': 42, '3': 42, '4': 42, '5': 42,
+     '6': 42, '7': 42, '8': 42, '9': 42},
     # 28
-    {'.': 14},
+    {'0': 28, '1': 28, '2': 28, '3': 28,
+     '4': 28, '5': 28, '6': 28, '7': 28,
+     '8': 28, '9': 28, 'E': 40, 'J': 15,
+     'e': 40, 'j': 15},
     # 29
+    {'.': 15},
+    # 30
     {},
-    # 30
-    {"'": 14},
     # 31
-    {automata.DEFAULT: 42, '\n': 14, '\r': 15},
+    {"'": 15},
     # 32
-    {automata.DEFAULT: 32, '\n': 29,
-     '\r': 29, "'": 14, '\\': 31},
+    {automata.DEFAULT: 43, '\n': 15, '\r': 16},
     # 33
-    {'"': 14},
+    {automata.DEFAULT: 33, '\n': 30,
+     '\r': 30, "'": 15, '\\': 32},
     # 34
-    {automata.DEFAULT: 43, '\n': 14, '\r': 15},
+    {'"': 15},
     # 35
-    {automata.DEFAULT: 35, '\n': 29,
-     '\r': 29, '"': 14, '\\': 34},
+    {automata.DEFAULT: 44, '\n': 15, '\r': 16},
     # 36
-    {'0': 36, '1': 36, '2': 36, '3': 36,
-     '4': 36, '5': 36, '6': 36, '7': 36,
-     '8': 36, '9': 36, 'A': 36, 'B': 36,
-     'C': 36, 'D': 36, 'E': 36, 'F': 36,
-     'a': 36, 'b': 36, 'c': 36, 'd': 36,
-     'e': 36, 'f': 36},
+    {automata.DEFAULT: 36, '\n': 30,
+     '\r': 30, '"': 15, '\\': 35},
     # 37
     {'0': 37, '1': 37, '2': 37, '3': 37,
-     '4': 37, '5': 37, '6': 37, '7': 37},
+     '4': 37, '5': 37, '6': 37, '7': 37,
+     '8': 37, '9': 37, 'A': 37, 'B': 37,
+     'C': 37, 'D': 37, 'E': 37, 'F': 37,
+     'a': 37, 'b': 37, 'c': 37, 'd': 37,
+     'e': 37, 'f': 37},
     # 38
-    {'0': 38, '1': 38},
+    {'0': 38, '1': 38, '2': 38, '3': 38,
+     '4': 38, '5': 38, '6': 38, '7': 38},
     # 39
-    {'+': 44, '-': 44, '0': 45, '1': 45,
-     '2': 45, '3': 45, '4': 45, '5': 45,
-     '6': 45, '7': 45, '8': 45, '9': 45},
+    {'0': 39, '1': 39},
     # 40
-    {'0': 41, '1': 41, '2': 41, '3': 41,
-     '4': 41, '5': 41, '6': 41, '7': 41,
-     '8': 41, '9': 41},
+    {'+': 45, '-': 45, '0': 46, '1': 46,
+     '2': 46, '3': 46, '4': 46, '5': 46,
+     '6': 46, '7': 46, '8': 46, '9': 46},
     # 41
-    {'0': 41, '1': 41, '2': 41, '3': 41,
-     '4': 41, '5': 41, '6': 41, '7': 41,
-     '8': 41, '9': 41, 'J': 14, 'j': 14},
+    {'0': 42, '1': 42, '2': 42, '3': 42,
+     '4': 42, '5': 42, '6': 42, '7': 42,
+     '8': 42, '9': 42},
     # 42
-    {automata.DEFAULT: 42, '\n': 29,
-     '\r': 29, "'": 14, '\\': 31},
+    {'0': 42, '1': 42, '2': 42, '3': 42,
+     '4': 42, '5': 42, '6': 42, '7': 42,
+     '8': 42, '9': 42, 'J': 15, 'j': 15},
     # 43
-    {automata.DEFAULT: 43, '\n': 29,
-     '\r': 29, '"': 14, '\\': 34},
+    {automata.DEFAULT: 43, '\n': 30,
+     '\r': 30, "'": 15, '\\': 32},
     # 44
-    {'0': 45, '1': 45, '2': 45, '3': 45,
-     '4': 45, '5': 45, '6': 45, '7': 45,
-     '8': 45, '9': 45},
+    {automata.DEFAULT: 44, '\n': 30,
+     '\r': 30, '"': 15, '\\': 35},
     # 45
-    {'0': 45, '1': 45, '2': 45, '3': 45,
-     '4': 45, '5': 45, '6': 45, '7': 45,
-     '8': 45, '9': 45, 'J': 14, 'j': 14},
+    {'0': 46, '1': 46, '2': 46, '3': 46,
+     '4': 46, '5': 46, '6': 46, '7': 46,
+     '8': 46, '9': 46},
+    # 46
+    {'0': 46, '1': 46, '2': 46, '3': 46,
+     '4': 46, '5': 46, '6': 46, '7': 46,
+     '8': 46, '9': 46, 'J': 15, 'j': 15},
     ]
 pseudoDFA = automata.DFA(states, accepts)
 
diff --git a/pypy/interpreter/pyparser/gendfa.py b/pypy/interpreter/pyparser/gendfa.py
--- a/pypy/interpreter/pyparser/gendfa.py
+++ b/pypy/interpreter/pyparser/gendfa.py
@@ -151,6 +151,9 @@
     def makeStrPrefix ():
         return group(states,
                      chain(states,
+                           maybe(states, groupStr(states, "rR")),
+                           maybe(states, groupStr(states, "bB"))),
+                     chain(states,
                            maybe(states, groupStr(states, "bB")),
                            maybe(states, groupStr(states, "rR"))),
                      maybe(states, groupStr(states, "uU")))
diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -33,10 +33,21 @@
         ps += 1
         quote = s[ps]
         saw_u = True
-    if not saw_u and quote == 'r' or quote == 'R':
+    elif quote == 'r' or quote == 'R':
         ps += 1
         quote = s[ps]
         rawmode = True
+
+    if not saw_u:
+        if quote == 'r' or quote == 'R':
+            ps += 1
+            quote = s[ps]
+            rawmode = True
+        elif quote == 'b' or quote == 'B':
+            ps += 1
+            quote = s[ps]
+            unicode_literal = False
+
     if quote != "'" and quote != '"':
         raise_app_valueerror(space,
                              'Internal error: parser passed unquoted literal')
diff --git a/pypy/interpreter/pyparser/pytokenize.py b/pypy/interpreter/pyparser/pytokenize.py
--- a/pypy/interpreter/pyparser/pytokenize.py
+++ b/pypy/interpreter/pyparser/pytokenize.py
@@ -32,9 +32,14 @@
 
 for uniPrefix in ("", "b", "B"):
     for rawPrefix in ("", "r", "R"):
-        prefix = uniPrefix + rawPrefix
-        endDFAs[prefix + "'''"] = single3DFA
-        endDFAs[prefix + '"""'] = double3DFA
+        prefix_1 = uniPrefix + rawPrefix
+        prefix_2 = rawPrefix + uniPrefix
+
+        endDFAs[prefix_1 + "'''"] = single3DFA
+        endDFAs[prefix_1 + '"""'] = double3DFA
+        endDFAs[prefix_2 + "'''"] = single3DFA
+        endDFAs[prefix_2 + '"""'] = double3DFA
+
 for uniPrefix in ("u", "U"):
     endDFAs[uniPrefix + "'''"] = single3DFA
     endDFAs[uniPrefix + '"""'] = double3DFA
@@ -52,7 +57,9 @@
           "u'''", 'u"""', "U'''", 'U"""',
           "b'''", 'b"""', "B'''", 'B"""',
           "br'''", 'br"""', "Br'''", 'Br"""',
-          "bR'''", 'bR"""', "BR'''", 'BR"""'):
+          "bR'''", 'bR"""', "BR'''", 'BR"""',
+          "rb'''", 'rb"""', "rB'''", 'rB"""',
+          "Rb'''", 'Rb"""', "RB'''", 'RB"""'):
     triple_quoted[t] = t
 single_quoted = {}
 for t in ("'", '"',
@@ -60,7 +67,10 @@
           "u'", 'u"', "U'", 'U"',
           "b'", 'b"', "B'", 'B"',
           "br'", 'br"', "Br'", 'Br"',
-          "bR'", 'bR"', "BR'", 'BR"'):
+          "bR'", 'bR"', "BR'", 'BR"',
+          "rb'", 'rb"', "rB'", 'rB"',
+          "Rb'", 'Rb"', "RB'", 'RB"'):
+
     single_quoted[t] = t
 
 tabsize = 8
diff --git a/pypy/interpreter/pyparser/test/test_parsestring.py b/pypy/interpreter/pyparser/test/test_parsestring.py
--- a/pypy/interpreter/pyparser/test/test_parsestring.py
+++ b/pypy/interpreter/pyparser/test/test_parsestring.py
@@ -23,6 +23,8 @@
 
         # octal
         self.parse_and_compare(r'b"\0"', chr(0))
+        self.parse_and_compare(r'br"\0"', '\\0')
+        self.parse_and_compare(r'rb"\0"', '\\0')
         self.parse_and_compare(r'b"\07"', chr(7))
         self.parse_and_compare(r'b"\123"', chr(0123))
         self.parse_and_compare(r'b"\400"', chr(0))
diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py
--- a/pypy/interpreter/test/test_syntax.py
+++ b/pypy/interpreter/test/test_syntax.py
@@ -259,13 +259,33 @@
 x = 'u'
 y = r'u'
 b = b'u'
-c = br'u'"""
+c = br'u'
+d = rb'u'
+"""
         ns = {}
         exec(s, ns)
         assert isinstance(ns["x"], str)
         assert isinstance(ns["y"], str)
         assert isinstance(ns["b"], bytes)
         assert isinstance(ns["c"], bytes)
+        assert isinstance(ns["d"], bytes)
+
+    def test_triple_quotes(self):
+        s = '''
+x = """u"""
+y = r"""u"""
+b = b"""u"""
+c = br"""u"""
+d = rb"""u"""
+'''
+
+        ns = {}
+        exec(s, ns)
+        assert isinstance(ns["x"], str)
+        assert isinstance(ns["y"], str)
+        assert isinstance(ns["b"], bytes)
+        assert isinstance(ns["c"], bytes)
+        assert isinstance(ns["d"], bytes)
 
 
 class AppTestComprehensions:

From noreply at buildbot.pypy.org  Sun Feb 22 18:52:42 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Feb 2015 18:52:42 +0100 (CET)
Subject: [pypy-commit] stmgc default: improve comment
Message-ID: <20150222175242.20AC21C03C6@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1642:5e496c71f2ed
Date: 2015-02-22 18:53 +0100
http://bitbucket.org/pypy/stmgc/changeset/5e496c71f2ed/

Log:	improve comment

diff --git a/c7/stm/extra.c b/c7/stm/extra.c
--- a/c7/stm/extra.c
+++ b/c7/stm/extra.c
@@ -72,8 +72,9 @@
         assert(key != NULL);
         assert(callback != NULL);
 
-        /* The callback may call stm_call_on_abort(key, NULL).  It is ignored,
-           because 'callbacks_on_commit_and_abort' was cleared already. */
+        /* The callback may call stm_call_on_abort(key, NULL)
+           (so with callback==NULL).  It is ignored, because
+           'callbacks_on_commit_and_abort' was cleared already. */
         callback(key);
 
     } TREE_LOOP_END;

From noreply at buildbot.pypy.org  Sun Feb 22 18:59:11 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Feb 2015 18:59:11 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: minor stuff
Message-ID: <20150222175911.1BF951C04CD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r76059:4f3da042f52d
Date: 2015-02-22 18:58 +0100
http://bitbucket.org/pypy/pypy/changeset/4f3da042f52d/

Log:	minor stuff

diff --git a/rpython/translator/c/src/dtoa.c b/rpython/translator/c/src/dtoa.c
--- a/rpython/translator/c/src/dtoa.c
+++ b/rpython/translator/c/src/dtoa.c
@@ -3002,7 +3002,7 @@
     _PyPy_SET_53BIT_PRECISION_END;
 #ifdef RPY_STM
     stm_call_on_abort(&stm_thread_local, result,
-                      (void(*)(void *))_PyPy_dg_freedtoa);
+                      (void(*)(void *))__Py_dg_freedtoa);
 #endif
     return result;
 }

From noreply at buildbot.pypy.org  Sun Feb 22 23:21:32 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Sun, 22 Feb 2015 23:21:32 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Skip array test in lib-python which
 doesn't make sense without flexible string representation.
Message-ID: <20150222222132.76DBF1C009D@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76060:829c84510c32
Date: 2015-02-22 23:21 +0100
http://bitbucket.org/pypy/pypy/changeset/829c84510c32/

Log:	Skip array test in lib-python which doesn't make sense without
	flexible string representation.

diff --git a/lib-python/3/test/test_array.py b/lib-python/3/test/test_array.py
--- a/lib-python/3/test/test_array.py
+++ b/lib-python/3/test/test_array.py
@@ -1075,6 +1075,8 @@
         self.assertRaises(TypeError, a.fromunicode)
 
     def test_issue17223(self):
+        if support.check_impl_detail(pypy=True):
+            self.skipTest("specific to flexible string representation")
         # this used to crash
         if sizeof_wchar == 4:
             # U+FFFFFFFF is an invalid code point in Unicode 6.0

From noreply at buildbot.pypy.org  Sun Feb 22 23:22:16 2015
From: noreply at buildbot.pypy.org (Alexander Schremmer)
Date: Sun, 22 Feb 2015 23:22:16 +0100 (CET)
Subject: [pypy-commit] benchmarks default: Fixed exit call in bottle
	benchmark.
Message-ID: <20150222222216.6C00A1C0271@cobra.cs.uni-duesseldorf.de>

Author: Alexander Schremmer 
Branch: 
Changeset: r306:fe3138fe8a6f
Date: 2015-02-21 17:51 +0100
http://bitbucket.org/pypy/benchmarks/changeset/fe3138fe8a6f/

Log:	Fixed exit call in bottle benchmark.

diff --git a/multithread/bottle/app.py b/multithread/bottle/app.py
--- a/multithread/bottle/app.py
+++ b/multithread/bottle/app.py
@@ -95,7 +95,7 @@
                              stderr=subprocess.PIPE)
     except OSError as e:
         sys.stderr.write("Error trying to execute 'openload'\n%s" % e)
-        os.exit(1)
+        os._exit(1)
 
     returncode = p.wait()
     out, err = p.communicate()

From noreply at buildbot.pypy.org  Sun Feb 22 23:22:17 2015
From: noreply at buildbot.pypy.org (Alexander Schremmer)
Date: Sun, 22 Feb 2015 23:22:17 +0100 (CET)
Subject: [pypy-commit] benchmarks default: Import wsgi_intercept into
 benchmarks repository to be able to write a web benchmark.
Message-ID: <20150222222217.D7E971C0271@cobra.cs.uni-duesseldorf.de>

Author: Alexander Schremmer 
Branch: 
Changeset: r307:67e1f07b6919
Date: 2015-02-21 17:52 +0100
http://bitbucket.org/pypy/benchmarks/changeset/67e1f07b6919/

Log:	Import wsgi_intercept into benchmarks repository to be able to write
	a web benchmark.

diff --git a/multithread/wsgi/wsgi_intercept/__init__.py b/multithread/wsgi/wsgi_intercept/__init__.py
new file mode 100644
--- /dev/null
+++ b/multithread/wsgi/wsgi_intercept/__init__.py
@@ -0,0 +1,581 @@
+
+"""installs a WSGI application in place of a real URI for testing.
+
+Introduction
+============
+
+Testing a WSGI application normally involves starting a server at a
+local host and port, then pointing your test code to that address.
+Instead, this library lets you intercept calls to any specific host/port
+combination and redirect them into a `WSGI application`_ importable by
+your test program. Thus, you can avoid spawning multiple processes or
+threads to test your Web app.
+
+How Does It Work?
+=================
+
+``wsgi_intercept`` works by replacing ``httplib.HTTPConnection`` with a
+subclass, ``wsgi_intercept.WSGI_HTTPConnection``. This class then
+redirects specific server/port combinations into a WSGI application by
+emulating a socket. If no intercept is registered for the host and port
+requested, those requests are passed on to the standard handler.
+
+The functions ``add_wsgi_intercept(host, port, app_create_fn,
+script_name='')`` and ``remove_wsgi_intercept(host,port)`` specify
+which URLs should be redirect into what applications. Note especially
+that ``app_create_fn`` is a *function object* returning a WSGI
+application; ``script_name`` becomes ``SCRIPT_NAME`` in the WSGI app's
+environment, if set.
+
+Install
+=======
+
+::
+
+    pip install -U wsgi_intercept
+
+Packages Intercepted
+====================
+
+Unfortunately each of the Web testing frameworks uses its own specific
+mechanism for making HTTP call-outs, so individual implementations are
+needed. At this time there are implementations for ``httplib2`` and
+``requests`` in both Python 2 and 3, ``urllib2`` and ``httplib``
+in Python 2 and ``urllib.request`` and ``http.client`` in Python 3.
+
+If you are using Python 2 and need support for a different HTTP
+client, require a version of ``wsgi_intercept<0.6``. Earlier versions
+include support for ``webtest``, ``webunit`` and ``zope.testbrowser``.
+It is quite likely that support for these versions will be relatively
+easy to add back in to the new version.
+
+The best way to figure out how to use interception is to inspect
+`the tests`_. More comprehensive documentation available upon
+request.
+
+.. _the tests: https://github.com/cdent/python3-wsgi-intercept/tree/master/test
+
+
+History
+=======
+
+Pursuant to Ian Bicking's `"best Web testing framework"`_ post, Titus
+Brown put together an `in-process HTTP-to-WSGI interception mechanism`_
+for his own Web testing system, twill_. Because the mechanism is pretty
+generic -- it works at the httplib level -- Titus decided to try adding
+it into all of the *other* Python Web testing frameworks.
+
+The Python 2 version of wsgi-intercept was the result. Kumar McMillan
+later took over maintenance.
+
+The current version works with Python 2.6, 2.7, 3.3 and 3.4 and was assembled
+by `Chris Dent`_. Testing and documentation improvements from `Sasha Hart`_.
+
+.. _twill: http://www.idyll.org/~t/www-tools/twill.html
+.. _"best Web testing framework": http://blog.ianbicking.org/best-of-the-web-app-test-frameworks.html
+.. _in-process HTTP-to-WSGI interception mechanism: http://www.advogato.org/person/titus/diary.html?start=119
+.. _WSGI application: http://www.python.org/peps/pep-3333.html
+.. _Chris Dent: https://github.com/cdent
+.. _Sasha Hart: https://github.com/sashahart
+
+Project Home
+============
+
+This project lives on `GitHub`_. Please submit all bugs, patches,
+failing tests, et cetera using the Issue Tracker.
+
+Additional documentation is available on `Read The Docs`_.
+
+.. _GitHub: http://github.com/cdent/python3-wsgi-intercept
+.. _Read The Docs: http://wsgi-intercept.readthedocs.org/en/latest/
+
+"""
+from __future__ import print_function
+
+__version__ = '0.9.1'
+
+
+import sys
+try:
+    from http.client import HTTPConnection, HTTPSConnection
+except ImportError:
+    from httplib import HTTPConnection, HTTPSConnection
+
+try:
+    from io import BytesIO
+except ImportError:
+    from StringIO import StringIO as BytesIO
+
+try:
+    from urllib.parse import unquote as url_unquote
+except ImportError:
+    from urllib import unquote as url_unquote
+
+import traceback
+
+debuglevel = 0
+# 1 basic
+# 2 verbose
+
+####
+
+#
+# Specify which hosts/ports to target for interception to a given WSGI app.
+#
+# For simplicity's sake, intercept ENTIRE host/port combinations;
+# intercepting only specific URL subtrees gets complicated, because we don't
+# have that information in the HTTPConnection.connect() function that does the
+# redirection.
+#
+# format: key=(host, port), value=(create_app, top_url)
+#
+# (top_url becomes the SCRIPT_NAME)
+
+_wsgi_intercept = {}
+
+
+def add_wsgi_intercept(host, port, app_create_fn, script_name=''):
+    """
+    Add a WSGI intercept call for host:port, using the app returned
+    by app_create_fn with a SCRIPT_NAME of 'script_name' (default '').
+    """
+    _wsgi_intercept[(host, port)] = (app_create_fn, script_name)
+
+
+def remove_wsgi_intercept(*args):
+    """
+    Remove the WSGI intercept call for (host, port).  If no arguments are
+    given, removes all intercepts
+    """
+    global _wsgi_intercept
+    if len(args) == 0:
+        _wsgi_intercept = {}
+    else:
+        key = (args[0], args[1])
+        if key in _wsgi_intercept:
+            del _wsgi_intercept[key]
+
+
+#
+# make_environ: behave like a Web server.  Take in 'input', and behave
+# as if you're bound to 'host' and 'port'; build an environment dict
+# for the WSGI app.
+#
+# This is where the magic happens, folks.
+#
+def make_environ(inp, host, port, script_name):
+    """
+    Take 'inp' as if it were HTTP-speak being received on host:port,
+    and parse it into a WSGI-ok environment dictionary.  Return the
+    dictionary.
+
+    Set 'SCRIPT_NAME' from the 'script_name' input, and, if present,
+    remove it from the beginning of the PATH_INFO variable.
+    """
+    #
+    # parse the input up to the first blank line (or its end).
+    #
+
+    environ = {}
+
+    method_line = inp.readline()
+    if sys.version_info[0] > 2:
+        method_line = method_line.decode('ISO-8859-1')
+
+    content_type = None
+    content_length = None
+    cookies = []
+
+    for line in inp:
+        if not line.strip():
+            break
+
+        k, v = line.strip().split(b':', 1)
+        v = v.lstrip()
+        v = v.decode('ISO-8859-1')
+
+        #
+        # take care of special headers, and for the rest, put them
+        # into the environ with HTTP_ in front.
+        #
+
+        if k.lower() == b'content-type':
+            content_type = v
+        elif k.lower() == b'content-length':
+            content_length = v
+        elif k.lower() == b'cookie' or k.lower() == b'cookie2':
+            cookies.append(v)
+        else:
+            h = k.upper()
+            h = h.replace(b'-', b'_')
+            environ['HTTP_' + h.decode('ISO-8859-1')] = v
+
+        if debuglevel >= 2:
+            print('HEADER:', k, v)
+
+    #
+    # decode the method line
+    #
+
+    if debuglevel >= 2:
+        print('METHOD LINE:', method_line)
+
+    method, url, protocol = method_line.split(' ')
+
+    # Store the URI as requested by the user, without modification
+    # so that PATH_INFO munging can be corrected.
+    environ['REQUEST_URI'] = url
+    environ['RAW_URI'] = url
+
+    # clean the script_name off of the url, if it's there.
+    if not url.startswith(script_name):
+        script_name = ''                # @CTB what to do -- bad URL.  scrap?
+    else:
+        url = url[len(script_name):]
+
+    url = url.split('?', 1)
+    path_info = url_unquote(url[0])
+    query_string = ""
+    if len(url) == 2:
+        query_string = url[1]
+
+    if debuglevel:
+        print("method: %s; script_name: %s; path_info: %s; query_string: %s" %
+                (method, script_name, path_info, query_string))
+
+    r = inp.read()
+    inp = BytesIO(r)
+
+    #
+    # fill out our dictionary.
+    #
+
+    environ.update({
+        "wsgi.version": (1, 0),
+        "wsgi.url_scheme": "http",
+        "wsgi.input": inp,  # to read for POSTs
+        "wsgi.errors": BytesIO(),
+        "wsgi.multithread": 0,
+        "wsgi.multiprocess": 0,
+        "wsgi.run_once": 0,
+
+        "PATH_INFO": path_info,
+        "REMOTE_ADDR": '127.0.0.1',
+        "REQUEST_METHOD": method,
+        "SCRIPT_NAME": script_name,
+        "SERVER_NAME": host,
+        "SERVER_PORT": port,
+        "SERVER_PROTOCOL": protocol,
+    })
+
+    #
+    # query_string, content_type & length are optional.
+    #
+
+    if query_string:
+        environ['QUERY_STRING'] = query_string
+
+    if content_type:
+        environ['CONTENT_TYPE'] = content_type
+        if debuglevel >= 2:
+            print('CONTENT-TYPE:', content_type)
+    if content_length:
+        environ['CONTENT_LENGTH'] = content_length
+        if debuglevel >= 2:
+            print('CONTENT-LENGTH:', content_length)
+
+    #
+    # handle cookies.
+    #
+    if cookies:
+        environ['HTTP_COOKIE'] = "; ".join(cookies)
+
+    if debuglevel:
+        print('WSGI environ dictionary:', environ)
+
+    return environ
+
+
+class WSGIAppError(Exception):
+    """
+    An exception that wraps any Exception raised by the WSGI app
+    that is called. This is done for two reasons: it ensures that
+    intercepted libraries (such as requests) which use exceptions
+    to trigger behaviors are not interfered with by exceptions from
+    the WSGI app. It also helps to define a solid boundary, akin
+    to the network boundary between server and client, in the
+    testing environment.
+    """
+    def __init__(self, error, exc_info):
+        Exception.__init__(self)
+        self.error = error
+        self.exception_type = exc_info[0]
+        self.exception_value = exc_info[1]
+        self.traceback = exc_info[2]
+
+    def __str__(self):
+        frame = traceback.extract_tb(self.traceback)[-1]
+        formatted = "{0!r} at {1}:{2}".format(
+            self.error,
+            frame[0],
+            frame[1],
+        )
+        return formatted
+
+
+#
+# fake socket for WSGI intercept stuff.
+#
+class wsgi_fake_socket:
+    """
+    Handle HTTP traffic and stuff into a WSGI application object instead.
+
+    Note that this class assumes:
+
+     1. 'makefile' is called (by the response class) only after all of the
+        data has been sent to the socket by the request class;
+     2. non-persistent (i.e. non-HTTP/1.1) connections.
+    """
+    def __init__(self, app, host, port, script_name, https=False):
+        self.app = app                  # WSGI app object
+        self.host = host
+        self.port = port
+        self.script_name = script_name  # SCRIPT_NAME (app mount point)
+
+        self.inp = BytesIO()           # stuff written into this "socket"
+        self.write_results = []          # results from the 'write_fn'
+        self.results = None             # results from running the app
+        self.output = BytesIO()        # all output from the app, incl headers
+        self.https = https
+
+    def makefile(self, *args, **kwargs):
+        """
+        'makefile' is called by the HTTPResponse class once all of the
+        data has been written.  So, in this interceptor class, we need to:
+
+          1. build a start_response function that grabs all the headers
+             returned by the WSGI app;
+          2. create a wsgi.input file object 'inp', containing all of the
+             traffic;
+          3. build an environment dict out of the traffic in inp;
+          4. run the WSGI app & grab the result object;
+          5. concatenate & return the result(s) read from the result object.
+
+        @CTB: 'start_response' should return a function that writes
+        directly to self.result, too.
+        """
+
+        # dynamically construct the start_response function for no good reason.
+
+        def start_response(status, headers, exc_info=None):
+            # construct the HTTP request.
+            self.output.write(b"HTTP/1.0 " + status.encode('utf-8') + b"\n")
+
+            for k, v in headers:
+                try:
+                    k = k.encode('utf-8')
+                except AttributeError:
+                    pass
+                try:
+                    v = v.encode('utf-8')
+                except AttributeError:
+                    pass
+                self.output.write(k + b': ' + v + b"\n")
+            self.output.write(b'\n')
+
+            def write_fn(s):
+                self.write_results.append(s)
+            return write_fn
+
+        # construct the wsgi.input file from everything that's been
+        # written to this "socket".
+        inp = BytesIO(self.inp.getvalue())
+
+        # build the environ dictionary.
+        environ = make_environ(inp, self.host, self.port, self.script_name)
+        if self.https:
+            environ['wsgi.url_scheme'] = 'https'
+
+        # run the application.
+        try:
+            app_result = self.app(environ, start_response)
+        except Exception as error:
+            raise WSGIAppError(error, sys.exc_info())
+        self.result = iter(app_result)
+
+        ###
+
+        # read all of the results.  the trick here is to get the *first*
+        # bit of data from the app via the generator, *then* grab & return
+        # the data passed back from the 'write' function, and then return
+        # the generator data.  this is because the 'write' fn doesn't
+        # necessarily get called until the first result is requested from
+        # the app function.
+
+        try:
+            generator_data = None
+            try:
+                generator_data = next(self.result)
+
+            finally:
+                for data in self.write_results:
+                    self.output.write(data)
+
+            if generator_data:
+                try:
+                    self.output.write(generator_data)
+                except TypeError as exc:
+                    raise TypeError('bytes required in response: %s' % exc)
+
+                while 1:
+                    data = next(self.result)
+                    self.output.write(data)
+
+        except StopIteration:
+            pass
+
+        if hasattr(app_result, 'close'):
+            app_result.close()
+
+        if debuglevel >= 2:
+            print("***", self.output.getvalue(), "***")
+
+        # return the concatenated results.
+        return BytesIO(self.output.getvalue())
+
+    def sendall(self, content):
+        """
+        Save all the traffic to self.inp.
+        """
+        if debuglevel >= 2:
+            print(">>>", content, ">>>")
+
+        try:
+            self.inp.write(content)
+        except TypeError:
+            self.inp.write(content.encode('utf-8'))
+
+    def close(self):
+        "Do nothing, for now."
+        pass
+
+
+#
+# WSGI_HTTPConnection
+#
+class WSGI_HTTPConnection(HTTPConnection):
+    """
+    Intercept all traffic to certain hosts & redirect into a WSGI
+    application object.
+    """
+    def get_app(self, host, port):
+        """
+        Return the app object for the given (host, port).
+        """
+        key = (host, int(port))
+
+        app, script_name = None, None
+
+        if key in _wsgi_intercept:
+            (app_fn, script_name) = _wsgi_intercept[key]
+            app = app_fn()
+
+        return app, script_name
+
+    def connect(self):
+        """
+        Override the connect() function to intercept calls to certain
+        host/ports.
+
+        If no app at host/port has been registered for interception then
+        a normal HTTPConnection is made.
+        """
+        if debuglevel:
+            sys.stderr.write('connect: %s, %s\n' % (self.host, self.port,))
+
+        try:
+            (app, script_name) = self.get_app(self.host, self.port)
+            if app:
+                if debuglevel:
+                    sys.stderr.write('INTERCEPTING call to %s:%s\n' %
+                                     (self.host, self.port,))
+                self.sock = wsgi_fake_socket(app, self.host, self.port,
+                                             script_name)
+            else:
+                HTTPConnection.connect(self)
+
+        except Exception:
+            if debuglevel:              # intercept & print out tracebacks
+                traceback.print_exc()
+            raise
+
+
+#
+# WSGI_HTTPSConnection
+#
+
+
+class WSGI_HTTPSConnection(HTTPSConnection, WSGI_HTTPConnection):
+    """
+    Intercept all traffic to certain hosts & redirect into a WSGI
+    application object.
+    """
+    def get_app(self, host, port):
+        """
+        Return the app object for the given (host, port).
+        """
+        key = (host, int(port))
+
+        app, script_name = None, None
+
+        if key in _wsgi_intercept:
+            (app_fn, script_name) = _wsgi_intercept[key]
+            app = app_fn()
+
+        return app, script_name
+
+    def connect(self):
+        """
+        Override the connect() function to intercept calls to certain
+        host/ports.
+
+        If no app at host/port has been registered for interception then
+        a normal HTTPSConnection is made.
+        """
+        if debuglevel:
+            sys.stderr.write('connect: %s, %s\n' % (self.host, self.port,))
+
+        try:
+            (app, script_name) = self.get_app(self.host, self.port)
+            if app:
+                if debuglevel:
+                    sys.stderr.write('INTERCEPTING call to %s:%s\n' %
+                                     (self.host, self.port,))
+                self.sock = wsgi_fake_socket(app, self.host, self.port,
+                                             script_name, https=True)
+            else:
+                try:
+                    import ssl
+                    if not hasattr(self, 'key_file'):
+                        self.key_file = None
+                    if not hasattr(self, 'cert_file'):
+                        self.cert_file = None
+                    if not hasattr(self, '_context'):
+                        try:
+                            self._context = ssl.create_default_context()
+                        except AttributeError:
+                            self._context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+                            self._context.options |= ssl.OP_NO_SSLv2
+                            if not hasattr(self, 'check_hostname'):
+                                self._check_hostname = (self._context.verify_mode
+                                        != ssl.CERT_NONE)
+                            else:
+                                self._check_hostname = self.check_hostname
+                except (ImportError, AttributeError):
+                    pass
+                HTTPSConnection.connect(self)
+
+        except Exception:
+            if debuglevel:              # intercept & print out tracebacks
+                traceback.print_exc()
+            raise
diff --git a/multithread/wsgi/wsgi_intercept/http_client_intercept.py b/multithread/wsgi/wsgi_intercept/http_client_intercept.py
new file mode 100644
--- /dev/null
+++ b/multithread/wsgi/wsgi_intercept/http_client_intercept.py
@@ -0,0 +1,49 @@
+"""Intercept HTTP connections that use httplib (Py2) or http.client (Py3).
+"""
+
+try:
+    import http.client as http_lib
+except ImportError:
+    import httplib as http_lib
+
+from . import WSGI_HTTPConnection, WSGI_HTTPSConnection
+
+try:
+    from http.client import (
+            HTTPConnection as OriginalHTTPConnection,
+            HTTPSConnection as OriginalHTTPSConnection
+    )
+except ImportError:
+    from httplib import (
+            HTTPConnection as OriginalHTTPConnection,
+            HTTPSConnection as OriginalHTTPSConnection
+    )
+
+HTTPInterceptorMixin = WSGI_HTTPConnection
+HTTPSInterceptorMixin = WSGI_HTTPSConnection
+
+
+class HTTP_WSGIInterceptor(HTTPInterceptorMixin, http_lib.HTTPConnection):
+    pass
+
+
+class HTTPS_WSGIInterceptor(HTTPSInterceptorMixin, http_lib.HTTPSConnection,
+        HTTP_WSGIInterceptor):
+
+    def __init__(self, host, **kwargs):
+        self.host = host
+        try:
+            self.port = kwargs['port']
+        except KeyError:
+            self.port = None
+        HTTP_WSGIInterceptor.__init__(self, host, **kwargs)
+
+
+def install():
+    http_lib.HTTPConnection = HTTP_WSGIInterceptor
+    http_lib.HTTPSConnection = HTTPS_WSGIInterceptor
+
+
+def uninstall():
+    http_lib.HTTPConnection = OriginalHTTPConnection
+    http_lib.HTTPSConnection = OriginalHTTPSConnection
diff --git a/multithread/wsgi/wsgi_intercept/httplib2_intercept.py b/multithread/wsgi/wsgi_intercept/httplib2_intercept.py
new file mode 100644
--- /dev/null
+++ b/multithread/wsgi/wsgi_intercept/httplib2_intercept.py
@@ -0,0 +1,61 @@
+"""Intercept HTTP connections that use `httplib2 `_.
+"""
+
+import sys
+
+from httplib2 import (SCHEME_TO_CONNECTION, HTTPConnectionWithTimeout,
+                      HTTPSConnectionWithTimeout)
+
+from . import (HTTPConnection, HTTPSConnection, WSGI_HTTPConnection,
+               WSGI_HTTPSConnection)
+
+HTTPInterceptorMixin = WSGI_HTTPConnection
+HTTPSInterceptorMixin = WSGI_HTTPSConnection
+
+
+class HTTP_WSGIInterceptorWithTimeout(HTTPInterceptorMixin,
+        HTTPConnectionWithTimeout):
+    def __init__(self, host, port=None, strict=None, timeout=None,
+            proxy_info=None, source_address=None):
+
+        # In Python3 strict is deprecated
+        if sys.version_info[0] < 3:
+            try:
+                HTTPConnection.__init__(self, host, port=port, strict=strict,
+                        timeout=timeout, source_address=source_address)
+            except TypeError:  # Python 2.6 doesn't have source_address
+                HTTPConnection.__init__(self, host, port=port, strict=strict,
+                        timeout=timeout)
+        else:
+            HTTPConnection.__init__(self, host, port=port,
+                    timeout=timeout, source_address=source_address)
+
+
+class HTTPS_WSGIInterceptorWithTimeout(HTTPSInterceptorMixin,
+        HTTPSConnectionWithTimeout):
+    def __init__(self, host, port=None, strict=None, timeout=None,
+            proxy_info=None, ca_certs=None, source_address=None,
+            disable_ssl_certificate_validation=False):
+
+        # ignore proxy_info and ca_certs
+        # In Python3 strict is deprecated
+        if sys.version_info[0] < 3:
+            try:
+                HTTPSConnection.__init__(self, host, port=port, strict=strict,
+                        timeout=timeout, source_address=source_address)
+            except TypeError:  # Python 2.6 doesn't have source_address
+                HTTPSConnection.__init__(self, host, port=port, strict=strict,
+                        timeout=timeout)
+        else:
+            HTTPSConnection.__init__(self, host, port=port,
+                    timeout=timeout, source_address=source_address)
+
+
+def install():
+    SCHEME_TO_CONNECTION['http'] = HTTP_WSGIInterceptorWithTimeout
+    SCHEME_TO_CONNECTION['https'] = HTTPS_WSGIInterceptorWithTimeout
+
+
+def uninstall():
+    SCHEME_TO_CONNECTION['http'] = HTTPConnectionWithTimeout
+    SCHEME_TO_CONNECTION['https'] = HTTPSConnectionWithTimeout
diff --git a/multithread/wsgi/wsgi_intercept/requests_intercept.py b/multithread/wsgi/wsgi_intercept/requests_intercept.py
new file mode 100644
--- /dev/null
+++ b/multithread/wsgi/wsgi_intercept/requests_intercept.py
@@ -0,0 +1,41 @@
+"""Intercept HTTP connections that use `requests `_.
+"""
+
+import sys
+
+from . import WSGI_HTTPConnection, WSGI_HTTPSConnection, wsgi_fake_socket
+from requests.packages.urllib3.connectionpool import (HTTPConnectionPool,
+        HTTPSConnectionPool)
+from requests.packages.urllib3.connection import (HTTPConnection,
+        HTTPSConnection)
+
+
+wsgi_fake_socket.settimeout = lambda self, timeout: None
+
+
+class HTTP_WSGIInterceptor(WSGI_HTTPConnection, HTTPConnection):
+    def __init__(self, *args, **kwargs):
+        if 'strict' in kwargs and sys.version_info > (3, 0):
+            kwargs.pop('strict')
+        WSGI_HTTPConnection.__init__(self, *args, **kwargs)
+        HTTPConnection.__init__(self, *args, **kwargs)
+
+
+class HTTPS_WSGIInterceptor(WSGI_HTTPSConnection, HTTPSConnection):
+    is_verified = True
+
+    def __init__(self, *args, **kwargs):
+        if 'strict' in kwargs and sys.version_info > (3, 0):
+            kwargs.pop('strict')
+        WSGI_HTTPSConnection.__init__(self, *args, **kwargs)
+        HTTPSConnection.__init__(self, *args, **kwargs)
+
+
+def install():
+    HTTPConnectionPool.ConnectionCls = HTTP_WSGIInterceptor
+    HTTPSConnectionPool.ConnectionCls = HTTPS_WSGIInterceptor
+
+
+def uninstall():
+    HTTPConnectionPool.ConnectionCls = HTTPConnection
+    HTTPSConnectionPool.ConnectionCls = HTTPSConnection
diff --git a/multithread/wsgi/wsgi_intercept/urllib_intercept.py b/multithread/wsgi/wsgi_intercept/urllib_intercept.py
new file mode 100644
--- /dev/null
+++ b/multithread/wsgi/wsgi_intercept/urllib_intercept.py
@@ -0,0 +1,40 @@
+"""Intercept HTTP connections that use urllib.request (Py3) aka urllib2 (Python 2).
+"""
+try:
+    import urllib.request as url_lib
+except ImportError:
+    import urllib2 as url_lib
+
+from . import WSGI_HTTPConnection, WSGI_HTTPSConnection
+
+
+class WSGI_HTTPHandler(url_lib.HTTPHandler):
+    """
+    Override the default HTTPHandler class with one that uses the
+    WSGI_HTTPConnection class to open HTTP URLs.
+    """
+    def http_open(self, req):
+        return self.do_open(WSGI_HTTPConnection, req)
+
+
+class WSGI_HTTPSHandler(url_lib.HTTPSHandler):
+    """
+    Override the default HTTPSHandler class with one that uses the
+    WSGI_HTTPConnection class to open HTTPS URLs.
+    """
+    def https_open(self, req):
+        return self.do_open(WSGI_HTTPSConnection, req)
+
+
+def install_opener():
+    handlers = [WSGI_HTTPHandler()]
+    if WSGI_HTTPSHandler is not None:
+        handlers.append(WSGI_HTTPSHandler())
+    opener = url_lib.build_opener(*handlers)
+    url_lib.install_opener(opener)
+
+    return opener
+
+
+def uninstall_opener():
+    url_lib.install_opener(None)

From noreply at buildbot.pypy.org  Sun Feb 22 23:22:19 2015
From: noreply at buildbot.pypy.org (Quim Sanchez)
Date: Sun, 22 Feb 2015 23:22:19 +0100 (CET)
Subject: [pypy-commit] benchmarks default: (focs,
 xorAxAx) add benchmark runner for readthedocs and webapp benchmark
 framework
Message-ID: <20150222222219.2B9471C0271@cobra.cs.uni-duesseldorf.de>

Author: Quim Sanchez 
Branch: 
Changeset: r308:635504ab8d29
Date: 2015-02-22 16:51 +0100
http://bitbucket.org/pypy/benchmarks/changeset/635504ab8d29/

Log:	(focs, xorAxAx) add benchmark runner for readthedocs and webapp
	benchmark framework

	 Usage: python bench.py wsgi/webapp.py readthedocs [request_count
	[thread_count]]

diff --git a/multithread/bench.py b/multithread/bench.py
--- a/multithread/bench.py
+++ b/multithread/bench.py
@@ -49,6 +49,10 @@
     os.chdir(folder)
     sys.path.insert(0, os.path.abspath('.'))
     test = import_file(os.path.basename(args.file))
+    more_args = args.more
+
+    if hasattr(test, 'init'):
+        more_args = test.init(*more_args)
 
     times = []
     results = []
@@ -61,10 +65,10 @@
 
             test_time = time.time()
             if args.p:
-                results.append(test.run(*args.more))
+                results.append(test.run(*more_args))
             else:
                 with nostdout():
-                    results.append(test.run(*args.more))
+                    results.append(test.run(*more_args))
             times.append(time.time() - test_time)
 
             if not args.q:
diff --git a/multithread/wsgi/readthedocs.py b/multithread/wsgi/readthedocs.py
new file mode 100644
--- /dev/null
+++ b/multithread/wsgi/readthedocs.py
@@ -0,0 +1,10 @@
+
+
+REQUEST_LIST = [
+        '/',
+        ]
+
+def make_app():
+    from readthedocs import wsgi
+
+    return wsgi.application
diff --git a/multithread/wsgi/webapp.py b/multithread/wsgi/webapp.py
new file mode 100644
--- /dev/null
+++ b/multithread/wsgi/webapp.py
@@ -0,0 +1,57 @@
+
+import sys
+import threading
+
+from urllib2 import urlopen
+from wsgi_intercept import (
+    urllib_intercept, add_wsgi_intercept, remove_wsgi_intercept
+)
+
+host, port = 'localhost', 80
+url = 'http://{0}:{1}/'.format(host, port)
+settings = {}
+
+def init(benchmark, request_count=100, *args):
+    settings["request_count"] = request_count
+
+    bm_module = __import__(benchmark)
+
+    if not hasattr(bm_module, 'REQUEST_LIST'):
+        print 'request list not defined'
+        sys.exit(-1)
+    if not hasattr(bm_module, 'make_app'):
+        print 'app maker not defined'
+        sys.exit(-1)
+
+    settings["bm_module"] = bm_module
+
+    # set up intercepter
+    urllib_intercept.install_opener()
+    add_wsgi_intercept(host, port, bm_module.make_app)
+
+    return args
+
+def run(thread_count=2):
+    # test
+    threads = []
+    for i in range(thread_count):
+        thread = threading.Thread(target=do_requests, args=(settings['request_count'],))
+        thread.start()
+        threads.append(thread)
+
+    for thread in threads:
+        thread.join()
+
+    remove_wsgi_intercept()
+
+def do_requests(request_count):
+    for i in range(request_count):
+        for request in settings["bm_module"].REQUEST_LIST:
+            request_url = url + request
+            stream = urlopen(url)
+            content = stream.read()
+
+
+if __name__ == '__main__':
+    init(sys.argv[1])
+    run()

From noreply at buildbot.pypy.org  Sun Feb 22 23:22:20 2015
From: noreply at buildbot.pypy.org (Alexander Schremmer)
Date: Sun, 22 Feb 2015 23:22:20 +0100 (CET)
Subject: [pypy-commit] benchmarks default: (focs,
 xoraxax) Make import logic more robust,
 also add readthedocs package dir to python path for settings package.
Message-ID: <20150222222220.7046A1C0271@cobra.cs.uni-duesseldorf.de>

Author: Alexander Schremmer 
Branch: 
Changeset: r309:4f9c89d1a521
Date: 2015-02-22 17:12 +0100
http://bitbucket.org/pypy/benchmarks/changeset/4f9c89d1a521/

Log:	(focs, xoraxax) Make import logic more robust, also add readthedocs
	package dir to python path for settings package.

diff --git a/multithread/wsgi/readthedocs.py b/multithread/wsgi/read_the_docs.py
rename from multithread/wsgi/readthedocs.py
rename to multithread/wsgi/read_the_docs.py
--- a/multithread/wsgi/readthedocs.py
+++ b/multithread/wsgi/read_the_docs.py
@@ -1,10 +1,16 @@
+import os
+import sys
+
+import readthedocs
+from readthedocs import wsgi
 
 
 REQUEST_LIST = [
         '/',
         ]
 
+
 def make_app():
-    from readthedocs import wsgi
+    sys.path.append(os.path.dirname(readthedocs.__file__))
 
     return wsgi.application
diff --git a/multithread/wsgi/webapp.py b/multithread/wsgi/webapp.py
--- a/multithread/wsgi/webapp.py
+++ b/multithread/wsgi/webapp.py
@@ -1,4 +1,4 @@
-
+import os
 import sys
 import threading
 
@@ -14,7 +14,9 @@
 def init(benchmark, request_count=100, *args):
     settings["request_count"] = request_count
 
-    bm_module = __import__(benchmark)
+    bm_module = type(sys)("bm_module")
+    file_name = os.path.join(os.path.dirname(__file__), benchmark + ".py")
+    execfile(file_name, bm_module.__dict__)
 
     if not hasattr(bm_module, 'REQUEST_LIST'):
         print 'request list not defined'

From noreply at buildbot.pypy.org  Sun Feb 22 23:22:21 2015
From: noreply at buildbot.pypy.org (Alexander Schremmer)
Date: Sun, 22 Feb 2015 23:22:21 +0100 (CET)
Subject: [pypy-commit] benchmarks default: Fix urllib errors on subsequent
 calls to test.run, convert arguments correctly.
Message-ID: <20150222222221.B47A41C0271@cobra.cs.uni-duesseldorf.de>

Author: Alexander Schremmer 
Branch: 
Changeset: r310:797b4f8c65ed
Date: 2015-02-22 23:22 +0100
http://bitbucket.org/pypy/benchmarks/changeset/797b4f8c65ed/

Log:	Fix urllib errors on subsequent calls to test.run, convert arguments
	correctly.

diff --git a/multithread/wsgi/webapp.py b/multithread/wsgi/webapp.py
--- a/multithread/wsgi/webapp.py
+++ b/multithread/wsgi/webapp.py
@@ -11,8 +11,8 @@
 url = 'http://{0}:{1}/'.format(host, port)
 settings = {}
 
-def init(benchmark, request_count=100, *args):
-    settings["request_count"] = request_count
+def init(benchmark, request_count="100", *args):
+    settings["request_count"] = int(request_count)
 
     bm_module = type(sys)("bm_module")
     file_name = os.path.join(os.path.dirname(__file__), benchmark + ".py")
@@ -33,10 +33,10 @@
 
     return args
 
-def run(thread_count=2):
+def run(thread_count="2"):
     # test
     threads = []
-    for i in range(thread_count):
+    for i in range(int(thread_count)):
         thread = threading.Thread(target=do_requests, args=(settings['request_count'],))
         thread.start()
         threads.append(thread)
@@ -44,7 +44,6 @@
     for thread in threads:
         thread.join()
 
-    remove_wsgi_intercept()
 
 def do_requests(request_count):
     for i in range(request_count):

From noreply at buildbot.pypy.org  Mon Feb 23 10:13:18 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Feb 2015 10:13:18 +0100 (CET)
Subject: [pypy-commit] pypy default: Ups, fix SyntaxError
Message-ID: <20150223091318.EC6DE1C1478@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r76061:c2c30e39d094
Date: 2015-02-23 11:13 +0200
http://bitbucket.org/pypy/pypy/changeset/c2c30e39d094/

Log:	Ups, fix SyntaxError

diff --git a/lib_pypy/gdbm.py b/lib_pypy/gdbm.py
--- a/lib_pypy/gdbm.py
+++ b/lib_pypy/gdbm.py
@@ -127,7 +127,7 @@
     def __contains__(self, key):
         self._check_closed()
         key = _checkstr(key)
-        return lib.pygdbm_exists(self.ll_dbm, key, len(key)
+        return lib.pygdbm_exists(self.ll_dbm, key, len(key))
     has_key = __contains__
 
     def __getitem__(self, key):

From noreply at buildbot.pypy.org  Mon Feb 23 10:49:42 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Mon, 23 Feb 2015 10:49:42 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Remove duplicate code probably introduced
 by a merge of py3k.
Message-ID: <20150223094942.C543E1C01CD@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76062:c38d94c15fd6
Date: 2015-02-23 08:49 +0100
http://bitbucket.org/pypy/pypy/changeset/c38d94c15fd6/

Log:	Remove duplicate code probably introduced by a merge of py3k.

diff --git a/lib_pypy/audioop.py b/lib_pypy/audioop.py
--- a/lib_pypy/audioop.py
+++ b/lib_pypy/audioop.py
@@ -1118,452 +1118,6 @@
         sample <<= 16
     _put_sample(result, size, i, sample)
 
-ffi = FFI()
-ffi.cdef("""
-typedef short PyInt16;
-
-/* 2's complement (14-bit range) */
-unsigned char
-st_14linear2ulaw(PyInt16 pcm_val);
-PyInt16 st_ulaw2linear16(unsigned char);
-
-/* 2's complement (13-bit range) */
-unsigned char
-st_linear2alaw(PyInt16 pcm_val);
-PyInt16 st_alaw2linear16(unsigned char);
-
-
-void lin2adcpm(unsigned char* rv, unsigned char* cp, size_t len,
-               size_t size, int* state);
-void adcpm2lin(unsigned char* rv, unsigned char* cp, size_t len,
-               size_t size, int* state);
-""")
-
-# This code is directly copied from CPython file: Modules/audioop.c
-_AUDIOOP_C_MODULE = """
-typedef short PyInt16;
-typedef int Py_Int32;
-
-/* Code shamelessly stolen from sox, 12.17.7, g711.c
-** (c) Craig Reese, Joe Campbell and Jeff Poskanzer 1989 */
-
-/* From g711.c:
- *
- * December 30, 1994:
- * Functions linear2alaw, linear2ulaw have been updated to correctly
- * convert unquantized 16 bit values.
- * Tables for direct u- to A-law and A- to u-law conversions have been
- * corrected.
- * Borge Lindberg, Center for PersonKommunikation, Aalborg University.
- * bli at cpk.auc.dk
- *
- */
-#define BIAS 0x84   /* define the add-in bias for 16 bit samples */
-#define CLIP 32635
-#define SIGN_BIT        (0x80)          /* Sign bit for a A-law byte. */
-#define QUANT_MASK      (0xf)           /* Quantization field mask. */
-#define SEG_SHIFT       (4)             /* Left shift for segment number. */
-#define SEG_MASK        (0x70)          /* Segment field mask. */
-
-static PyInt16 seg_aend[8] = {0x1F, 0x3F, 0x7F, 0xFF,
-                              0x1FF, 0x3FF, 0x7FF, 0xFFF};
-static PyInt16 seg_uend[8] = {0x3F, 0x7F, 0xFF, 0x1FF,
-                              0x3FF, 0x7FF, 0xFFF, 0x1FFF};
-
-static PyInt16
-search(PyInt16 val, PyInt16 *table, int size)
-{
-    int i;
-
-    for (i = 0; i < size; i++) {
-        if (val <= *table++)
-            return (i);
-    }
-    return (size);
-}
-#define st_ulaw2linear16(uc) (_st_ulaw2linear16[uc])
-#define st_alaw2linear16(uc) (_st_alaw2linear16[uc])
-
-static PyInt16 _st_ulaw2linear16[256] = {
-    -32124,  -31100,  -30076,  -29052,  -28028,  -27004,  -25980,
-    -24956,  -23932,  -22908,  -21884,  -20860,  -19836,  -18812,
-    -17788,  -16764,  -15996,  -15484,  -14972,  -14460,  -13948,
-    -13436,  -12924,  -12412,  -11900,  -11388,  -10876,  -10364,
-     -9852,   -9340,   -8828,   -8316,   -7932,   -7676,   -7420,
-     -7164,   -6908,   -6652,   -6396,   -6140,   -5884,   -5628,
-     -5372,   -5116,   -4860,   -4604,   -4348,   -4092,   -3900,
-     -3772,   -3644,   -3516,   -3388,   -3260,   -3132,   -3004,
-     -2876,   -2748,   -2620,   -2492,   -2364,   -2236,   -2108,
-     -1980,   -1884,   -1820,   -1756,   -1692,   -1628,   -1564,
-     -1500,   -1436,   -1372,   -1308,   -1244,   -1180,   -1116,
-     -1052,    -988,    -924,    -876,    -844,    -812,    -780,
-      -748,    -716,    -684,    -652,    -620,    -588,    -556,
-      -524,    -492,    -460,    -428,    -396,    -372,    -356,
-      -340,    -324,    -308,    -292,    -276,    -260,    -244,
-      -228,    -212,    -196,    -180,    -164,    -148,    -132,
-      -120,    -112,    -104,     -96,     -88,     -80,     -72,
-       -64,     -56,     -48,     -40,     -32,     -24,     -16,
-    -8,       0,   32124,   31100,   30076,   29052,   28028,
-     27004,   25980,   24956,   23932,   22908,   21884,   20860,
-     19836,   18812,   17788,   16764,   15996,   15484,   14972,
-     14460,   13948,   13436,   12924,   12412,   11900,   11388,
-     10876,   10364,    9852,    9340,    8828,    8316,    7932,
-      7676,    7420,    7164,    6908,    6652,    6396,    6140,
-      5884,    5628,    5372,    5116,    4860,    4604,    4348,
-      4092,    3900,    3772,    3644,    3516,    3388,    3260,
-      3132,    3004,    2876,    2748,    2620,    2492,    2364,
-      2236,    2108,    1980,    1884,    1820,    1756,    1692,
-      1628,    1564,    1500,    1436,    1372,    1308,    1244,
-      1180,    1116,    1052,     988,     924,     876,     844,
-       812,     780,     748,     716,     684,     652,     620,
-       588,     556,     524,     492,     460,     428,     396,
-       372,     356,     340,     324,     308,     292,     276,
-       260,     244,     228,     212,     196,     180,     164,
-       148,     132,     120,     112,     104,      96,      88,
-    80,      72,      64,      56,      48,      40,      32,
-    24,      16,       8,       0
-};
-
-/*
- * linear2ulaw() accepts a 14-bit signed integer and encodes it as u-law data
- * stored in a unsigned char.  This function should only be called with
- * the data shifted such that it only contains information in the lower
- * 14-bits.
- *
- * In order to simplify the encoding process, the original linear magnitude
- * is biased by adding 33 which shifts the encoding range from (0 - 8158) to
- * (33 - 8191). The result can be seen in the following encoding table:
- *
- *      Biased Linear Input Code        Compressed Code
- *      ------------------------        ---------------
- *      00000001wxyza                   000wxyz
- *      0000001wxyzab                   001wxyz
- *      000001wxyzabc                   010wxyz
- *      00001wxyzabcd                   011wxyz
- *      0001wxyzabcde                   100wxyz
- *      001wxyzabcdef                   101wxyz
- *      01wxyzabcdefg                   110wxyz
- *      1wxyzabcdefgh                   111wxyz
- *
- * Each biased linear code has a leading 1 which identifies the segment
- * number. The value of the segment number is equal to 7 minus the number
- * of leading 0's. The quantization interval is directly available as the
- * four bits wxyz.  * The trailing bits (a - h) are ignored.
- *
- * Ordinarily the complement of the resulting code word is used for
- * transmission, and so the code word is complemented before it is returned.
- *
- * For further information see John C. Bellamy's Digital Telephony, 1982,
- * John Wiley & Sons, pps 98-111 and 472-476.
- */
-static unsigned char
-st_14linear2ulaw(PyInt16 pcm_val)       /* 2's complement (14-bit range) */
-{
-    PyInt16         mask;
-    PyInt16         seg;
-    unsigned char   uval;
-
-    /* The original sox code does this in the calling function, not here */
-    pcm_val = pcm_val >> 2;
-
-    /* u-law inverts all bits */
-    /* Get the sign and the magnitude of the value. */
-    if (pcm_val < 0) {
-        pcm_val = -pcm_val;
-        mask = 0x7F;
-    } else {
-        mask = 0xFF;
-    }
-    if ( pcm_val > CLIP ) pcm_val = CLIP;           /* clip the magnitude */
-    pcm_val += (BIAS >> 2);
-
-    /* Convert the scaled magnitude to segment number. */
-    seg = search(pcm_val, seg_uend, 8);
-
-    /*
-     * Combine the sign, segment, quantization bits;
-     * and complement the code word.
-     */
-    if (seg >= 8)           /* out of range, return maximum value. */
-        return (unsigned char) (0x7F ^ mask);
-    else {
-        uval = (unsigned char) (seg << 4) | ((pcm_val >> (seg + 1)) & 0xF);
-        return (uval ^ mask);
-    }
-
-}
-
-static PyInt16 _st_alaw2linear16[256] = {
-     -5504,   -5248,   -6016,   -5760,   -4480,   -4224,   -4992,
-     -4736,   -7552,   -7296,   -8064,   -7808,   -6528,   -6272,
-     -7040,   -6784,   -2752,   -2624,   -3008,   -2880,   -2240,
-     -2112,   -2496,   -2368,   -3776,   -3648,   -4032,   -3904,
-     -3264,   -3136,   -3520,   -3392,  -22016,  -20992,  -24064,
-    -23040,  -17920,  -16896,  -19968,  -18944,  -30208,  -29184,
-    -32256,  -31232,  -26112,  -25088,  -28160,  -27136,  -11008,
-    -10496,  -12032,  -11520,   -8960,   -8448,   -9984,   -9472,
-    -15104,  -14592,  -16128,  -15616,  -13056,  -12544,  -14080,
-    -13568,    -344,    -328,    -376,    -360,    -280,    -264,
-      -312,    -296,    -472,    -456,    -504,    -488,    -408,
-      -392,    -440,    -424,     -88,     -72,    -120,    -104,
-       -24,      -8,     -56,     -40,    -216,    -200,    -248,
-      -232,    -152,    -136,    -184,    -168,   -1376,   -1312,
-     -1504,   -1440,   -1120,   -1056,   -1248,   -1184,   -1888,
-     -1824,   -2016,   -1952,   -1632,   -1568,   -1760,   -1696,
-      -688,    -656,    -752,    -720,    -560,    -528,    -624,
-      -592,    -944,    -912,   -1008,    -976,    -816,    -784,
-      -880,    -848,    5504,    5248,    6016,    5760,    4480,
-      4224,    4992,    4736,    7552,    7296,    8064,    7808,
-      6528,    6272,    7040,    6784,    2752,    2624,    3008,
-      2880,    2240,    2112,    2496,    2368,    3776,    3648,
-      4032,    3904,    3264,    3136,    3520,    3392,   22016,
-     20992,   24064,   23040,   17920,   16896,   19968,   18944,
-     30208,   29184,   32256,   31232,   26112,   25088,   28160,
-     27136,   11008,   10496,   12032,   11520,    8960,    8448,
-      9984,    9472,   15104,   14592,   16128,   15616,   13056,
-     12544,   14080,   13568,     344,     328,     376,     360,
-       280,     264,     312,     296,     472,     456,     504,
-       488,     408,     392,     440,     424,      88,      72,
-       120,     104,      24,       8,      56,      40,     216,
-       200,     248,     232,     152,     136,     184,     168,
-      1376,    1312,    1504,    1440,    1120,    1056,    1248,
-      1184,    1888,    1824,    2016,    1952,    1632,    1568,
-      1760,    1696,     688,     656,     752,     720,     560,
-       528,     624,     592,     944,     912,    1008,     976,
-       816,     784,     880,     848
-};
-
-/*
- * linear2alaw() accepts an 13-bit signed integer and encodes it as A-law data
- * stored in a unsigned char.  This function should only be called with
- * the data shifted such that it only contains information in the lower
- * 13-bits.
- *
- *              Linear Input Code       Compressed Code
- *      ------------------------        ---------------
- *      0000000wxyza                    000wxyz
- *      0000001wxyza                    001wxyz
- *      000001wxyzab                    010wxyz
- *      00001wxyzabc                    011wxyz
- *      0001wxyzabcd                    100wxyz
- *      001wxyzabcde                    101wxyz
- *      01wxyzabcdef                    110wxyz
- *      1wxyzabcdefg                    111wxyz
- *
- * For further information see John C. Bellamy's Digital Telephony, 1982,
- * John Wiley & Sons, pps 98-111 and 472-476.
- */
-static unsigned char
-st_linear2alaw(PyInt16 pcm_val) /* 2's complement (13-bit range) */
-{
-    PyInt16         mask;
-    short           seg;
-    unsigned char   aval;
-
-    /* The original sox code does this in the calling function, not here */
-    pcm_val = pcm_val >> 3;
-
-    /* A-law using even bit inversion */
-    if (pcm_val >= 0) {
-        mask = 0xD5;            /* sign (7th) bit = 1 */
-    } else {
-        mask = 0x55;            /* sign bit = 0 */
-        pcm_val = -pcm_val - 1;
-    }
-
-    /* Convert the scaled magnitude to segment number. */
-    seg = search(pcm_val, seg_aend, 8);
-
-    /* Combine the sign, segment, and quantization bits. */
-
-    if (seg >= 8)           /* out of range, return maximum value. */
-        return (unsigned char) (0x7F ^ mask);
-    else {
-        aval = (unsigned char) seg << SEG_SHIFT;
-        if (seg < 2)
-            aval |= (pcm_val >> 1) & QUANT_MASK;
-        else
-            aval |= (pcm_val >> seg) & QUANT_MASK;
-        return (aval ^ mask);
-    }
-}
-/* End of code taken from sox */
-
-/* Intel ADPCM step variation table */
-static int indexTable[16] = {
-    -1, -1, -1, -1, 2, 4, 6, 8,
-    -1, -1, -1, -1, 2, 4, 6, 8,
-};
-
-static int stepsizeTable[89] = {
-    7, 8, 9, 10, 11, 12, 13, 14, 16, 17,
-    19, 21, 23, 25, 28, 31, 34, 37, 41, 45,
-    50, 55, 60, 66, 73, 80, 88, 97, 107, 118,
-    130, 143, 157, 173, 190, 209, 230, 253, 279, 307,
-    337, 371, 408, 449, 494, 544, 598, 658, 724, 796,
-    876, 963, 1060, 1166, 1282, 1411, 1552, 1707, 1878, 2066,
-    2272, 2499, 2749, 3024, 3327, 3660, 4026, 4428, 4871, 5358,
-    5894, 6484, 7132, 7845, 8630, 9493, 10442, 11487, 12635, 13899,
-    15289, 16818, 18500, 20350, 22385, 24623, 27086, 29794, 32767
-};
-
-#define CHARP(cp, i) ((signed char *)(cp+i))
-#define SHORTP(cp, i) ((short *)(cp+i))
-#define LONGP(cp, i) ((Py_Int32 *)(cp+i))
-"""
-
-lib = ffi.verify(_AUDIOOP_C_MODULE + """
-void lin2adcpm(unsigned char* ncp, unsigned char* cp, size_t len,
-               size_t size, int* state)
-{
-    int step, outputbuffer = 0, bufferstep;
-    int val = 0;
-    int diff, vpdiff, sign, delta;
-    size_t i;
-    int valpred = state[0];
-    int index = state[1];
-
-    step = stepsizeTable[index];
-    bufferstep = 1;
-
-    for ( i=0; i < len; i += size ) {
-        if ( size == 1 )      val = ((int)*CHARP(cp, i)) << 8;
-        else if ( size == 2 ) val = (int)*SHORTP(cp, i);
-        else if ( size == 4 ) val = ((int)*LONGP(cp, i)) >> 16;
-
-        /* Step 1 - compute difference with previous value */
-        diff = val - valpred;
-        sign = (diff < 0) ? 8 : 0;
-        if ( sign ) diff = (-diff);
-
-        /* Step 2 - Divide and clamp */
-        /* Note:
-        ** This code *approximately* computes:
-        **    delta = diff*4/step;
-        **    vpdiff = (delta+0.5)*step/4;
-        ** but in shift step bits are dropped. The net result of this
-        ** is that even if you have fast mul/div hardware you cannot
-        ** put it to good use since the fixup would be too expensive.
-        */
-        delta = 0;
-        vpdiff = (step >> 3);
-
-        if ( diff >= step ) {
-            delta = 4;
-            diff -= step;
-            vpdiff += step;
-        }
-        step >>= 1;
-        if ( diff >= step  ) {
-            delta |= 2;
-            diff -= step;
-            vpdiff += step;
-        }
-        step >>= 1;
-        if ( diff >= step ) {
-            delta |= 1;
-            vpdiff += step;
-        }
-
-        /* Step 3 - Update previous value */
-        if ( sign )
-            valpred -= vpdiff;
-        else
-            valpred += vpdiff;
-
-        /* Step 4 - Clamp previous value to 16 bits */
-        if ( valpred > 32767 )
-            valpred = 32767;
-        else if ( valpred < -32768 )
-            valpred = -32768;
-
-        /* Step 5 - Assemble value, update index and step values */
-        delta |= sign;
-
-        index += indexTable[delta];
-        if ( index < 0 ) index = 0;
-        if ( index > 88 ) index = 88;
-        step = stepsizeTable[index];
-
-        /* Step 6 - Output value */
-        if ( bufferstep ) {
-            outputbuffer = (delta << 4) & 0xf0;
-        } else {
-            *ncp++ = (delta & 0x0f) | outputbuffer;
-        }
-        bufferstep = !bufferstep;
-    }
-    state[0] = valpred;
-    state[1] = index;
-}
-
-
-void adcpm2lin(unsigned char* ncp, unsigned char* cp, size_t len,
-               size_t size, int* state)
-{
-    int step, inputbuffer = 0, bufferstep;
-    int val = 0;
-    int diff, vpdiff, sign, delta;
-    size_t i;
-    int valpred = state[0];
-    int index = state[1];
-
-    step = stepsizeTable[index];
-    bufferstep = 0;
-
-    for ( i=0; i < len*size*2; i += size ) {
-        /* Step 1 - get the delta value and compute next index */
-        if ( bufferstep ) {
-            delta = inputbuffer & 0xf;
-        } else {
-            inputbuffer = *cp++;
-            delta = (inputbuffer >> 4) & 0xf;
-        }
-
-        bufferstep = !bufferstep;
-
-        /* Step 2 - Find new index value (for later) */
-        index += indexTable[delta];
-        if ( index < 0 ) index = 0;
-        if ( index > 88 ) index = 88;
-
-        /* Step 3 - Separate sign and magnitude */
-        sign = delta & 8;
-        delta = delta & 7;
-
-        /* Step 4 - Compute difference and new predicted value */
-        /*
-        ** Computes 'vpdiff = (delta+0.5)*step/4', but see comment
-        ** in adpcm_coder.
-        */
-        vpdiff = step >> 3;
-        if ( delta & 4 ) vpdiff += step;
-        if ( delta & 2 ) vpdiff += step>>1;
-        if ( delta & 1 ) vpdiff += step>>2;
-
-        if ( sign )
-            valpred -= vpdiff;
-        else
-            valpred += vpdiff;
-
-        /* Step 5 - clamp output value */
-        if ( valpred > 32767 )
-            valpred = 32767;
-        else if ( valpred < -32768 )
-            valpred = -32768;
-
-        /* Step 6 - Update step value */
-        step = stepsizeTable[index];
-
-        /* Step 6 - Output value */
-        if ( size == 1 ) *CHARP(ncp, i) = (signed char)(valpred >> 8);
-        else if ( size == 2 ) *SHORTP(ncp, i) = (short)(valpred);
-        else if ( size == 4 ) *LONGP(ncp, i) = (Py_Int32)(valpred<<16);
-    }
-    state[0] = valpred;
-    state[1] = index;
-}
-""")
 
 def _get_lin_samples(cp, size):
     for sample in _get_samples(cp, size):

From noreply at buildbot.pypy.org  Mon Feb 23 10:49:44 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Mon, 23 Feb 2015 10:49:44 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Make unicodeobjects' encode_object and
 decode_object more symmetric (in preparation of a soon-to-be-committed
 change).
Message-ID: <20150223094944.350F51C01CD@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76063:7a3e3b7ec94c
Date: 2015-02-23 09:49 +0100
http://bitbucket.org/pypy/pypy/changeset/7a3e3b7ec94c/

Log:	Make unicodeobjects' encode_object and decode_object more symmetric
	(in preparation of a soon-to-be-committed change).

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -539,13 +539,19 @@
             eh = unicodehelper.decode_error_handler(space)
             return space.wrap(str_decode_utf_8(
                     s, len(s), None, final=True, errorhandler=eh)[0])
-    w_codecs = space.getbuiltinmodule("_codecs")
-    w_decode = space.getattr(w_codecs, space.wrap("decode"))
+
+    from pypy.module._codecs.interp_codecs import lookup_codec
+    w_decoder = space.getitem(lookup_codec(space, encoding), space.wrap(1))
     if errors is None:
-        w_retval = space.call_function(w_decode, w_obj, space.wrap(encoding))
+        w_errors = space.wrap('strict')
     else:
-        w_retval = space.call_function(w_decode, w_obj, space.wrap(encoding),
-                                       space.wrap(errors))
+        w_errors = space.wrap(errors)
+    w_restuple = space.call_function(w_decoder, w_obj, w_errors)
+    w_retval = space.getitem(w_restuple, space.wrap(0))
+    if not space.isinstance_w(w_retval, space.w_unicode):
+        raise oefmt(space.w_TypeError,
+                    "decoder did not return a bytes object (type '%T')",
+                    w_retval)
     return w_retval
 
 

From noreply at buildbot.pypy.org  Mon Feb 23 10:49:45 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Mon, 23 Feb 2015 10:49:45 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Raise an error if str.encode() or
 bytes.decode() is called with a non-text encoding.
Message-ID: <20150223094945.6B3631C01CD@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76064:0ae2fcdca348
Date: 2015-02-23 10:49 +0100
http://bitbucket.org/pypy/pypy/changeset/0ae2fcdca348/

Log:	Raise an error if str.encode() or bytes.decode() is called with a
	non-text encoding.

	This is necessary because non-text encodings (this is a slightly
	confusing term for things like the base64 encoding) were
	reintroduced in Python 3.3, but have to be used with
	codecs.encode()/decode().

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -500,7 +500,20 @@
             raise wrap_encode_error(space, ue)
 
     from pypy.module._codecs.interp_codecs import lookup_codec
-    w_encoder = space.getitem(lookup_codec(space, encoding), space.wrap(0))
+    codec_info = lookup_codec(space, encoding)
+    try:
+        is_text_encoding = space.is_true(
+                space.getattr(codec_info, space.wrap('_is_text_encoding')))
+    except OperationError as e:
+        if e.match(space.w_AttributeError):
+            is_text_encoding = True
+        else:
+            raise
+    if not is_text_encoding:
+        raise oefmt(space.w_LookupError,
+                    "'%s' is not a text encoding; "
+                    "use codecs.encode() to handle arbitrary codecs", encoding)
+    w_encoder = space.getitem(codec_info, space.wrap(0))
     if errors is None:
         w_errors = space.wrap('strict')
     else:
@@ -541,7 +554,20 @@
                     s, len(s), None, final=True, errorhandler=eh)[0])
 
     from pypy.module._codecs.interp_codecs import lookup_codec
-    w_decoder = space.getitem(lookup_codec(space, encoding), space.wrap(1))
+    codec_info = lookup_codec(space, encoding)
+    try:
+        is_text_encoding = space.is_true(
+                space.getattr(codec_info, space.wrap('_is_text_encoding')))
+    except OperationError as e:
+        if e.match(space.w_AttributeError):
+            is_text_encoding = True
+        else:
+            raise
+    if not is_text_encoding:
+        raise oefmt(space.w_LookupError,
+                    "'%s' is not a text encoding; "
+                    "use codecs.decode() to handle arbitrary codecs", encoding)
+    w_decoder = space.getitem(codec_info, space.wrap(1))
     if errors is None:
         w_errors = space.wrap('strict')
     else:

From noreply at buildbot.pypy.org  Mon Feb 23 12:13:03 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 23 Feb 2015 12:13:03 +0100 (CET)
Subject: [pypy-commit] stmgc default: Need to check more carefully that the
 transaction is running
Message-ID: <20150223111303.665E31C03D0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1643:ebab634a6517
Date: 2015-02-23 12:13 +0100
http://bitbucket.org/pypy/stmgc/changeset/ebab634a6517/

Log:	Need to check more carefully that the transaction is running

diff --git a/c7/stm/extra.c b/c7/stm/extra.c
--- a/c7/stm/extra.c
+++ b/c7/stm/extra.c
@@ -6,13 +6,13 @@
 static long register_callbacks(stm_thread_local_t *tl,
                                void *key, void callback(void *), long index)
 {
-    if (!_stm_in_transaction(tl)) {
-        /* check that the current thread-local is really running a
-           transaction, and do nothing otherwise. */
+    if (!(_stm_in_transaction(tl) && tl == STM_SEGMENT->running_thread)) {
+        /* check that the current thread-local is really running the
+           transaction in STM_SEGMENT, and do nothing otherwise. */
         return -1;
     }
 
-    if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {
+    if (STM_PSEGMENT->transaction_state != TS_REGULAR) {
         /* ignore callbacks if we're in an inevitable transaction
            (which cannot abort) */
         return -1;
@@ -27,6 +27,7 @@
     }
     else {
         /* double-registering the same key will crash */
+        dprintf(("register_callbacks: tl=%p key=%p callback=%p index=%ld\n", tl, key, callback, index));
         tree_insert(callbacks, (uintptr_t)key, (uintptr_t)callback);
         return 1;
     }

From noreply at buildbot.pypy.org  Mon Feb 23 12:55:24 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Feb 2015 12:55:24 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: (fijal,
 arigo) try a bit harder with lists to be signal-safe
Message-ID: <20150223115524.0C0D21C12B8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r76065:466b2e195fc7
Date: 2015-02-23 13:54 +0200
http://bitbucket.org/pypy/pypy/changeset/466b2e195fc7/

Log:	(fijal, arigo) try a bit harder with lists to be signal-safe

diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -102,7 +102,7 @@
         self.store_reg(mc, r.r0, r.fp, ofs)
         mc.MOV_rr(r.r0.value, r.fp.value)
         self.gen_func_epilog(mc)
-        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
+        rawstart = mc.materialize(self.cpu, [])
         self.propagate_exception_path = rawstart
 
     def _store_and_reset_exception(self, mc, excvalloc=None, exctploc=None,
@@ -198,7 +198,7 @@
         mc.ADD_ri(r.sp.value, r.sp.value, (len(r.argument_regs) + 2) * WORD)
         mc.B(self.propagate_exception_path)
         #
-        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
+        rawstart = mc.materialize(self.cpu, [])
         self.stack_check_slowpath = rawstart
 
     def _build_wb_slowpath(self, withcards, withfloats=False, for_frame=False):
@@ -255,7 +255,7 @@
         #
         mc.POP([r.ip.value, r.pc.value])
         #
-        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
+        rawstart = mc.materialize(self.cpu, [])
         if for_frame:
             self.wb_slowpath[4] = rawstart
         else:
@@ -276,7 +276,7 @@
                                       callee_only)
         # return
         mc.POP([r.ip.value, r.pc.value])
-        return mc.materialize(self.cpu.asmmemmgr, [])
+        return mc.materialize(self.cpu, [])
 
     def _build_malloc_slowpath(self, kind):
         """ While arriving on slowpath, we have a gcpattern on stack 0.
@@ -352,7 +352,7 @@
         mc.POP([r.ip.value, r.pc.value])
 
         #
-        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
+        rawstart = mc.materialize(self.cpu, [])
         return rawstart
 
     def _reload_frame_if_necessary(self, mc):
@@ -473,7 +473,7 @@
         mc.MOV_rr(r.r0.value, r.fp.value)
         #
         self.gen_func_epilog(mc)
-        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
+        rawstart = mc.materialize(self.cpu, [])
         self.failure_recovery_code[exc + 2 * withfloats] = rawstart
 
     def generate_quick_failure(self, guardtok):
@@ -855,7 +855,7 @@
         # restore registers
         self._pop_all_regs_from_jitframe(mc, [], self.cpu.supports_floats)
         mc.POP([r.ip.value, r.pc.value])  # return
-        self._frame_realloc_slowpath = mc.materialize(self.cpu.asmmemmgr, [])
+        self._frame_realloc_slowpath = mc.materialize(self.cpu, [])
 
     def _load_shadowstack_top(self, mc, reg, gcrootmap):
         rst = gcrootmap.get_root_stack_top_addr()
@@ -885,7 +885,7 @@
         self.datablockwrapper = None
         allblocks = self.get_asmmemmgr_blocks(looptoken)
         size = self.mc.get_relative_pos() 
-        res = self.mc.materialize(self.cpu.asmmemmgr, allblocks,
+        res = self.mc.materialize(self.cpu, allblocks,
                                    self.cpu.gc_ll_descr.gcrootmap)
         self.cpu.asmmemmgr.register_codemap(
             self.codemap.get_final_bytecode(res, size))
diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py
--- a/rpython/jit/backend/arm/codebuilder.py
+++ b/rpython/jit/backend/arm/codebuilder.py
@@ -468,9 +468,9 @@
             f.close()
 
     # XXX remove and setup aligning in llsupport
-    def materialize(self, asmmemmgr, allblocks, gcrootmap=None):
+    def materialize(self, cpu, allblocks, gcrootmap=None):
         size = self.get_relative_pos() + WORD
-        malloced = asmmemmgr.malloc(size, size + 7)
+        malloced = cpu.asmmemmgr.malloc(size, size + 7)
         allblocks.append(malloced)
         rawstart = malloced[0]
         while(rawstart % FUNC_ALIGN != 0):
diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py
--- a/rpython/jit/backend/arm/runner.py
+++ b/rpython/jit/backend/arm/runner.py
@@ -50,6 +50,7 @@
     def setup_once(self):
         self.cpuinfo.arch_version = detect_arch_version()
         self.cpuinfo.hf_abi = detect_hardfloat()
+        self.codemap.setup()
         self.assembler.setup_once()
 
     def finish_once(self):
diff --git a/rpython/jit/backend/arm/test/support.py b/rpython/jit/backend/arm/test/support.py
--- a/rpython/jit/backend/arm/test/support.py
+++ b/rpython/jit/backend/arm/test/support.py
@@ -24,7 +24,7 @@
 
 def run_asm(asm):
     BOOTSTRAP_TP = lltype.FuncType([], lltype.Signed)
-    addr = asm.mc.materialize(asm.cpu.asmmemmgr, [], None)
+    addr = asm.mc.materialize(asm.cpu, [], None)
     assert addr % 8 == 0
     func = rffi.cast(lltype.Ptr(BOOTSTRAP_TP), addr)
     asm.mc._dump_trace(addr, 'test.asm')
diff --git a/rpython/jit/backend/arm/test/test_calling_convention.py b/rpython/jit/backend/arm/test/test_calling_convention.py
--- a/rpython/jit/backend/arm/test/test_calling_convention.py
+++ b/rpython/jit/backend/arm/test/test_calling_convention.py
@@ -29,7 +29,7 @@
         mc = InstrBuilder()
         mc.MOV_rr(r.r0.value, r.sp.value)
         mc.MOV_rr(r.pc.value, r.lr.value)
-        return mc.materialize(self.cpu.asmmemmgr, [])
+        return mc.materialize(self.cpu, [])
 
     def get_alignment_requirements(self):
         return 8
diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py
--- a/rpython/jit/backend/llsupport/asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/asmmemmgr.py
@@ -5,9 +5,6 @@
 from rpython.rlib.debug import debug_start, debug_print, debug_stop
 from rpython.rlib.debug import have_debug_prints
 from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib.rbisect import bisect_left, bisect_left_tuple
-
-_memmngr = None # global reference so we can use @entrypoint :/
 
 
 class AsmMemoryManager(object):
@@ -27,12 +24,6 @@
         self.free_blocks = {}      # map {start: stop}
         self.free_blocks_end = {}  # map {stop: start}
         self.blocks_by_size = [[] for i in range(self.num_indices)]
-        # two lists of jit addresses (sorted) and the corresponding stack
-        # depths
-        self.jit_addr_map = []
-        self.jit_frame_depth_map = []
-        self.jit_codemap = []
-        # see codemap.py
 
     def malloc(self, minsize, maxsize):
         """Allocate executable memory, between minsize and maxsize bytes,
@@ -54,17 +45,6 @@
         if r_uint is not None:
             self.total_mallocs -= r_uint(stop - start)
         self._add_free_block(start, stop)
-        # fix up jit_addr_map
-        jit_adr_start = bisect_left(self.jit_addr_map, start)
-        jit_adr_stop = bisect_left(self.jit_addr_map, stop)
-        del self.jit_addr_map[jit_adr_start:jit_adr_stop]
-        del self.jit_frame_depth_map[jit_adr_start:jit_adr_stop]
-        # fix up codemap
-        # (there should only be zero or one codemap entry in that range,
-        # but still we use a range to distinguish between zero and one)
-        codemap_adr_start = bisect_left_tuple(self.jit_codemap, start)
-        codemap_adr_stop = bisect_left_tuple(self.jit_codemap, stop)
-        del self.jit_codemap[codemap_adr_start:codemap_adr_stop]
 
     def open_malloc(self, minsize):
         """Allocate at least minsize bytes.  Returns (start, stop)."""
@@ -171,31 +151,6 @@
         del self.free_blocks_end[stop]
         return (start, stop)
 
-    def register_frame_depth_map(self, rawstart, frame_positions,
-                                 frame_assignments):
-        if not frame_positions:
-            return
-        if not self.jit_addr_map or rawstart > self.jit_addr_map[-1]:
-            start = len(self.jit_addr_map)
-            self.jit_addr_map += [0] * len(frame_positions)
-            self.jit_frame_depth_map += [0] * len(frame_positions)
-        else:
-            start = bisect_left(self.jit_addr_map, rawstart)
-            self.jit_addr_map = (self.jit_addr_map[:start] +
-                                 [0] * len(frame_positions) +
-                                 self.jit_addr_map[start:])
-            self.jit_frame_depth_map = (self.jit_frame_depth_map[:start] +
-                                 [0] * len(frame_positions) +
-                                 self.jit_frame_depth_map[start:])
-        for i, pos in enumerate(frame_positions):
-            self.jit_addr_map[i + start] = pos + rawstart
-            self.jit_frame_depth_map[i + start] = frame_assignments[i]
-
-    def register_codemap(self, codemap):
-        start = codemap[0]
-        pos = bisect_left_tuple(self.jit_codemap, start)
-        self.jit_codemap.insert(pos, codemap)
-
     def _delete(self):
         "NOT_RPYTHON"
         if self._allocated:
@@ -349,9 +304,9 @@
             #
         debug_stop(logname)
 
-    def materialize(self, asmmemmgr, allblocks, gcrootmap=None):
+    def materialize(self, cpu, allblocks, gcrootmap=None):
         size = self.get_relative_pos()
-        malloced = asmmemmgr.malloc(size, size)
+        malloced = cpu.asmmemmgr.malloc(size, size)
         allblocks.append(malloced)
         rawstart = malloced[0]
         self.copy_to_raw_memory(rawstart)
@@ -359,8 +314,8 @@
             assert gcrootmap is not None
             for pos, mark in self.gcroot_markers:
                 gcrootmap.register_asm_addr(rawstart + pos, mark)
-        asmmemmgr.register_frame_depth_map(rawstart, self.frame_positions,
-                                           self.frame_assignments)
+        cpu.codemap.register_frame_depth_map(rawstart, self.frame_positions,
+                                             self.frame_assignments)
         self.frame_positions = None
         self.frame_assignments = None
         return rawstart
diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -130,7 +130,7 @@
         self.gcmap_for_finish[0] = r_uint(1)
 
     def setup(self, looptoken):
-        self.codemap = CodemapBuilder()
+        self.codemap_builder = CodemapBuilder()
         self._finish_gcmap = lltype.nullptr(jitframe.GCMAP)
 
     def set_debug(self, v):
@@ -200,7 +200,9 @@
         return fail_descr, target
 
     def debug_merge_point(self, op):
-        self.codemap.debug_merge_point(op, self.mc.get_relative_pos())
+        self.codemap_builder.debug_merge_point(op.getarg(1).getint(),
+                                               op.getarg(3).getint(),
+                                               self.mc.get_relative_pos())
 
     def call_assembler(self, op, guard_op, argloc, vloc, result_loc, tmploc):
         self._store_force_index(guard_op)
diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py
--- a/rpython/jit/backend/llsupport/codemap.py
+++ b/rpython/jit/backend/llsupport/codemap.py
@@ -10,40 +10,238 @@
 """
 
 from rpython.rlib import rgc
+from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rlib.entrypoint import jit_entrypoint
-from rpython.jit.backend.llsupport import asmmemmgr
-from rpython.rlib.rbisect import bisect_right, bisect_right_tuple
+from rpython.rlib.rbisect import bisect_right, bisect_right_addr
+from rpython.rlib.rbisect import bisect_left, bisect_left_addr
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+
+INITIAL_SIZE = 1000
+GROWTH_FACTOR = 4
+
+INT_LIST = lltype.Array(lltype.Signed) # raw, but with length
+
+CODEMAP = lltype.Struct(
+    'codemap',
+    ('addr', lltype.Signed),
+    ('machine_code_size', lltype.Signed),
+    ('bytecode_info', lltype.Ptr(INT_LIST)))
+CODEMAP_LIST = lltype.Array(CODEMAP)
+
+CODEMAP_GCARRAY = lltype.GcArray(CODEMAP)
+
+_codemap = None
+
+eci = ExternalCompilationInfo(post_include_bits=["""
+volatile int pypy_codemap_currently_invalid = 0;
+void pypy_codemap_invalid_set(int);
+"""], separate_module_sources=["""
+void pypy_codemap_invalid_set(int value)
+{
+    pypy_codemap_currently_invalid = value;
+}
+"""])
+
+ll_pypy_codemap_invalid_set = rffi.llexternal('pypy_codemap_invalid_set',
+                                              [rffi.INT], lltype.Void,
+                                              compilation_info=eci)
+
+def pypy_codemap_invalid_set(val):
+    if we_are_translated():
+        ll_pypy_codemap_invalid_set(val)
+
+ at specialize.ll()
+def copy_item(source, dest, si, di, baseline=0):
+    TP = lltype.typeOf(dest)
+    if isinstance(TP.TO.OF, lltype.Struct):
+        rgc.copy_struct_item(source, dest, si, di)
+    else:
+        dest[di] = source[si] + baseline
+
+class ListStorageMixin(object):
+    # XXX this code has wrong complexity, we should come up with a better
+    #     data structure ideally
+    _mixin_ = True
+
+    @specialize.arg(1)
+    def extend_with(self, name, to_insert, pos, baseline=0):
+        # first check if we need to reallocate
+        used = getattr(self, name + '_used')
+        allocated = len(getattr(self, name))
+        lst = getattr(self, name)
+        if pos + len(to_insert) >= allocated or pos != used:
+            old_lst = lst
+            if pos == used:
+                new_size = max(4 * len(old_lst),
+                               (len(old_lst) + len(to_insert)) * 2)
+                lst = lltype.malloc(lltype.typeOf(lst).TO, new_size,
+                                    flavor='raw',
+                                    track_allocation=self.track_allocation)
+            else:
+                lst = lltype.malloc(lltype.typeOf(lst).TO, len(old_lst),
+                                    flavor='raw',
+                                    track_allocation=self.track_allocation)
+            for i in range(0, pos):
+                copy_item(old_lst, lst, i, i)
+            j = 0
+            for i in range(pos, pos + len(to_insert)):
+                copy_item(to_insert, lst, j, i, baseline)
+                j += 1
+            j = pos
+            for i in range(pos + len(to_insert), len(to_insert) + used):
+                copy_item(old_lst, lst, j, i)
+                j += 1
+            self.free_lst(name, old_lst)
+        else:
+            for i in range(len(to_insert)):
+                copy_item(to_insert, lst, i, i + pos, baseline)
+        pypy_codemap_invalid_set(1)
+        setattr(self, name, lst)
+        setattr(self, name + '_used', len(to_insert) + used)
+        pypy_codemap_invalid_set(0)
+
+    @specialize.arg(1)
+    def remove(self, name, start, end):
+        pypy_codemap_invalid_set(1)
+        lst = getattr(self, name)
+        used = getattr(self, name + '_used')
+        j = end
+        for i in range(start, used - (end - start)):
+            info = lltype.nullptr(INT_LIST)
+            if name == 'jit_codemap':
+                if i < end:
+                    info = lst[i].bytecode_info
+            copy_item(lst, lst, j, i)
+            if name == 'jit_codemap':
+                if info:
+                    lltype.free(info, flavor='raw', track_allocation=False)
+            j += 1
+        setattr(self, name + '_used', used - (end - start))
+        pypy_codemap_invalid_set(0)
+
+class CodemapStorage(ListStorageMixin):
+    """ An immortal wrapper around underlaying jit codemap data
+    """
+    track_allocation = False
+    
+    def __init__(self):
+        global _codemap
+
+        _codemap = self # a global singleton, for @entrypoint, self it's
+        # a prebuilt constant anyway
+
+    def setup(self):
+        self.jit_addr_map = lltype.malloc(INT_LIST, INITIAL_SIZE, flavor='raw',
+                                          track_allocation=False)
+        self.jit_addr_map_used = 0
+        self.jit_codemap = lltype.malloc(CODEMAP_LIST, INITIAL_SIZE,
+                                         flavor='raw',
+                                         track_allocation=False)
+        self.jit_codemap_used = 0
+        self.jit_frame_depth_map = lltype.malloc(INT_LIST, INITIAL_SIZE,
+                                                 flavor='raw',
+                                                 track_allocation=False)
+        self.jit_frame_depth_map_used = 0
+
+    @specialize.arg(1)
+    def free_lst(self, name, lst):
+        lltype.free(lst, flavor='raw', track_allocation=False)
+
+    def __del__(self):
+        self.free()
+
+    def free(self):
+        # if setup has not been called
+        if not hasattr(self, 'jit_addr_map') or not self.jit_addr_map:
+            return
+        lltype.free(self.jit_addr_map, flavor='raw')
+        i = 0
+        while i < self.jit_codemap_used:
+            lltype.free(self.jit_codemap[i].bytecode_info, flavor='raw')
+            i += 1
+        lltype.free(self.jit_codemap, flavor='raw')
+        lltype.free(self.jit_frame_depth_map, flavor='raw')
+        self.jit_adr_map = lltype.nullptr(INT_LIST)
+
+    def free_asm_block(self, start, stop):
+        # fix up jit_addr_map
+        jit_adr_start = bisect_left(self.jit_addr_map, start,
+                                    self.jit_addr_map_used)
+        jit_adr_stop = bisect_left(self.jit_addr_map, stop,
+                                   self.jit_addr_map_used)
+        self.remove('jit_addr_map', jit_adr_start, jit_adr_stop)
+        self.remove('jit_frame_depth_map', jit_adr_start, jit_adr_stop)
+        # fix up codemap
+        # (there should only be zero or one codemap entry in that range,
+        # but still we use a range to distinguish between zero and one)
+        codemap_adr_start = bisect_left_addr(self.jit_codemap, start,
+                                             self.jit_codemap_used)
+        codemap_adr_stop = bisect_left_addr(self.jit_codemap, stop,
+                                            self.jit_codemap_used)
+        self.remove('jit_codemap', codemap_adr_start, codemap_adr_stop)
+
+    def register_frame_depth_map(self, rawstart, frame_positions,
+                                 frame_assignments):
+        if not frame_positions:
+            return
+        if (not self.jit_addr_map_used or
+            rawstart > self.jit_addr_map[self.jit_addr_map_used - 1]):
+            start = self.jit_addr_map_used
+            self.extend_with('jit_addr_map', frame_positions,
+                             self.jit_addr_map_used, rawstart)
+            self.extend_with('jit_frame_depth_map', frame_assignments,
+                             self.jit_frame_depth_map_used)
+        else:
+            start = bisect_left(self.jit_addr_map, rawstart,
+                                self.jit_addr_map_used)
+            self.extend_with('jit_addr_map', frame_positions, start, rawstart)
+            self.extend_with('jit_frame_depth_map', frame_assignments,
+                             start)
+
+    def register_codemap(self, codemap):
+        start = codemap[0]
+        pos = bisect_left_addr(self.jit_codemap, start, self.jit_codemap_used)
+        items = lltype.malloc(INT_LIST, len(codemap[2]), flavor='raw',
+                             track_allocation=False)
+        for i in range(len(codemap[2])):
+            items[i] = codemap[2][i]
+        s = lltype.malloc(CODEMAP_GCARRAY, 1)
+        s[0].addr = codemap[0]
+        s[0].machine_code_size = codemap[1]
+        s[0].bytecode_info = items
+        self.extend_with('jit_codemap', s, pos)
 
 @jit_entrypoint([lltype.Signed], lltype.Signed,
                 c_name='pypy_jit_stack_depth_at_loc')
 @rgc.no_collect
 def stack_depth_at_loc(loc):
-    _memmngr = asmmemmgr._memmngr
+    global _codemap
 
-    pos = bisect_right(_memmngr.jit_addr_map, loc)
-    if pos == 0 or pos == len(_memmngr.jit_addr_map):
+    pos = bisect_right(_codemap.jit_addr_map, loc, _codemap.jit_addr_map_used)
+    if pos == 0 or pos == _codemap.jit_addr_map_used:
         return -1
-    return _memmngr.jit_frame_depth_map[pos - 1]
+    return _codemap.jit_frame_depth_map[pos - 1]
 
 @jit_entrypoint([], lltype.Signed, c_name='pypy_jit_start_addr')
 def jit_start_addr():
-    _memmngr = asmmemmgr._memmngr
+    global _codemap
 
-    return _memmngr.jit_addr_map[0]
+    return _codemap.jit_addr_map[0]
 
 @jit_entrypoint([], lltype.Signed, c_name='pypy_jit_end_addr')
 def jit_end_addr():
-    _memmngr = asmmemmgr._memmngr
+    global _codemap
 
-    return _memmngr.jit_addr_map[-1]
+    return _codemap.jit_addr_map[_codemap.jit_addr_map_used - 1]
 
 @jit_entrypoint([lltype.Signed], lltype.Signed,
                 c_name='pypy_find_codemap_at_addr')
 def find_codemap_at_addr(addr):
-    _memmngr = asmmemmgr._memmngr
+    global _codemap
 
-    res = bisect_right_tuple(_memmngr.jit_codemap, addr) - 1
+    res = bisect_right_addr(_codemap.jit_codemap, addr,
+                            _codemap.jit_codemap_used) - 1
     return res
 
 @jit_entrypoint([lltype.Signed, lltype.Signed,
@@ -53,24 +251,24 @@
     """ will return consecutive unique_ids from codemap, starting from position
     `pos` until addr
     """
-    _memmngr = asmmemmgr._memmngr
+    global _codemap
 
-    codemap = _memmngr.jit_codemap[codemap_no]
+    codemap = _codemap.jit_codemap[codemap_no]
     current_pos = current_pos_addr[0]
-    start_addr = codemap[0]
+    start_addr = codemap.addr
     rel_addr = addr - start_addr
     while True:
-        if current_pos >= len(codemap[2]):
+        if current_pos >= len(codemap.bytecode_info):
             return 0
-        next_start = codemap[2][current_pos + 1]
+        next_start = codemap.bytecode_info[current_pos + 1]
         if next_start > rel_addr:
             return 0
-        next_stop = codemap[2][current_pos + 2]
+        next_stop = codemap.bytecode_info[current_pos + 2]
         if next_stop > rel_addr:
             current_pos_addr[0] = current_pos + 4
-            return codemap[2][current_pos]
+            return codemap.bytecode_info[current_pos]
         # we need to skip potentially more than one
-        current_pos = codemap[2][current_pos + 3]
+        current_pos = codemap.bytecode_info[current_pos + 3]
 
 def unpack_traceback(addr):
     codemap_pos = find_codemap_at_addr(addr)
@@ -94,10 +292,8 @@
         self.patch_position = []
         self.last_call_depth = -1
 
-    def debug_merge_point(self, op, pos):
-        call_depth = op.getarg(1).getint()
+    def debug_merge_point(self, call_depth, unique_id, pos):
         if call_depth != self.last_call_depth:
-            unique_id = op.getarg(3).getint()
             if unique_id == 0: # uninteresting case
                 return
             assert unique_id & 1 == 0
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -16,7 +16,7 @@
     FieldDescr, ArrayDescr, CallDescr, InteriorFieldDescr,
     FLAG_POINTER, FLAG_FLOAT)
 from rpython.jit.backend.llsupport.memcpy import memset_fn
-from rpython.jit.backend.llsupport import asmmemmgr
+from rpython.jit.backend.llsupport import asmmemmgr, codemap
 from rpython.rlib.unroll import unrolling_iterable
 
 
@@ -49,7 +49,7 @@
         else:
             self._setup_exception_handling_untranslated()
         self.asmmemmgr = asmmemmgr.AsmMemoryManager()
-        asmmemmgr._memmngr = self.asmmemmgr
+        self.codemap = codemap.CodemapStorage()
         self._setup_frame_realloc(translate_support_code)
         ad = self.gc_ll_descr.getframedescrs(self).arraydescr
         self.signedarraydescr = ad
@@ -213,6 +213,7 @@
             for rawstart, rawstop in blocks:
                 self.gc_ll_descr.freeing_block(rawstart, rawstop)
                 self.asmmemmgr.free(rawstart, rawstop)
+                self.codemap.free_asm_block(rawstart, rawstop)
 
     def force(self, addr_of_force_token):
         frame = rffi.cast(jitframe.JITFRAMEPTR, addr_of_force_token)
diff --git a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -2,8 +2,7 @@
 from rpython.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
 from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
 from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin
-from rpython.jit.backend.llsupport import asmmemmgr
-from rpython.jit.backend.llsupport.codemap import stack_depth_at_loc
+from rpython.jit.backend.llsupport.codemap import CodemapStorage
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib import debug
 
@@ -97,20 +96,21 @@
 class TestAsmMemoryManager:
 
     def setup_method(self, _):
-        self.memmgr = AsmMemoryManager(min_fragment=8,
+        self.asmmemmgr = AsmMemoryManager(min_fragment=8,
                                        num_indices=10,
                                        large_alloc_size=8192)
+        self.codemap = CodemapStorage()
 
     def teardown_method(self, _):
-        self.memmgr._delete()
+        self.asmmemmgr._delete()
 
     def test_malloc_simple(self):
         for i in range(100):
-            while self.memmgr.total_memory_allocated < 16384:
+            while self.asmmemmgr.total_memory_allocated < 16384:
                 reqsize = random.randrange(1, 200)
-                (start, stop) = self.memmgr.malloc(reqsize, reqsize)
+                (start, stop) = self.asmmemmgr.malloc(reqsize, reqsize)
                 assert reqsize <= stop - start < reqsize + 8
-                assert self.memmgr.total_memory_allocated in [8192, 16384]
+                assert self.asmmemmgr.total_memory_allocated in [8192, 16384]
             self.teardown_method(None)
             self.setup_method(None)
 
@@ -124,7 +124,7 @@
             if got and (random.random() < 0.4 or len(got) == 1000):
                 # free
                 start, stop = got.pop(random.randrange(0, len(got)))
-                self.memmgr.free(start, stop)
+                self.asmmemmgr.free(start, stop)
                 real_use -= (stop - start)
                 assert real_use >= 0
             #
@@ -135,18 +135,18 @@
                     reqmaxsize = reqsize
                 else:
                     reqmaxsize = reqsize + random.randrange(0, 200)
-                (start, stop) = self.memmgr.malloc(reqsize, reqmaxsize)
+                (start, stop) = self.asmmemmgr.malloc(reqsize, reqmaxsize)
                 assert reqsize <= stop - start < reqmaxsize + 8
                 for otherstart, otherstop in got:           # no overlap
                     assert otherstop <= start or stop <= otherstart
                 got.append((start, stop))
                 real_use += (stop - start)
-                if self.memmgr.total_memory_allocated == prev_total:
+                if self.asmmemmgr.total_memory_allocated == prev_total:
                     iterations_without_allocating_more += 1
                     if iterations_without_allocating_more == 40000:
                         break    # ok
                 else:
-                    new_total = self.memmgr.total_memory_allocated
+                    new_total = self.asmmemmgr.total_memory_allocated
                     iterations_without_allocating_more = 0
                     print real_use, new_total
                     # We seem to never see a printed value greater
@@ -173,7 +173,7 @@
         #
         gcrootmap = FakeGcRootMap()
         allblocks = []
-        rawstart = mc.materialize(self.memmgr, allblocks, gcrootmap)
+        rawstart = mc.materialize(self, allblocks, gcrootmap)
         p = rffi.cast(rffi.CArrayPtr(lltype.Char), rawstart)
         assert p[0] == 'X'
         assert p[1] == 'x'
@@ -265,16 +265,3 @@
     md.done()
     assert allblocks == [(1597, 1697), (1797, 1835)]
     assert ops == [('free', 1835, 1897)]
-
-def test_find_jit_frame_depth():
-    mgr = AsmMemoryManager()
-    mgr.register_frame_depth_map(11, [0, 5, 10], [1, 2, 3])
-    mgr.register_frame_depth_map(30, [0, 5, 10], [4, 5, 6])
-    mgr.register_frame_depth_map(0, [0, 5, 10], [7, 8, 9])
-    asmmemmgr._memmngr = mgr
-    assert stack_depth_at_loc(13) == 1
-    assert stack_depth_at_loc(-3) == -1
-    assert stack_depth_at_loc(41) == -1
-    assert stack_depth_at_loc(5) == 8
-    assert stack_depth_at_loc(17) == 2
-    assert stack_depth_at_loc(38) == 5
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -149,7 +149,7 @@
         mc.MOV_bi(gcmap_ofs, 0)
         self._pop_all_regs_from_frame(mc, [], self.cpu.supports_floats)
         mc.RET()
-        self._frame_realloc_slowpath = mc.materialize(self.cpu.asmmemmgr, [])
+        self._frame_realloc_slowpath = mc.materialize(self.cpu, [])
 
     def _build_cond_call_slowpath(self, supports_floats, callee_only):
         """ This builds a general call slowpath, for whatever call happens to
@@ -184,7 +184,7 @@
         self._pop_all_regs_from_frame(mc, [], supports_floats, callee_only)
         self.pop_gcmap(mc)   # push_gcmap(store=True) done by the caller
         mc.RET()
-        return mc.materialize(self.cpu.asmmemmgr, [])
+        return mc.materialize(self.cpu, [])
 
     def _build_malloc_slowpath(self, kind):
         """ While arriving on slowpath, we have a gcpattern on stack 0.
@@ -274,7 +274,7 @@
         mc.ADD_ri(esp.value, WORD)
         mc.JMP(imm(self.propagate_exception_path))
         #
-        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
+        rawstart = mc.materialize(self.cpu, [])
         return rawstart
 
     def _build_propagate_exception_path(self):
@@ -295,7 +295,7 @@
         self.mc.MOV(RawEbpLoc(ofs), imm(propagate_exception_descr))
         #
         self._call_footer()
-        rawstart = self.mc.materialize(self.cpu.asmmemmgr, [])
+        rawstart = self.mc.materialize(self.cpu, [])
         self.propagate_exception_path = rawstart
         self.mc = None
 
@@ -343,7 +343,7 @@
         mc.ADD_ri(esp.value, WORD)
         mc.JMP(imm(self.propagate_exception_path))
         #
-        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
+        rawstart = mc.materialize(self.cpu, [])
         self.stack_check_slowpath = rawstart
 
     def _build_wb_slowpath(self, withcards, withfloats=False, for_frame=False):
@@ -430,7 +430,7 @@
             mc.LEA_rs(esp.value, 7 * WORD)
             mc.RET()
 
-        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
+        rawstart = mc.materialize(self.cpu, [])
         if for_frame:
             self.wb_slowpath[4] = rawstart
         else:
@@ -524,7 +524,8 @@
             assert len(set(inputargs)) == len(inputargs)
 
         self.setup(original_loop_token)
-        self.codemap.inherit_code_from_position(faildescr.adr_jump_offset)
+        self.codemap_builder.inherit_code_from_position(
+            faildescr.adr_jump_offset)
         self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         descr_number = compute_unique_id(faildescr)
         if log:
@@ -687,10 +688,10 @@
         self.datablockwrapper = None
         allblocks = self.get_asmmemmgr_blocks(looptoken)
         size = self.mc.get_relative_pos()
-        res = self.mc.materialize(self.cpu.asmmemmgr, allblocks,
+        res = self.mc.materialize(self.cpu, allblocks,
                                   self.cpu.gc_ll_descr.gcrootmap)
-        self.cpu.asmmemmgr.register_codemap(
-            self.codemap.get_final_bytecode(res, size))
+        self.cpu.codemap.register_codemap(
+            self.codemap_builder.get_final_bytecode(res, size))
         return res
 
     def patch_jump_for_descr(self, faildescr, adr_new_target):
@@ -1885,7 +1886,7 @@
         # now we return from the complete frame, which starts from
         # _call_header_with_stack_check().  The _call_footer below does it.
         self._call_footer()
-        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
+        rawstart = mc.materialize(self.cpu, [])
         self.failure_recovery_code[exc + 2 * withfloats] = rawstart
         self.mc = None
 
diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -67,6 +67,7 @@
     @rgc.no_release_gil
     def setup_once(self):
         self.profile_agent.startup()
+        self.codemap.setup()
         self.assembler.setup_once()
 
     @rgc.no_release_gil
diff --git a/rpython/jit/backend/x86/test/test_assembler.py b/rpython/jit/backend/x86/test/test_assembler.py
--- a/rpython/jit/backend/x86/test/test_assembler.py
+++ b/rpython/jit/backend/x86/test/test_assembler.py
@@ -51,6 +51,7 @@
                 asmmemmgr_blocks = None
         cpu = ACTUAL_CPU(None, None)
         cpu.setup()
+        cpu.codemap.setup()
         looptoken = FakeToken()
         asm = cpu.assembler
         asm.setup_once()
diff --git a/rpython/jit/backend/x86/test/test_calling_convention.py b/rpython/jit/backend/x86/test/test_calling_convention.py
--- a/rpython/jit/backend/x86/test/test_calling_convention.py
+++ b/rpython/jit/backend/x86/test/test_calling_convention.py
@@ -10,7 +10,7 @@
         mc.MOV(eax, esp)
         mc.ADD_ri(eax.value, WORD)
         mc.RET()
-        return mc.materialize(self.cpu.asmmemmgr, [])
+        return mc.materialize(self.cpu, [])
 
     def get_alignment_requirements(self):
         return 16
diff --git a/rpython/jit/backend/x86/test/test_recursive.py b/rpython/jit/backend/x86/test/test_recursive.py
--- a/rpython/jit/backend/x86/test/test_recursive.py
+++ b/rpython/jit/backend/x86/test/test_recursive.py
@@ -1,7 +1,7 @@
 
 from rpython.jit.metainterp.test.test_recursive import RecursiveTests
 from rpython.jit.backend.x86.test.test_basic import Jit386Mixin
-from rpython.jit.backend.llsupport import asmmemmgr
+from rpython.jit.backend.llsupport import codemap
 from rpython.jit.backend.llsupport.codemap import unpack_traceback
 from rpython.jit.backend.x86.arch import WORD
 
@@ -11,9 +11,11 @@
     def check_get_unique_id(self):
         if WORD == 4:
             return # this is 64 bit only check
-        codemaps = asmmemmgr._memmngr.jit_codemap[:] # ups, sorting later
-        assert len(codemaps) == 3
-        codemaps.sort(lambda arg0, arg1: cmp(arg0[1], arg1[1]))
+        jit_codemap = codemap._codemap.jit_codemap
+        codemaps = [(jit_codemap[i].addr, jit_codemap[i].machine_code_size)
+                    for i in range(3)]
+        assert codemap._codemap.jit_codemap_used == 3
+        codemaps.sort(lambda a, b: cmp(a[1], b[1]))
         # biggest is the big loop, smallest is the bridge
         assert unpack_traceback(codemaps[1][0]) == []
         # XXX very specific ASM addresses, very fragile test, but what we can
diff --git a/rpython/jit/backend/x86/test/test_runner.py b/rpython/jit/backend/x86/test/test_runner.py
--- a/rpython/jit/backend/x86/test/test_runner.py
+++ b/rpython/jit/backend/x86/test/test_runner.py
@@ -461,7 +461,7 @@
                 mc.RET()
             else:
                 mc.RET16_i(40)
-            rawstart = mc.materialize(cpu.asmmemmgr, [])
+            rawstart = mc.materialize(cpu, [])
             #
             calldescr = cpu._calldescr_dynamic_for_tests([types.slong] * 10,
                                                          types.slong)
diff --git a/rpython/rlib/rbisect.py b/rpython/rlib/rbisect.py
--- a/rpython/rlib/rbisect.py
+++ b/rpython/rlib/rbisect.py
@@ -1,39 +1,43 @@
 
-def bisect_left(a, x):
+def bisect_left(a, x, hi=-1):
     """Return the index in the sorted list 'a' of 'x'.  If 'x' is not in 'a',
     return the index where it can be inserted."""
     lo = 0
-    hi = len(a)
+    if hi == -1:
+        hi = len(a)
     while lo < hi:
         mid = (lo+hi)//2
         if a[mid] < x: lo = mid+1
         else: hi = mid
     return lo
 
-def bisect_right(a, x):
+def bisect_right(a, x, hi=-1):
     lo = 0
-    hi = len(a)
+    if hi == -1:
+        hi = len(a)
     while lo < hi:
         mid = (lo+hi)//2
         if x < a[mid]: hi = mid
         else: lo = mid+1
     return lo
 
-# a copy of the above, but compares the first item of a tuple only
-def bisect_left_tuple(a, x):
+# a copy of the above, but compares the item called 'addr' only
+def bisect_left_addr(a, x, hi=-1):
     lo = 0
-    hi = len(a)
+    if hi == -1:
+        hi = len(a)
     while lo < hi:
         mid = (lo+hi)//2
-        if a[mid][0] < x: lo = mid+1
+        if a[mid].addr < x: lo = mid+1
         else: hi = mid
     return lo
 
-def bisect_right_tuple(a, x):
+def bisect_right_addr(a, x, hi=-1):
     lo = 0
-    hi = len(a)
+    if hi == -1:
+        hi = len(a)
     while lo < hi:
         mid = (lo+hi)//2
-        if x < a[mid][0]: hi = mid
+        if x < a[mid].addr: hi = mid
         else: lo = mid+1
     return lo

From noreply at buildbot.pypy.org  Mon Feb 23 13:00:32 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Feb 2015 13:00:32 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: try to please C
Message-ID: <20150223120032.708651C12E9@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r76066:2833fe32547c
Date: 2015-02-23 13:59 +0200
http://bitbucket.org/pypy/pypy/changeset/2833fe32547c/

Log:	try to please C

diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py
--- a/rpython/jit/backend/llsupport/codemap.py
+++ b/rpython/jit/backend/llsupport/codemap.py
@@ -34,9 +34,11 @@
 _codemap = None
 
 eci = ExternalCompilationInfo(post_include_bits=["""
+RPY_EXTERN volatile int pypy_codemap_currently_invalid;
+RPY_EXTERN void pypy_codemap_invalid_set(int);
+"""], separate_module_sources=["""
 volatile int pypy_codemap_currently_invalid = 0;
-void pypy_codemap_invalid_set(int);
-"""], separate_module_sources=["""
+
 void pypy_codemap_invalid_set(int value)
 {
     pypy_codemap_currently_invalid = value;

From noreply at buildbot.pypy.org  Mon Feb 23 13:02:24 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Feb 2015 13:02:24 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: pfff
Message-ID: <20150223120224.6E55B1C12B8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r76067:b035e778d5d4
Date: 2015-02-23 14:01 +0200
http://bitbucket.org/pypy/pypy/changeset/b035e778d5d4/

Log:	pfff

diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py
--- a/rpython/jit/backend/llsupport/codemap.py
+++ b/rpython/jit/backend/llsupport/codemap.py
@@ -126,6 +126,7 @@
     """ An immortal wrapper around underlaying jit codemap data
     """
     track_allocation = False
+    jit_addr_map = lltype.nullptr(INT_LIST)
     
     def __init__(self):
         global _codemap
@@ -155,7 +156,7 @@
 
     def free(self):
         # if setup has not been called
-        if not hasattr(self, 'jit_addr_map') or not self.jit_addr_map:
+        if not self.jit_addr_map:
             return
         lltype.free(self.jit_addr_map, flavor='raw')
         i = 0

From noreply at buildbot.pypy.org  Mon Feb 23 13:09:38 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Feb 2015 13:09:38 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: have a custom sanity check
Message-ID: <20150223120938.968121C01CD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r76068:2e668504f253
Date: 2015-02-23 14:08 +0200
http://bitbucket.org/pypy/pypy/changeset/2e668504f253/

Log:	have a custom sanity check

diff --git a/pypy/module/_vmprof/src/get_custom_offset.c b/pypy/module/_vmprof/src/get_custom_offset.c
--- a/pypy/module/_vmprof/src/get_custom_offset.c
+++ b/pypy/module/_vmprof/src/get_custom_offset.c
@@ -5,10 +5,17 @@
 long pypy_find_codemap_at_addr(long);
 long pypy_yield_codemap_at_addr(long, long, long*);
 
+extern volatile int pypy_codemap_currently_invalid;
+
 void vmprof_set_tramp_range(void* start, void* end)
 {
 }
 
+int custom_sanity_check()
+{
+	return !pypy_codemap_currently_invalid;
+}
+
 static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, unw_cursor_t *cp) {
 	intptr_t ip_l = (intptr_t)ip;
 
diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
--- a/pypy/module/_vmprof/src/vmprof.c
+++ b/pypy/module/_vmprof/src/vmprof.c
@@ -141,6 +141,9 @@
     if (recursive) {
         return 0;
     }
+	if (!custom_sanity_check()) {
+		return 0;
+	}
     ++recursive;
 
     int ret = unw_init_local(&cursor, &uc);

From noreply at buildbot.pypy.org  Mon Feb 23 13:12:32 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Feb 2015 13:12:32 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: silence RPython warnings
Message-ID: <20150223121232.163B41C03D0@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r76069:23f57f89a9c4
Date: 2015-02-23 14:11 +0200
http://bitbucket.org/pypy/pypy/changeset/23f57f89a9c4/

Log:	silence RPython warnings

diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py
--- a/rpython/jit/backend/llsupport/codemap.py
+++ b/rpython/jit/backend/llsupport/codemap.py
@@ -127,6 +127,11 @@
     """
     track_allocation = False
     jit_addr_map = lltype.nullptr(INT_LIST)
+    jit_addr_map_used = 0
+    jit_codemap = lltype.nullptr(CODEMAP_LIST)
+    jit_codemap_used = 0
+    jit_frame_depth_map = lltype.nullptr(INT_LIST)
+    jit_frame_depth_map_used = 0
     
     def __init__(self):
         global _codemap

From noreply at buildbot.pypy.org  Mon Feb 23 13:26:48 2015
From: noreply at buildbot.pypy.org (xando)
Date: Mon, 23 Feb 2015 13:26:48 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: (xando,
 mjacob) Raise SyntaxError when compiling multiple statements as
 single interactive statement
Message-ID: <20150223122648.836FD1C01CD@cobra.cs.uni-duesseldorf.de>

Author: Sebastian Pawlu? 
Branch: py3.3
Changeset: r76070:b3ad7de8410c
Date: 2015-02-23 13:26 +0100
http://bitbucket.org/pypy/pypy/changeset/b3ad7de8410c/

Log:	(xando, mjacob) Raise SyntaxError when compiling multiple statements
	as single interactive statement

diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py
--- a/pypy/interpreter/pyparser/pyparse.py
+++ b/pypy/interpreter/pyparser/pyparse.py
@@ -166,10 +166,27 @@
                 compile_info.last_future_import = last_future_import
                 compile_info.flags |= newflags
                 self.grammar = pygram.python_grammar
-
-                for tp, value, lineno, column, line in tokens:
+                tokens_stream = iter(tokens)
+                for tp, value, lineno, column, line in tokens_stream:
                     if self.add_token(tp, value, lineno, column, line):
                         break
+
+                if compile_info.mode == 'single':
+                    for tp, _, _, _, _ in tokens_stream:
+                        if tp == pygram.tokens.NEWLINE:
+                            continue
+
+                        if tp == pygram.tokens.COMMENT:
+                            for tp, _, _, _, _ in tokens_stream:
+                                if tp == pygram.tokens.NEWLINE:
+                                    break
+                        else:
+                            new_err = error.SyntaxError
+                            msg = ("multiple statements found while "
+                                   "compiling a single statement")
+                            raise new_err(msg, lineno, column,
+                                          line, compile_info.filename)
+
             except error.TokenError, e:
                 e.filename = compile_info.filename
                 raise
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -152,6 +152,16 @@
         self.parse("u''''''")
         self.parse("U''''''")
 
+    def test_bad_single_statement(self):
+        py.test.raises(SyntaxError, self.parse, '1\n2', "single")
+        py.test.raises(SyntaxError, self.parse, 'def f(): pass', "single")
+        py.test.raises(SyntaxError, self.parse, 'a = 13\nb = 187', "single")
+        py.test.raises(SyntaxError, self.parse, 'del x\ndel y', "single")
+        py.test.raises(SyntaxError, self.parse, 'f()\ng()', "single")
+        py.test.raises(SyntaxError, self.parse, 'f()\n# blah\nblah()', "single")
+        py.test.raises(SyntaxError, self.parse, 'f()\nxy # blah\nblah()', "single")
+        py.test.raises(SyntaxError, self.parse, 'x = 5 # comment\nx = 6\n', "single")
+
 
 class TestPythonParserWithSpace:
 

From noreply at buildbot.pypy.org  Mon Feb 23 13:38:27 2015
From: noreply at buildbot.pypy.org (xando)
Date: Mon, 23 Feb 2015 13:38:27 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: fix
Message-ID: <20150223123827.153D71C01CD@cobra.cs.uni-duesseldorf.de>

Author: Sebastian Pawlu? 
Branch: py3.3
Changeset: r76071:e142a637dd0c
Date: 2015-02-23 13:38 +0100
http://bitbucket.org/pypy/pypy/changeset/e142a637dd0c/

Log:	fix

diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py
--- a/pypy/interpreter/pyparser/pyparse.py
+++ b/pypy/interpreter/pyparser/pyparse.py
@@ -172,7 +172,7 @@
                         break
 
                 if compile_info.mode == 'single':
-                    for tp, _, _, _, _ in tokens_stream:
+                    for tp, value, lineno, column, line in tokens_stream:
                         if tp == pygram.tokens.NEWLINE:
                             continue
 

From noreply at buildbot.pypy.org  Mon Feb 23 13:47:06 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Feb 2015 13:47:06 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: try to fix translation
Message-ID: <20150223124706.86BA31C01CD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r76072:5a7fa33f27a8
Date: 2015-02-23 14:46 +0200
http://bitbucket.org/pypy/pypy/changeset/5a7fa33f27a8/

Log:	try to fix translation

diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py
--- a/rpython/jit/backend/llsupport/codemap.py
+++ b/rpython/jit/backend/llsupport/codemap.py
@@ -47,7 +47,8 @@
 
 ll_pypy_codemap_invalid_set = rffi.llexternal('pypy_codemap_invalid_set',
                                               [rffi.INT], lltype.Void,
-                                              compilation_info=eci)
+                                              compilation_info=eci,
+                                              _nowrapper=True)
 
 def pypy_codemap_invalid_set(val):
     if we_are_translated():

From noreply at buildbot.pypy.org  Mon Feb 23 14:03:17 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Feb 2015 14:03:17 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: (fijal,
 arigo) fix the frame handling race conditions in unjitted trampoline
Message-ID: <20150223130317.EA7581C054A@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r76073:e6bed8caddf8
Date: 2015-02-23 15:03 +0200
http://bitbucket.org/pypy/pypy/changeset/e6bed8caddf8/

Log:	(fijal, arigo) fix the frame handling race conditions in unjitted
	trampoline

diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -56,7 +56,7 @@
 
 pypy_execute_frame_trampoline = rffi.llexternal(
     "pypy_execute_frame_trampoline",
-    [llmemory.GCREF, llmemory.GCREF, llmemory.GCREF],
+    [llmemory.GCREF, llmemory.GCREF, llmemory.GCREF, lltype.Signed],
     llmemory.GCREF,
     compilation_info=eci,
     _nowrapper=True, sandboxsafe=True,
@@ -96,23 +96,18 @@
             gc_frame = cast_instance_to_gcref(frame)
             gc_inputvalue = cast_instance_to_gcref(w_inputvalue)
             gc_operr = cast_instance_to_gcref(operr)
-            gc_result = pypy_execute_frame_trampoline(gc_frame, gc_inputvalue, gc_operr)
+            unique_id = frame.pycode._unique_id
+            gc_result = pypy_execute_frame_trampoline(gc_frame, gc_inputvalue,
+                                                      gc_operr, unique_id)
             return cast_base_ptr_to_instance(W_Root, gc_result)
         else:
             return original_execute_frame(frame, w_inputvalue, operr)
 
 
- at entrypoint.entrypoint_lowlevel('main', [llmemory.GCREF],
+ at entrypoint.entrypoint_lowlevel('main', [lltype.Signed],
                                 'pypy_vmprof_get_virtual_ip', True)
-def get_virtual_ip(gc_frame):
-    frame = cast_base_ptr_to_instance(PyFrame, gc_frame)
-    if jit._get_virtualizable_token(frame):
-        return rffi.cast(rffi.VOIDP, 0)
-    virtual_ip = do_get_virtual_ip(frame)
-    return rffi.cast(rffi.VOIDP, virtual_ip)
-
-def do_get_virtual_ip(frame):
-    return frame.pycode._unique_id
+def get_virtual_ip(unique_id):
+    return rffi.cast(rffi.VOIDP, unique_id)
 
 def write_long_to_string_builder(l, b):
     if sys.maxint == 2147483647:
diff --git a/pypy/module/_vmprof/src/trampoline.asmgcc.s b/pypy/module/_vmprof/src/trampoline.asmgcc.s
--- a/pypy/module/_vmprof/src/trampoline.asmgcc.s
+++ b/pypy/module/_vmprof/src/trampoline.asmgcc.s
@@ -6,11 +6,11 @@
 	.type	pypy_execute_frame_trampoline, @function
 pypy_execute_frame_trampoline:
 	.cfi_startproc
-	pushq	%rdi
+	pushq	%rcx
 	.cfi_def_cfa_offset 16
 	call pypy_pyframe_execute_frame at PLT
 	/* GCROOT 0(%rsp) */
-	popq	%rdi
+	popq	%rcx
 	.cfi_def_cfa_offset 8
 	ret
 	.cfi_endproc

From noreply at buildbot.pypy.org  Mon Feb 23 14:04:28 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Feb 2015 14:04:28 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: reenable vmprof by default
Message-ID: <20150223130428.468111C054A@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r76074:c028c55c7a58
Date: 2015-02-23 15:04 +0200
http://bitbucket.org/pypy/pypy/changeset/c028c55c7a58/

Log:	reenable vmprof by default

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -38,8 +38,8 @@
     "_csv", "cppyy", "_pypyjson"
 ])
 
-#if sys.platform.startswith('linux') and sys.maxint > 2147483647:
-#    working_modules.add('_vmprof')
+if sys.platform.startswith('linux') and sys.maxint > 2147483647:
+    working_modules.add('_vmprof')
 
 translation_modules = default_modules.copy()
 translation_modules.update([

From noreply at buildbot.pypy.org  Mon Feb 23 14:06:02 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Feb 2015 14:06:02 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: fix the import
Message-ID: <20150223130602.D840B1C054A@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r76075:3bc22debc8ac
Date: 2015-02-23 15:05 +0200
http://bitbucket.org/pypy/pypy/changeset/3bc22debc8ac/

Log:	fix the import

diff --git a/pypy/module/_vmprof/src/fake_pypy_api.c b/pypy/module/_vmprof/src/fake_pypy_api.c
--- a/pypy/module/_vmprof/src/fake_pypy_api.c
+++ b/pypy/module/_vmprof/src/fake_pypy_api.c
@@ -27,3 +27,5 @@
 void pypy_pyframe_execute_frame(void)
 {
 }
+
+volatile int pypy_codemap_currently_invalid = 0;

From noreply at buildbot.pypy.org  Mon Feb 23 14:23:16 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Mon, 23 Feb 2015 14:23:16 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: oops
Message-ID: <20150223132316.1BAE81C03D0@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76076:bba15e337db2
Date: 2015-02-23 13:49 +0100
http://bitbucket.org/pypy/pypy/changeset/bba15e337db2/

Log:	oops

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -505,7 +505,7 @@
         is_text_encoding = space.is_true(
                 space.getattr(codec_info, space.wrap('_is_text_encoding')))
     except OperationError as e:
-        if e.match(space.w_AttributeError):
+        if e.match(space, space.w_AttributeError):
             is_text_encoding = True
         else:
             raise
@@ -559,7 +559,7 @@
         is_text_encoding = space.is_true(
                 space.getattr(codec_info, space.wrap('_is_text_encoding')))
     except OperationError as e:
-        if e.match(space.w_AttributeError):
+        if e.match(space, space.w_AttributeError):
             is_text_encoding = True
         else:
             raise

From noreply at buildbot.pypy.org  Mon Feb 23 14:30:42 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Feb 2015 14:30:42 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: bleh
Message-ID: <20150223133042.5247E1C01CD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r76077:a08cabccecc0
Date: 2015-02-23 15:30 +0200
http://bitbucket.org/pypy/pypy/changeset/a08cabccecc0/

Log:	bleh

diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py
--- a/rpython/jit/backend/llsupport/codemap.py
+++ b/rpython/jit/backend/llsupport/codemap.py
@@ -48,7 +48,7 @@
 ll_pypy_codemap_invalid_set = rffi.llexternal('pypy_codemap_invalid_set',
                                               [rffi.INT], lltype.Void,
                                               compilation_info=eci,
-                                              _nowrapper=True)
+                                              releasegil=False)
 
 def pypy_codemap_invalid_set(val):
     if we_are_translated():

From noreply at buildbot.pypy.org  Mon Feb 23 14:40:40 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 23 Feb 2015 14:40:40 +0100 (CET)
Subject: [pypy-commit] stmgc default: Change some dprints
Message-ID: <20150223134040.B9ACF1C01CD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1644:856d24b4ea37
Date: 2015-02-23 14:41 +0100
http://bitbucket.org/pypy/stmgc/changeset/856d24b4ea37/

Log:	Change some dprints

diff --git a/c7/stm/extra.c b/c7/stm/extra.c
--- a/c7/stm/extra.c
+++ b/c7/stm/extra.c
@@ -40,6 +40,7 @@
     if (result < 0 && callback != NULL) {
         /* no regular transaction running, invoke the callback
            immediately */
+        dprintf(("stm_call_on_commit calls now: %p(%p)\n", callback, key));
         callback(key);
     }
     return result;
@@ -76,6 +77,8 @@
         /* The callback may call stm_call_on_abort(key, NULL)
            (so with callback==NULL).  It is ignored, because
            'callbacks_on_commit_and_abort' was cleared already. */
+        dprintf(("invoke_and_clear_user_callbacks(%ld): %p(%p)\n",
+                 index, callback, key));
         callback(key);
 
     } TREE_LOOP_END;
diff --git a/c7/stm/nursery.c b/c7/stm/nursery.c
--- a/c7/stm/nursery.c
+++ b/c7/stm/nursery.c
@@ -263,9 +263,9 @@
     OPT_ASSERT(write_locks[first_card_index] <= NB_SEGMENTS
                || write_locks[first_card_index] == 255); /* see gcpage.c */
 
-    dprintf(("mark cards of %p, size %lu with %d, all: %d\n",
+  /*dprintf(("mark cards of %p, size %lu with %d, all: %d\n",
              obj, size, mark_value, mark_all));
-    dprintf(("obj has %lu cards\n", last_card_index));
+    dprintf(("obj has %lu cards\n", last_card_index));*/
     while (card_index <= last_card_index) {
         uintptr_t card_lock_idx = first_card_index + card_index;
 
diff --git a/c7/stm/pages.c b/c7/stm/pages.c
--- a/c7/stm/pages.c
+++ b/c7/stm/pages.c
@@ -63,12 +63,12 @@
 
 static void d_remap_file_pages(char *addr, size_t size, ssize_t pgoff)
 {
-    dprintf(("remap_file_pages: 0x%lx bytes: (seg%ld %p) --> (seg%ld %p)\n",
+    /*dprintf(("remap_file_pages: 0x%lx bytes: (seg%ld %p) --> (seg%ld %p)\n",
              (long)size,
              (long)((addr - stm_object_pages) / 4096UL) / NB_PAGES,
              (void *)((addr - stm_object_pages) % (4096UL * NB_PAGES)),
              (long)pgoff / NB_PAGES,
-             (void *)((pgoff % NB_PAGES) * 4096UL)));
+             (void *)((pgoff % NB_PAGES) * 4096UL)));*/
     assert(size % 4096 == 0);
     assert(size <= TOTAL_MEMORY);
     assert(((uintptr_t)addr) % 4096 == 0);

From noreply at buildbot.pypy.org  Mon Feb 23 14:42:05 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 23 Feb 2015 14:42:05 +0100 (CET)
Subject: [pypy-commit] pypy default: Raaaah I hate libffi.
Message-ID: <20150223134205.F050C1C052B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76078:8e51496c7822
Date: 2015-02-23 14:37 +0100
http://bitbucket.org/pypy/pypy/changeset/8e51496c7822/

Log:	Raaaah I hate libffi.

diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py
--- a/pypy/module/_rawffi/array.py
+++ b/pypy/module/_rawffi/array.py
@@ -15,7 +15,7 @@
 from pypy.module._rawffi.interp_rawffi import unpack_shape_with_length
 from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr
 from rpython.rlib.rarithmetic import r_uint
-from rpython.rlib import rgc
+from rpython.rlib import rgc, clibffi
 
 
 class W_Array(W_DataShape):
@@ -84,14 +84,11 @@
 
 class W_ArrayInstance(W_DataInstance):
     def __init__(self, space, shape, length, address=r_uint(0)):
-        # Workaround for a strange behavior of libffi: make sure that
-        # we always have at least 8 bytes.  For W_ArrayInstances that are
-        # used as the result value of a function call, ffi_call() writes
-        # 8 bytes into it even if the function's result type asks for less.
-        # This strange behavior is documented.
         memsize = shape.size * length
-        if memsize < 8:
-            memsize = 8
+        # For W_ArrayInstances that are used as the result value of a
+        # function call, ffi_call() writes 8 bytes into it even if the
+        # function's result type asks for less.
+        memsize = clibffi.adjust_return_size(memsize)
         W_DataInstance.__init__(self, space, memsize, address)
         self.length = length
         self.shape = shape
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -495,6 +495,7 @@
         try:
             if self.resshape is not None:
                 result = self.resshape.allocate(space, 1, autofree=True)
+                # adjust_return_size() was used here on result.ll_buffer
                 self.ptr.call(args_ll, result.ll_buffer)
                 return space.wrap(result)
             else:
diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py
--- a/rpython/rlib/clibffi.py
+++ b/rpython/rlib/clibffi.py
@@ -564,6 +564,7 @@
         self.funcsym = funcsym
 
     def call(self, args_ll, ll_result):
+        # adjust_return_size() should always be used here on ll_result
         assert len(args_ll) == len(self.argtypes), (
             "wrong number of arguments in call to %s(): "
             "%d instead of %d" % (self.name, len(args_ll), len(self.argtypes)))
@@ -594,8 +595,8 @@
                                             intmask(argtypes[i].c_size),
                                             flavor='raw')
         if restype != ffi_type_void:
-            self.ll_result = lltype.malloc(rffi.VOIDP.TO,
-                                           intmask(restype.c_size),
+            size = adjust_return_size(intmask(restype.c_size))
+            self.ll_result = lltype.malloc(rffi.VOIDP.TO, size,
                                            flavor='raw')
 
     def push_arg(self, value):
@@ -693,3 +694,12 @@
             dlclose(self.lib)
             self.lib = rffi.cast(DLLHANDLE, -1)
 
+
+def adjust_return_size(memsize):
+    # Workaround for a strange behavior of libffi: make sure that
+    # we always have at least 8 bytes.  ffi_call() writes 8 bytes
+    # into the buffer even if the function's result type asks for
+    # less.  This strange behavior is documented.
+    if memsize < 8:
+        memsize = 8
+    return memsize
diff --git a/rpython/rlib/libffi.py b/rpython/rlib/libffi.py
--- a/rpython/rlib/libffi.py
+++ b/rpython/rlib/libffi.py
@@ -9,7 +9,8 @@
 from rpython.rlib import jit
 from rpython.rlib import clibffi
 from rpython.rlib.clibffi import FUNCFLAG_CDECL, FUNCFLAG_STDCALL, \
-        AbstractFuncPtr, push_arg_as_ffiptr, c_ffi_call, FFI_TYPE_STRUCT
+        AbstractFuncPtr, push_arg_as_ffiptr, c_ffi_call, FFI_TYPE_STRUCT, \
+        adjust_return_size
 from rpython.rlib.rdynload import dlopen, dlclose, dlsym, dlsym_byordinal
 from rpython.rlib.rdynload import DLLHANDLE
 
@@ -369,8 +370,8 @@
         # XXX: check len(args)?
         ll_result = lltype.nullptr(rffi.CCHARP.TO)
         if self.restype != types.void:
-            ll_result = lltype.malloc(rffi.CCHARP.TO,
-                                      intmask(self.restype.c_size),
+            size = adjust_return_size(intmask(self.restype.c_size))
+            ll_result = lltype.malloc(rffi.CCHARP.TO, size,
                                       flavor='raw')
         ffires = c_ffi_call(self.ll_cif,
                             self.funcsym,

From noreply at buildbot.pypy.org  Mon Feb 23 14:42:07 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 23 Feb 2015 14:42:07 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: Raaaah I hate libffi.
Message-ID: <20150223134207.277C41C052B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: stmgc-c7
Changeset: r76079:5c6438dc2ec7
Date: 2015-02-23 14:37 +0100
http://bitbucket.org/pypy/pypy/changeset/5c6438dc2ec7/

Log:	Raaaah I hate libffi. (grafted from
	8e51496c782298ffe274ce4e64aa33d98be65c0b)

diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py
--- a/pypy/module/_rawffi/array.py
+++ b/pypy/module/_rawffi/array.py
@@ -15,7 +15,7 @@
 from pypy.module._rawffi.interp_rawffi import unpack_shape_with_length
 from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr
 from rpython.rlib.rarithmetic import r_uint
-from rpython.rlib import rgc
+from rpython.rlib import rgc, clibffi
 
 
 class W_Array(W_DataShape):
@@ -84,14 +84,11 @@
 
 class W_ArrayInstance(W_DataInstance):
     def __init__(self, space, shape, length, address=r_uint(0)):
-        # Workaround for a strange behavior of libffi: make sure that
-        # we always have at least 8 bytes.  For W_ArrayInstances that are
-        # used as the result value of a function call, ffi_call() writes
-        # 8 bytes into it even if the function's result type asks for less.
-        # This strange behavior is documented.
         memsize = shape.size * length
-        if memsize < 8:
-            memsize = 8
+        # For W_ArrayInstances that are used as the result value of a
+        # function call, ffi_call() writes 8 bytes into it even if the
+        # function's result type asks for less.
+        memsize = clibffi.adjust_return_size(memsize)
         W_DataInstance.__init__(self, space, memsize, address)
         self.length = length
         self.shape = shape
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -495,6 +495,7 @@
         try:
             if self.resshape is not None:
                 result = self.resshape.allocate(space, 1, autofree=True)
+                # adjust_return_size() was used here on result.ll_buffer
                 self.ptr.call(args_ll, result.ll_buffer)
                 return space.wrap(result)
             else:
diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py
--- a/rpython/rlib/clibffi.py
+++ b/rpython/rlib/clibffi.py
@@ -563,6 +563,7 @@
         self.funcsym = funcsym
 
     def call(self, args_ll, ll_result):
+        # adjust_return_size() should always be used here on ll_result
         assert len(args_ll) == len(self.argtypes), (
             "wrong number of arguments in call to %s(): "
             "%d instead of %d" % (self.name, len(args_ll), len(self.argtypes)))
@@ -593,8 +594,8 @@
                                             intmask(argtypes[i].c_size),
                                             flavor='raw')
         if restype != ffi_type_void:
-            self.ll_result = lltype.malloc(rffi.VOIDP.TO,
-                                           intmask(restype.c_size),
+            size = adjust_return_size(intmask(restype.c_size))
+            self.ll_result = lltype.malloc(rffi.VOIDP.TO, size,
                                            flavor='raw')
 
     def push_arg(self, value):
@@ -692,3 +693,12 @@
             dlclose(self.lib)
             self.lib = rffi.cast(DLLHANDLE, -1)
 
+
+def adjust_return_size(memsize):
+    # Workaround for a strange behavior of libffi: make sure that
+    # we always have at least 8 bytes.  ffi_call() writes 8 bytes
+    # into the buffer even if the function's result type asks for
+    # less.  This strange behavior is documented.
+    if memsize < 8:
+        memsize = 8
+    return memsize
diff --git a/rpython/rlib/libffi.py b/rpython/rlib/libffi.py
--- a/rpython/rlib/libffi.py
+++ b/rpython/rlib/libffi.py
@@ -9,7 +9,8 @@
 from rpython.rlib import jit
 from rpython.rlib import clibffi
 from rpython.rlib.clibffi import FUNCFLAG_CDECL, FUNCFLAG_STDCALL, \
-        AbstractFuncPtr, push_arg_as_ffiptr, c_ffi_call, FFI_TYPE_STRUCT
+        AbstractFuncPtr, push_arg_as_ffiptr, c_ffi_call, FFI_TYPE_STRUCT, \
+        adjust_return_size
 from rpython.rlib.rdynload import dlopen, dlclose, dlsym, dlsym_byordinal
 from rpython.rlib.rdynload import DLLHANDLE
 
@@ -369,8 +370,8 @@
         # XXX: check len(args)?
         ll_result = lltype.nullptr(rffi.CCHARP.TO)
         if self.restype != types.void:
-            ll_result = lltype.malloc(rffi.CCHARP.TO,
-                                      intmask(self.restype.c_size),
+            size = adjust_return_size(intmask(self.restype.c_size))
+            ll_result = lltype.malloc(rffi.CCHARP.TO, size,
                                       flavor='raw')
         ffires = c_ffi_call(self.ll_cif,
                             self.funcsym,

From noreply at buildbot.pypy.org  Mon Feb 23 14:42:08 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 23 Feb 2015 14:42:08 +0100 (CET)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20150223134208.43FFB1C052B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76080:a5f3df81f370
Date: 2015-02-23 14:41 +0100
http://bitbucket.org/pypy/pypy/changeset/a5f3df81f370/

Log:	merge heads

diff --git a/lib_pypy/gdbm.py b/lib_pypy/gdbm.py
--- a/lib_pypy/gdbm.py
+++ b/lib_pypy/gdbm.py
@@ -127,7 +127,7 @@
     def __contains__(self, key):
         self._check_closed()
         key = _checkstr(key)
-        return lib.pygdbm_exists(self.ll_dbm, key, len(key)
+        return lib.pygdbm_exists(self.ll_dbm, key, len(key))
     has_key = __contains__
 
     def __getitem__(self, key):

From noreply at buildbot.pypy.org  Mon Feb 23 15:11:47 2015
From: noreply at buildbot.pypy.org (Alexander Schremmer)
Date: Mon, 23 Feb 2015 15:11:47 +0100 (CET)
Subject: [pypy-commit] benchmarks default: Fix webapp.py to adhere to
	command line arguments.
Message-ID: <20150223141147.DDFD71C03D0@cobra.cs.uni-duesseldorf.de>

Author: Alexander Schremmer 
Branch: 
Changeset: r311:a40d3b116f07
Date: 2015-02-23 15:12 +0100
http://bitbucket.org/pypy/benchmarks/changeset/a40d3b116f07/

Log:	Fix webapp.py to adhere to command line arguments.

diff --git a/multithread/wsgi/webapp.py b/multithread/wsgi/webapp.py
--- a/multithread/wsgi/webapp.py
+++ b/multithread/wsgi/webapp.py
@@ -54,5 +54,4 @@
 
 
 if __name__ == '__main__':
-    init(sys.argv[1])
-    run()
+    run(*init(*sys.argv[1:]))

From noreply at buildbot.pypy.org  Mon Feb 23 15:23:58 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Feb 2015 15:23:58 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: oops
Message-ID: <20150223142358.4CE581C054A@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r76081:e1127a96291d
Date: 2015-02-23 16:23 +0200
http://bitbucket.org/pypy/pypy/changeset/e1127a96291d/

Log:	oops

diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -28,7 +28,7 @@
     libraries = ['unwind'],
     
     post_include_bits=["""
-        void* pypy_vmprof_get_virtual_ip(void*);
+        void* pypy_vmprof_get_virtual_ip(long);
         void pypy_vmprof_init(void);
     """],
     

From noreply at buildbot.pypy.org  Mon Feb 23 15:26:01 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Feb 2015 15:26:01 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: another one
Message-ID: <20150223142601.A35441C0579@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r76082:44c72ab59bf7
Date: 2015-02-23 16:25 +0200
http://bitbucket.org/pypy/pypy/changeset/44c72ab59bf7/

Log:	another one

diff --git a/pypy/module/_vmprof/src/trampoline.h b/pypy/module/_vmprof/src/trampoline.h
--- a/pypy/module/_vmprof/src/trampoline.h
+++ b/pypy/module/_vmprof/src/trampoline.h
@@ -1,1 +1,1 @@
-void* pypy_execute_frame_trampoline(void*, void*, void*);
+void* pypy_execute_frame_trampoline(void*, void*, void*, long);

From noreply at buildbot.pypy.org  Mon Feb 23 15:59:50 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Feb 2015 15:59:50 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: grr grr grr
Message-ID: <20150223145950.444CE1C03D0@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r76083:509694031e71
Date: 2015-02-23 16:59 +0200
http://bitbucket.org/pypy/pypy/changeset/509694031e71/

Log:	grr grr grr

diff --git a/pypy/module/_vmprof/src/trampoline.asmgcc.s b/pypy/module/_vmprof/src/trampoline.asmgcc.s
--- a/pypy/module/_vmprof/src/trampoline.asmgcc.s
+++ b/pypy/module/_vmprof/src/trampoline.asmgcc.s
@@ -9,7 +9,6 @@
 	pushq	%rcx
 	.cfi_def_cfa_offset 16
 	call pypy_pyframe_execute_frame at PLT
-	/* GCROOT 0(%rsp) */
 	popq	%rcx
 	.cfi_def_cfa_offset 8
 	ret

From noreply at buildbot.pypy.org  Mon Feb 23 16:26:55 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 23 Feb 2015 16:26:55 +0100 (CET)
Subject: [pypy-commit] stmgc default: (remi, arigo)
Message-ID: <20150223152655.C26BB1C01CD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1645:61743f8ebe26
Date: 2015-02-23 16:27 +0100
http://bitbucket.org/pypy/stmgc/changeset/61743f8ebe26/

Log:	(remi, arigo)

	Maybe fix a bug, by making it explicit in each stm_thread_local if
	it is running or not (associated_segment_num >= 1 or == -1).

diff --git a/c7/stm/extra.c b/c7/stm/extra.c
--- a/c7/stm/extra.c
+++ b/c7/stm/extra.c
@@ -6,15 +6,21 @@
 static long register_callbacks(stm_thread_local_t *tl,
                                void *key, void callback(void *), long index)
 {
-    if (!(_stm_in_transaction(tl) && tl == STM_SEGMENT->running_thread)) {
-        /* check that the current thread-local is really running the
-           transaction in STM_SEGMENT, and do nothing otherwise. */
+    if (tl->associated_segment_num == -1) {
+        /* check that the provided thread-local is really running a
+           transaction, and do nothing otherwise. */
         return -1;
     }
-
+    /* The tl was only here to check that.  We're really using
+       STM_PSEGMENT below, which is often but not always the
+       segment corresponding to the tl.  One case where it's not
+       the case is if this gets called from stmcb_light_finalizer()
+       from abort_finalizers() from major collections or contention.
+    */
     if (STM_PSEGMENT->transaction_state != TS_REGULAR) {
         /* ignore callbacks if we're in an inevitable transaction
            (which cannot abort) */
+        assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE);
         return -1;
     }
 
diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c
--- a/c7/stm/gcpage.c
+++ b/c7/stm/gcpage.c
@@ -418,12 +418,12 @@
 
     stm_thread_local_t *tl = stm_all_thread_locals;
     do {
-        /* If 'tl' is currently running, its 'associated_segment_num'
+        /* If 'tl' is currently running, its 'last_associated_segment_num'
            field is the segment number that contains the correct
            version of its overflowed objects.  If not, then the
            field is still some correct segment number, and it doesn't
            matter which one we pick. */
-        char *segment_base = get_segment_base(tl->associated_segment_num);
+        char *segment_base = get_segment_base(tl->last_associated_segment_num);
 
         struct stm_shadowentry_s *current = tl->shadowstack;
         struct stm_shadowentry_s *base = tl->shadowstack_base;
diff --git a/c7/stm/marker.c b/c7/stm/marker.c
--- a/c7/stm/marker.c
+++ b/c7/stm/marker.c
@@ -58,6 +58,7 @@
     */
     long i;
     int in_segment_num = out_marker->tl->associated_segment_num;
+    assert(in_segment_num >= 1);
     struct stm_priv_segment_info_s *pseg = get_priv_segment(in_segment_num);
     struct list_s *mlst = pseg->modified_old_objects;
     struct list_s *mlstm = pseg->modified_old_objects_markers;
diff --git a/c7/stm/setup.c b/c7/stm/setup.c
--- a/c7/stm/setup.c
+++ b/c7/stm/setup.c
@@ -255,7 +255,7 @@
         tl->prev = stm_all_thread_locals->prev;
         stm_all_thread_locals->prev->next = tl;
         stm_all_thread_locals->prev = tl;
-        num = tl->prev->associated_segment_num;
+        num = tl->prev->last_associated_segment_num;
     }
     tl->thread_local_obj = NULL;
 
@@ -263,7 +263,8 @@
        assign the same number to all of them and they would get their own
        numbers automatically. */
     num = (num % NB_SEGMENTS) + 1;
-    tl->associated_segment_num = num;
+    tl->associated_segment_num = -1;
+    tl->last_associated_segment_num = num;
     tl->thread_local_counter = ++thread_local_counters;
     *_get_cpth(tl) = pthread_self();
     _init_shadow_stack(tl);
diff --git a/c7/stm/sync.c b/c7/stm/sync.c
--- a/c7/stm/sync.c
+++ b/c7/stm/sync.c
@@ -148,7 +148,7 @@
     assert(_has_mutex());
     assert(_is_tl_registered(tl));
 
-    int num = tl->associated_segment_num;
+    int num = tl->last_associated_segment_num;
     if (sync_ctl.in_use1[num - 1] == 0) {
         /* fast-path: we can get the same segment number than the one
            we had before.  The value stored in GS is still valid. */
@@ -167,8 +167,9 @@
         num = (num % NB_SEGMENTS) + 1;
         if (sync_ctl.in_use1[num - 1] == 0) {
             /* we're getting 'num', a different number. */
-            dprintf(("acquired different segment: %d->%d\n", tl->associated_segment_num, num));
-            tl->associated_segment_num = num;
+            dprintf(("acquired different segment: %d->%d\n",
+                     tl->last_associated_segment_num, num));
+            tl->last_associated_segment_num = num;
             set_gs_register(get_segment_base(num));
             goto got_num;
         }
@@ -186,6 +187,7 @@
     sync_ctl.in_use1[num - 1] = 1;
     assert(STM_SEGMENT->segment_num == num);
     assert(STM_SEGMENT->running_thread == NULL);
+    tl->associated_segment_num = tl->last_associated_segment_num;
     STM_SEGMENT->running_thread = tl;
     return true;
 }
@@ -204,10 +206,12 @@
     }
 
     assert(STM_SEGMENT->running_thread == tl);
+    assert(tl->associated_segment_num == tl->last_associated_segment_num);
+    tl->associated_segment_num = -1;
     STM_SEGMENT->running_thread = NULL;
 
-    assert(sync_ctl.in_use1[tl->associated_segment_num - 1] == 1);
-    sync_ctl.in_use1[tl->associated_segment_num - 1] = 0;
+    assert(sync_ctl.in_use1[tl->last_associated_segment_num - 1] == 1);
+    sync_ctl.in_use1[tl->last_associated_segment_num - 1] = 0;
 }
 
 __attribute__((unused))
@@ -218,9 +222,16 @@
 
 bool _stm_in_transaction(stm_thread_local_t *tl)
 {
-    int num = tl->associated_segment_num;
-    assert(1 <= num && num <= NB_SEGMENTS);
-    return get_segment(num)->running_thread == tl;
+    if (tl->associated_segment_num == -1) {
+        return false;
+    }
+    else {
+        int num = tl->associated_segment_num;
+        OPT_ASSERT(1 <= num && num <= NB_SEGMENTS);
+        OPT_ASSERT(num == tl->last_associated_segment_num);
+        OPT_ASSERT(get_segment(num)->running_thread == tl);
+        return true;
+    }
 }
 
 void _stm_test_switch(stm_thread_local_t *tl)
diff --git a/c7/stmgc.h b/c7/stmgc.h
--- a/c7/stmgc.h
+++ b/c7/stmgc.h
@@ -69,6 +69,7 @@
     long last_abort__bytes_in_nursery;
     /* the next fields are handled internally by the library */
     int associated_segment_num;
+    int last_associated_segment_num;
     int thread_local_counter;
     struct stm_thread_local_s *prev, *next;
     void *creating_pthread[2];

From noreply at buildbot.pypy.org  Mon Feb 23 16:30:28 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 23 Feb 2015 16:30:28 +0100 (CET)
Subject: [pypy-commit] stmgc default: Add some dprints
Message-ID: <20150223153028.696501C01CD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1646:cc001ffe20bb
Date: 2015-02-23 16:30 +0100
http://bitbucket.org/pypy/stmgc/changeset/cc001ffe20bb/

Log:	Add some dprints

diff --git a/c7/stm/extra.c b/c7/stm/extra.c
--- a/c7/stm/extra.c
+++ b/c7/stm/extra.c
@@ -6,9 +6,12 @@
 static long register_callbacks(stm_thread_local_t *tl,
                                void *key, void callback(void *), long index)
 {
+    dprintf(("register_callbacks: tl=%p key=%p callback=%p index=%ld\n",
+             tl, key, callback, index));
     if (tl->associated_segment_num == -1) {
         /* check that the provided thread-local is really running a
            transaction, and do nothing otherwise. */
+        dprintf(("  NOT IN TRANSACTION\n"));
         return -1;
     }
     /* The tl was only here to check that.  We're really using
@@ -20,6 +23,7 @@
     if (STM_PSEGMENT->transaction_state != TS_REGULAR) {
         /* ignore callbacks if we're in an inevitable transaction
            (which cannot abort) */
+        dprintf(("  STATE = %d\n", (int)STM_PSEGMENT->transaction_state));
         assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE);
         return -1;
     }
@@ -29,11 +33,13 @@
 
     if (callback == NULL) {
         /* double-unregistering works, but return 0 */
-        return tree_delete_item(callbacks, (uintptr_t)key);
+        long res = tree_delete_item(callbacks, (uintptr_t)key);
+        dprintf(("  DELETED %ld\n", res));
+        return res;
     }
     else {
         /* double-registering the same key will crash */
-        dprintf(("register_callbacks: tl=%p key=%p callback=%p index=%ld\n", tl, key, callback, index));
+        dprintf(("  INSERTING\n"));
         tree_insert(callbacks, (uintptr_t)key, (uintptr_t)callback);
         return 1;
     }

From noreply at buildbot.pypy.org  Mon Feb 23 16:42:13 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Feb 2015 16:42:13 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: (arigo,fijal) bleh, fix the logic
Message-ID: <20150223154213.237E61C03D0@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r76084:d4794a390ef8
Date: 2015-02-23 17:41 +0200
http://bitbucket.org/pypy/pypy/changeset/d4794a390ef8/

Log:	(arigo,fijal) bleh, fix the logic

diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py
--- a/rpython/jit/backend/llsupport/codemap.py
+++ b/rpython/jit/backend/llsupport/codemap.py
@@ -73,18 +73,16 @@
         used = getattr(self, name + '_used')
         allocated = len(getattr(self, name))
         lst = getattr(self, name)
-        if pos + len(to_insert) >= allocated or pos != used:
+        if pos + len(to_insert) > allocated or pos != used:
             old_lst = lst
-            if pos == used:
+            if pos + len(to_insert) > allocated:
                 new_size = max(4 * len(old_lst),
                                (len(old_lst) + len(to_insert)) * 2)
-                lst = lltype.malloc(lltype.typeOf(lst).TO, new_size,
-                                    flavor='raw',
-                                    track_allocation=self.track_allocation)
             else:
-                lst = lltype.malloc(lltype.typeOf(lst).TO, len(old_lst),
-                                    flavor='raw',
-                                    track_allocation=self.track_allocation)
+                new_size = len(old_lst)
+            lst = lltype.malloc(lltype.typeOf(lst).TO, new_size,
+                                flavor='raw',
+                                track_allocation=self.track_allocation)
             for i in range(0, pos):
                 copy_item(old_lst, lst, i, i)
             j = 0

From noreply at buildbot.pypy.org  Mon Feb 23 16:42:18 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Feb 2015 16:42:18 +0100 (CET)
Subject: [pypy-commit] benchmarks default: one more warmupish benchmark
Message-ID: <20150223154218.01A991C03D0@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r312:a580dcb818dc
Date: 2015-02-23 17:31 +0200
http://bitbucket.org/pypy/benchmarks/changeset/a580dcb818dc/

Log:	one more warmupish benchmark

diff --git a/warmup/pypy-annotate.py b/warmup/pypy-annotate.py
new file mode 100644
--- /dev/null
+++ b/warmup/pypy-annotate.py
@@ -0,0 +1,18 @@
+
+from rpython.annotator.annrpython import RPythonAnnotator
+from rpython.translator.goal.targetrpystonedalone import pystones_main
+from rpython.rtyper.rtyper import RPythonTyper
+
+import time
+l = []
+
+for i in range(1):
+    print i
+    t0 = time.time()
+    a = RPythonAnnotator()
+    a.build_types(pystones_main, [int])
+    rtyper = RPythonTyper(a)
+    rtyper.specialize()
+    l.append(time.time() - t0)
+
+print l

From noreply at buildbot.pypy.org  Mon Feb 23 16:42:19 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Feb 2015 16:42:19 +0100 (CET)
Subject: [pypy-commit] benchmarks default: merge
Message-ID: <20150223154219.79B891C03D0@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r313:878f71f4bccc
Date: 2015-02-23 17:42 +0200
http://bitbucket.org/pypy/benchmarks/changeset/878f71f4bccc/

Log:	merge

diff --git a/multithread/bench.py b/multithread/bench.py
--- a/multithread/bench.py
+++ b/multithread/bench.py
@@ -49,6 +49,10 @@
     os.chdir(folder)
     sys.path.insert(0, os.path.abspath('.'))
     test = import_file(os.path.basename(args.file))
+    more_args = args.more
+
+    if hasattr(test, 'init'):
+        more_args = test.init(*more_args)
 
     times = []
     results = []
@@ -61,10 +65,10 @@
 
             test_time = time.time()
             if args.p:
-                results.append(test.run(*args.more))
+                results.append(test.run(*more_args))
             else:
                 with nostdout():
-                    results.append(test.run(*args.more))
+                    results.append(test.run(*more_args))
             times.append(time.time() - test_time)
 
             if not args.q:
diff --git a/multithread/bottle/app.py b/multithread/bottle/app.py
--- a/multithread/bottle/app.py
+++ b/multithread/bottle/app.py
@@ -95,7 +95,7 @@
                              stderr=subprocess.PIPE)
     except OSError as e:
         sys.stderr.write("Error trying to execute 'openload'\n%s" % e)
-        os.exit(1)
+        os._exit(1)
 
     returncode = p.wait()
     out, err = p.communicate()
diff --git a/multithread/wsgi/read_the_docs.py b/multithread/wsgi/read_the_docs.py
new file mode 100644
--- /dev/null
+++ b/multithread/wsgi/read_the_docs.py
@@ -0,0 +1,16 @@
+import os
+import sys
+
+import readthedocs
+from readthedocs import wsgi
+
+
+REQUEST_LIST = [
+        '/',
+        ]
+
+
+def make_app():
+    sys.path.append(os.path.dirname(readthedocs.__file__))
+
+    return wsgi.application
diff --git a/multithread/wsgi/webapp.py b/multithread/wsgi/webapp.py
new file mode 100644
--- /dev/null
+++ b/multithread/wsgi/webapp.py
@@ -0,0 +1,57 @@
+import os
+import sys
+import threading
+
+from urllib2 import urlopen
+from wsgi_intercept import (
+    urllib_intercept, add_wsgi_intercept, remove_wsgi_intercept
+)
+
+host, port = 'localhost', 80
+url = 'http://{0}:{1}/'.format(host, port)
+settings = {}
+
+def init(benchmark, request_count="100", *args):
+    settings["request_count"] = int(request_count)
+
+    bm_module = type(sys)("bm_module")
+    file_name = os.path.join(os.path.dirname(__file__), benchmark + ".py")
+    execfile(file_name, bm_module.__dict__)
+
+    if not hasattr(bm_module, 'REQUEST_LIST'):
+        print 'request list not defined'
+        sys.exit(-1)
+    if not hasattr(bm_module, 'make_app'):
+        print 'app maker not defined'
+        sys.exit(-1)
+
+    settings["bm_module"] = bm_module
+
+    # set up intercepter
+    urllib_intercept.install_opener()
+    add_wsgi_intercept(host, port, bm_module.make_app)
+
+    return args
+
+def run(thread_count="2"):
+    # test
+    threads = []
+    for i in range(int(thread_count)):
+        thread = threading.Thread(target=do_requests, args=(settings['request_count'],))
+        thread.start()
+        threads.append(thread)
+
+    for thread in threads:
+        thread.join()
+
+
+def do_requests(request_count):
+    for i in range(request_count):
+        for request in settings["bm_module"].REQUEST_LIST:
+            request_url = url + request
+            stream = urlopen(url)
+            content = stream.read()
+
+
+if __name__ == '__main__':
+    run(*init(*sys.argv[1:]))
diff --git a/multithread/wsgi/wsgi_intercept/__init__.py b/multithread/wsgi/wsgi_intercept/__init__.py
new file mode 100644
--- /dev/null
+++ b/multithread/wsgi/wsgi_intercept/__init__.py
@@ -0,0 +1,581 @@
+
+"""installs a WSGI application in place of a real URI for testing.
+
+Introduction
+============
+
+Testing a WSGI application normally involves starting a server at a
+local host and port, then pointing your test code to that address.
+Instead, this library lets you intercept calls to any specific host/port
+combination and redirect them into a `WSGI application`_ importable by
+your test program. Thus, you can avoid spawning multiple processes or
+threads to test your Web app.
+
+How Does It Work?
+=================
+
+``wsgi_intercept`` works by replacing ``httplib.HTTPConnection`` with a
+subclass, ``wsgi_intercept.WSGI_HTTPConnection``. This class then
+redirects specific server/port combinations into a WSGI application by
+emulating a socket. If no intercept is registered for the host and port
+requested, those requests are passed on to the standard handler.
+
+The functions ``add_wsgi_intercept(host, port, app_create_fn,
+script_name='')`` and ``remove_wsgi_intercept(host,port)`` specify
+which URLs should be redirect into what applications. Note especially
+that ``app_create_fn`` is a *function object* returning a WSGI
+application; ``script_name`` becomes ``SCRIPT_NAME`` in the WSGI app's
+environment, if set.
+
+Install
+=======
+
+::
+
+    pip install -U wsgi_intercept
+
+Packages Intercepted
+====================
+
+Unfortunately each of the Web testing frameworks uses its own specific
+mechanism for making HTTP call-outs, so individual implementations are
+needed. At this time there are implementations for ``httplib2`` and
+``requests`` in both Python 2 and 3, ``urllib2`` and ``httplib``
+in Python 2 and ``urllib.request`` and ``http.client`` in Python 3.
+
+If you are using Python 2 and need support for a different HTTP
+client, require a version of ``wsgi_intercept<0.6``. Earlier versions
+include support for ``webtest``, ``webunit`` and ``zope.testbrowser``.
+It is quite likely that support for these versions will be relatively
+easy to add back in to the new version.
+
+The best way to figure out how to use interception is to inspect
+`the tests`_. More comprehensive documentation available upon
+request.
+
+.. _the tests: https://github.com/cdent/python3-wsgi-intercept/tree/master/test
+
+
+History
+=======
+
+Pursuant to Ian Bicking's `"best Web testing framework"`_ post, Titus
+Brown put together an `in-process HTTP-to-WSGI interception mechanism`_
+for his own Web testing system, twill_. Because the mechanism is pretty
+generic -- it works at the httplib level -- Titus decided to try adding
+it into all of the *other* Python Web testing frameworks.
+
+The Python 2 version of wsgi-intercept was the result. Kumar McMillan
+later took over maintenance.
+
+The current version works with Python 2.6, 2.7, 3.3 and 3.4 and was assembled
+by `Chris Dent`_. Testing and documentation improvements from `Sasha Hart`_.
+
+.. _twill: http://www.idyll.org/~t/www-tools/twill.html
+.. _"best Web testing framework": http://blog.ianbicking.org/best-of-the-web-app-test-frameworks.html
+.. _in-process HTTP-to-WSGI interception mechanism: http://www.advogato.org/person/titus/diary.html?start=119
+.. _WSGI application: http://www.python.org/peps/pep-3333.html
+.. _Chris Dent: https://github.com/cdent
+.. _Sasha Hart: https://github.com/sashahart
+
+Project Home
+============
+
+This project lives on `GitHub`_. Please submit all bugs, patches,
+failing tests, et cetera using the Issue Tracker.
+
+Additional documentation is available on `Read The Docs`_.
+
+.. _GitHub: http://github.com/cdent/python3-wsgi-intercept
+.. _Read The Docs: http://wsgi-intercept.readthedocs.org/en/latest/
+
+"""
+from __future__ import print_function
+
+__version__ = '0.9.1'
+
+
+import sys
+try:
+    from http.client import HTTPConnection, HTTPSConnection
+except ImportError:
+    from httplib import HTTPConnection, HTTPSConnection
+
+try:
+    from io import BytesIO
+except ImportError:
+    from StringIO import StringIO as BytesIO
+
+try:
+    from urllib.parse import unquote as url_unquote
+except ImportError:
+    from urllib import unquote as url_unquote
+
+import traceback
+
+debuglevel = 0
+# 1 basic
+# 2 verbose
+
+####
+
+#
+# Specify which hosts/ports to target for interception to a given WSGI app.
+#
+# For simplicity's sake, intercept ENTIRE host/port combinations;
+# intercepting only specific URL subtrees gets complicated, because we don't
+# have that information in the HTTPConnection.connect() function that does the
+# redirection.
+#
+# format: key=(host, port), value=(create_app, top_url)
+#
+# (top_url becomes the SCRIPT_NAME)
+
+_wsgi_intercept = {}
+
+
+def add_wsgi_intercept(host, port, app_create_fn, script_name=''):
+    """
+    Add a WSGI intercept call for host:port, using the app returned
+    by app_create_fn with a SCRIPT_NAME of 'script_name' (default '').
+    """
+    _wsgi_intercept[(host, port)] = (app_create_fn, script_name)
+
+
+def remove_wsgi_intercept(*args):
+    """
+    Remove the WSGI intercept call for (host, port).  If no arguments are
+    given, removes all intercepts
+    """
+    global _wsgi_intercept
+    if len(args) == 0:
+        _wsgi_intercept = {}
+    else:
+        key = (args[0], args[1])
+        if key in _wsgi_intercept:
+            del _wsgi_intercept[key]
+
+
+#
+# make_environ: behave like a Web server.  Take in 'input', and behave
+# as if you're bound to 'host' and 'port'; build an environment dict
+# for the WSGI app.
+#
+# This is where the magic happens, folks.
+#
+def make_environ(inp, host, port, script_name):
+    """
+    Take 'inp' as if it were HTTP-speak being received on host:port,
+    and parse it into a WSGI-ok environment dictionary.  Return the
+    dictionary.
+
+    Set 'SCRIPT_NAME' from the 'script_name' input, and, if present,
+    remove it from the beginning of the PATH_INFO variable.
+    """
+    #
+    # parse the input up to the first blank line (or its end).
+    #
+
+    environ = {}
+
+    method_line = inp.readline()
+    if sys.version_info[0] > 2:
+        method_line = method_line.decode('ISO-8859-1')
+
+    content_type = None
+    content_length = None
+    cookies = []
+
+    for line in inp:
+        if not line.strip():
+            break
+
+        k, v = line.strip().split(b':', 1)
+        v = v.lstrip()
+        v = v.decode('ISO-8859-1')
+
+        #
+        # take care of special headers, and for the rest, put them
+        # into the environ with HTTP_ in front.
+        #
+
+        if k.lower() == b'content-type':
+            content_type = v
+        elif k.lower() == b'content-length':
+            content_length = v
+        elif k.lower() == b'cookie' or k.lower() == b'cookie2':
+            cookies.append(v)
+        else:
+            h = k.upper()
+            h = h.replace(b'-', b'_')
+            environ['HTTP_' + h.decode('ISO-8859-1')] = v
+
+        if debuglevel >= 2:
+            print('HEADER:', k, v)
+
+    #
+    # decode the method line
+    #
+
+    if debuglevel >= 2:
+        print('METHOD LINE:', method_line)
+
+    method, url, protocol = method_line.split(' ')
+
+    # Store the URI as requested by the user, without modification
+    # so that PATH_INFO munging can be corrected.
+    environ['REQUEST_URI'] = url
+    environ['RAW_URI'] = url
+
+    # clean the script_name off of the url, if it's there.
+    if not url.startswith(script_name):
+        script_name = ''                # @CTB what to do -- bad URL.  scrap?
+    else:
+        url = url[len(script_name):]
+
+    url = url.split('?', 1)
+    path_info = url_unquote(url[0])
+    query_string = ""
+    if len(url) == 2:
+        query_string = url[1]
+
+    if debuglevel:
+        print("method: %s; script_name: %s; path_info: %s; query_string: %s" %
+                (method, script_name, path_info, query_string))
+
+    r = inp.read()
+    inp = BytesIO(r)
+
+    #
+    # fill out our dictionary.
+    #
+
+    environ.update({
+        "wsgi.version": (1, 0),
+        "wsgi.url_scheme": "http",
+        "wsgi.input": inp,  # to read for POSTs
+        "wsgi.errors": BytesIO(),
+        "wsgi.multithread": 0,
+        "wsgi.multiprocess": 0,
+        "wsgi.run_once": 0,
+
+        "PATH_INFO": path_info,
+        "REMOTE_ADDR": '127.0.0.1',
+        "REQUEST_METHOD": method,
+        "SCRIPT_NAME": script_name,
+        "SERVER_NAME": host,
+        "SERVER_PORT": port,
+        "SERVER_PROTOCOL": protocol,
+    })
+
+    #
+    # query_string, content_type & length are optional.
+    #
+
+    if query_string:
+        environ['QUERY_STRING'] = query_string
+
+    if content_type:
+        environ['CONTENT_TYPE'] = content_type
+        if debuglevel >= 2:
+            print('CONTENT-TYPE:', content_type)
+    if content_length:
+        environ['CONTENT_LENGTH'] = content_length
+        if debuglevel >= 2:
+            print('CONTENT-LENGTH:', content_length)
+
+    #
+    # handle cookies.
+    #
+    if cookies:
+        environ['HTTP_COOKIE'] = "; ".join(cookies)
+
+    if debuglevel:
+        print('WSGI environ dictionary:', environ)
+
+    return environ
+
+
+class WSGIAppError(Exception):
+    """
+    An exception that wraps any Exception raised by the WSGI app
+    that is called. This is done for two reasons: it ensures that
+    intercepted libraries (such as requests) which use exceptions
+    to trigger behaviors are not interfered with by exceptions from
+    the WSGI app. It also helps to define a solid boundary, akin
+    to the network boundary between server and client, in the
+    testing environment.
+    """
+    def __init__(self, error, exc_info):
+        Exception.__init__(self)
+        self.error = error
+        self.exception_type = exc_info[0]
+        self.exception_value = exc_info[1]
+        self.traceback = exc_info[2]
+
+    def __str__(self):
+        frame = traceback.extract_tb(self.traceback)[-1]
+        formatted = "{0!r} at {1}:{2}".format(
+            self.error,
+            frame[0],
+            frame[1],
+        )
+        return formatted
+
+
+#
+# fake socket for WSGI intercept stuff.
+#
+class wsgi_fake_socket:
+    """
+    Handle HTTP traffic and stuff into a WSGI application object instead.
+
+    Note that this class assumes:
+
+     1. 'makefile' is called (by the response class) only after all of the
+        data has been sent to the socket by the request class;
+     2. non-persistent (i.e. non-HTTP/1.1) connections.
+    """
+    def __init__(self, app, host, port, script_name, https=False):
+        self.app = app                  # WSGI app object
+        self.host = host
+        self.port = port
+        self.script_name = script_name  # SCRIPT_NAME (app mount point)
+
+        self.inp = BytesIO()           # stuff written into this "socket"
+        self.write_results = []          # results from the 'write_fn'
+        self.results = None             # results from running the app
+        self.output = BytesIO()        # all output from the app, incl headers
+        self.https = https
+
+    def makefile(self, *args, **kwargs):
+        """
+        'makefile' is called by the HTTPResponse class once all of the
+        data has been written.  So, in this interceptor class, we need to:
+
+          1. build a start_response function that grabs all the headers
+             returned by the WSGI app;
+          2. create a wsgi.input file object 'inp', containing all of the
+             traffic;
+          3. build an environment dict out of the traffic in inp;
+          4. run the WSGI app & grab the result object;
+          5. concatenate & return the result(s) read from the result object.
+
+        @CTB: 'start_response' should return a function that writes
+        directly to self.result, too.
+        """
+
+        # dynamically construct the start_response function for no good reason.
+
+        def start_response(status, headers, exc_info=None):
+            # construct the HTTP request.
+            self.output.write(b"HTTP/1.0 " + status.encode('utf-8') + b"\n")
+
+            for k, v in headers:
+                try:
+                    k = k.encode('utf-8')
+                except AttributeError:
+                    pass
+                try:
+                    v = v.encode('utf-8')
+                except AttributeError:
+                    pass
+                self.output.write(k + b': ' + v + b"\n")
+            self.output.write(b'\n')
+
+            def write_fn(s):
+                self.write_results.append(s)
+            return write_fn
+
+        # construct the wsgi.input file from everything that's been
+        # written to this "socket".
+        inp = BytesIO(self.inp.getvalue())
+
+        # build the environ dictionary.
+        environ = make_environ(inp, self.host, self.port, self.script_name)
+        if self.https:
+            environ['wsgi.url_scheme'] = 'https'
+
+        # run the application.
+        try:
+            app_result = self.app(environ, start_response)
+        except Exception as error:
+            raise WSGIAppError(error, sys.exc_info())
+        self.result = iter(app_result)
+
+        ###
+
+        # read all of the results.  the trick here is to get the *first*
+        # bit of data from the app via the generator, *then* grab & return
+        # the data passed back from the 'write' function, and then return
+        # the generator data.  this is because the 'write' fn doesn't
+        # necessarily get called until the first result is requested from
+        # the app function.
+
+        try:
+            generator_data = None
+            try:
+                generator_data = next(self.result)
+
+            finally:
+                for data in self.write_results:
+                    self.output.write(data)
+
+            if generator_data:
+                try:
+                    self.output.write(generator_data)
+                except TypeError as exc:
+                    raise TypeError('bytes required in response: %s' % exc)
+
+                while 1:
+                    data = next(self.result)
+                    self.output.write(data)
+
+        except StopIteration:
+            pass
+
+        if hasattr(app_result, 'close'):
+            app_result.close()
+
+        if debuglevel >= 2:
+            print("***", self.output.getvalue(), "***")
+
+        # return the concatenated results.
+        return BytesIO(self.output.getvalue())
+
+    def sendall(self, content):
+        """
+        Save all the traffic to self.inp.
+        """
+        if debuglevel >= 2:
+            print(">>>", content, ">>>")
+
+        try:
+            self.inp.write(content)
+        except TypeError:
+            self.inp.write(content.encode('utf-8'))
+
+    def close(self):
+        "Do nothing, for now."
+        pass
+
+
+#
+# WSGI_HTTPConnection
+#
+class WSGI_HTTPConnection(HTTPConnection):
+    """
+    Intercept all traffic to certain hosts & redirect into a WSGI
+    application object.
+    """
+    def get_app(self, host, port):
+        """
+        Return the app object for the given (host, port).
+        """
+        key = (host, int(port))
+
+        app, script_name = None, None
+
+        if key in _wsgi_intercept:
+            (app_fn, script_name) = _wsgi_intercept[key]
+            app = app_fn()
+
+        return app, script_name
+
+    def connect(self):
+        """
+        Override the connect() function to intercept calls to certain
+        host/ports.
+
+        If no app at host/port has been registered for interception then
+        a normal HTTPConnection is made.
+        """
+        if debuglevel:
+            sys.stderr.write('connect: %s, %s\n' % (self.host, self.port,))
+
+        try:
+            (app, script_name) = self.get_app(self.host, self.port)
+            if app:
+                if debuglevel:
+                    sys.stderr.write('INTERCEPTING call to %s:%s\n' %
+                                     (self.host, self.port,))
+                self.sock = wsgi_fake_socket(app, self.host, self.port,
+                                             script_name)
+            else:
+                HTTPConnection.connect(self)
+
+        except Exception:
+            if debuglevel:              # intercept & print out tracebacks
+                traceback.print_exc()
+            raise
+
+
+#
+# WSGI_HTTPSConnection
+#
+
+
+class WSGI_HTTPSConnection(HTTPSConnection, WSGI_HTTPConnection):
+    """
+    Intercept all traffic to certain hosts & redirect into a WSGI
+    application object.
+    """
+    def get_app(self, host, port):
+        """
+        Return the app object for the given (host, port).
+        """
+        key = (host, int(port))
+
+        app, script_name = None, None
+
+        if key in _wsgi_intercept:
+            (app_fn, script_name) = _wsgi_intercept[key]
+            app = app_fn()
+
+        return app, script_name
+
+    def connect(self):
+        """
+        Override the connect() function to intercept calls to certain
+        host/ports.
+
+        If no app at host/port has been registered for interception then
+        a normal HTTPSConnection is made.
+        """
+        if debuglevel:
+            sys.stderr.write('connect: %s, %s\n' % (self.host, self.port,))
+
+        try:
+            (app, script_name) = self.get_app(self.host, self.port)
+            if app:
+                if debuglevel:
+                    sys.stderr.write('INTERCEPTING call to %s:%s\n' %
+                                     (self.host, self.port,))
+                self.sock = wsgi_fake_socket(app, self.host, self.port,
+                                             script_name, https=True)
+            else:
+                try:
+                    import ssl
+                    if not hasattr(self, 'key_file'):
+                        self.key_file = None
+                    if not hasattr(self, 'cert_file'):
+                        self.cert_file = None
+                    if not hasattr(self, '_context'):
+                        try:
+                            self._context = ssl.create_default_context()
+                        except AttributeError:
+                            self._context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+                            self._context.options |= ssl.OP_NO_SSLv2
+                            if not hasattr(self, 'check_hostname'):
+                                self._check_hostname = (self._context.verify_mode
+                                        != ssl.CERT_NONE)
+                            else:
+                                self._check_hostname = self.check_hostname
+                except (ImportError, AttributeError):
+                    pass
+                HTTPSConnection.connect(self)
+
+        except Exception:
+            if debuglevel:              # intercept & print out tracebacks
+                traceback.print_exc()
+            raise
diff --git a/multithread/wsgi/wsgi_intercept/http_client_intercept.py b/multithread/wsgi/wsgi_intercept/http_client_intercept.py
new file mode 100644
--- /dev/null
+++ b/multithread/wsgi/wsgi_intercept/http_client_intercept.py
@@ -0,0 +1,49 @@
+"""Intercept HTTP connections that use httplib (Py2) or http.client (Py3).
+"""
+
+try:
+    import http.client as http_lib
+except ImportError:
+    import httplib as http_lib
+
+from . import WSGI_HTTPConnection, WSGI_HTTPSConnection
+
+try:
+    from http.client import (
+            HTTPConnection as OriginalHTTPConnection,
+            HTTPSConnection as OriginalHTTPSConnection
+    )
+except ImportError:
+    from httplib import (
+            HTTPConnection as OriginalHTTPConnection,
+            HTTPSConnection as OriginalHTTPSConnection
+    )
+
+HTTPInterceptorMixin = WSGI_HTTPConnection
+HTTPSInterceptorMixin = WSGI_HTTPSConnection
+
+
+class HTTP_WSGIInterceptor(HTTPInterceptorMixin, http_lib.HTTPConnection):
+    pass
+
+
+class HTTPS_WSGIInterceptor(HTTPSInterceptorMixin, http_lib.HTTPSConnection,
+        HTTP_WSGIInterceptor):
+
+    def __init__(self, host, **kwargs):
+        self.host = host
+        try:
+            self.port = kwargs['port']
+        except KeyError:
+            self.port = None
+        HTTP_WSGIInterceptor.__init__(self, host, **kwargs)
+
+
+def install():
+    http_lib.HTTPConnection = HTTP_WSGIInterceptor
+    http_lib.HTTPSConnection = HTTPS_WSGIInterceptor
+
+
+def uninstall():
+    http_lib.HTTPConnection = OriginalHTTPConnection
+    http_lib.HTTPSConnection = OriginalHTTPSConnection
diff --git a/multithread/wsgi/wsgi_intercept/httplib2_intercept.py b/multithread/wsgi/wsgi_intercept/httplib2_intercept.py
new file mode 100644
--- /dev/null
+++ b/multithread/wsgi/wsgi_intercept/httplib2_intercept.py
@@ -0,0 +1,61 @@
+"""Intercept HTTP connections that use `httplib2 `_.
+"""
+
+import sys
+
+from httplib2 import (SCHEME_TO_CONNECTION, HTTPConnectionWithTimeout,
+                      HTTPSConnectionWithTimeout)
+
+from . import (HTTPConnection, HTTPSConnection, WSGI_HTTPConnection,
+               WSGI_HTTPSConnection)
+
+HTTPInterceptorMixin = WSGI_HTTPConnection
+HTTPSInterceptorMixin = WSGI_HTTPSConnection
+
+
+class HTTP_WSGIInterceptorWithTimeout(HTTPInterceptorMixin,
+        HTTPConnectionWithTimeout):
+    def __init__(self, host, port=None, strict=None, timeout=None,
+            proxy_info=None, source_address=None):
+
+        # In Python3 strict is deprecated
+        if sys.version_info[0] < 3:
+            try:
+                HTTPConnection.__init__(self, host, port=port, strict=strict,
+                        timeout=timeout, source_address=source_address)
+            except TypeError:  # Python 2.6 doesn't have source_address
+                HTTPConnection.__init__(self, host, port=port, strict=strict,
+                        timeout=timeout)
+        else:
+            HTTPConnection.__init__(self, host, port=port,
+                    timeout=timeout, source_address=source_address)
+
+
+class HTTPS_WSGIInterceptorWithTimeout(HTTPSInterceptorMixin,
+        HTTPSConnectionWithTimeout):
+    def __init__(self, host, port=None, strict=None, timeout=None,
+            proxy_info=None, ca_certs=None, source_address=None,
+            disable_ssl_certificate_validation=False):
+
+        # ignore proxy_info and ca_certs
+        # In Python3 strict is deprecated
+        if sys.version_info[0] < 3:
+            try:
+                HTTPSConnection.__init__(self, host, port=port, strict=strict,
+                        timeout=timeout, source_address=source_address)
+            except TypeError:  # Python 2.6 doesn't have source_address
+                HTTPSConnection.__init__(self, host, port=port, strict=strict,
+                        timeout=timeout)
+        else:
+            HTTPSConnection.__init__(self, host, port=port,
+                    timeout=timeout, source_address=source_address)
+
+
+def install():
+    SCHEME_TO_CONNECTION['http'] = HTTP_WSGIInterceptorWithTimeout
+    SCHEME_TO_CONNECTION['https'] = HTTPS_WSGIInterceptorWithTimeout
+
+
+def uninstall():
+    SCHEME_TO_CONNECTION['http'] = HTTPConnectionWithTimeout
+    SCHEME_TO_CONNECTION['https'] = HTTPSConnectionWithTimeout
diff --git a/multithread/wsgi/wsgi_intercept/requests_intercept.py b/multithread/wsgi/wsgi_intercept/requests_intercept.py
new file mode 100644
--- /dev/null
+++ b/multithread/wsgi/wsgi_intercept/requests_intercept.py
@@ -0,0 +1,41 @@
+"""Intercept HTTP connections that use `requests `_.
+"""
+
+import sys
+
+from . import WSGI_HTTPConnection, WSGI_HTTPSConnection, wsgi_fake_socket
+from requests.packages.urllib3.connectionpool import (HTTPConnectionPool,
+        HTTPSConnectionPool)
+from requests.packages.urllib3.connection import (HTTPConnection,
+        HTTPSConnection)
+
+
+wsgi_fake_socket.settimeout = lambda self, timeout: None
+
+
+class HTTP_WSGIInterceptor(WSGI_HTTPConnection, HTTPConnection):
+    def __init__(self, *args, **kwargs):
+        if 'strict' in kwargs and sys.version_info > (3, 0):
+            kwargs.pop('strict')
+        WSGI_HTTPConnection.__init__(self, *args, **kwargs)
+        HTTPConnection.__init__(self, *args, **kwargs)
+
+
+class HTTPS_WSGIInterceptor(WSGI_HTTPSConnection, HTTPSConnection):
+    is_verified = True
+
+    def __init__(self, *args, **kwargs):
+        if 'strict' in kwargs and sys.version_info > (3, 0):
+            kwargs.pop('strict')
+        WSGI_HTTPSConnection.__init__(self, *args, **kwargs)
+        HTTPSConnection.__init__(self, *args, **kwargs)
+
+
+def install():
+    HTTPConnectionPool.ConnectionCls = HTTP_WSGIInterceptor
+    HTTPSConnectionPool.ConnectionCls = HTTPS_WSGIInterceptor
+
+
+def uninstall():
+    HTTPConnectionPool.ConnectionCls = HTTPConnection
+    HTTPSConnectionPool.ConnectionCls = HTTPSConnection
diff --git a/multithread/wsgi/wsgi_intercept/urllib_intercept.py b/multithread/wsgi/wsgi_intercept/urllib_intercept.py
new file mode 100644
--- /dev/null
+++ b/multithread/wsgi/wsgi_intercept/urllib_intercept.py
@@ -0,0 +1,40 @@
+"""Intercept HTTP connections that use urllib.request (Py3) aka urllib2 (Python 2).
+"""
+try:
+    import urllib.request as url_lib
+except ImportError:
+    import urllib2 as url_lib
+
+from . import WSGI_HTTPConnection, WSGI_HTTPSConnection
+
+
+class WSGI_HTTPHandler(url_lib.HTTPHandler):
+    """
+    Override the default HTTPHandler class with one that uses the
+    WSGI_HTTPConnection class to open HTTP URLs.
+    """
+    def http_open(self, req):
+        return self.do_open(WSGI_HTTPConnection, req)
+
+
+class WSGI_HTTPSHandler(url_lib.HTTPSHandler):
+    """
+    Override the default HTTPSHandler class with one that uses the
+    WSGI_HTTPConnection class to open HTTPS URLs.
+    """
+    def https_open(self, req):
+        return self.do_open(WSGI_HTTPSConnection, req)
+
+
+def install_opener():
+    handlers = [WSGI_HTTPHandler()]
+    if WSGI_HTTPSHandler is not None:
+        handlers.append(WSGI_HTTPSHandler())
+    opener = url_lib.build_opener(*handlers)
+    url_lib.install_opener(opener)
+
+    return opener
+
+
+def uninstall_opener():
+    url_lib.install_opener(None)

From noreply at buildbot.pypy.org  Mon Feb 23 16:54:43 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Mon, 23 Feb 2015 16:54:43 +0100 (CET)
Subject: [pypy-commit] stmgc default: (arigo, xoraxax,
	remi) remove (probably) bogus assert
Message-ID: <20150223155443.204EB1C052B@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: 
Changeset: r1647:ee1552efd127
Date: 2015-02-23 16:54 +0100
http://bitbucket.org/pypy/stmgc/changeset/ee1552efd127/

Log:	(arigo,xoraxax,remi) remove (probably) bogus assert

diff --git a/c7/stm/extra.c b/c7/stm/extra.c
--- a/c7/stm/extra.c
+++ b/c7/stm/extra.c
@@ -22,9 +22,8 @@
     */
     if (STM_PSEGMENT->transaction_state != TS_REGULAR) {
         /* ignore callbacks if we're in an inevitable transaction
-           (which cannot abort) */
+           (which cannot abort) or no transaction at all in this segment */
         dprintf(("  STATE = %d\n", (int)STM_PSEGMENT->transaction_state));
-        assert(STM_PSEGMENT->transaction_state == TS_INEVITABLE);
         return -1;
     }
 

From noreply at buildbot.pypy.org  Mon Feb 23 16:54:49 2015
From: noreply at buildbot.pypy.org (xando)
Date: Mon, 23 Feb 2015 16:54:49 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: (xando,
 mjacob) fixing recent changes in pyparser (b3ad7de8410c)
Message-ID: <20150223155449.A7A8B1C052B@cobra.cs.uni-duesseldorf.de>

Author: Sebastian Pawlu? 
Branch: py3.3
Changeset: r76085:e9662ca0f43f
Date: 2015-02-23 16:52 +0100
http://bitbucket.org/pypy/pypy/changeset/e9662ca0f43f/

Log:	(xando, mjacob) fixing recent changes in pyparser (b3ad7de8410c)

diff --git a/lib-python/3/test/test_compile.py b/lib-python/3/test/test_compile.py
--- a/lib-python/3/test/test_compile.py
+++ b/lib-python/3/test/test_compile.py
@@ -3,6 +3,7 @@
 import _ast
 import types
 from test import support
+from test.support import check_impl_detail
 
 class TestSpecifics(unittest.TestCase):
 
@@ -466,7 +467,9 @@
 
     def test_bad_single_statement(self):
         self.assertInvalidSingle('1\n2')
-        self.assertInvalidSingle('def f(): pass')
+        if check_impl_detail():
+            # it's a single statment in PyPy
+            self.assertInvalidSingle('def f(): pass')
         self.assertInvalidSingle('a = 13\nb = 187')
         self.assertInvalidSingle('del x\ndel y')
         self.assertInvalidSingle('f()\ng()')
diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py
--- a/pypy/interpreter/pyparser/pyparse.py
+++ b/pypy/interpreter/pyparser/pyparse.py
@@ -3,6 +3,7 @@
 from pypy.interpreter.astcompiler import consts
 from rpython.rlib import rstring
 
+
 def recode_to_utf8(space, bytes, encoding):
     if encoding == 'utf-8':
         return bytes
@@ -167,12 +168,15 @@
                 compile_info.flags |= newflags
                 self.grammar = pygram.python_grammar
                 tokens_stream = iter(tokens)
+
                 for tp, value, lineno, column, line in tokens_stream:
                     if self.add_token(tp, value, lineno, column, line):
                         break
 
                 if compile_info.mode == 'single':
                     for tp, value, lineno, column, line in tokens_stream:
+                        if tp == pygram.tokens.ENDMARKER:
+                            break
                         if tp == pygram.tokens.NEWLINE:
                             continue
 
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -154,7 +154,6 @@
 
     def test_bad_single_statement(self):
         py.test.raises(SyntaxError, self.parse, '1\n2', "single")
-        py.test.raises(SyntaxError, self.parse, 'def f(): pass', "single")
         py.test.raises(SyntaxError, self.parse, 'a = 13\nb = 187', "single")
         py.test.raises(SyntaxError, self.parse, 'del x\ndel y', "single")
         py.test.raises(SyntaxError, self.parse, 'f()\ng()', "single")

From noreply at buildbot.pypy.org  Mon Feb 23 16:54:50 2015
From: noreply at buildbot.pypy.org (xando)
Date: Mon, 23 Feb 2015 16:54:50 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: hg merge
Message-ID: <20150223155450.E279A1C052B@cobra.cs.uni-duesseldorf.de>

Author: Sebastian Pawlu? 
Branch: py3.3
Changeset: r76086:71d4155f0140
Date: 2015-02-23 16:53 +0100
http://bitbucket.org/pypy/pypy/changeset/71d4155f0140/

Log:	hg merge

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -505,7 +505,7 @@
         is_text_encoding = space.is_true(
                 space.getattr(codec_info, space.wrap('_is_text_encoding')))
     except OperationError as e:
-        if e.match(space.w_AttributeError):
+        if e.match(space, space.w_AttributeError):
             is_text_encoding = True
         else:
             raise
@@ -559,7 +559,7 @@
         is_text_encoding = space.is_true(
                 space.getattr(codec_info, space.wrap('_is_text_encoding')))
     except OperationError as e:
-        if e.match(space.w_AttributeError):
+        if e.match(space, space.w_AttributeError):
             is_text_encoding = True
         else:
             raise

From noreply at buildbot.pypy.org  Mon Feb 23 18:16:44 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 23 Feb 2015 18:16:44 +0100 (CET)
Subject: [pypy-commit] pypy default: I misunderstood this part,
 the escaping happens in mark_escaped
Message-ID: <20150223171644.7C2BE1C0579@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: 
Changeset: r76087:8c142885e09c
Date: 2015-02-23 18:14 +0100
http://bitbucket.org/pypy/pypy/changeset/8c142885e09c/

Log:	I misunderstood this part, the escaping happens in mark_escaped

diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -199,10 +199,6 @@
                 self._clear_caches_arraycopy(opnum, descr, argboxes, effectinfo)
                 return
             else:
-                # first escape arguments:
-                for argbox in argboxes:
-                    self._escape_box(argbox)
-
                 # Only invalidate things that are escaped
                 # XXX can do better, only do it for the descrs in the effectinfo
                 for descr, cache in self.heap_cache.iteritems():

From noreply at buildbot.pypy.org  Mon Feb 23 18:50:06 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Feb 2015 18:50:06 +0100 (CET)
Subject: [pypy-commit] pypy rewrite-unrolling: (arigo,
	fijal) small simplification
Message-ID: <20150223175006.8A2981C01CD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: rewrite-unrolling
Changeset: r76088:c4f4f1c04d75
Date: 2015-02-23 18:11 +0200
http://bitbucket.org/pypy/pypy/changeset/c4f4f1c04d75/

Log:	(arigo, fijal) small simplification

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -59,13 +59,10 @@
                                      op.result, pure_value)
                     self.optimizer.pure_reverse(op)
         for box in self.optimizer.loop.operations[0].getarglist():
-            try:
-                # XXX do the same thing for pure opt value
-                other = old_optimizer.values[box]
-                self.optimizer.getvalue(box).import_from(other,
-                                                         self.optimizer)
-            except KeyError:
-                pass
+            # XXX do the same thing for pure opt value
+            other = old_optimizer.values.get(box, None)
+            if other is not None:
+                self.optimizer.getvalue(box).import_from(other, self.optimizer)
         
     #         for opargs, value in old_optpure.pure_operations.items():
     #             if not value.is_virtual():

From noreply at buildbot.pypy.org  Mon Feb 23 18:50:07 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Feb 2015 18:50:07 +0100 (CET)
Subject: [pypy-commit] pypy optresult: (fijal, arigo) fix this one case
Message-ID: <20150223175007.DB59E1C01CD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r76089:2f22a201b4ea
Date: 2015-02-23 19:14 +0200
http://bitbucket.org/pypy/pypy/changeset/2f22a201b4ea/

Log:	(fijal, arigo) fix this one case

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -11,6 +11,7 @@
 from rpython.jit.metainterp.resoperation import rop, ResOperation, InputArgInt,\
      OpHelpers
 from rpython.rlib.rarithmetic import LONG_BIT
+from rpython.jit.tool.oparser import parse
 
 def test_store_final_boxes_in_guard():
     from rpython.jit.metainterp.compile import ResumeGuardDescr
@@ -105,7 +106,8 @@
                           loop.operations
         if loop.operations[-1].getopnum() == rop.JUMP:
             loop.operations[-1].setdescr(token)
-        expected = convert_old_style_to_targets(self.parse(optops), jump=True)
+        exp = parse(optops, namespace=self.namespace.copy())
+        expected = convert_old_style_to_targets(exp, jump=True)
         self._do_optimize_loop(loop, call_pure_results, export_state=False)
         #print '\n'.join([str(o) for o in loop.operations])
         self.assert_equal(loop, expected)

From noreply at buildbot.pypy.org  Mon Feb 23 18:50:09 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Feb 2015 18:50:09 +0100 (CET)
Subject: [pypy-commit] pypy optresult: intermediate checkin
Message-ID: <20150223175009.079971C01CD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r76090:a1041f0d5ff5
Date: 2015-02-23 19:21 +0200
http://bitbucket.org/pypy/pypy/changeset/a1041f0d5ff5/

Log:	intermediate checkin

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -521,7 +521,7 @@
             self.import_box(a, inputargs, short_jumpargs, jumpargs)
 
     def jump_to_already_compiled_trace(self, jumpop, patchguardop):
-        jumpop = jumpop.clone()
+        jumpop = jumpop.copy_and_change(jumpop.getopnum())
         assert jumpop.getopnum() == rop.JUMP
         cell_token = jumpop.getdescr()
 
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -120,7 +120,7 @@
     def clone(self, memo):
         args = [memo.get(arg, arg) for arg in self.getarglist()]
         descr = self.getdescr()
-        op = ResOperation(self.getopnum(), args[:], self.result, descr)
+        op = ResOperation(self.getopnum(), args[:], descr)
         if not we_are_translated():
             op.name = self.name
             op.pc = self.pc
@@ -306,8 +306,7 @@
         return newop
 
     def clone(self, memo):
-        xxx
-        newop = AbstractResOp.clone(self)
+        newop = AbstractResOp.clone(self, memo)
         assert isinstance(newop, GuardResOp)
         newop.setfailargs(self.getfailargs())
         newop.rd_snapshot = self.rd_snapshot

From noreply at buildbot.pypy.org  Mon Feb 23 18:50:10 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Feb 2015 18:50:10 +0100 (CET)
Subject: [pypy-commit] pypy optresult: simple progress
Message-ID: <20150223175010.23CE11C01CD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r76091:46763a4d640e
Date: 2015-02-23 19:49 +0200
http://bitbucket.org/pypy/pypy/changeset/46763a4d640e/

Log:	simple progress

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -658,7 +658,9 @@
         newop = op.copy_and_change(newopnum, args, descr)
         if newop.type != 'v':
             val = self.getvalue(op)
-            val.box = newop
+            if val.box is not None:
+                assert val.box is op
+                val.box = newop
             self.values[newop] = val
         return newop
 
@@ -731,8 +733,8 @@
     @specialize.argtype(0)
     def _emit_operation(self, op):
         assert not op.is_call_pure()
-        changed = False
         orig_op = op
+        op = self.replace_op_with(op, op.getopnum())
         for i in range(op.numargs()):
             arg = op.getarg(i)
             try:
@@ -743,9 +745,6 @@
                 self.ensure_imported(value)
                 newbox = value.force_box(self)
                 if arg is not newbox:
-                    if not changed:
-                        op = self.replace_op_with(op, op.getopnum())
-                        changed = True
                     op.setarg(i, newbox)
         self.metainterp_sd.profiler.count(jitprof.Counters.OPT_OPS)
         if op.is_guard():

From noreply at buildbot.pypy.org  Mon Feb 23 19:12:50 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Feb 2015 19:12:50 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: small fix
Message-ID: <20150223181250.187831C03D0@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r76092:f9fa525ca0f0
Date: 2015-02-23 20:12 +0200
http://bitbucket.org/pypy/pypy/changeset/f9fa525ca0f0/

Log:	small fix

diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py
--- a/rpython/jit/backend/llsupport/codemap.py
+++ b/rpython/jit/backend/llsupport/codemap.py
@@ -73,9 +73,9 @@
         used = getattr(self, name + '_used')
         allocated = len(getattr(self, name))
         lst = getattr(self, name)
-        if pos + len(to_insert) > allocated or pos != used:
+        if used + len(to_insert) > allocated or pos != used:
             old_lst = lst
-            if pos + len(to_insert) > allocated:
+            if used + len(to_insert) > allocated:
                 new_size = max(4 * len(old_lst),
                                (len(old_lst) + len(to_insert)) * 2)
             else:

From noreply at buildbot.pypy.org  Mon Feb 23 22:34:27 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Feb 2015 22:34:27 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: failed to add the test file
Message-ID: <20150223213427.610081C0F98@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r76093:e4dcf8816972
Date: 2015-02-23 23:34 +0200
http://bitbucket.org/pypy/pypy/changeset/e4dcf8816972/

Log:	failed to add the test file

diff --git a/rpython/jit/backend/llsupport/test/test_codemap.py b/rpython/jit/backend/llsupport/test/test_codemap.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/llsupport/test/test_codemap.py
@@ -0,0 +1,95 @@
+
+from rpython.jit.backend.llsupport.codemap import stack_depth_at_loc
+from rpython.jit.backend.llsupport.codemap import CodemapStorage,\
+     ListStorageMixin, INT_LIST, CodemapBuilder, unpack_traceback
+from rpython.rtyper.lltypesystem import lltype
+
+
+def test_list_storage_mixin():
+    class X(ListStorageMixin):
+        track_allocation = True
+        
+        def __init__(self):
+            self.x = lltype.malloc(INT_LIST, 4, flavor='raw')
+            self.x_used = 0
+
+        def unpack(self):
+            return [self.x[i] for i in range(self.x_used)]
+
+        def free_lst(self, name, lst):
+            lltype.free(lst, flavor='raw')
+        
+        def free(self):
+            lltype.free(self.x, flavor='raw')
+
+    x = X()
+    x.extend_with('x', [1, 2, 3], 0)
+    assert x.unpack() == [1, 2, 3]
+    x.extend_with('x', [4, 5, 6], 3)
+    assert x.unpack() == [1, 2, 3, 4, 5, 6]
+    x.extend_with('x', [7, 8, 9], 2, baseline=10)
+    assert x.unpack() == [1, 2, 17, 18, 19, 3, 4, 5, 6]
+    x.remove('x', 3, 6)
+    assert x.unpack() == [1, 2, 17, 4, 5, 6]
+    x.extend_with('x', [1] * 6, 6)
+    assert x.unpack() == [1, 2, 17, 4, 5, 6, 1, 1, 1, 1, 1, 1]
+    x.extend_with('x', [10] * 4, 5)
+    assert x.unpack() == [1, 2, 17, 4, 5, 10, 10, 10, 10, 6,
+                          1, 1, 1, 1, 1, 1]
+    x.free()
+
+def test_find_jit_frame_depth():
+    codemap = CodemapStorage()
+    codemap.setup()
+    codemap.register_frame_depth_map(11, [0, 5, 10], [1, 2, 3])
+    codemap.register_frame_depth_map(30, [0, 5, 10], [4, 5, 6])
+    codemap.register_frame_depth_map(0, [0, 5, 10], [7, 8, 9])
+    assert stack_depth_at_loc(13) == 1
+    assert stack_depth_at_loc(-3) == -1
+    assert stack_depth_at_loc(41) == -1
+    assert stack_depth_at_loc(5) == 8
+    assert stack_depth_at_loc(17) == 2
+    assert stack_depth_at_loc(38) == 5
+    codemap.free_asm_block(11, 22)
+    assert stack_depth_at_loc(13) == 9
+    assert stack_depth_at_loc(-3) == -1
+    assert stack_depth_at_loc(41) == -1
+    assert stack_depth_at_loc(5) == 8
+    assert stack_depth_at_loc(17) == 9
+    assert stack_depth_at_loc(38) == 5
+
+def test_codemaps():
+    builder = CodemapBuilder()
+    builder.debug_merge_point(0, 102, 0)
+    builder.debug_merge_point(0, 102, 13)
+    builder.debug_merge_point(1, 104, 15)
+    builder.debug_merge_point(1, 104, 16)
+    builder.debug_merge_point(2, 106, 20)
+    builder.debug_merge_point(2, 106, 25)
+    builder.debug_merge_point(1, 104, 30)
+    builder.debug_merge_point(0, 102, 35)
+    codemap = CodemapStorage()
+    codemap.setup()
+    codemap.register_codemap(builder.get_final_bytecode(100, 40))
+    builder = CodemapBuilder()
+    builder.debug_merge_point(0, 202, 0)
+    builder.debug_merge_point(0, 202, 10)
+    builder.debug_merge_point(1, 204, 20)
+    builder.debug_merge_point(1, 204, 30)
+    builder.debug_merge_point(2, 206, 40)
+    builder.debug_merge_point(2, 206, 50)
+    builder.debug_merge_point(1, 204, 60)
+    builder.debug_merge_point(0, 202, 70)
+    codemap.register_codemap(builder.get_final_bytecode(200, 100))
+    assert unpack_traceback(110) == [102]
+    assert unpack_traceback(117) == [102, 104]
+    assert unpack_traceback(121) == [102, 104, 106]
+    assert unpack_traceback(131) == [102, 104]
+    assert unpack_traceback(137) == [102]
+    assert unpack_traceback(205) == [202]
+    assert unpack_traceback(225) == [202, 204]
+    assert unpack_traceback(245) == [202, 204, 206]
+    assert unpack_traceback(265) == [202, 204]
+    assert unpack_traceback(275) == [202]
+    codemap.free_asm_block(200, 300)
+    assert unpack_traceback(225) == []

From noreply at buildbot.pypy.org  Tue Feb 24 10:08:59 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 24 Feb 2015 10:08:59 +0100 (CET)
Subject: [pypy-commit] pypy optresult: merge default
Message-ID: <20150224090859.30DA01C1C56@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r76094:38dd74d4ff19
Date: 2015-02-24 11:08 +0200
http://bitbucket.org/pypy/pypy/changeset/38dd74d4ff19/

Log:	merge default

diff too long, truncating to 2000 out of 54547 lines

diff --git a/.gitignore b/.gitignore
--- a/.gitignore
+++ b/.gitignore
@@ -7,7 +7,10 @@
 
 bin/pypy-c
 include/*.h
+include/numpy/
 lib_pypy/ctypes_config_cache/_[^_]*_*.py
+libpypy-c.*
+pypy-c
 pypy/_cache
 pypy/doc/*.html
 pypy/doc/config/*.html
@@ -18,4 +21,5 @@
 pypy/translator/c/src/dtoa.o
 pypy/translator/goal/pypy-c
 pypy/translator/goal/target*-c
-release/
\ No newline at end of file
+release/
+rpython/_cache/
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -7,10 +7,7 @@
 9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
 ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm
 20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0
-20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0
-0000000000000000000000000000000000000000 release-2.3.0
 394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3
 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1
 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1
-32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1
-0000000000000000000000000000000000000000 release-2.2=3.1
+10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -28,7 +28,7 @@
     DEALINGS IN THE SOFTWARE.
 
 
-PyPy Copyright holders 2003-2014
+PyPy Copyright holders 2003-2015
 ----------------------------------- 
 
 Except when otherwise stated (look for LICENSE files or information at
@@ -42,19 +42,19 @@
   Amaury Forgeot d'Arc
   Samuele Pedroni
   Alex Gaynor
+  Brian Kearns
+  Matti Picus
+  Philip Jenvey
   Michael Hudson
   David Schneider
-  Matti Picus
-  Brian Kearns
-  Philip Jenvey
   Holger Krekel
   Christian Tismer
   Hakan Ardo
   Benjamin Peterson
   Manuel Jacob
+  Ronan Lamy
   Anders Chrigstrom
   Eric van Riet Paap
-  Ronan Lamy
   Wim Lavrijsen
   Richard Emslie
   Alexander Schremmer
@@ -68,9 +68,9 @@
   Camillo Bruni
   Laura Creighton
   Toon Verwaest
+  Romain Guillebert
   Leonardo Santagada
   Seo Sanghyeon
-  Romain Guillebert
   Justin Peel
   Ronny Pfannschmidt
   David Edelsohn
@@ -91,15 +91,16 @@
   Michal Bendowski
   Jan de Mooij
   stian
+  Tyler Wade
   Michael Foord
   Stephan Diehl
-  Tyler Wade
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
   Patrick Maupin
   Bob Ippolito
   Bruno Gola
+  David Malcolm
   Jean-Paul Calderone
   Timo Paulssen
   Squeaky
@@ -108,18 +109,19 @@
   Marius Gedminas
   Martin Matusiak
   Konstantin Lopuhin
+  Wenzhu Man
   John Witulski
-  Wenzhu Man
+  Laurence Tratt
+  Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
   Mark Pearse
   Simon Cross
-  Ivan Sichmann Freitas
   Andreas Stührk
+  Stefano Rivera
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Stefano Rivera
   Paweł Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -129,7 +131,6 @@
   tav
   Taavi Burns
   Georg Brandl
-  Laurence Tratt
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -141,13 +142,12 @@
   Jeremy Thurgood
   Rami Chowdhury
   Tobias Pape
-  David Malcolm
   Eugene Oden
   Henry Mason
   Vasily Kuznetsov
   Preston Timmons
+  David Ripton
   Jeff Terrace
-  David Ripton
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
@@ -166,13 +166,16 @@
   Gintautas Miliauskas
   Michael Twomey
   Lucian Branescu Mihaila
+  Yichao Yu
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
   Karl Bartel
+  Wouter van Heyst
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  anatoly techtonik
   Stuart Williams
   Jasper Schulz
   Christian Hudon
@@ -182,12 +185,11 @@
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
-  anatoly techtonik
   Neil Shepperd
+  Stanislaw Halik
   Mikael Schönenberg
   Elmo M?ntynen
   Jonathan David Riehl
-  Stanislaw Halik
   Anders Qvist
   Corbin Simpson
   Chirag Jadwani
@@ -196,10 +198,13 @@
   Vincent Legoll
   Alan McIntyre
   Alexander Sedov
+  Attila Gobi
   Christopher Pope
   Christian Tismer 
   Marc Abramowitz
   Dan Stromberg
+  Arjun Naik
+  Valentina Mukhamedzhanova
   Stefano Parmesan
   Alexis Daboville
   Jens-Uwe Mager
@@ -213,8 +218,6 @@
   Sylvain Thenault
   Nathan Taylor
   Vladimir Kryachko
-  Arjun Naik
-  Attila Gobi
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -222,22 +225,23 @@
   Ryan Gonzalez
   Ian Foote
   Kristjan Valur Jonsson
+  David Lievens
   Neil Blakey-Milner
   Lutz Paelike
   Lucio Torre
   Lars Wassermann
-  Valentina Mukhamedzhanova
   Henrik Vendelbo
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
   Sergey Kishchenko
-  Yichao Yu
   Ignas Mikalajunas
   Christoph Gerum
   Martin Blais
   Lene Wagner
   Tomo Cocoa
+  Toni Mattis
+  Lucas Stadler
   roberto at goyle
   Yury V. Zaytsev
   Anna Katrina Dominguez
@@ -265,23 +269,30 @@
   Stephan Busemann
   Rafał Gałczyński
   Christian Muirhead
+  Berker Peksag
   James Lan
   shoma hosaka
-  Daniel Neuh?user
-  Matthew Miller
+  Daniel Neuhäuser
+  Ben Mather
+  halgari
+  Boglarka Vezer
+  Chris Pressey
   Buck Golemon
   Konrad Delong
   Dinu Gherman
   Chris Lambacher
   coolbutuseless at gmail.com
+  Jim Baker
   Rodrigo Araújo
-  Jim Baker
+  Nikolaos-Digenis Karagiannis
   James Robert
   Armin Ronacher
   Brett Cannon
+  Donald Stufft
   yrttyr
   aliceinwire
   OlivierBlanvillain
+  Dan Sanders
   Zooko Wilcox-O Hearn
   Tomer Chachamu
   Christopher Groskopf
@@ -295,6 +306,7 @@
   Markus Unterwaditzer
   Even Wiik Thomassen
   jbs
+  squeaky
   soareschen
   Kurt Griffiths
   Mike Bayer
@@ -306,6 +318,7 @@
   Anna Ravencroft
   Dan Crosta
   Julien Phalip
+  Roman Podoliaka
   Dan Loewenherz
 
   Heinrich-Heine University, Germany 
diff --git a/lib-python/2.7/CGIHTTPServer.py b/lib-python/2.7/CGIHTTPServer.py
--- a/lib-python/2.7/CGIHTTPServer.py
+++ b/lib-python/2.7/CGIHTTPServer.py
@@ -106,16 +106,16 @@
     def run_cgi(self):
         """Execute a CGI script."""
         dir, rest = self.cgi_info
-
-        i = rest.find('/')
+        path = dir + '/' + rest
+        i = path.find('/', len(dir)+1)
         while i >= 0:
-            nextdir = rest[:i]
-            nextrest = rest[i+1:]
+            nextdir = path[:i]
+            nextrest = path[i+1:]
 
             scriptdir = self.translate_path(nextdir)
             if os.path.isdir(scriptdir):
                 dir, rest = nextdir, nextrest
-                i = rest.find('/')
+                i = path.find('/', len(dir)+1)
             else:
                 break
 
diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py
--- a/lib-python/2.7/Cookie.py
+++ b/lib-python/2.7/Cookie.py
@@ -56,7 +56,7 @@
    >>> C = Cookie.SmartCookie()
 
 [Note: Long-time users of Cookie.py will remember using
-Cookie.Cookie() to create an Cookie object.  Although deprecated, it
+Cookie.Cookie() to create a Cookie object.  Although deprecated, it
 is still supported by the code.  See the Backward Compatibility notes
 for more information.]
 
@@ -426,6 +426,8 @@
                    "version" : "Version",
                    }
 
+    _flags = {'secure', 'httponly'}
+
     def __init__(self):
         # Set defaults
         self.key = self.value = self.coded_value = None
@@ -529,9 +531,11 @@
 _LegalCharsPatt  = r"[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=]"
 _CookiePattern = re.compile(
     r"(?x)"                       # This is a Verbose pattern
+    r"\s*"                        # Optional whitespace at start of cookie
     r"(?P"                   # Start of group 'key'
     ""+ _LegalCharsPatt +"+?"     # Any word of at least one letter, nongreedy
     r")"                          # End of group 'key'
+    r"("                          # Optional group: there may not be a value.
     r"\s*=\s*"                    # Equal Sign
     r"(?P"                   # Start of group 'val'
     r'"(?:[^\\"]|\\.)*"'            # Any doublequoted string
@@ -540,7 +544,9 @@
     r"|"                            # or
     ""+ _LegalCharsPatt +"*"        # Any word or empty string
     r")"                          # End of group 'val'
-    r"\s*;?"                      # Probably ending in a semi-colon
+    r")?"                         # End of optional value group
+    r"\s*"                        # Any number of spaces.
+    r"(\s+|;|$)"                  # Ending either at space, semicolon, or EOS.
     )
 
 
@@ -585,8 +591,12 @@
 
     def __setitem__(self, key, value):
         """Dictionary style assignment."""
-        rval, cval = self.value_encode(value)
-        self.__set(key, rval, cval)
+        if isinstance(value, Morsel):
+            # allow assignment of constructed Morsels (e.g. for pickling)
+            dict.__setitem__(self, key, value)
+        else:
+            rval, cval = self.value_encode(value)
+            self.__set(key, rval, cval)
     # end __setitem__
 
     def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"):
@@ -641,7 +651,7 @@
 
         while 0 <= i < n:
             # Start looking for a cookie
-            match = patt.search(str, i)
+            match = patt.match(str, i)
             if not match: break          # No more cookies
 
             K,V = match.group("key"), match.group("val")
@@ -656,8 +666,12 @@
                     M[ K[1:] ] = V
             elif K.lower() in Morsel._reserved:
                 if M:
-                    M[ K ] = _unquote(V)
-            else:
+                    if V is None:
+                        if K.lower() in Morsel._flags:
+                            M[K] = True
+                    else:
+                        M[K] = _unquote(V)
+            elif V is not None:
                 rval, cval = self.value_decode(V)
                 self.__set(K, rval, cval)
                 M = self[K]
diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py
--- a/lib-python/2.7/SocketServer.py
+++ b/lib-python/2.7/SocketServer.py
@@ -416,8 +416,12 @@
         self.socket = socket.socket(self.address_family,
                                     self.socket_type)
         if bind_and_activate:
-            self.server_bind()
-            self.server_activate()
+            try:
+                self.server_bind()
+                self.server_activate()
+            except:
+                self.server_close()
+                raise
 
     def server_bind(self):
         """Called by constructor to bind the socket.
diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py
--- a/lib-python/2.7/_abcoll.py
+++ b/lib-python/2.7/_abcoll.py
@@ -143,7 +143,7 @@
     methods except for __contains__, __iter__ and __len__.
 
     To override the comparisons (presumably for speed, as the
-    semantics are fixed), all you have to do is redefine __le__ and
+    semantics are fixed), redefine __le__ and __ge__,
     then the other operations will automatically follow suit.
     """
 
diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py
--- a/lib-python/2.7/argparse.py
+++ b/lib-python/2.7/argparse.py
@@ -1089,7 +1089,14 @@
         # parse all the remaining options into the namespace
         # store any unrecognized options on the object, so that the top
         # level parser can decide what to do with them
-        namespace, arg_strings = parser.parse_known_args(arg_strings, namespace)
+
+        # In case this subparser defines new defaults, we parse them
+        # in a new namespace object and then update the original
+        # namespace for the relevant parts.
+        subnamespace, arg_strings = parser.parse_known_args(arg_strings, None)
+        for key, value in vars(subnamespace).items():
+            setattr(namespace, key, value)
+
         if arg_strings:
             vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, [])
             getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings)
diff --git a/lib-python/2.7/asynchat.py b/lib-python/2.7/asynchat.py
--- a/lib-python/2.7/asynchat.py
+++ b/lib-python/2.7/asynchat.py
@@ -46,12 +46,17 @@
 you - by calling your self.found_terminator() method.
 """
 
+import asyncore
+import errno
 import socket
-import asyncore
 from collections import deque
 from sys import py3kwarning
 from warnings import filterwarnings, catch_warnings
 
+_BLOCKING_IO_ERRORS = (errno.EAGAIN, errno.EALREADY, errno.EINPROGRESS,
+                       errno.EWOULDBLOCK)
+
+
 class async_chat (asyncore.dispatcher):
     """This is an abstract class.  You must derive from this class, and add
     the two methods collect_incoming_data() and found_terminator()"""
@@ -109,6 +114,8 @@
         try:
             data = self.recv (self.ac_in_buffer_size)
         except socket.error, why:
+            if why.args[0] in _BLOCKING_IO_ERRORS:
+                return
             self.handle_error()
             return
 
diff --git a/lib-python/2.7/bsddb/test/test_queue.py b/lib-python/2.7/bsddb/test/test_queue.py
--- a/lib-python/2.7/bsddb/test/test_queue.py
+++ b/lib-python/2.7/bsddb/test/test_queue.py
@@ -10,6 +10,7 @@
 
 #----------------------------------------------------------------------
 
+ at unittest.skip("fails on Windows; see issue 22943")
 class SimpleQueueTestCase(unittest.TestCase):
     def setUp(self):
         self.filename = get_new_database_path()
diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -17,6 +17,10 @@
 except ImportError:
     assert '__pypy__' not in _sys.builtin_module_names
     newdict = lambda _ : {}
+try:
+    from __pypy__ import reversed_dict
+except ImportError:
+    reversed_dict = lambda d: reversed(d.keys())
 
 try:
     from thread import get_ident as _get_ident
@@ -29,142 +33,35 @@
 ################################################################################
 
 class OrderedDict(dict):
-    'Dictionary that remembers insertion order'
-    # An inherited dict maps keys to values.
-    # The inherited dict provides __getitem__, __len__, __contains__, and get.
-    # The remaining methods are order-aware.
-    # Big-O running times for all methods are the same as regular dictionaries.
+    '''Dictionary that remembers insertion order.
 
-    # The internal self.__map dict maps keys to links in a doubly linked list.
-    # The circular doubly linked list starts and ends with a sentinel element.
-    # The sentinel element never gets deleted (this simplifies the algorithm).
-    # Each link is stored as a list of length three:  [PREV, NEXT, KEY].
+    In PyPy all dicts are ordered anyway.  This is mostly useful as a
+    placeholder to mean "this dict must be ordered even on CPython".
 
-    def __init__(self, *args, **kwds):
-        '''Initialize an ordered dictionary.  The signature is the same as
-        regular dictionaries, but keyword arguments are not recommended because
-        their insertion order is arbitrary.
-
-        '''
-        if len(args) > 1:
-            raise TypeError('expected at most 1 arguments, got %d' % len(args))
-        try:
-            self.__root
-        except AttributeError:
-            self.__root = root = []                     # sentinel node
-            root[:] = [root, root, None]
-            self.__map = {}
-        self.__update(*args, **kwds)
-
-    def __setitem__(self, key, value, dict_setitem=dict.__setitem__):
-        'od.__setitem__(i, y) <==> od[i]=y'
-        # Setting a new item creates a new link at the end of the linked list,
-        # and the inherited dictionary is updated with the new key/value pair.
-        if key not in self:
-            root = self.__root
-            last = root[0]
-            last[1] = root[0] = self.__map[key] = [last, root, key]
-        return dict_setitem(self, key, value)
-
-    def __delitem__(self, key, dict_delitem=dict.__delitem__):
-        'od.__delitem__(y) <==> del od[y]'
-        # Deleting an existing item uses self.__map to find the link which gets
-        # removed by updating the links in the predecessor and successor nodes.
-        dict_delitem(self, key)
-        link_prev, link_next, _ = self.__map.pop(key)
-        link_prev[1] = link_next                        # update link_prev[NEXT]
-        link_next[0] = link_prev                        # update link_next[PREV]
-
-    def __iter__(self):
-        'od.__iter__() <==> iter(od)'
-        # Traverse the linked list in order.
-        root = self.__root
-        curr = root[1]                                  # start at the first node
-        while curr is not root:
-            yield curr[2]                               # yield the curr[KEY]
-            curr = curr[1]                              # move to next node
+    Known difference: iterating over an OrderedDict which is being
+    concurrently modified raises RuntimeError in PyPy.  In CPython
+    instead we get some behavior that appears reasonable in some
+    cases but is nonsensical in other cases.  This is officially
+    forbidden by the CPython docs, so we forbid it explicitly for now.
+    '''
 
     def __reversed__(self):
-        'od.__reversed__() <==> reversed(od)'
-        # Traverse the linked list in reverse order.
-        root = self.__root
-        curr = root[0]                                  # start at the last node
-        while curr is not root:
-            yield curr[2]                               # yield the curr[KEY]
-            curr = curr[0]                              # move to previous node
-
-    def clear(self):
-        'od.clear() -> None.  Remove all items from od.'
-        root = self.__root
-        root[:] = [root, root, None]
-        self.__map.clear()
-        dict.clear(self)
-
-    # -- the following methods do not depend on the internal structure --
-
-    def keys(self):
-        'od.keys() -> list of keys in od'
-        return list(self)
-
-    def values(self):
-        'od.values() -> list of values in od'
-        return [self[key] for key in self]
-
-    def items(self):
-        'od.items() -> list of (key, value) pairs in od'
-        return [(key, self[key]) for key in self]
-
-    def iterkeys(self):
-        'od.iterkeys() -> an iterator over the keys in od'
-        return iter(self)
-
-    def itervalues(self):
-        'od.itervalues -> an iterator over the values in od'
-        for k in self:
-            yield self[k]
-
-    def iteritems(self):
-        'od.iteritems -> an iterator over the (key, value) pairs in od'
-        for k in self:
-            yield (k, self[k])
-
-    update = MutableMapping.update
-
-    __update = update # let subclasses override update without breaking __init__
-
-    __marker = object()
-
-    def pop(self, key, default=__marker):
-        '''od.pop(k[,d]) -> v, remove specified key and return the corresponding
-        value.  If key is not found, d is returned if given, otherwise KeyError
-        is raised.
-
-        '''
-        if key in self:
-            result = self[key]
-            del self[key]
-            return result
-        if default is self.__marker:
-            raise KeyError(key)
-        return default
-
-    def setdefault(self, key, default=None):
-        'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
-        if key in self:
-            return self[key]
-        self[key] = default
-        return default
+        return reversed_dict(self)
 
     def popitem(self, last=True):
         '''od.popitem() -> (k, v), return and remove a (key, value) pair.
         Pairs are returned in LIFO order if last is true or FIFO order if false.
 
         '''
-        if not self:
-            raise KeyError('dictionary is empty')
-        key = next(reversed(self) if last else iter(self))
-        value = self.pop(key)
-        return key, value
+        if last:
+            return dict.popitem(self)
+        else:
+            it = dict.__iter__(self)
+            try:
+                k = it.next()
+            except StopIteration:
+                raise KeyError('dictionary is empty')
+            return (k, self.pop(k))
 
     def __repr__(self, _repr_running={}):
         'od.__repr__() <==> repr(od)'
@@ -183,8 +80,6 @@
         'Return state information for pickling'
         items = [[k, self[k]] for k in self]
         inst_dict = vars(self).copy()
-        for k in vars(OrderedDict()):
-            inst_dict.pop(k, None)
         if inst_dict:
             return (self.__class__, (items,), inst_dict)
         return self.__class__, (items,)
@@ -193,17 +88,6 @@
         'od.copy() -> a shallow copy of od'
         return self.__class__(self)
 
-    @classmethod
-    def fromkeys(cls, iterable, value=None):
-        '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S.
-        If not specified, the value defaults to None.
-
-        '''
-        self = cls()
-        for key in iterable:
-            self[key] = value
-        return self
-
     def __eq__(self, other):
         '''od.__eq__(y) <==> od==y.  Comparison to another OD is order-sensitive
         while comparison to a regular mapping is order-insensitive.
diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py
--- a/lib-python/2.7/cookielib.py
+++ b/lib-python/2.7/cookielib.py
@@ -1719,12 +1719,12 @@
     def __repr__(self):
         r = []
         for cookie in self: r.append(repr(cookie))
-        return "<%s[%s]>" % (self.__class__, ", ".join(r))
+        return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r))
 
     def __str__(self):
         r = []
         for cookie in self: r.append(str(cookie))
-        return "<%s[%s]>" % (self.__class__, ", ".join(r))
+        return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r))
 
 
 # derives from IOError for backwards-compatibility with Python 2.4.0
diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py
@@ -2,7 +2,6 @@
 import array
 import gc
 import unittest
-from ctypes.test import xfail
 
 class X(Structure):
     _fields_ = [("c_int", c_int)]
@@ -11,7 +10,6 @@
         self._init_called = True
 
 class Test(unittest.TestCase):
-    @xfail
     def test_fom_buffer(self):
         a = array.array("i", range(16))
         x = (c_int * 16).from_buffer(a)
@@ -34,10 +32,9 @@
         del a; gc.collect(); gc.collect(); gc.collect()
         self.assertEqual(x[:], expected)
 
-        self.assertRaises(TypeError,
+        self.assertRaises((TypeError, ValueError),
                           (c_char * 16).from_buffer, "a" * 16)
 
-    @xfail
     def test_fom_buffer_with_offset(self):
         a = array.array("i", range(16))
         x = (c_int * 15).from_buffer(a, sizeof(c_int))
@@ -46,7 +43,6 @@
         self.assertRaises(ValueError, lambda: (c_int * 16).from_buffer(a, sizeof(c_int)))
         self.assertRaises(ValueError, lambda: (c_int * 1).from_buffer(a, 16 * sizeof(c_int)))
 
-    @xfail
     def test_from_buffer_copy(self):
         a = array.array("i", range(16))
         x = (c_int * 16).from_buffer_copy(a)
@@ -71,7 +67,6 @@
         x = (c_char * 16).from_buffer_copy("a" * 16)
         self.assertEqual(x[:], "a" * 16)
 
-    @xfail
     def test_fom_buffer_copy_with_offset(self):
         a = array.array("i", range(16))
         x = (c_int * 15).from_buffer_copy(a, sizeof(c_int))
diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py
--- a/lib-python/2.7/ctypes/test/test_pointers.py
+++ b/lib-python/2.7/ctypes/test/test_pointers.py
@@ -7,6 +7,8 @@
                  c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float]
 python_types = [int, int, int, int, int, long,
                 int, long, long, long, float, float]
+LargeNamedType = type('T' * 2 ** 25, (Structure,), {})
+large_string = 'T' * 2 ** 25
 
 class PointersTestCase(unittest.TestCase):
 
@@ -188,5 +190,11 @@
             mth = WINFUNCTYPE(None)(42, "name", (), None)
             self.assertEqual(bool(mth), True)
 
+    def test_pointer_type_name(self):
+        self.assertTrue(POINTER(LargeNamedType))
+
+    def test_pointer_type_str_name(self):
+        self.assertTrue(POINTER(large_string))
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_python_api.py b/lib-python/2.7/ctypes/test/test_python_api.py
--- a/lib-python/2.7/ctypes/test/test_python_api.py
+++ b/lib-python/2.7/ctypes/test/test_python_api.py
@@ -46,8 +46,8 @@
     # This test is unreliable, because it is possible that code in
     # unittest changes the refcount of the '42' integer.  So, it
     # is disabled by default.
-    @requires("refcount")
     def test_PyInt_Long(self):
+        requires("refcount")
         ref42 = grc(42)
         pythonapi.PyInt_FromLong.restype = py_object
         self.assertEqual(pythonapi.PyInt_FromLong(42), 42)
diff --git a/lib-python/2.7/ctypes/test/test_win32.py b/lib-python/2.7/ctypes/test/test_win32.py
--- a/lib-python/2.7/ctypes/test/test_win32.py
+++ b/lib-python/2.7/ctypes/test/test_win32.py
@@ -38,8 +38,11 @@
 
 @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test')
 class FunctionCallTestCase(unittest.TestCase):
-    @requires("SEH")
+    @unittest.skipUnless('MSC' in sys.version, "SEH only supported by MSC")
+    @unittest.skipIf(sys.executable.endswith('_d.exe'),
+                     "SEH not enabled in debug builds")
     def test_SEH(self):
+        requires("SEH")
         # Call functions with invalid arguments, and make sure
         # that access violations are trapped and raise an
         # exception.
@@ -87,9 +90,29 @@
 
         dll = CDLL(_ctypes_test.__file__)
 
-        pt = POINT(10, 10)
-        rect = RECT(0, 0, 20, 20)
-        self.assertEqual(1, dll.PointInRect(byref(rect), pt))
+        pt = POINT(15, 25)
+        left = c_long.in_dll(dll, 'left')
+        top = c_long.in_dll(dll, 'top')
+        right = c_long.in_dll(dll, 'right')
+        bottom = c_long.in_dll(dll, 'bottom')
+        rect = RECT(left, top, right, bottom)
+        PointInRect = dll.PointInRect
+        PointInRect.argtypes = [POINTER(RECT), POINT]
+        self.assertEqual(1, PointInRect(byref(rect), pt))
+
+        ReturnRect = dll.ReturnRect
+        ReturnRect.argtypes = [c_int, RECT, POINTER(RECT), POINT, RECT,
+                               POINTER(RECT), POINT, RECT]
+        ReturnRect.restype = RECT
+        for i in range(4):
+            ret = ReturnRect(i, rect, pointer(rect), pt, rect,
+                         byref(rect), pt, rect)
+            # the c function will check and modify ret if something is
+            # passed in improperly
+            self.assertEqual(ret.left, left.value)
+            self.assertEqual(ret.right, right.value)
+            self.assertEqual(ret.top, top.value)
+            self.assertEqual(ret.bottom, bottom.value)
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/decimal.py b/lib-python/2.7/decimal.py
--- a/lib-python/2.7/decimal.py
+++ b/lib-python/2.7/decimal.py
@@ -136,7 +136,6 @@
 
 __version__ = '1.70'    # Highest version of the spec this complies with
 
-import copy as _copy
 import math as _math
 import numbers as _numbers
 
@@ -3665,6 +3664,8 @@
         if self._is_special:
             sign = _format_sign(self._sign, spec)
             body = str(self.copy_abs())
+            if spec['type'] == '%':
+                body += '%'
             return _format_align(sign, body, spec)
 
         # a type of None defaults to 'g' or 'G', depending on context
@@ -6033,7 +6034,10 @@
         format_dict['decimal_point'] = '.'
 
     # record whether return type should be str or unicode
-    format_dict['unicode'] = isinstance(format_spec, unicode)
+    try:
+        format_dict['unicode'] = isinstance(format_spec, unicode)
+    except NameError:
+        format_dict['unicode'] = False
 
     return format_dict
 
diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py
--- a/lib-python/2.7/distutils/__init__.py
+++ b/lib-python/2.7/distutils/__init__.py
@@ -15,5 +15,5 @@
 # Updated automatically by the Python release process.
 #
 #--start constants--
-__version__ = "2.7.8"
+__version__ = "2.7.9"
 #--end constants--
diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py
--- a/lib-python/2.7/distutils/command/build_ext.py
+++ b/lib-python/2.7/distutils/command/build_ext.py
@@ -245,7 +245,7 @@
         # Python's library directory must be appended to library_dirs
         # See Issues: #1600860, #4366
         if (sysconfig.get_config_var('Py_ENABLE_SHARED')):
-            if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")):
+            if not sysconfig.python_build:
                 # building third party extensions
                 self.library_dirs.append(sysconfig.get_config_var('LIBDIR'))
             else:
diff --git a/lib-python/2.7/distutils/command/upload.py b/lib-python/2.7/distutils/command/upload.py
--- a/lib-python/2.7/distutils/command/upload.py
+++ b/lib-python/2.7/distutils/command/upload.py
@@ -136,8 +136,8 @@
 
         # Build up the MIME payload for the POST data
         boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'
-        sep_boundary = '\n--' + boundary
-        end_boundary = sep_boundary + '--'
+        sep_boundary = '\r\n--' + boundary
+        end_boundary = sep_boundary + '--\r\n'
         body = StringIO.StringIO()
         for key, value in data.items():
             # handle multiple entries for the same name
@@ -151,14 +151,13 @@
                     fn = ""
 
                 body.write(sep_boundary)
-                body.write('\nContent-Disposition: form-data; name="%s"'%key)
+                body.write('\r\nContent-Disposition: form-data; name="%s"' % key)
                 body.write(fn)
-                body.write("\n\n")
+                body.write("\r\n\r\n")
                 body.write(value)
                 if value and value[-1] == '\r':
                     body.write('\n')  # write an extra newline (lurve Macs)
         body.write(end_boundary)
-        body.write("\n")
         body = body.getvalue()
 
         self.announce("Submitting %s to %s" % (filename, self.repository), log.INFO)
diff --git a/lib-python/2.7/distutils/file_util.py b/lib-python/2.7/distutils/file_util.py
--- a/lib-python/2.7/distutils/file_util.py
+++ b/lib-python/2.7/distutils/file_util.py
@@ -85,7 +85,8 @@
     (os.symlink) instead of copying: set it to "hard" or "sym"; if it is
     None (the default), files are copied.  Don't set 'link' on systems that
     don't support it: 'copy_file()' doesn't check if hard or symbolic
-    linking is available.
+    linking is available. If hardlink fails, falls back to
+    _copy_file_contents().
 
     Under Mac OS, uses the native file copy function in macostools; on
     other systems, uses '_copy_file_contents()' to copy file contents.
@@ -137,24 +138,31 @@
     # (Unix only, of course, but that's the caller's responsibility)
     if link == 'hard':
         if not (os.path.exists(dst) and os.path.samefile(src, dst)):
-            os.link(src, dst)
+            try:
+                os.link(src, dst)
+                return (dst, 1)
+            except OSError:
+                # If hard linking fails, fall back on copying file
+                # (some special filesystems don't support hard linking
+                #  even under Unix, see issue #8876).
+                pass
     elif link == 'sym':
         if not (os.path.exists(dst) and os.path.samefile(src, dst)):
             os.symlink(src, dst)
+            return (dst, 1)
 
     # Otherwise (non-Mac, not linking), copy the file contents and
     # (optionally) copy the times and mode.
-    else:
-        _copy_file_contents(src, dst)
-        if preserve_mode or preserve_times:
-            st = os.stat(src)
+    _copy_file_contents(src, dst)
+    if preserve_mode or preserve_times:
+        st = os.stat(src)
 
-            # According to David Ascher , utime() should be done
-            # before chmod() (at least under NT).
-            if preserve_times:
-                os.utime(dst, (st[ST_ATIME], st[ST_MTIME]))
-            if preserve_mode:
-                os.chmod(dst, S_IMODE(st[ST_MODE]))
+        # According to David Ascher , utime() should be done
+        # before chmod() (at least under NT).
+        if preserve_times:
+            os.utime(dst, (st[ST_ATIME], st[ST_MTIME]))
+        if preserve_mode:
+            os.chmod(dst, S_IMODE(st[ST_MODE]))
 
     return (dst, 1)
 
diff --git a/lib-python/2.7/distutils/sysconfig_cpython.py b/lib-python/2.7/distutils/sysconfig_cpython.py
--- a/lib-python/2.7/distutils/sysconfig_cpython.py
+++ b/lib-python/2.7/distutils/sysconfig_cpython.py
@@ -165,7 +165,8 @@
             # version and build tools may not support the same set
             # of CPU architectures for universal builds.
             global _config_vars
-            if not _config_vars.get('CUSTOMIZED_OSX_COMPILER', ''):
+            # Use get_config_var() to ensure _config_vars is initialized.
+            if not get_config_var('CUSTOMIZED_OSX_COMPILER'):
                 import _osx_support
                 _osx_support.customize_compiler(_config_vars)
                 _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True'
diff --git a/lib-python/2.7/distutils/tests/test_bdist_rpm.py b/lib-python/2.7/distutils/tests/test_bdist_rpm.py
--- a/lib-python/2.7/distutils/tests/test_bdist_rpm.py
+++ b/lib-python/2.7/distutils/tests/test_bdist_rpm.py
@@ -25,6 +25,7 @@
 """
 
 class BuildRpmTestCase(support.TempdirManager,
+                       support.EnvironGuard,
                        support.LoggingSilencer,
                        unittest.TestCase):
 
@@ -50,6 +51,7 @@
     def test_quiet(self):
         # let's create a package
         tmp_dir = self.mkdtemp()
+        os.environ['HOME'] = tmp_dir   # to confine dir '.rpmdb' creation
         pkg_dir = os.path.join(tmp_dir, 'foo')
         os.mkdir(pkg_dir)
         self.write_file((pkg_dir, 'setup.py'), SETUP_PY)
@@ -92,6 +94,7 @@
     def test_no_optimize_flag(self):
         # let's create a package that brakes bdist_rpm
         tmp_dir = self.mkdtemp()
+        os.environ['HOME'] = tmp_dir   # to confine dir '.rpmdb' creation
         pkg_dir = os.path.join(tmp_dir, 'foo')
         os.mkdir(pkg_dir)
         self.write_file((pkg_dir, 'setup.py'), SETUP_PY)
diff --git a/lib-python/2.7/distutils/tests/test_dist.py b/lib-python/2.7/distutils/tests/test_dist.py
--- a/lib-python/2.7/distutils/tests/test_dist.py
+++ b/lib-python/2.7/distutils/tests/test_dist.py
@@ -11,7 +11,7 @@
 from distutils.dist import Distribution, fix_help_options
 from distutils.cmd import Command
 import distutils.dist
-from test.test_support import TESTFN, captured_stdout, run_unittest
+from test.test_support import TESTFN, captured_stdout, run_unittest, unlink
 from distutils.tests import support
 
 
@@ -64,6 +64,7 @@
         with open(TESTFN, "w") as f:
             f.write("[global]\n")
             f.write("command_packages = foo.bar, splat")
+        self.addCleanup(unlink, TESTFN)
 
         files = [TESTFN]
         sys.argv.append("build")
diff --git a/lib-python/2.7/distutils/tests/test_file_util.py b/lib-python/2.7/distutils/tests/test_file_util.py
--- a/lib-python/2.7/distutils/tests/test_file_util.py
+++ b/lib-python/2.7/distutils/tests/test_file_util.py
@@ -8,6 +8,11 @@
 from distutils.tests import support
 from test.test_support import run_unittest
 
+
+requires_os_link = unittest.skipUnless(hasattr(os, "link"),
+                                       "test requires os.link()")
+
+
 class FileUtilTestCase(support.TempdirManager, unittest.TestCase):
 
     def _log(self, msg, *args):
@@ -74,6 +79,44 @@
         copy_file(foo, dst_dir)
         self.assertTrue(os.path.exists(os.path.join(dst_dir, 'foo')))
 
+    @requires_os_link
+    def test_copy_file_hard_link(self):
+        with open(self.source, 'w') as f:
+            f.write('some content')
+        st = os.stat(self.source)
+        copy_file(self.source, self.target, link='hard')
+        st2 = os.stat(self.source)
+        st3 = os.stat(self.target)
+        self.assertTrue(os.path.samestat(st, st2), (st, st2))
+        self.assertTrue(os.path.samestat(st2, st3), (st2, st3))
+        with open(self.source, 'r') as f:
+            self.assertEqual(f.read(), 'some content')
+
+    @requires_os_link
+    def test_copy_file_hard_link_failure(self):
+        # If hard linking fails, copy_file() falls back on copying file
+        # (some special filesystems don't support hard linking even under
+        #  Unix, see issue #8876).
+        with open(self.source, 'w') as f:
+            f.write('some content')
+        st = os.stat(self.source)
+        def _os_link(*args):
+            raise OSError(0, "linking unsupported")
+        old_link = os.link
+        os.link = _os_link
+        try:
+            copy_file(self.source, self.target, link='hard')
+        finally:
+            os.link = old_link
+        st2 = os.stat(self.source)
+        st3 = os.stat(self.target)
+        self.assertTrue(os.path.samestat(st, st2), (st, st2))
+        self.assertFalse(os.path.samestat(st2, st3), (st2, st3))
+        for fn in (self.source, self.target):
+            with open(fn, 'r') as f:
+                self.assertEqual(f.read(), 'some content')
+
+
 def test_suite():
     return unittest.makeSuite(FileUtilTestCase)
 
diff --git a/lib-python/2.7/distutils/tests/test_sysconfig.py b/lib-python/2.7/distutils/tests/test_sysconfig.py
--- a/lib-python/2.7/distutils/tests/test_sysconfig.py
+++ b/lib-python/2.7/distutils/tests/test_sysconfig.py
@@ -3,6 +3,9 @@
 import test
 import unittest
 import shutil
+import subprocess
+import sys
+import textwrap
 
 from distutils import sysconfig
 from distutils.tests import support
@@ -99,6 +102,24 @@
         self.assertEqual(global_sysconfig.get_config_var('LDSHARED'), sysconfig.get_config_var('LDSHARED'))
         self.assertEqual(global_sysconfig.get_config_var('CC'), sysconfig.get_config_var('CC'))
 
+    def test_customize_compiler_before_get_config_vars(self):
+        # Issue #21923: test that a Distribution compiler
+        # instance can be called without an explicit call to
+        # get_config_vars().
+        with open(TESTFN, 'w') as f:
+            f.writelines(textwrap.dedent('''\
+                from distutils.core import Distribution
+                config = Distribution().get_command_obj('config')
+                # try_compile may pass or it may fail if no compiler
+                # is found but it should not raise an exception.
+                rc = config.try_compile('int x;')
+                '''))
+        p = subprocess.Popen([str(sys.executable), TESTFN],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                universal_newlines=True)
+        outs, errs = p.communicate()
+        self.assertEqual(0, p.returncode, "Subprocess failed: " + outs)
 
 
 def test_suite():
diff --git a/lib-python/2.7/distutils/tests/test_upload.py b/lib-python/2.7/distutils/tests/test_upload.py
--- a/lib-python/2.7/distutils/tests/test_upload.py
+++ b/lib-python/2.7/distutils/tests/test_upload.py
@@ -119,7 +119,7 @@
         # what did we send ?
         self.assertIn('dédé', self.last_open.req.data)
         headers = dict(self.last_open.req.headers)
-        self.assertEqual(headers['Content-length'], '2085')
+        self.assertEqual(headers['Content-length'], '2159')
         self.assertTrue(headers['Content-type'].startswith('multipart/form-data'))
         self.assertEqual(self.last_open.req.get_method(), 'POST')
         self.assertEqual(self.last_open.req.get_full_url(),
diff --git a/lib-python/2.7/distutils/unixccompiler.py b/lib-python/2.7/distutils/unixccompiler.py
--- a/lib-python/2.7/distutils/unixccompiler.py
+++ b/lib-python/2.7/distutils/unixccompiler.py
@@ -58,7 +58,7 @@
     executables = {'preprocessor' : None,
                    'compiler'     : ["cc"],
                    'compiler_so'  : ["cc"],
-                   'compiler_cxx' : ["cc"],
+                   'compiler_cxx' : ["c++"],  # pypy: changed, 'cc' is bogus
                    'linker_so'    : ["cc", "-shared"],
                    'linker_exe'   : ["cc"],
                    'archiver'     : ["ar", "-cr"],
diff --git a/lib-python/2.7/doctest.py b/lib-python/2.7/doctest.py
--- a/lib-python/2.7/doctest.py
+++ b/lib-python/2.7/doctest.py
@@ -216,7 +216,7 @@
                 # get_data() opens files as 'rb', so one must do the equivalent
                 # conversion as universal newlines would do.
                 return file_contents.replace(os.linesep, '\n'), filename
-    with open(filename) as f:
+    with open(filename, 'U') as f:
         return f.read(), filename
 
 # Use sys.stdout encoding for ouput.
diff --git a/lib-python/2.7/email/feedparser.py b/lib-python/2.7/email/feedparser.py
--- a/lib-python/2.7/email/feedparser.py
+++ b/lib-python/2.7/email/feedparser.py
@@ -49,8 +49,8 @@
     simple abstraction -- it parses until EOF closes the current message.
     """
     def __init__(self):
-        # The last partial line pushed into this object.
-        self._partial = ''
+        # Chunks of the last partial line pushed into this object.
+        self._partial = []
         # The list of full, pushed lines, in reverse order
         self._lines = []
         # The stack of false-EOF checking predicates.
@@ -66,8 +66,8 @@
 
     def close(self):
         # Don't forget any trailing partial line.
-        self._lines.append(self._partial)
-        self._partial = ''
+        self.pushlines(''.join(self._partial).splitlines(True))
+        self._partial = []
         self._closed = True
 
     def readline(self):
@@ -95,8 +95,29 @@
 
     def push(self, data):
         """Push some new data into this object."""
-        # Handle any previous leftovers
-        data, self._partial = self._partial + data, ''
+        # Crack into lines, but preserve the linesep characters on the end of each
+        parts = data.splitlines(True)
+
+        if not parts or not parts[0].endswith(('\n', '\r')):
+            # No new complete lines, so just accumulate partials
+            self._partial += parts
+            return
+
+        if self._partial:
+            # If there are previous leftovers, complete them now
+            self._partial.append(parts[0])
+            parts[0:1] = ''.join(self._partial).splitlines(True)
+            del self._partial[:]
+
+        # If the last element of the list does not end in a newline, then treat
+        # it as a partial line.  We only check for '\n' here because a line
+        # ending with '\r' might be a line that was split in the middle of a
+        # '\r\n' sequence (see bugs 1555570 and 1721862).
+        if not parts[-1].endswith('\n'):
+            self._partial = [parts.pop()]
+        self.pushlines(parts)
+
+    def pushlines(self, lines):
         # Crack into lines, but preserve the newlines on the end of each
         parts = NLCRE_crack.split(data)
         # The *ahem* interesting behaviour of re.split when supplied grouping
diff --git a/lib-python/2.7/email/mime/nonmultipart.py b/lib-python/2.7/email/mime/nonmultipart.py
--- a/lib-python/2.7/email/mime/nonmultipart.py
+++ b/lib-python/2.7/email/mime/nonmultipart.py
@@ -12,7 +12,7 @@
 
 

 class MIMENonMultipart(MIMEBase):
-    """Base class for MIME multipart/* type messages."""
+    """Base class for MIME non-multipart type messages."""
 
     def attach(self, payload):
         # The public API prohibits attaching multiple subparts to MIMEBase
diff --git a/lib-python/2.7/email/test/test_email.py b/lib-python/2.7/email/test/test_email.py
--- a/lib-python/2.7/email/test/test_email.py
+++ b/lib-python/2.7/email/test/test_email.py
@@ -11,6 +11,7 @@
 import warnings
 import textwrap
 from cStringIO import StringIO
+from random import choice
 
 import email
 
@@ -2578,16 +2579,64 @@
             bsf.push(il)
             nt += n
             n1 = 0
-            while True:
-                ol = bsf.readline()
-                if ol == NeedMoreData:
-                    break
+            for ol in iter(bsf.readline, NeedMoreData):
                 om.append(ol)
                 n1 += 1
             self.assertEqual(n, n1)
         self.assertEqual(len(om), nt)
         self.assertEqual(''.join([il for il, n in imt]), ''.join(om))
 
+    def test_push_random(self):
+        from email.feedparser import BufferedSubFile, NeedMoreData
+
+        n = 10000
+        chunksize = 5
+        chars = 'abcd \t\r\n'
+
+        s = ''.join(choice(chars) for i in range(n)) + '\n'
+        target = s.splitlines(True)
+
+        bsf = BufferedSubFile()
+        lines = []
+        for i in range(0, len(s), chunksize):
+            chunk = s[i:i+chunksize]
+            bsf.push(chunk)
+            lines.extend(iter(bsf.readline, NeedMoreData))
+        self.assertEqual(lines, target)
+
+
+class TestFeedParsers(TestEmailBase):
+
+    def parse(self, chunks):
+        from email.feedparser import FeedParser
+        feedparser = FeedParser()
+        for chunk in chunks:
+            feedparser.feed(chunk)
+        return feedparser.close()
+
+    def test_newlines(self):
+        m = self.parse(['a:\nb:\rc:\r\nd:\n'])
+        self.assertEqual(m.keys(), ['a', 'b', 'c', 'd'])
+        m = self.parse(['a:\nb:\rc:\r\nd:'])
+        self.assertEqual(m.keys(), ['a', 'b', 'c', 'd'])
+        m = self.parse(['a:\rb', 'c:\n'])
+        self.assertEqual(m.keys(), ['a', 'bc'])
+        m = self.parse(['a:\r', 'b:\n'])
+        self.assertEqual(m.keys(), ['a', 'b'])
+        m = self.parse(['a:\r', '\nb:\n'])
+        self.assertEqual(m.keys(), ['a', 'b'])
+
+    def test_long_lines(self):
+        # Expected peak memory use on 32-bit platform: 4*N*M bytes.
+        M, N = 1000, 20000
+        m = self.parse(['a:b\n\n'] + ['x'*M] * N)
+        self.assertEqual(m.items(), [('a', 'b')])
+        self.assertEqual(m.get_payload(), 'x'*M*N)
+        m = self.parse(['a:b\r\r'] + ['x'*M] * N)
+        self.assertEqual(m.items(), [('a', 'b')])
+        self.assertEqual(m.get_payload(), 'x'*M*N)
+        m = self.parse(['a:\r', 'b: '] + ['x'*M] * N)
+        self.assertEqual(m.items(), [('a', ''), ('b', 'x'*M*N)])
 
 
 class TestParsers(TestEmailBase):
@@ -3180,7 +3229,6 @@
         self.assertEqual(res, '=?iso-8859-2?q?abc?=')
         self.assertIsInstance(res, str)
 
-
 # Test RFC 2231 header parameters (en/de)coding
 class TestRFC2231(TestEmailBase):
     def test_get_param(self):
diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/ensurepip/__init__.py
@@ -0,0 +1,227 @@
+#!/usr/bin/env python2
+from __future__ import print_function
+
+import os
+import os.path
+import pkgutil
+import shutil
+import sys
+import tempfile
+
+
+__all__ = ["version", "bootstrap"]
+
+
+_SETUPTOOLS_VERSION = "7.0"
+
+_PIP_VERSION = "1.5.6"
+
+# pip currently requires ssl support, so we try to provide a nicer
+# error message when that is missing (http://bugs.python.org/issue19744)
+_MISSING_SSL_MESSAGE = ("pip {} requires SSL/TLS".format(_PIP_VERSION))
+try:
+    import ssl
+except ImportError:
+    ssl = None
+
+    def _require_ssl_for_pip():
+        raise RuntimeError(_MISSING_SSL_MESSAGE)
+else:
+    def _require_ssl_for_pip():
+        pass
+
+_PROJECTS = [
+    ("setuptools", _SETUPTOOLS_VERSION),
+    ("pip", _PIP_VERSION),
+]
+
+
+def _run_pip(args, additional_paths=None):
+    # Add our bundled software to the sys.path so we can import it
+    if additional_paths is not None:
+        sys.path = additional_paths + sys.path
+
+    # Install the bundled software
+    import pip
+    pip.main(args)
+
+
+def version():
+    """
+    Returns a string specifying the bundled version of pip.
+    """
+    return _PIP_VERSION
+
+
+def _disable_pip_configuration_settings():
+    # We deliberately ignore all pip environment variables
+    # when invoking pip
+    # See http://bugs.python.org/issue19734 for details
+    keys_to_remove = [k for k in os.environ if k.startswith("PIP_")]
+    for k in keys_to_remove:
+        del os.environ[k]
+    # We also ignore the settings in the default pip configuration file
+    # See http://bugs.python.org/issue20053 for details
+    os.environ['PIP_CONFIG_FILE'] = os.devnull
+
+
+def bootstrap(root=None, upgrade=False, user=False,
+              altinstall=False, default_pip=True,
+              verbosity=0):
+    """
+    Bootstrap pip into the current Python installation (or the given root
+    directory).
+
+    Note that calling this function will alter both sys.path and os.environ.
+    """
+    if altinstall and default_pip:
+        raise ValueError("Cannot use altinstall and default_pip together")
+
+    _require_ssl_for_pip()
+    _disable_pip_configuration_settings()
+
+    # By default, installing pip and setuptools installs all of the
+    # following scripts (X.Y == running Python version):
+    #
+    #   pip, pipX, pipX.Y, easy_install, easy_install-X.Y
+    #
+    # pip 1.5+ allows ensurepip to request that some of those be left out
+    if altinstall:
+        # omit pip, pipX and easy_install
+        os.environ["ENSUREPIP_OPTIONS"] = "altinstall"
+    elif not default_pip:
+        # omit pip and easy_install
+        os.environ["ENSUREPIP_OPTIONS"] = "install"
+
+    tmpdir = tempfile.mkdtemp()
+    try:
+        # Put our bundled wheels into a temporary directory and construct the
+        # additional paths that need added to sys.path
+        additional_paths = []
+        for project, version in _PROJECTS:
+            wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version)
+            whl = pkgutil.get_data(
+                "ensurepip",
+                "_bundled/{}".format(wheel_name),
+            )
+            with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
+                fp.write(whl)
+
+            additional_paths.append(os.path.join(tmpdir, wheel_name))
+
+        # Construct the arguments to be passed to the pip command
+        args = ["install", "--no-index", "--find-links", tmpdir]
+        if root:
+            args += ["--root", root]
+        if upgrade:
+            args += ["--upgrade"]
+        if user:
+            args += ["--user"]
+        if verbosity:
+            args += ["-" + "v" * verbosity]
+
+        _run_pip(args + [p[0] for p in _PROJECTS], additional_paths)
+    finally:
+        shutil.rmtree(tmpdir, ignore_errors=True)
+
+
+def _uninstall_helper(verbosity=0):
+    """Helper to support a clean default uninstall process on Windows
+
+    Note that calling this function may alter os.environ.
+    """
+    # Nothing to do if pip was never installed, or has been removed
+    try:
+        import pip
+    except ImportError:
+        return
+
+    # If the pip version doesn't match the bundled one, leave it alone
+    if pip.__version__ != _PIP_VERSION:
+        msg = ("ensurepip will only uninstall a matching version "
+               "({!r} installed, {!r} bundled)")
+        print(msg.format(pip.__version__, _PIP_VERSION), file=sys.stderr)
+        return
+
+    _require_ssl_for_pip()
+    _disable_pip_configuration_settings()
+
+    # Construct the arguments to be passed to the pip command
+    args = ["uninstall", "-y"]
+    if verbosity:
+        args += ["-" + "v" * verbosity]
+
+    _run_pip(args + [p[0] for p in reversed(_PROJECTS)])
+
+
+def _main(argv=None):
+    if ssl is None:
+        print("Ignoring ensurepip failure: {}".format(_MISSING_SSL_MESSAGE),
+              file=sys.stderr)
+        return
+
+    import argparse
+    parser = argparse.ArgumentParser(prog="python -m ensurepip")
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="pip {}".format(version()),
+        help="Show the version of pip that is bundled with this Python.",
+    )
+    parser.add_argument(
+        "-v", "--verbose",
+        action="count",
+        default=0,
+        dest="verbosity",
+        help=("Give more output. Option is additive, and can be used up to 3 "
+              "times."),
+    )
+    parser.add_argument(
+        "-U", "--upgrade",
+        action="store_true",
+        default=False,
+        help="Upgrade pip and dependencies, even if already installed.",
+    )
+    parser.add_argument(
+        "--user",
+        action="store_true",
+        default=False,
+        help="Install using the user scheme.",
+    )
+    parser.add_argument(
+        "--root",
+        default=None,
+        help="Install everything relative to this alternate root directory.",
+    )
+    parser.add_argument(
+        "--altinstall",
+        action="store_true",
+        default=False,
+        help=("Make an alternate install, installing only the X.Y versioned"
+              "scripts (Default: pipX, pipX.Y, easy_install-X.Y)"),
+    )
+    parser.add_argument(
+        "--default-pip",
+        action="store_true",
+        default=True,
+        dest="default_pip",
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--no-default-pip",
+        action="store_false",
+        dest="default_pip",
+        help=("Make a non default install, installing only the X and X.Y "
+              "versioned scripts."),
+    )
+
+    args = parser.parse_args(argv)
+
+    bootstrap(
+        root=args.root,
+        upgrade=args.upgrade,
+        user=args.user,
+        verbosity=args.verbosity,
+        altinstall=args.altinstall,
+        default_pip=args.default_pip,
+    )
diff --git a/lib-python/2.7/ensurepip/__main__.py b/lib-python/2.7/ensurepip/__main__.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/ensurepip/__main__.py
@@ -0,0 +1,4 @@
+import ensurepip
+
+if __name__ == "__main__":
+    ensurepip._main()
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..097ab43430d4c1302b0be353a8c16407c370693b
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..fa1d1054da1dab98f8906555d31a9fda271b3a85
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_uninstall.py b/lib-python/2.7/ensurepip/_uninstall.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/ensurepip/_uninstall.py
@@ -0,0 +1,30 @@
+"""Basic pip uninstallation support, helper for the Windows uninstaller"""
+
+import argparse
+import ensurepip
+
+
+def _main(argv=None):
+    parser = argparse.ArgumentParser(prog="python -m ensurepip._uninstall")
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="pip {}".format(ensurepip.version()),
+        help="Show the version of pip this will attempt to uninstall.",
+    )
+    parser.add_argument(
+        "-v", "--verbose",
+        action="count",
+        default=0,
+        dest="verbosity",
+        help=("Give more output. Option is additive, and can be used up to 3 "
+              "times."),
+    )
+
+    args = parser.parse_args(argv)
+
+    ensurepip._uninstall_helper(verbosity=args.verbosity)
+
+
+if __name__ == "__main__":
+    _main()
diff --git a/lib-python/2.7/glob.py b/lib-python/2.7/glob.py
--- a/lib-python/2.7/glob.py
+++ b/lib-python/2.7/glob.py
@@ -35,11 +35,16 @@
     patterns.
 
     """
+    dirname, basename = os.path.split(pathname)
     if not has_magic(pathname):
-        if os.path.lexists(pathname):
-            yield pathname
+        if basename:
+            if os.path.lexists(pathname):
+                yield pathname
+        else:
+            # Patterns ending with a slash should match only directories
+            if os.path.isdir(dirname):
+                yield pathname
         return
-    dirname, basename = os.path.split(pathname)
     if not dirname:
         for name in glob1(os.curdir, basename):
             yield name
diff --git a/lib-python/2.7/gzip.py b/lib-python/2.7/gzip.py
--- a/lib-python/2.7/gzip.py
+++ b/lib-python/2.7/gzip.py
@@ -164,9 +164,16 @@
     def _write_gzip_header(self):
         self.fileobj.write('\037\213')             # magic header
         self.fileobj.write('\010')                 # compression method
-        fname = os.path.basename(self.name)
-        if fname.endswith(".gz"):
-            fname = fname[:-3]
+        try:
+            # RFC 1952 requires the FNAME field to be Latin-1. Do not
+            # include filenames that cannot be represented that way.
+            fname = os.path.basename(self.name)
+            if not isinstance(fname, str):
+                fname = fname.encode('latin-1')
+            if fname.endswith('.gz'):
+                fname = fname[:-3]
+        except UnicodeEncodeError:
+            fname = ''
         flags = 0
         if fname:
             flags = FNAME
diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py
--- a/lib-python/2.7/hashlib.py
+++ b/lib-python/2.7/hashlib.py
@@ -15,8 +15,9 @@
 
 md5(), sha1(), sha224(), sha256(), sha384(), and sha512()
 
-More algorithms may be available on your platform but the above are
-guaranteed to exist.
+More algorithms may be available on your platform but the above are guaranteed
+to exist.  See the algorithms_guaranteed and algorithms_available attributes
+to find out what algorithm names can be passed to new().
 
 NOTE: If you want the adler32 or crc32 hash functions they are available in
 the zlib module.
@@ -58,9 +59,14 @@
 # always available algorithm is added.
 __always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')
 
+algorithms_guaranteed = set(__always_supported)
+algorithms_available = set(__always_supported)
+
 algorithms = __always_supported
 
-__all__ = __always_supported + ('new', 'algorithms', 'pbkdf2_hmac')
+__all__ = __always_supported + ('new', 'algorithms_guaranteed',
+                                'algorithms_available', 'algorithms',
+                                'pbkdf2_hmac')
 
 
 def __get_builtin_constructor(name):
@@ -128,6 +134,8 @@
     import _hashlib
     new = __hash_new
     __get_hash = __get_openssl_constructor
+    algorithms_available = algorithms_available.union(
+        _hashlib.openssl_md_meth_names)
 except ImportError:
     new = __py_new
     __get_hash = __get_builtin_constructor
diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py
--- a/lib-python/2.7/httplib.py
+++ b/lib-python/2.7/httplib.py
@@ -215,6 +215,10 @@
 # maximal line length when calling readline().
 _MAXLINE = 65536
 
+# maximum amount of headers accepted
+_MAXHEADERS = 100
+
+
 class HTTPMessage(mimetools.Message):
 
     def addheader(self, key, value):
@@ -271,6 +275,8 @@
         elif self.seekable:
             tell = self.fp.tell
         while True:
+            if len(hlist) > _MAXHEADERS:
+                raise HTTPException("got more than %d headers" % _MAXHEADERS)
             if tell:
                 try:
                     startofline = tell()
@@ -1185,21 +1191,29 @@
 
         def __init__(self, host, port=None, key_file=None, cert_file=None,
                      strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
-                     source_address=None):
+                     source_address=None, context=None):
             HTTPConnection.__init__(self, host, port, strict, timeout,
                                     source_address)
             self.key_file = key_file
             self.cert_file = cert_file
+            if context is None:
+                context = ssl._create_default_https_context()
+            if key_file or cert_file:
+                context.load_cert_chain(cert_file, key_file)
+            self._context = context
 
         def connect(self):
             "Connect to a host on a given (SSL) port."
 
-            sock = self._create_connection((self.host, self.port),
-                                          self.timeout, self.source_address)
+            HTTPConnection.connect(self)
+
             if self._tunnel_host:
-                self.sock = sock
-                self._tunnel()
-            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+                server_hostname = self._tunnel_host
+            else:
+                server_hostname = self.host
+
+            self.sock = self._context.wrap_socket(self.sock,
+                                                  server_hostname=server_hostname)
 
     __all__.append("HTTPSConnection")
 
@@ -1214,14 +1228,15 @@
         _connection_class = HTTPSConnection
 
         def __init__(self, host='', port=None, key_file=None, cert_file=None,
-                     strict=None):
+                     strict=None, context=None):
             # provide a default host, pass the X509 cert info
 
             # urf. compensate for bad input.
             if port == 0:
                 port = None
             self._setup(self._connection_class(host, port, key_file,
-                                               cert_file, strict))
+                                               cert_file, strict,
+                                               context=context))
 
             # we never actually use these for anything, but we keep them
             # here for compatibility with post-1.5.2 CVS.
diff --git a/lib-python/2.7/idlelib/Bindings.py b/lib-python/2.7/idlelib/Bindings.py
--- a/lib-python/2.7/idlelib/Bindings.py
+++ b/lib-python/2.7/idlelib/Bindings.py
@@ -75,7 +75,8 @@
    ('!_Auto-open Stack Viewer', '<>'),
    ]),
  ('options', [
-   ('_Configure IDLE...', '<>'),
+   ('Configure _IDLE', '<>'),
+   ('Configure _Extensions', '<>'),
    None,
    ]),
  ('help', [
diff --git a/lib-python/2.7/idlelib/CallTipWindow.py b/lib-python/2.7/idlelib/CallTipWindow.py
--- a/lib-python/2.7/idlelib/CallTipWindow.py
+++ b/lib-python/2.7/idlelib/CallTipWindow.py
@@ -2,9 +2,8 @@
 
 After ToolTip.py, which uses ideas gleaned from PySol
 Used by the CallTips IDLE extension.
-
 """
-from Tkinter import *
+from Tkinter import Toplevel, Label, LEFT, SOLID, TclError
 
 HIDE_VIRTUAL_EVENT_NAME = "<>"
 HIDE_SEQUENCES = ("", "")
@@ -133,35 +132,28 @@
         return bool(self.tipwindow)
 
 
-def _calltip_window(parent):
-    root = Tk()
-    root.title("Test calltips")
-    width, height, x, y = list(map(int, re.split('[x+]', parent.geometry())))
-    root.geometry("+%d+%d"%(x, y + 150))
+def _calltip_window(parent):  # htest #
+    from Tkinter import Toplevel, Text, LEFT, BOTH
 
-    class MyEditWin: # comparenceptually an editor_window
-        def __init__(self):
-            text = self.text = Text(root)
-            text.pack(side=LEFT, fill=BOTH, expand=1)
-            text.insert("insert", "string.split")
-            root.update()
-            self.calltip = CallTip(text)
+    top = Toplevel(parent)
+    top.title("Test calltips")
+    top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200,
+                  parent.winfo_rooty() + 150))
+    text = Text(top)
+    text.pack(side=LEFT, fill=BOTH, expand=1)
+    text.insert("insert", "string.split")
+    top.update()
+    calltip = CallTip(text)
 
-            text.event_add("<>", "(")
-            text.event_add("<>", ")")
-            text.bind("<>", self.calltip_show)
-            text.bind("<>", self.calltip_hide)
-
-            text.focus_set()
-            root.mainloop()
-
-        def calltip_show(self, event):
-            self.calltip.showtip("Hello world", "insert", "end")
-
-        def calltip_hide(self, event):
-            self.calltip.hidetip()
-
-    editwin = MyEditWin()
+    def calltip_show(event):
+        calltip.showtip("(s=Hello world)", "insert", "end")
+    def calltip_hide(event):
+        calltip.hidetip()
+    text.event_add("<>", "(")
+    text.event_add("<>", ")")
+    text.bind("<>", calltip_show)
+    text.bind("<>", calltip_hide)
+    text.focus_set()
 
 if __name__=='__main__':
     from idlelib.idle_test.htest import run
diff --git a/lib-python/2.7/idlelib/ClassBrowser.py b/lib-python/2.7/idlelib/ClassBrowser.py
--- a/lib-python/2.7/idlelib/ClassBrowser.py
+++ b/lib-python/2.7/idlelib/ClassBrowser.py
@@ -19,6 +19,9 @@
 from idlelib.TreeWidget import TreeNode, TreeItem, ScrolledCanvas
 from idlelib.configHandler import idleConf
 
+file_open = None  # Method...Item and Class...Item use this.
+# Normally PyShell.flist.open, but there is no PyShell.flist for htest.
+
 class ClassBrowser:
 
     def __init__(self, flist, name, path, _htest=False):
@@ -27,6 +30,9 @@
         """
         _htest - bool, change box when location running htest.
         """
+        global file_open
+        if not _htest:
+            file_open = PyShell.flist.open
         self.name = name
         self.file = os.path.join(path[0], self.name + ".py")
         self._htest = _htest
@@ -101,7 +107,7 @@
             return []
         try:
             dict = pyclbr.readmodule_ex(name, [dir] + sys.path)
-        except ImportError, msg:
+        except ImportError:
             return []
         items = []
         self.classes = {}
@@ -170,7 +176,7 @@
     def OnDoubleClick(self):
         if not os.path.exists(self.file):
             return
-        edit = PyShell.flist.open(self.file)
+        edit = file_open(self.file)
         if hasattr(self.cl, 'lineno'):
             lineno = self.cl.lineno
             edit.gotoline(lineno)
@@ -206,7 +212,7 @@
     def OnDoubleClick(self):
         if not os.path.exists(self.file):
             return
-        edit = PyShell.flist.open(self.file)
+        edit = file_open(self.file)
         edit.gotoline(self.cl.methods[self.name])
 
 def _class_browser(parent): #Wrapper for htest
@@ -221,8 +227,9 @@
     dir, file = os.path.split(file)
     name = os.path.splitext(file)[0]
     flist = PyShell.PyShellFileList(parent)
+    global file_open
+    file_open = flist.open
     ClassBrowser(flist, name, [dir], _htest=True)
-    parent.mainloop()
 
 if __name__ == "__main__":
     from idlelib.idle_test.htest import run
diff --git a/lib-python/2.7/idlelib/ColorDelegator.py b/lib-python/2.7/idlelib/ColorDelegator.py
--- a/lib-python/2.7/idlelib/ColorDelegator.py
+++ b/lib-python/2.7/idlelib/ColorDelegator.py
@@ -2,7 +2,6 @@
 import re
 import keyword
 import __builtin__
-from Tkinter import *
 from idlelib.Delegator import Delegator
 from idlelib.configHandler import idleConf
 
@@ -34,7 +33,6 @@
 
 prog = re.compile(make_pat(), re.S)
 idprog = re.compile(r"\s+(\w+)", re.S)
-asprog = re.compile(r".*?\b(as)\b")
 
 class ColorDelegator(Delegator):
 
@@ -42,7 +40,6 @@
         Delegator.__init__(self)
         self.prog = prog
         self.idprog = idprog
-        self.asprog = asprog
         self.LoadTagDefs()
 
     def setdelegate(self, delegate):
@@ -74,7 +71,6 @@
             "DEFINITION": idleConf.GetHighlight(theme, "definition"),
             "SYNC": {'background':None,'foreground':None},
             "TODO": {'background':None,'foreground':None},
-            "BREAK": idleConf.GetHighlight(theme, "break"),
             "ERROR": idleConf.GetHighlight(theme, "error"),
             # The following is used by ReplaceDialog:
             "hit": idleConf.GetHighlight(theme, "hit"),
@@ -216,22 +212,6 @@
                                     self.tag_add("DEFINITION",
                                                  head + "+%dc" % a,
                                                  head + "+%dc" % b)
-                            elif value == "import":
-                                # color all the "as" words on same line, except
-                                # if in a comment; cheap approximation to the
-                                # truth
-                                if '#' in chars:
-                                    endpos = chars.index('#')
-                                else:
-                                    endpos = len(chars)
-                                while True:
-                                    m1 = self.asprog.match(chars, b, endpos)
-                                    if not m1:
-                                        break
-                                    a, b = m1.span(1)
-                                    self.tag_add("KEYWORD",
-                                                 head + "+%dc" % a,
-                                                 head + "+%dc" % b)
                     m = self.prog.search(chars, m.end())
                 if "SYNC" in self.tag_names(next + "-1c"):
                     head = next
@@ -255,20 +235,23 @@
         for tag in self.tagdefs.keys():
             self.tag_remove(tag, "1.0", "end")
 
-def _color_delegator(parent):
+def _color_delegator(parent):  # htest #
+    from Tkinter import Toplevel, Text
     from idlelib.Percolator import Percolator
-    root = Tk()
-    root.title("Test ColorDelegator")
-    width, height, x, y = list(map(int, re.split('[x+]', parent.geometry())))
-    root.geometry("+%d+%d"%(x, y + 150))
-    source = "if somename: x = 'abc' # comment\nprint"
-    text = Text(root, background="white")
+
+    top = Toplevel(parent)
+    top.title("Test ColorDelegator")
+    top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200,
+                  parent.winfo_rooty() + 150))
+    source = "if somename: x = 'abc' # comment\nprint\n"
+    text = Text(top, background="white")
+    text.pack(expand=1, fill="both")
     text.insert("insert", source)

From noreply at buildbot.pypy.org  Tue Feb 24 12:46:12 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Tue, 24 Feb 2015 12:46:12 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Fix lib-python test: getrefcount is not
	available on PyPy.
Message-ID: <20150224114612.EAEB01C0107@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76095:14e2febbcc2c
Date: 2015-02-24 12:45 +0100
http://bitbucket.org/pypy/pypy/changeset/14e2febbcc2c/

Log:	Fix lib-python test: getrefcount is not available on PyPy.

diff --git a/lib-python/3/ctypes/test/test_python_api.py b/lib-python/3/ctypes/test/test_python_api.py
--- a/lib-python/3/ctypes/test/test_python_api.py
+++ b/lib-python/3/ctypes/test/test_python_api.py
@@ -10,7 +10,8 @@
 
 ################################################################
 
-from sys import getrefcount as grc
+if is_resource_enabled("refcount"):
+    from sys import getrefcount as grc
 if sys.version_info > (2, 4):
     c_py_ssize_t = c_size_t
 else:

From noreply at buildbot.pypy.org  Tue Feb 24 13:04:41 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Tue, 24 Feb 2015 13:04:41 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Add a test case for 0ae2fcdca348.
Message-ID: <20150224120441.21A261C04A7@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76096:575325d6c64c
Date: 2015-02-24 13:04 +0100
http://bitbucket.org/pypy/pypy/changeset/575325d6c64c/

Log:	Add a test case for 0ae2fcdca348.

diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -695,6 +695,10 @@
         assert 'hello'.encode() == b'hello'
         assert type('hello'.encode()) is bytes
 
+    def test_non_text_encoding(self):
+        raises(LookupError, b'hello'.decode, 'base64')
+        raises(LookupError, 'hello'.encode, 'base64')
+
     def test_hash(self):
         # check that we have the same hash as CPython for at least 31 bits
         # (but don't go checking CPython's special case -1)

From noreply at buildbot.pypy.org  Tue Feb 24 13:41:58 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 24 Feb 2015 13:41:58 +0100 (CET)
Subject: [pypy-commit] stmgc gc-small-uniform: merge default
Message-ID: <20150224124158.EBA1C1C014F@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: gc-small-uniform
Changeset: r1648:3dc8c734e257
Date: 2015-02-24 12:47 +0100
http://bitbucket.org/pypy/stmgc/changeset/3dc8c734e257/

Log:	merge default

diff too long, truncating to 2000 out of 22436 lines

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -7,3 +7,5 @@
 *.orig
 */__pycache__
 *.out.*
+*/\#*\#
+*/.\#*
diff --git a/c7/demo/demo2.c b/c7/demo/demo2.c
--- a/c7/demo/demo2.c
+++ b/c7/demo/demo2.c
@@ -3,6 +3,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #ifdef USE_HTM
 #  include "../../htm-c7/stmgc.h"
@@ -59,12 +60,25 @@
 }
 void stmcb_commit_soon() {}
 
-static void expand_marker(char *base, uintptr_t odd_number,
-                          object_t *following_object,
-                          char *outputbuf, size_t outputbufsize)
+static void timing_event(stm_thread_local_t *tl, /* the local thread */
+                         enum stm_event_e event,
+                         stm_loc_marker_t *markers)
 {
-    assert(following_object == NULL);
-    snprintf(outputbuf, outputbufsize, "<%p %lu>", base, odd_number);
+    static char *event_names[] = { STM_EVENT_NAMES };
+
+    char buf[1024], *p;
+    struct timespec tp;
+    clock_gettime(CLOCK_MONOTONIC, &tp);
+
+    p = buf;
+    p += sprintf(p, "{%.9f} %p %s", tp.tv_sec + 0.000000001 * tp.tv_nsec,
+                 tl, event_names[event]);
+    if (markers != NULL) {
+        p += sprintf(p, ", markers: %lu, %lu",
+                     markers[0].odd_number, markers[1].odd_number);
+    }
+    sprintf(p, "\n");
+    fputs(buf, stderr);
 }
 
 
@@ -108,18 +122,6 @@
 
     stm_start_transaction(&stm_thread_local);
 
-    if (stm_thread_local.longest_marker_state != 0) {
-        fprintf(stderr, "[%p] marker %d for %.6f seconds:\n",
-                &stm_thread_local,
-                stm_thread_local.longest_marker_state,
-                stm_thread_local.longest_marker_time);
-        fprintf(stderr, "\tself:\t\"%s\"\n\tother:\t\"%s\"\n",
-                stm_thread_local.longest_marker_self,
-                stm_thread_local.longest_marker_other);
-        stm_thread_local.longest_marker_state = 0;
-        stm_thread_local.longest_marker_time = 0.0;
-    }
-
     nodeptr_t prev = initial;
     stm_read((objptr_t)prev);
 
@@ -223,7 +225,6 @@
 
 void unregister_thread_local(void)
 {
-    stm_flush_timing(&stm_thread_local, 1);
     stm_unregister_thread_local(&stm_thread_local);
 }
 
@@ -295,9 +296,15 @@
 
     stm_setup();
     stm_register_thread_local(&stm_thread_local);
+
+    /* check that we can use stm_start_inevitable_transaction() without
+       any rjbuf on the stack */
+    stm_start_inevitable_transaction(&stm_thread_local);
+    stm_commit_transaction();
+
+
     stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf);
-    stmcb_expand_marker = expand_marker;
-
+    stmcb_timing_event = timing_event;
 
     setup_list();
 
diff --git a/c7/demo/demo_hashtable1.c b/c7/demo/demo_hashtable1.c
new file mode 100644
--- /dev/null
+++ b/c7/demo/demo_hashtable1.c
@@ -0,0 +1,217 @@
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "stmgc.h"
+
+#define NUMTHREADS  4
+
+
+typedef TLPREFIX struct node_s node_t;
+typedef TLPREFIX struct dict_s dict_t;
+
+
+struct node_s {
+    struct object_s header;
+    int typeid;
+    intptr_t freevalue;
+};
+
+struct dict_s {
+    struct node_s hdr;
+    stm_hashtable_t *hashtable;
+};
+
+#define TID_NODE       0x01234567
+#define TID_DICT       0x56789ABC
+#define TID_DICTENTRY  0x6789ABCD
+
+
+static sem_t done;
+__thread stm_thread_local_t stm_thread_local;
+
+// global and per-thread-data
+time_t default_seed;
+dict_t *global_dict;
+
+struct thread_data {
+    unsigned int thread_seed;
+};
+__thread struct thread_data td;
+
+
+ssize_t stmcb_size_rounded_up(struct object_s *ob)
+{
+    if (((struct node_s*)ob)->typeid == TID_NODE)
+        return sizeof(struct node_s);
+    if (((struct node_s*)ob)->typeid == TID_DICT)
+        return sizeof(struct dict_s);
+    if (((struct node_s*)ob)->typeid == TID_DICTENTRY)
+        return sizeof(struct stm_hashtable_entry_s);
+    abort();
+}
+
+void stmcb_trace(struct object_s *obj, void visit(object_t **))
+{
+    struct node_s *n;
+    n = (struct node_s*)obj;
+    if (n->typeid == TID_NODE) {
+        return;
+    }
+    if (n->typeid == TID_DICT) {
+        stm_hashtable_tracefn(((struct dict_s *)n)->hashtable, visit);
+        return;
+    }
+    if (n->typeid == TID_DICTENTRY) {
+        object_t **ref = &((struct stm_hashtable_entry_s *)obj)->object;
+        visit(ref);
+        return;
+    }
+    abort();
+}
+
+void stmcb_commit_soon() {}
+long stmcb_obj_supports_cards(struct object_s *obj)
+{
+    return 0;
+}
+void stmcb_trace_cards(struct object_s *obj, void cb(object_t **),
+                       uintptr_t start, uintptr_t stop) {
+    abort();
+}
+void stmcb_get_card_base_itemsize(struct object_s *obj,
+                                  uintptr_t offset_itemsize[2]) {
+    abort();
+}
+
+int get_rand(int max)
+{
+    if (max == 0)
+        return 0;
+    return (int)(rand_r(&td.thread_seed) % (unsigned int)max);
+}
+
+
+void populate_hashtable(int keymin, int keymax)
+{
+    int i;
+    int diff = get_rand(keymax - keymin);
+    for (i = 0; i < keymax - keymin; i++) {
+        int key = keymin + i + diff;
+        if (key >= keymax)
+            key -= (keymax - keymin);
+        object_t *o = stm_allocate(sizeof(struct node_s));
+        ((node_t *)o)->typeid = TID_NODE;
+        ((node_t *)o)->freevalue = key;
+        assert(global_dict->hdr.freevalue == 42);
+        stm_hashtable_write((object_t *)global_dict, global_dict->hashtable,
+                            key, o, &stm_thread_local);
+    }
+}
+
+void setup_thread(void)
+{
+    memset(&td, 0, sizeof(struct thread_data));
+    td.thread_seed = default_seed++;
+}
+
+void *demo_random(void *arg)
+{
+    int threadnum = (uintptr_t)arg;
+    int status;
+    rewind_jmp_buf rjbuf;
+    stm_register_thread_local(&stm_thread_local);
+    stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf);
+
+    setup_thread();
+
+    volatile int start_count = 0;
+
+    stm_start_transaction(&stm_thread_local);
+    ++start_count;
+    assert(start_count == 1);  // all the writes that follow must not conflict
+    populate_hashtable(1291 * threadnum, 1291 * (threadnum + 1));
+    stm_commit_transaction();
+
+    stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
+    stm_unregister_thread_local(&stm_thread_local);
+
+    status = sem_post(&done); assert(status == 0);
+    return NULL;
+}
+
+void newthread(void*(*func)(void*), void *arg)
+{
+    pthread_t th;
+    int status = pthread_create(&th, NULL, func, arg);
+    if (status != 0)
+        abort();
+    pthread_detach(th);
+    printf("started new thread\n");
+}
+
+void setup_globals(void)
+{
+    stm_hashtable_t *my_hashtable = stm_hashtable_create();
+    struct dict_s new_templ = {
+        .hdr = {
+            .typeid = TID_DICT,
+            .freevalue = 42,
+        },
+        .hashtable = my_hashtable,
+    };
+
+    stm_start_inevitable_transaction(&stm_thread_local);
+    global_dict = (dict_t *)stm_setup_prebuilt(
+                      (object_t* )(uintptr_t)&new_templ);
+    assert(global_dict->hashtable);
+    stm_commit_transaction();
+}
+
+
+int main(void)
+{
+    int i, status;
+    rewind_jmp_buf rjbuf;
+
+    stm_hashtable_entry_userdata = TID_DICTENTRY;
+
+    /* pick a random seed from the time in seconds.
+       A bit pointless for now... because the interleaving of the
+       threads is really random. */
+    default_seed = time(NULL);
+    printf("running with seed=%lld\n", (long long)default_seed);
+
+    status = sem_init(&done, 0, 0);
+    assert(status == 0);
+
+
+    stm_setup();
+    stm_register_thread_local(&stm_thread_local);
+    stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf);
+
+    setup_globals();
+
+    for (i = 0; i < NUMTHREADS; i++) {
+        newthread(demo_random, (void *)(uintptr_t)i);
+    }
+
+    for (i=0; i < NUMTHREADS; i++) {
+        status = sem_wait(&done);
+        assert(status == 0);
+        printf("thread finished\n");
+    }
+
+    printf("Test OK!\n");
+
+    stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
+    stm_unregister_thread_local(&stm_thread_local);
+    stm_teardown();
+
+    return 0;
+}
diff --git a/c7/llvmfix/README.txt b/c7/llvmfix/README.txt
--- a/c7/llvmfix/README.txt
+++ b/c7/llvmfix/README.txt
@@ -1,3 +1,22 @@
+Apply these patches to llvm, svn revision 201645,
+which you get from:
+
+ svn co http://llvm.org/svn/llvm-project/llvm/trunk llvm -r 201645
+ cd llvm/tools
+ svn co http://llvm.org/svn/llvm-project/cfe/trunk clang -r 201645
+ cd ../..
+ cd llvm/projects
+ svn co http://llvm.org/svn/llvm-project/compiler-rt/trunk compiler-rt -r 201645
+ cd ../..
+ cd llvm
+ patch -p0 < ~/.../c7/llvmfix/...diff
+ # ^^^ repeat that line for all patches in this directory
+ cd ..
+ mkdir llvm-build
+ cd llvm-build
+ ../llvm/configure --enable-optimized      # requires gcc >= 4.7!
+ make
+
 
 no-introduce-bogus-cast-in-combine.diff
 
diff --git a/c7/stm/contention.c b/c7/stm/contention.c
--- a/c7/stm/contention.c
+++ b/c7/stm/contention.c
@@ -3,34 +3,50 @@
 #endif
 
 
-enum contention_kind_e {
+/* Here are the possible kinds of contention:
 
-    /* A write-write contention occurs when we running our transaction
-       and detect that we are about to write to an object that another
-       thread is also writing to.  This kind of contention must be
-       resolved before continuing.  This *must* abort one of the two
-       threads: the caller's thread is not at a safe-point, so cannot
-       wait! */
-    WRITE_WRITE_CONTENTION,
+   STM_CONTENTION_WRITE_WRITE
 
-    /* A write-read contention occurs when we are trying to commit: it
+       A write-write contention occurs when we are running our
+       transaction and detect that we are about to write to an object
+       that another thread is also writing to.  This kind of
+       contention must be resolved before continuing.  This *must*
+       abort one of the two threads: the caller's thread is not at a
+       safe-point, so cannot wait!
+
+       It is reported as a timing event with the following two markers:
+       the current thread (i.e. where the second-in-time write occurs);
+       and the other thread (from its 'modified_old_objects_markers',
+       where the first-in-time write occurred).
+
+   STM_CONTENTION_WRITE_READ
+
+       A write-read contention occurs when we are trying to commit: it
        means that an object we wrote to was also read by another
        transaction.  Even though it would seem obvious that we should
        just abort the other thread and proceed in our commit, a more
        subtle answer would be in some cases to wait for the other thread
        to commit first.  It would commit having read the old value, and
-       then we can commit our change to it. */
-    WRITE_READ_CONTENTION,
+       then we can commit our change to it.
 
-    /* An inevitable contention occurs when we're trying to become
+       It is reported as a timing event with only one marker: the
+       older location of the write that was done by the current thread.
+
+    STM_CONTENTION_INEVITABLE
+
+       An inevitable contention occurs when we're trying to become
        inevitable but another thread already is.  We can never abort the
        other thread in this case, but we still have the choice to abort
-       ourselves or pause until the other thread commits. */
-    INEVITABLE_CONTENTION,
-};
+       ourselves or pause until the other thread commits.
+
+       It is reported with two markers, one for the current thread and
+       one for the other thread.  Each marker gives the location that
+       attempts to make the transaction inevitable.
+*/
+
 
 struct contmgr_s {
-    enum contention_kind_e kind;
+    enum stm_event_e kind;
     struct stm_priv_segment_info_s *other_pseg;
     bool abort_other;
     bool try_sleep;  // XXX add a way to timeout, but should handle repeated
@@ -99,7 +115,7 @@
 
 
 static bool contention_management(uint8_t other_segment_num,
-                                  enum contention_kind_e kind,
+                                  enum stm_event_e kind,
                                   object_t *obj)
 {
     assert(_has_mutex());
@@ -109,6 +125,9 @@
     if (must_abort())
         abort_with_mutex();
 
+    /* Report the contention */
+    timing_contention(kind, other_segment_num, obj);
+
     /* Who should abort here: this thread, or the other thread? */
     struct contmgr_s contmgr;
     contmgr.kind = kind;
@@ -138,20 +157,9 @@
         contmgr.abort_other = false;
     }
 
-
-    int wait_category =
-        kind == WRITE_READ_CONTENTION ? STM_TIME_WAIT_WRITE_READ :
-        kind == INEVITABLE_CONTENTION ? STM_TIME_WAIT_INEVITABLE :
-        STM_TIME_WAIT_OTHER;
-
-    int abort_category =
-        kind == WRITE_WRITE_CONTENTION ? STM_TIME_RUN_ABORTED_WRITE_WRITE :
-        kind == WRITE_READ_CONTENTION ? STM_TIME_RUN_ABORTED_WRITE_READ :
-        kind == INEVITABLE_CONTENTION ? STM_TIME_RUN_ABORTED_INEVITABLE :
-        STM_TIME_RUN_ABORTED_OTHER;
-
-
-    if (contmgr.try_sleep && kind != WRITE_WRITE_CONTENTION &&
+    /* Do one of three things here...
+     */
+    if (contmgr.try_sleep && kind != STM_CONTENTION_WRITE_WRITE &&
         contmgr.other_pseg->safe_point != SP_WAIT_FOR_C_TRANSACTION_DONE) {
         others_may_have_run = true;
         /* Sleep.
@@ -164,30 +172,24 @@
              itself already paused here.
         */
         contmgr.other_pseg->signal_when_done = true;
-        marker_contention(kind, false, other_segment_num, obj);
-
-        change_timing_state(wait_category);
 
         /* tell the other to commit ASAP */
         signal_other_to_commit_soon(contmgr.other_pseg);
 
         dprintf(("pausing...\n"));
+
+        timing_event(STM_SEGMENT->running_thread, STM_WAIT_CONTENTION);
+
         cond_signal(C_AT_SAFE_POINT);
         STM_PSEGMENT->safe_point = SP_WAIT_FOR_C_TRANSACTION_DONE;
         cond_wait(C_TRANSACTION_DONE);
         STM_PSEGMENT->safe_point = SP_RUNNING;
         dprintf(("pausing done\n"));
 
+        timing_event(STM_SEGMENT->running_thread, STM_WAIT_DONE);
+
         if (must_abort())
             abort_with_mutex();
-
-        struct stm_priv_segment_info_s *pseg =
-            get_priv_segment(STM_SEGMENT->segment_num);
-        double elapsed =
-            change_timing_state_tl(pseg->pub.running_thread,
-                                   STM_TIME_RUN_CURRENT);
-        marker_copy(pseg->pub.running_thread, pseg,
-                    wait_category, elapsed);
     }
 
     else if (!contmgr.abort_other) {
@@ -195,16 +197,16 @@
         signal_other_to_commit_soon(contmgr.other_pseg);
 
         dprintf(("abort in contention: kind %d\n", kind));
-        STM_SEGMENT->nursery_end = abort_category;
-        marker_contention(kind, false, other_segment_num, obj);
         abort_with_mutex();
     }
 
     else {
         /* We have to signal the other thread to abort, and wait until
            it does. */
-        contmgr.other_pseg->pub.nursery_end = abort_category;
-        marker_contention(kind, true, other_segment_num, obj);
+        contmgr.other_pseg->pub.nursery_end = NSE_SIGABORT;
+
+        timing_event(STM_SEGMENT->running_thread,
+                     STM_ABORTING_OTHER_CONTENTION);
 
         int sp = contmgr.other_pseg->safe_point;
         switch (sp) {
@@ -296,7 +298,8 @@
         assert(get_priv_segment(other_segment_num)->write_lock_num ==
                prev_owner);
 
-        contention_management(other_segment_num, WRITE_WRITE_CONTENTION, obj);
+        contention_management(other_segment_num,
+                              STM_CONTENTION_WRITE_WRITE, obj);
 
         /* now we return into _stm_write_slowpath() and will try again
            to acquire the write lock on our object. */
@@ -308,10 +311,12 @@
 static bool write_read_contention_management(uint8_t other_segment_num,
                                              object_t *obj)
 {
-    return contention_management(other_segment_num, WRITE_READ_CONTENTION, obj);
+    return contention_management(other_segment_num,
+                                 STM_CONTENTION_WRITE_READ, obj);
 }
 
 static void inevitable_contention_management(uint8_t other_segment_num)
 {
-    contention_management(other_segment_num, INEVITABLE_CONTENTION, NULL);
+    contention_management(other_segment_num,
+                          STM_CONTENTION_INEVITABLE, NULL);
 }
diff --git a/c7/stm/core.c b/c7/stm/core.c
--- a/c7/stm/core.c
+++ b/c7/stm/core.c
@@ -124,17 +124,13 @@
 
         dprintf_test(("write_slowpath %p -> mod_old\n", obj));
 
-        /* First change to this old object from this transaction.
+        /* Add the current marker, recording where we wrote to this object */
+        timing_record_write();
+
+        /* Change to this old object from this transaction.
            Add it to the list 'modified_old_objects'. */
         LIST_APPEND(STM_PSEGMENT->modified_old_objects, obj);
 
-        /* Add the current marker, recording where we wrote to this object */
-        uintptr_t marker[2];
-        marker_fetch(STM_SEGMENT->running_thread, marker);
-        STM_PSEGMENT->modified_old_objects_markers =
-            list_append2(STM_PSEGMENT->modified_old_objects_markers,
-                         marker[0], marker[1]);
-
         release_marker_lock(STM_SEGMENT->segment_base);
 
         /* We need to privatize the pages containing the object, if they
@@ -313,44 +309,41 @@
     /* force-reset all read markers to 0 */
 
     char *readmarkers = REAL_ADDRESS(STM_SEGMENT->segment_base,
-                                     FIRST_READMARKER_PAGE * 4096UL);
+                                     FIRST_OLD_RM_PAGE * 4096UL);
+    uintptr_t num_bytes = 4096UL *
+        (NB_READMARKER_PAGES - (FIRST_OLD_RM_PAGE - FIRST_READMARKER_PAGE));
+
     dprintf(("reset_transaction_read_version: %p %ld\n", readmarkers,
-             (long)(NB_READMARKER_PAGES * 4096UL)));
-    if (mmap(readmarkers, NB_READMARKER_PAGES * 4096UL,
+             (long)num_bytes));
+
+    if (mmap(readmarkers, num_bytes,
              PROT_READ | PROT_WRITE,
-             MAP_FIXED | MAP_PAGES_FLAGS, -1, 0) != readmarkers) {
-        /* fall-back */
-#if STM_TESTS
+             MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE,
+             -1, 0) != readmarkers) {
+        /* failed */
         stm_fatalerror("reset_transaction_read_version: %m");
-#endif
-        memset(readmarkers, 0, NB_READMARKER_PAGES * 4096UL);
     }
     STM_SEGMENT->transaction_read_version = 1;
 }
 
-static void _stm_start_transaction(stm_thread_local_t *tl, bool inevitable)
+static uint64_t _global_start_time = 0;
+
+static void _stm_start_transaction(stm_thread_local_t *tl)
 {
     assert(!_stm_in_transaction(tl));
 
-  retry:
-    if (inevitable) {
-        wait_for_end_of_inevitable_transaction(tl);
-    }
-
-    if (!acquire_thread_segment(tl))
-        goto retry;
+    while (!acquire_thread_segment(tl))
+        ;
     /* GS invalid before this point! */
 
     assert(STM_PSEGMENT->safe_point == SP_NO_TRANSACTION);
     assert(STM_PSEGMENT->transaction_state == TS_NONE);
-    change_timing_state(STM_TIME_RUN_CURRENT);
-    STM_PSEGMENT->start_time = tl->_timing_cur_start;
+    timing_event(tl, STM_TRANSACTION_START);
+    STM_PSEGMENT->start_time = _global_start_time++;
     STM_PSEGMENT->signalled_to_commit_soon = false;
     STM_PSEGMENT->safe_point = SP_RUNNING;
-    STM_PSEGMENT->marker_inev[1] = 0;
-    if (inevitable)
-        marker_fetch_inev();
-    STM_PSEGMENT->transaction_state = (inevitable ? TS_INEVITABLE : TS_REGULAR);
+    STM_PSEGMENT->marker_inev.object = NULL;
+    STM_PSEGMENT->transaction_state = TS_REGULAR;
 #ifndef NDEBUG
     STM_PSEGMENT->running_pthread = pthread_self();
 #endif
@@ -376,13 +369,16 @@
 
     assert(list_is_empty(STM_PSEGMENT->modified_old_objects));
     assert(list_is_empty(STM_PSEGMENT->modified_old_objects_markers));
+    assert(list_is_empty(STM_PSEGMENT->modified_old_hashtables));
     assert(list_is_empty(STM_PSEGMENT->young_weakrefs));
     assert(tree_is_cleared(STM_PSEGMENT->young_outside_nursery));
     assert(tree_is_cleared(STM_PSEGMENT->nursery_objects_shadows));
     assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[0]));
     assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[1]));
+    assert(list_is_empty(STM_PSEGMENT->young_objects_with_light_finalizers));
     assert(STM_PSEGMENT->objects_pointing_to_nursery == NULL);
     assert(STM_PSEGMENT->large_overflow_objects == NULL);
+    assert(STM_PSEGMENT->finalizers == NULL);
 #ifndef NDEBUG
     /* this should not be used when objects_pointing_to_nursery == NULL */
     STM_PSEGMENT->modified_old_objects_markers_num_old = 99999999999999999L;
@@ -399,14 +395,21 @@
 #else
     long repeat_count = stm_rewind_jmp_setjmp(tl);
 #endif
-    _stm_start_transaction(tl, false);
+    _stm_start_transaction(tl);
     return repeat_count;
 }
 
 void stm_start_inevitable_transaction(stm_thread_local_t *tl)
 {
-    s_mutex_lock();
-    _stm_start_transaction(tl, true);
+    /* used to be more efficient, starting directly an inevitable transaction,
+       but there is no real point any more, I believe */
+    rewind_jmp_buf rjbuf;
+    stm_rewind_jmp_enterframe(tl, &rjbuf);
+
+    stm_start_transaction(tl);
+    stm_become_inevitable(tl, "start_inevitable_transaction");
+
+    stm_rewind_jmp_leaveframe(tl, &rjbuf);
 }
 
 
@@ -431,27 +434,50 @@
             continue;    /* no need to check: is pending immediate abort */
 
         char *remote_base = get_segment_base(i);
-        uint8_t remote_version = get_segment(i)->transaction_read_version;
+        object_t *conflicting_obj;
+        uintptr_t j, limit;
+        struct list_s *lst;
 
-        LIST_FOREACH_R(
-            STM_PSEGMENT->modified_old_objects,
-            object_t * /*item*/,
-            ({
-                if (was_read_remote(remote_base, item, remote_version)) {
-                    /* A write-read conflict! */
-                    dprintf(("write-read conflict on %p, our seg: %d, other: %ld\n",
-                             item, STM_SEGMENT->segment_num, i));
-                    if (write_read_contention_management(i, item)) {
-                        /* If we reach this point, we didn't abort, but we
-                           had to wait for the other thread to commit.  If we
-                           did, then we have to restart committing from our call
-                           to synchronize_all_threads(). */
-                        return true;
-                    }
-                    /* we aborted the other transaction without waiting, so
-                       we can just continue */
-                }
-            }));
+        /* Look in forward order: this is an attempt to report the _first_
+           write that conflicts with another segment's reads
+        */
+        lst = STM_PSEGMENT->modified_old_objects;
+        limit = list_count(lst);
+        for (j = 0; j < limit; j++) {
+            object_t *obj = (object_t *)list_item(lst, j);
+            if (was_read_remote(remote_base, obj)) {
+                conflicting_obj = obj;
+                goto found_conflict;
+            }
+        }
+
+        lst = STM_PSEGMENT->modified_old_hashtables;
+        limit = list_count(lst);
+        for (j = 0; j < limit; j += 2) {
+            object_t *hobj = (object_t *)list_item(lst, j);
+            if (was_read_remote(remote_base, hobj)) {
+                conflicting_obj = (object_t *)list_item(lst, j + 1);
+                goto found_conflict;
+            }
+        }
+
+        continue;
+
+     found_conflict:
+        /* A write-read conflict! */
+        dprintf(("write-read conflict on %p, our seg: %d, other: %ld\n",
+                 conflicting_obj, STM_SEGMENT->segment_num, i));
+        if (write_read_contention_management(i, conflicting_obj)) {
+            /* If we reach this point, we didn't abort, but we
+               had to wait for the other thread to commit.  If we
+               did, then we have to restart committing from our call
+               to synchronize_all_threads(). */
+            return true;
+        }
+        /* we aborted the other transaction without waiting, so we can
+           just ignore the rest of this (now aborted) segment.  Let's
+           move on to the next one. */
+        continue;
     }
 
     return false;
@@ -588,7 +614,6 @@
     uintptr_t first_card_index = get_write_lock_idx((uintptr_t)obj);
     uintptr_t card_index = 1;
     uintptr_t last_card_index = get_index_to_card_index(real_idx_count - 1); /* max valid index */
-    long myself = STM_SEGMENT->segment_num;
 
     /* simple heuristic to check if probably the whole object is
        marked anyway so we should do page-wise synchronize */
@@ -662,7 +687,7 @@
             /* dprintf(("copy %lu bytes\n", copy_size)); */
 
             /* since we have marked cards, at least one page here must be private */
-            assert(_has_private_page_in_range(myself, start, copy_size));
+            assert(_has_private_page_in_range(STM_SEGMENT->segment_num, start, copy_size));
 
             /* push to seg0 and enqueue for synchronization */
             _synchronize_fragment((stm_char *)start, copy_size);
@@ -806,15 +831,16 @@
     synchronize_objects_flush();
     list_clear(STM_PSEGMENT->modified_old_objects);
     list_clear(STM_PSEGMENT->modified_old_objects_markers);
+    list_clear(STM_PSEGMENT->modified_old_hashtables);
 }
 
-static void _finish_transaction(int attribute_to)
+static void _finish_transaction(enum stm_event_e event)
 {
     STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
     STM_PSEGMENT->transaction_state = TS_NONE;
 
     /* marker_inev is not needed anymore */
-    STM_PSEGMENT->marker_inev[1] = 0;
+    STM_PSEGMENT->marker_inev.object = NULL;
 
     /* reset these lists to NULL for the next transaction */
     _verify_cards_cleared_in_all_lists(get_priv_segment(STM_SEGMENT->segment_num));
@@ -822,24 +848,24 @@
     list_clear(STM_PSEGMENT->old_objects_with_cards);
     LIST_FREE(STM_PSEGMENT->large_overflow_objects);
 
-    timing_end_transaction(attribute_to);
+    stm_thread_local_t *tl = STM_SEGMENT->running_thread;
+    timing_event(tl, event);
 
-    stm_thread_local_t *tl = STM_SEGMENT->running_thread;
     release_thread_segment(tl);
     /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */
 }
 
 void stm_commit_transaction(void)
 {
+ restart_all:
+    exec_local_finalizers();
+
     assert(!_has_mutex());
     assert(STM_PSEGMENT->safe_point == SP_RUNNING);
     assert(STM_PSEGMENT->running_pthread == pthread_self());
 
     minor_collection(/*commit=*/ true);
 
-    /* the call to minor_collection() above leaves us with
-       STM_TIME_BOOKKEEPING */
-
     /* synchronize overflow objects living in privatized pages */
     push_overflow_objects_from_privatized_pages();
 
@@ -851,6 +877,11 @@
        Important: we should not call cond_wait() in the meantime. */
     synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK);
 
+    if (any_local_finalizers()) {
+        s_mutex_unlock();
+        goto restart_all;
+    }
+
     /* detect conflicts */
     if (detect_write_read_conflicts())
         goto restart;
@@ -863,15 +894,17 @@
 
     /* if a major collection is required, do it here */
     if (is_major_collection_requested()) {
-        int oldstate = change_timing_state(STM_TIME_MAJOR_GC);
+        timing_event(STM_SEGMENT->running_thread, STM_GC_MAJOR_START);
         major_collection_now_at_safe_point();
-        change_timing_state(oldstate);
+        timing_event(STM_SEGMENT->running_thread, STM_GC_MAJOR_DONE);
     }
 
     /* synchronize modified old objects to other threads */
     push_modified_to_other_segments();
     _verify_cards_cleared_in_all_lists(get_priv_segment(STM_SEGMENT->segment_num));
 
+    commit_finalizers();
+
     /* update 'overflow_number' if needed */
     if (STM_PSEGMENT->overflow_number_has_been_used) {
         highest_overflow_number += GCFLAG_OVERFLOW_NUMBER_bit0;
@@ -892,10 +925,13 @@
     }
 
     /* done */
-    _finish_transaction(STM_TIME_RUN_COMMITTED);
+    stm_thread_local_t *tl = STM_SEGMENT->running_thread;
+    _finish_transaction(STM_TRANSACTION_COMMIT);
     /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */
 
     s_mutex_unlock();
+
+    invoke_general_finalizers(tl);
 }
 
 void stm_abort_transaction(void)
@@ -957,6 +993,7 @@
 
     list_clear(pseg->modified_old_objects);
     list_clear(pseg->modified_old_objects_markers);
+    list_clear(pseg->modified_old_hashtables);
 }
 
 static void abort_data_structures_from_segment_num(int segment_num)
@@ -985,9 +1022,7 @@
                        (int)pseg->transaction_state);
     }
 
-    /* if we don't have marker information already, look up and preserve
-       the marker information from the shadowstack as a string */
-    marker_default_for_abort(pseg);
+    abort_finalizers(pseg);
 
     /* throw away the content of the nursery */
     long bytes_in_nursery = throw_away_nursery(pseg);
@@ -1077,16 +1112,13 @@
     /* invoke the callbacks */
     invoke_and_clear_user_callbacks(1);   /* for abort */
 
-    int attribute_to = STM_TIME_RUN_ABORTED_OTHER;
-
     if (is_abort(STM_SEGMENT->nursery_end)) {
         /* done aborting */
-        attribute_to = STM_SEGMENT->nursery_end;
         STM_SEGMENT->nursery_end = pause_signalled ? NSE_SIGPAUSE
                                                    : NURSERY_END;
     }
 
-    _finish_transaction(attribute_to);
+    _finish_transaction(STM_TRANSACTION_ABORT);
     /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */
 
     /* Broadcast C_ABORTED to wake up contention.c */
@@ -1128,8 +1160,10 @@
     if (STM_PSEGMENT->transaction_state == TS_REGULAR) {
         dprintf(("become_inevitable: %s\n", msg));
 
-        marker_fetch_inev();
-        wait_for_end_of_inevitable_transaction(NULL);
+        timing_fetch_inev();
+        write_fence();    /* make sure others see a correct 'marker_inev'
+                             if they see TS_INEVITABLE */
+        wait_for_end_of_inevitable_transaction();
         STM_PSEGMENT->transaction_state = TS_INEVITABLE;
         stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
         invoke_and_clear_user_callbacks(0);   /* for commit */
@@ -1150,3 +1184,23 @@
     synchronize_all_threads(STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE);
     s_mutex_unlock();
 }
+
+void stm_stop_all_other_threads(void)
+{
+    if (!stm_is_inevitable())         /* may still abort */
+        _stm_become_inevitable("stop_all_other_threads");
+
+    s_mutex_lock();
+    synchronize_all_threads(STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE);
+    s_mutex_unlock();
+}
+
+void stm_resume_all_other_threads(void)
+{
+    /* this calls 'committed_globally_unique_transaction()' even though
+       we're not committing now.  It's a way to piggyback on the existing
+       implementation for stm_become_globally_unique_transaction(). */
+    s_mutex_lock();
+    committed_globally_unique_transaction();
+    s_mutex_unlock();
+}
diff --git a/c7/stm/core.h b/c7/stm/core.h
--- a/c7/stm/core.h
+++ b/c7/stm/core.h
@@ -43,13 +43,18 @@
     */
     GCFLAG_WRITE_BARRIER = _STM_GCFLAG_WRITE_BARRIER,
 
+    /* This flag is set by gcpage.c for all objects living in
+       uniformly-sized pages of small objects.
+    */
+    GCFLAG_SMALL_UNIFORM = 0x02,
+
     /* The following flag is set on nursery objects of which we asked
        the id or the identityhash.  It means that a space of the size of
        the object has already been allocated in the nonmovable part.
        The same flag is abused to mark prebuilt objects whose hash has
        been taken during translation and is statically recorded just
        after the object. */
-    GCFLAG_HAS_SHADOW = 0x02,
+    GCFLAG_HAS_SHADOW = 0x04,
 
     /* Set on objects that are large enough (_STM_MIN_CARD_OBJ_SIZE)
        to have multiple cards (at least _STM_MIN_CARD_COUNT), and that
@@ -64,7 +69,7 @@
        current transaction that have been flushed out of the nursery,
        which occurs if the same transaction allocates too many objects.
     */
-    GCFLAG_OVERFLOW_NUMBER_bit0 = 0x8   /* must be last */
+    GCFLAG_OVERFLOW_NUMBER_bit0 = 0x10   /* must be last */
 };
 
 #define SYNC_QUEUE_SIZE    31
@@ -92,6 +97,15 @@
     struct list_s *modified_old_objects_markers;
     uintptr_t modified_old_objects_markers_num_old;
 
+    /* This list contains all old hashtables that have entries that we
+       modified.  It's actually a list of pairs hashtable/sample_entry.
+       Note that several transactions can all commit if
+       they have the same hashtable listed here.  The point of this
+       list is only that if another segment does a global "read" of
+       the hashtable (stm_hashtable_list), then it conflicts with this
+       segment if it has got any change to the hashtable. */
+    struct list_s *modified_old_hashtables;
+
     /* List of out-of-nursery objects that may contain pointers to
        nursery objects.  This is used to track the GC status: they are
        all objects outside the nursery on which an stm_write() occurred
@@ -135,7 +149,7 @@
 
     /* Start time: to know approximately for how long a transaction has
        been running, in contention management */
-    double start_time;
+    uint64_t start_time;
 
     /* This is the number stored in the overflowed objects (a multiple of
        GCFLAG_OVERFLOW_NUMBER_bit0).  It is incremented when the
@@ -202,10 +216,15 @@
     int sq_fragsizes[SYNC_QUEUE_SIZE];
     int sq_len;
 
-    /* Temporarily stores the marker information */
-    char marker_self[_STM_MARKER_LEN];
-    char marker_other[_STM_MARKER_LEN];
-    uintptr_t marker_inev[2];  /* marker where this thread became inevitable */
+    /* marker where this thread became inevitable */
+    stm_loc_marker_t marker_inev;
+
+    /* light finalizers */
+    struct list_s *young_objects_with_light_finalizers;
+    struct list_s *old_objects_with_light_finalizers;
+
+    /* regular finalizers (objs from the current transaction only) */
+    struct finalizers_s *finalizers;
 };
 
 enum /* safe_point */ {
@@ -285,9 +304,11 @@
 static stm_thread_local_t *abort_with_mutex_no_longjmp(void);
 static void abort_data_structures_from_segment_num(int segment_num);
 
-static inline bool was_read_remote(char *base, object_t *obj,
-                                   uint8_t other_transaction_read_version)
+static inline bool was_read_remote(char *base, object_t *obj)
 {
+    uint8_t other_transaction_read_version =
+        ((struct stm_segment_info_s *)REAL_ADDRESS(base, STM_PSEGMENT))
+            ->transaction_read_version;
     uint8_t rm = ((struct stm_read_marker_s *)
                   (base + (((uintptr_t)obj) >> 4)))->rm;
     assert(rm <= other_transaction_read_version);
diff --git a/c7/stm/extra.c b/c7/stm/extra.c
--- a/c7/stm/extra.c
+++ b/c7/stm/extra.c
@@ -6,15 +6,24 @@
 static long register_callbacks(stm_thread_local_t *tl,
                                void *key, void callback(void *), long index)
 {
-    if (!_stm_in_transaction(tl)) {
-        /* check that the current thread-local is really running a
+    dprintf(("register_callbacks: tl=%p key=%p callback=%p index=%ld\n",
+             tl, key, callback, index));
+    if (tl->associated_segment_num == -1) {
+        /* check that the provided thread-local is really running a
            transaction, and do nothing otherwise. */
+        dprintf(("  NOT IN TRANSACTION\n"));
         return -1;
     }
-
-    if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {
+    /* The tl was only here to check that.  We're really using
+       STM_PSEGMENT below, which is often but not always the
+       segment corresponding to the tl.  One case where it's not
+       the case is if this gets called from stmcb_light_finalizer()
+       from abort_finalizers() from major collections or contention.
+    */
+    if (STM_PSEGMENT->transaction_state != TS_REGULAR) {
         /* ignore callbacks if we're in an inevitable transaction
-           (which cannot abort) */
+           (which cannot abort) or no transaction at all in this segment */
+        dprintf(("  STATE = %d\n", (int)STM_PSEGMENT->transaction_state));
         return -1;
     }
 
@@ -23,10 +32,13 @@
 
     if (callback == NULL) {
         /* double-unregistering works, but return 0 */
-        return tree_delete_item(callbacks, (uintptr_t)key);
+        long res = tree_delete_item(callbacks, (uintptr_t)key);
+        dprintf(("  DELETED %ld\n", res));
+        return res;
     }
     else {
         /* double-registering the same key will crash */
+        dprintf(("  INSERTING\n"));
         tree_insert(callbacks, (uintptr_t)key, (uintptr_t)callback);
         return 1;
     }
@@ -39,6 +51,7 @@
     if (result < 0 && callback != NULL) {
         /* no regular transaction running, invoke the callback
            immediately */
+        dprintf(("stm_call_on_commit calls now: %p(%p)\n", callback, key));
         callback(key);
     }
     return result;
@@ -72,8 +85,11 @@
         assert(key != NULL);
         assert(callback != NULL);
 
-        /* The callback may call stm_call_on_abort(key, NULL).  It is ignored,
-           because 'callbacks_on_commit_and_abort' was cleared already. */
+        /* The callback may call stm_call_on_abort(key, NULL)
+           (so with callback==NULL).  It is ignored, because
+           'callbacks_on_commit_and_abort' was cleared already. */
+        dprintf(("invoke_and_clear_user_callbacks(%ld): %p(%p)\n",
+                 index, callback, key));
         callback(key);
 
     } TREE_LOOP_END;
diff --git a/c7/stm/finalizer.c b/c7/stm/finalizer.c
new file mode 100644
--- /dev/null
+++ b/c7/stm/finalizer.c
@@ -0,0 +1,473 @@
+
+
+/* callbacks */
+void (*stmcb_light_finalizer)(object_t *);
+void (*stmcb_finalizer)(object_t *);
+
+
+static void init_finalizers(struct finalizers_s *f)
+{
+    f->objects_with_finalizers = list_create();
+    f->count_non_young = 0;
+    f->run_finalizers = NULL;
+    f->running_next = NULL;
+}
+
+static void setup_finalizer(void)
+{
+    init_finalizers(&g_finalizers);
+}
+
+static void teardown_finalizer(void)
+{
+    if (g_finalizers.run_finalizers != NULL)
+        list_free(g_finalizers.run_finalizers);
+    list_free(g_finalizers.objects_with_finalizers);
+    memset(&g_finalizers, 0, sizeof(g_finalizers));
+}
+
+static void _commit_finalizers(void)
+{
+    if (STM_PSEGMENT->finalizers->run_finalizers != NULL) {
+        /* copy 'STM_PSEGMENT->finalizers->run_finalizers' into
+           'g_finalizers.run_finalizers', dropping any initial NULLs
+           (finalizers already called) */
+        struct list_s *src = STM_PSEGMENT->finalizers->run_finalizers;
+        uintptr_t frm = 0;
+        if (STM_PSEGMENT->finalizers->running_next != NULL) {
+            frm = *STM_PSEGMENT->finalizers->running_next;
+            assert(frm <= list_count(src));
+            *STM_PSEGMENT->finalizers->running_next = (uintptr_t)-1;
+        }
+        if (frm < list_count(src)) {
+            if (g_finalizers.run_finalizers == NULL)
+                g_finalizers.run_finalizers = list_create();
+            g_finalizers.run_finalizers = list_extend(
+                g_finalizers.run_finalizers,
+                src, frm);
+        }
+        list_free(src);
+    }
+
+    /* copy the whole 'STM_PSEGMENT->finalizers->objects_with_finalizers'
+       into 'g_finalizers.objects_with_finalizers' */
+    g_finalizers.objects_with_finalizers = list_extend(
+        g_finalizers.objects_with_finalizers,
+        STM_PSEGMENT->finalizers->objects_with_finalizers, 0);
+    list_free(STM_PSEGMENT->finalizers->objects_with_finalizers);
+
+    free(STM_PSEGMENT->finalizers);
+    STM_PSEGMENT->finalizers = NULL;
+}
+
+static void abort_finalizers(struct stm_priv_segment_info_s *pseg)
+{
+    /* like _commit_finalizers(), but forget everything from the
+       current transaction */
+    if (pseg->finalizers != NULL) {
+        if (pseg->finalizers->run_finalizers != NULL) {
+            if (pseg->finalizers->running_next != NULL) {
+                *pseg->finalizers->running_next = (uintptr_t)-1;
+            }
+            list_free(pseg->finalizers->run_finalizers);
+        }
+        list_free(pseg->finalizers->objects_with_finalizers);
+        free(pseg->finalizers);
+        pseg->finalizers = NULL;
+    }
+
+    /* call the light finalizers for objects that are about to
+       be forgotten from the current transaction */
+    char *old_gs_register = STM_SEGMENT->segment_base;
+    bool must_fix_gs = old_gs_register != pseg->pub.segment_base;
+
+    struct list_s *lst = pseg->young_objects_with_light_finalizers;
+    long i, count = list_count(lst);
+    if (lst > 0) {
+        for (i = 0; i < count; i++) {
+            object_t *obj = (object_t *)list_item(lst, i);
+            assert(_is_young(obj));
+            if (must_fix_gs) {
+                set_gs_register(pseg->pub.segment_base);
+                must_fix_gs = false;
+            }
+            stmcb_light_finalizer(obj);
+        }
+        list_clear(lst);
+    }
+
+    /* also deals with overflow objects: they are at the tail of
+       old_objects_with_light_finalizers (this list is kept in order
+       and we cannot add any already-committed object) */
+    lst = pseg->old_objects_with_light_finalizers;
+    count = list_count(lst);
+    while (count > 0) {
+        object_t *obj = (object_t *)list_item(lst, --count);
+        if (!IS_OVERFLOW_OBJ(pseg, obj))
+            break;
+        lst->count = count;
+        if (must_fix_gs) {
+            set_gs_register(pseg->pub.segment_base);
+            must_fix_gs = false;
+        }
+        stmcb_light_finalizer(obj);
+    }
+
+    if (STM_SEGMENT->segment_base != old_gs_register)
+        set_gs_register(old_gs_register);
+}
+
+
+void stm_enable_light_finalizer(object_t *obj)
+{
+    if (_is_young(obj)) {
+        LIST_APPEND(STM_PSEGMENT->young_objects_with_light_finalizers, obj);
+    }
+    else {
+        assert(_is_from_same_transaction(obj));
+        LIST_APPEND(STM_PSEGMENT->old_objects_with_light_finalizers, obj);
+    }
+}
+
+object_t *stm_allocate_with_finalizer(ssize_t size_rounded_up)
+{
+    object_t *obj = _stm_allocate_external(size_rounded_up);
+
+    if (STM_PSEGMENT->finalizers == NULL) {
+        struct finalizers_s *f = malloc(sizeof(struct finalizers_s));
+        if (f == NULL)
+            stm_fatalerror("out of memory in create_finalizers");   /* XXX */
+        init_finalizers(f);
+        STM_PSEGMENT->finalizers = f;
+    }
+    assert(STM_PSEGMENT->finalizers->count_non_young
+           <= list_count(STM_PSEGMENT->finalizers->objects_with_finalizers));
+    LIST_APPEND(STM_PSEGMENT->finalizers->objects_with_finalizers, obj);
+    return obj;
+}
+
+
+/************************************************************/
+/*  Light finalizers
+*/
+
+static void deal_with_young_objects_with_finalizers(void)
+{
+    /* for light finalizers */
+    struct list_s *lst = STM_PSEGMENT->young_objects_with_light_finalizers;
+    long i, count = list_count(lst);
+    for (i = 0; i < count; i++) {
+        object_t *obj = (object_t *)list_item(lst, i);
+        assert(_is_young(obj));
+
+        object_t *TLPREFIX *pforwarded_array = (object_t *TLPREFIX *)obj;
+        if (pforwarded_array[0] != GCWORD_MOVED) {
+            /* not moved: the object dies */
+            stmcb_light_finalizer(obj);
+        }
+        else {
+            obj = pforwarded_array[1]; /* moved location */
+            assert(!_is_young(obj));
+            LIST_APPEND(STM_PSEGMENT->old_objects_with_light_finalizers, obj);
+        }
+    }
+    list_clear(lst);
+}
+
+static void deal_with_old_objects_with_finalizers(void)
+{
+    /* for light finalizers */
+    int old_gs_register = STM_SEGMENT->segment_num;
+    int current_gs_register = old_gs_register;
+    long j;
+    for (j = 1; j <= NB_SEGMENTS; j++) {
+        struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
+
+        struct list_s *lst = pseg->old_objects_with_light_finalizers;
+        long i, count = list_count(lst);
+        lst->count = 0;
+        for (i = 0; i < count; i++) {
+            object_t *obj = (object_t *)list_item(lst, i);
+            if (!mark_visited_test(obj)) {
+                /* not marked: object dies */
+                /* we're calling the light finalizer in the same
+                   segment as where it was originally registered.  For
+                   objects that existed since a long time, it doesn't
+                   change anything: any thread should see the same old
+                   content (because if it wasn't the case, the object
+                   would be in a 'modified_old_objects' list
+                   somewhere, and so it wouldn't be dead).  But it's
+                   important if the object was created by the same
+                   transaction: then only that segment sees valid
+                   content.
+                */
+                if (j != current_gs_register) {
+                    set_gs_register(get_segment_base(j));
+                    current_gs_register = j;
+                }
+                stmcb_light_finalizer(obj);
+            }
+            else {
+                /* object survives */
+                list_set_item(lst, lst->count++, (uintptr_t)obj);
+            }
+        }
+    }
+    if (old_gs_register != current_gs_register)
+        set_gs_register(get_segment_base(old_gs_register));
+}
+
+
+/************************************************************/
+/*  Algorithm for regular (non-light) finalizers.
+    Follows closely pypy/doc/discussion/finalizer-order.rst
+    as well as rpython/memory/gc/minimark.py.
+*/
+
+static inline int _finalization_state(object_t *obj)
+{
+    /* Returns the state, "0", 1, 2 or 3, as per finalizer-order.rst.
+       One difference is that the official state 0 is returned here
+       as a number that is <= 0. */
+    uintptr_t lock_idx = mark_loc(obj);
+    return write_locks[lock_idx] - (WL_FINALIZ_ORDER_1 - 1);
+}
+
+static void _bump_finalization_state_from_0_to_1(object_t *obj)
+{
+    uintptr_t lock_idx = mark_loc(obj);
+    assert(write_locks[lock_idx] < WL_FINALIZ_ORDER_1);
+    write_locks[lock_idx] = WL_FINALIZ_ORDER_1;
+}
+
+static struct list_s *_finalizer_tmpstack;
+static struct list_s *_finalizer_emptystack;
+static struct list_s *_finalizer_pending;
+
+static inline void _append_to_finalizer_tmpstack(object_t **pobj)
+{
+    object_t *obj = *pobj;
+    if (obj != NULL)
+        LIST_APPEND(_finalizer_tmpstack, obj);
+}
+
+static inline struct list_s *finalizer_trace(char *base, object_t *obj,
+                                             struct list_s *lst)
+{
+    struct object_s *realobj = (struct object_s *)REAL_ADDRESS(base, obj);
+    _finalizer_tmpstack = lst;
+    stmcb_trace(realobj, &_append_to_finalizer_tmpstack);
+    return _finalizer_tmpstack;
+}
+
+static void _recursively_bump_finalization_state(char *base, object_t *obj,
+                                                 int to_state)
+{
+    struct list_s *tmpstack = _finalizer_emptystack;
+    assert(list_is_empty(tmpstack));
+
+    while (1) {
+        if (_finalization_state(obj) == to_state - 1) {
+            /* bump to the next state */
+            write_locks[mark_loc(obj)]++;
+
+            /* trace */
+            tmpstack = finalizer_trace(base, obj, tmpstack);
+        }
+
+        if (list_is_empty(tmpstack))
+            break;
+
+        obj = (object_t *)list_pop_item(tmpstack);
+    }
+    _finalizer_emptystack = tmpstack;
+}
+
+static struct list_s *mark_finalize_step1(char *base, struct finalizers_s *f)
+{
+    if (f == NULL)
+        return NULL;
+
+    struct list_s *marked = list_create();
+
+    struct list_s *lst = f->objects_with_finalizers;
+    long i, count = list_count(lst);
+    lst->count = 0;
+    f->count_non_young = 0;
+
+    for (i = 0; i < count; i++) {
+        object_t *x = (object_t *)list_item(lst, i);
+
+        assert(_finalization_state(x) != 1);
+        if (_finalization_state(x) >= 2) {
+            list_set_item(lst, lst->count++, (uintptr_t)x);
+            continue;
+        }
+        LIST_APPEND(marked, x);
+
+        struct list_s *pending = _finalizer_pending;
+        LIST_APPEND(pending, x);
+        while (!list_is_empty(pending)) {
+            object_t *y = (object_t *)list_pop_item(pending);
+            int state = _finalization_state(y);
+            if (state <= 0) {
+                _bump_finalization_state_from_0_to_1(y);
+                pending = finalizer_trace(base, y, pending);
+            }
+            else if (state == 2) {
+                _recursively_bump_finalization_state(base, y, 3);
+            }
+        }
+        _finalizer_pending = pending;
+        assert(_finalization_state(x) == 1);
+        _recursively_bump_finalization_state(base, x, 2);
+    }
+    return marked;
+}
+
+static void mark_finalize_step2(char *base, struct finalizers_s *f,
+                                struct list_s *marked)
+{
+    if (f == NULL)
+        return;
+
+    struct list_s *run_finalizers = f->run_finalizers;
+
+    long i, count = list_count(marked);
+    for (i = 0; i < count; i++) {
+        object_t *x = (object_t *)list_item(marked, i);
+
+        int state = _finalization_state(x);
+        assert(state >= 2);
+        if (state == 2) {
+            if (run_finalizers == NULL)
+                run_finalizers = list_create();
+            LIST_APPEND(run_finalizers, x);
+            _recursively_bump_finalization_state(base, x, 3);
+        }
+        else {
+            struct list_s *lst = f->objects_with_finalizers;
+            list_set_item(lst, lst->count++, (uintptr_t)x);
+        }
+    }
+    list_free(marked);
+
+    f->run_finalizers = run_finalizers;
+}
+
+static void deal_with_objects_with_finalizers(void)
+{
+    /* for non-light finalizers */
+
+    /* there is one 'objects_with_finalizers' list per segment.
+       Objects that die at a major collection running in the same
+       transaction as they were created will be put in the
+       'run_finalizers' list of that segment.  Objects that survive at
+       least one commit move to the global g_objects_with_finalizers,
+       and when they die they go to g_run_finalizers.  The former kind
+       of dying object must have its finalizer called in the correct
+       thread; the latter kind can be called in any thread, through
+       any segment, because they all should see the same old content
+       anyway.  (If the content was different between segments at this
+       point, the object would be in a 'modified_old_objects' list
+       somewhere, and so it wouldn't be dead).
+    */
+    struct list_s *marked_seg[NB_SEGMENTS + 1];
+    LIST_CREATE(_finalizer_emptystack);
+    LIST_CREATE(_finalizer_pending);
+
+    long j;
+    for (j = 1; j <= NB_SEGMENTS; j++) {
+        struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
+        marked_seg[j] = mark_finalize_step1(pseg->pub.segment_base,
+                                            pseg->finalizers);
+    }
+    marked_seg[0] = mark_finalize_step1(stm_object_pages, &g_finalizers);
+
+    LIST_FREE(_finalizer_pending);
+
+    for (j = 1; j <= NB_SEGMENTS; j++) {
+        struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
+        mark_finalize_step2(pseg->pub.segment_base, pseg->finalizers,
+                            marked_seg[j]);
+    }
+    mark_finalize_step2(stm_object_pages, &g_finalizers, marked_seg[0]);
+
+    LIST_FREE(_finalizer_emptystack);
+}
+
+static void mark_visit_from_finalizer1(char *base, struct finalizers_s *f)
+{
+    if (f != NULL && f->run_finalizers != NULL) {
+        LIST_FOREACH_R(f->run_finalizers, object_t * /*item*/,
+                       mark_visit_object(item, base));
+    }
+}
+
+static void mark_visit_from_finalizer_pending(void)
+{
+    long j;
+    for (j = 1; j <= NB_SEGMENTS; j++) {
+        struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
+        mark_visit_from_finalizer1(pseg->pub.segment_base, pseg->finalizers);
+    }
+    mark_visit_from_finalizer1(stm_object_pages, &g_finalizers);
+}
+
+static void _execute_finalizers(struct finalizers_s *f)
+{
+    if (f->run_finalizers == NULL)
+        return;   /* nothing to do */
+
+ restart:
+    if (f->running_next != NULL)
+        return;   /* in a nested invocation of execute_finalizers() */
+
+    uintptr_t next = 0, total = list_count(f->run_finalizers);
+    f->running_next = &next;
+
+    while (next < total) {
+        object_t *obj = (object_t *)list_item(f->run_finalizers, next);
+        list_set_item(f->run_finalizers, next, 0);
+        next++;
+
+        stmcb_finalizer(obj);
+    }
+    if (next == (uintptr_t)-1) {
+        /* transaction committed: the whole 'f' was freed */
+        return;
+    }
+    f->running_next = NULL;
+
+    if (f->run_finalizers->count > total) {
+        memmove(f->run_finalizers->items,
+                f->run_finalizers->items + total,
+                (f->run_finalizers->count - total) * sizeof(uintptr_t));
+        goto restart;
+    }
+
+    LIST_FREE(f->run_finalizers);
+}
+
+static void _invoke_general_finalizers(stm_thread_local_t *tl)
+{
+    /* called between transactions */
+    static int lock = 0;
+
+    if (__sync_lock_test_and_set(&lock, 1) != 0) {
+        /* can't acquire the lock: someone else is likely already
+           running this function, so don't wait. */
+        return;
+    }
+
+    rewind_jmp_buf rjbuf;
+    stm_rewind_jmp_enterframe(tl, &rjbuf);
+    stm_start_transaction(tl);
+
+    _execute_finalizers(&g_finalizers);
+
+    stm_commit_transaction();
+    stm_rewind_jmp_leaveframe(tl, &rjbuf);
+
+    __sync_lock_release(&lock);
+}
diff --git a/c7/stm/finalizer.h b/c7/stm/finalizer.h
new file mode 100644
--- /dev/null
+++ b/c7/stm/finalizer.h
@@ -0,0 +1,43 @@
+
+struct finalizers_s {
+    struct list_s *objects_with_finalizers;
+    uintptr_t count_non_young;
+    struct list_s *run_finalizers;
+    uintptr_t *running_next;
+};
+
+static void mark_visit_from_finalizer_pending(void);
+static void deal_with_young_objects_with_finalizers(void);
+static void deal_with_old_objects_with_finalizers(void);
+static void deal_with_objects_with_finalizers(void);
+
+static void setup_finalizer(void);
+static void teardown_finalizer(void);
+
+static void _commit_finalizers(void);
+static void abort_finalizers(struct stm_priv_segment_info_s *);
+
+#define commit_finalizers()   do {              \
+    if (STM_PSEGMENT->finalizers != NULL)       \
+        _commit_finalizers();                   \
+} while (0)
+
+
+/* regular finalizers (objs from already-committed transactions) */
+static struct finalizers_s g_finalizers;
+
+static void _invoke_general_finalizers(stm_thread_local_t *tl);
+
+#define invoke_general_finalizers(tl)    do {   \
+    if (g_finalizers.run_finalizers != NULL)    \
+        _invoke_general_finalizers(tl);         \
+} while (0)
+
+static void _execute_finalizers(struct finalizers_s *f);
+
+#define any_local_finalizers() (STM_PSEGMENT->finalizers != NULL &&         \
+                               STM_PSEGMENT->finalizers->run_finalizers != NULL)
+#define exec_local_finalizers()  do {                   \
+    if (any_local_finalizers())                         \
+        _execute_finalizers(STM_PSEGMENT->finalizers);  \
+} while (0)
diff --git a/c7/stm/forksupport.c b/c7/stm/forksupport.c
--- a/c7/stm/forksupport.c
+++ b/c7/stm/forksupport.c
@@ -12,16 +12,6 @@
 static stm_thread_local_t *fork_this_tl;
 static bool fork_was_in_transaction;
 
-static bool page_is_null(char *p)
-{
-    long *q = (long *)p;
-    long i;
-    for (i = 0; i < 4096 / sizeof(long); i++)
-        if (q[i] != 0)
-            return false;
-    return true;
-}
-
 
 static void forksupport_prepare(void)
 {
@@ -55,14 +45,12 @@
     s_mutex_unlock();
 
     bool was_in_transaction = _stm_in_transaction(this_tl);
-    if (was_in_transaction) {
-        stm_become_inevitable(this_tl, "fork");
-        /* Note that the line above can still fail and abort, which should
-           be fine */
-    }
-    else {
-        stm_start_inevitable_transaction(this_tl);
-    }
+    if (!was_in_transaction)
+        stm_start_transaction(this_tl);
+
+    stm_become_inevitable(this_tl, "fork");
+    /* Note that the line above can still fail and abort, which should
+       be fine */
 
     s_mutex_lock();
     synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK);
@@ -73,27 +61,6 @@
     int big_copy_fd;
     char *big_copy = setup_mmap("stmgc's fork support", &big_copy_fd);
 
-    /* Copy each of the segment infos into the new mmap, nurseries,
-       and associated read markers
-     */
-    long i;
-    for (i = 1; i <= NB_SEGMENTS; i++) {
-        char *src, *dst;
-        struct stm_priv_segment_info_s *psrc = get_priv_segment(i);
-        dst = big_copy + (((char *)psrc) - stm_object_pages);
-        *(struct stm_priv_segment_info_s *)dst = *psrc;
-
-        src = get_segment_base(i) + FIRST_READMARKER_PAGE * 4096UL;
-        dst = big_copy + (src - stm_object_pages);
-        long j;
-        for (j = 0; j < END_NURSERY_PAGE - FIRST_READMARKER_PAGE; j++) {
-            if (!page_is_null(src))
-                pagecopy(dst, src);
-            src += 4096;
-            dst += 4096;
-        }
-    }
-
     /* Copy all the data from the two ranges of objects (large, small)
        into the new mmap
     */
@@ -187,7 +154,6 @@
 #ifndef NDEBUG
     pr->running_pthread = pthread_self();
 #endif
-    strcpy(pr->marker_self, "fork");
     tl->shadowstack = NULL;
     pr->shadowstack_at_start_of_transaction = NULL;
     stm_rewind_jmp_forget(tl);
@@ -204,16 +170,24 @@
        just release these locks early */
     s_mutex_unlock();
 
-    /* Move the copy of the mmap over the old one, overwriting it
-       and thus freeing the old mapping in this process
+    /* Move the copy of the mmap over the old one, overwriting it,
+       with "holes" for each segment's read markers (which are already
+       MAP_PRIVATE and shouldn't be overwritten).  Then free the copy.
     */
     assert(fork_big_copy != NULL);
     assert(stm_object_pages != NULL);
-    void *res = mremap(fork_big_copy, TOTAL_MEMORY, TOTAL_MEMORY,
-                       MREMAP_MAYMOVE | MREMAP_FIXED,
-                       stm_object_pages);
-    if (res != stm_object_pages)
-        stm_fatalerror("after fork: mremap failed: %m");
+
+    long j;
+    for (j = 0; j <= NB_SEGMENTS; j++) {
+        char *dst = get_segment_base(j) + END_NURSERY_PAGE * 4096UL;
+        char *src = fork_big_copy + (dst - stm_object_pages);
+        uintptr_t num_bytes = (NB_PAGES - END_NURSERY_PAGE) * 4096UL;
+        void *res = mremap(src, num_bytes, num_bytes,
+                           MREMAP_MAYMOVE | MREMAP_FIXED, dst);
+        if (res != dst)
+            stm_fatalerror("after fork: mremap failed: %m");
+    }
+    munmap(fork_big_copy, TOTAL_MEMORY);
     fork_big_copy = NULL;
     close_fd_mmap(stm_object_pages_fd);
     stm_object_pages_fd = fork_big_copy_fd;
diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c
--- a/c7/stm/gcpage.c
+++ b/c7/stm/gcpage.c
@@ -84,6 +84,34 @@
     return o;
 }
 
+object_t *stm_allocate_preexisting(ssize_t size_rounded_up,
+                                   const char *initial_data)
+{
+    acquire_privatization_lock();
+
+    char *p = allocate_outside_nursery_large(size_rounded_up);
+    uintptr_t nobj = p - stm_object_pages;
+    dprintf(("allocate_preexisting: %p\n", (object_t *)nobj));
+    long j;
+    for (j = 0; j <= NB_SEGMENTS; j++) {
+        char *dest = get_segment_base(j) + nobj;
+        memcpy(dest, initial_data, size_rounded_up);
+        ((struct object_s *)dest)->stm_flags = GCFLAG_WRITE_BARRIER;
+#ifdef STM_TESTS
+        /* can't really enable this check outside tests, because there is
+           a change that the transaction_state changes in parallel */
+        if (j && get_priv_segment(j)->transaction_state != TS_NONE) {
+            assert(!was_read_remote(get_segment_base(j), (object_t *)nobj));
+        }
+#endif
+    }
+
+    release_privatization_lock();
+
+    write_fence();     /* make sure 'nobj' is fully initialized from
+                          all threads here */
+    return (object_t *)nobj;
+}
 
 /************************************************************/
 
@@ -98,7 +126,7 @@
 
     if (is_major_collection_requested()) {   /* if still true */
 
-        int oldstate = change_timing_state(STM_TIME_MAJOR_GC);
+        timing_event(STM_SEGMENT->running_thread, STM_GC_MAJOR_START);
 
         synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK);
 
@@ -106,10 +134,11 @@
             major_collection_now_at_safe_point();
         }
 
-        change_timing_state(oldstate);
+        timing_event(STM_SEGMENT->running_thread, STM_GC_MAJOR_DONE);
     }
 
     s_mutex_unlock();
+    exec_local_finalizers();
 }
 
 
@@ -118,7 +147,11 @@
 
 static struct list_s *mark_objects_to_trace;
 
-#define WL_VISITED   255
+#define WL_FINALIZ_ORDER_1    253
+#define WL_FINALIZ_ORDER_2    254
+#define WL_FINALIZ_ORDER_3    WL_VISITED
+
+#define WL_VISITED            255
 
 
 static inline uintptr_t mark_loc(object_t *obj)
@@ -296,6 +329,8 @@
     LIST_APPEND(mark_objects_to_trace, obj);
 }
 
+#define TRACE_FOR_MAJOR_COLLECTION  (&mark_record_trace)
+
 static void mark_trace(object_t *obj, char *segment_base)
 {
     assert(list_is_empty(mark_objects_to_trace));
@@ -304,7 +339,7 @@
         /* trace into the object (the version from 'segment_base') */
         struct object_s *realobj =
             (struct object_s *)REAL_ADDRESS(segment_base, obj);
-        stmcb_trace(realobj, &mark_record_trace);
+        stmcb_trace(realobj, TRACE_FOR_MAJOR_COLLECTION);
 
         if (list_is_empty(mark_objects_to_trace))
             break;
@@ -340,12 +375,12 @@
 
     stm_thread_local_t *tl = stm_all_thread_locals;
     do {
-        /* If 'tl' is currently running, its 'associated_segment_num'
+        /* If 'tl' is currently running, its 'last_associated_segment_num'
            field is the segment number that contains the correct
            version of its overflowed objects.  If not, then the
            field is still some correct segment number, and it doesn't
            matter which one we pick. */
-        char *segment_base = get_segment_base(tl->associated_segment_num);
+        char *segment_base = get_segment_base(tl->last_associated_segment_num);
 
         struct stm_shadowentry_s *current = tl->shadowstack;
         struct stm_shadowentry_s *base = tl->shadowstack_base;
@@ -375,18 +410,33 @@
 {
     /* The modified objects are the ones that may exist in two different
        versions: one in the segment that modified it, and another in all
-       other segments.  (It can also be more than two if we don't have
-       eager write locking.)
+       other segments.  (It could also be more than two if we did't have
+       eager write locking, but for now we do.)
     */
     long i;
     for (i = 1; i <= NB_SEGMENTS; i++) {
+        LIST_FOREACH_R(
+            get_priv_segment(i)->modified_old_objects,
+            object_t * /*item*/,
+            ({
+                /* This function is called first, and there should not be
+                   any duplicate in modified_old_objects. */
+                if (mark_visited_test_and_set(item)) {
+                    assert(!"duplicate in modified_old_objects!");
+                }
+            }));
+    }
+
+    /* Now that we have marked all modified_old_objects, trace them
+       (which will mark more objects).
+    */
+    for (i = 1; i <= NB_SEGMENTS; i++) {
         char *base = get_segment_base(i);
 
         LIST_FOREACH_R(
             get_priv_segment(i)->modified_old_objects,
             object_t * /*item*/,
             ({
-                mark_visited_test_and_set(item);
                 mark_trace(item, stm_object_pages);  /* shared version */
                 mark_trace(item, base);              /* private version */
             }));
@@ -403,9 +453,9 @@
         for (i = list_count(lst); i > 0; i -= 2) {
             mark_visit_object((object_t *)list_item(lst, i - 1), base);
         }
-        if (get_priv_segment(j)->marker_inev[1]) {
-            uintptr_t marker_inev_obj = get_priv_segment(j)->marker_inev[1];
-            mark_visit_object((object_t *)marker_inev_obj, base);
+        if (get_priv_segment(j)->marker_inev.segment_base) {
+            object_t *marker_inev_obj = get_priv_segment(j)->marker_inev.object;
+            mark_visit_object(marker_inev_obj, base);
         }
     }
 }
@@ -481,10 +531,28 @@
             uintptr_t n = list_count(lst);
             while (n > 0) {
                 object_t *obj = (object_t *)list_item(lst, --n);
-                if (!mark_visited_test(obj)) {
+                if (!mark_visited_test(obj))
                     list_set_item(lst, n, list_pop_item(lst));
+            }
+        }
+
+        /* Remove from 'modified_old_hashtables' all old hashtables that
+           die, but keeping the order */
+        {
+            lst = pseg->modified_old_hashtables;
+            uintptr_t j, k = 0, limit = list_count(lst);
+            for (j = 0; j < limit; j += 2) {
+                object_t *hobj = (object_t *)list_item(lst, j);
+                if (mark_visited_test(hobj)) {
+                    /* hobj does not die */
+                    if (j != k) {
+                        list_set_item(lst, k, (uintptr_t)hobj);
+                        list_set_item(lst, k + 1, list_item(lst, j + 1));
+                    }
+                    k += 2;
                 }
             }
+            lst->count = k;
         }
     }
 #pragma pop_macro("STM_SEGMENT")
@@ -581,10 +649,17 @@
     mark_visit_from_modified_objects();
     mark_visit_from_markers();
     mark_visit_from_roots();
+    mark_visit_from_finalizer_pending();
     LIST_FREE(mark_objects_to_trace);
 
-    /* weakrefs: */
+    /* finalizer support: will mark as WL_VISITED all objects with a
+       finalizer and all objects reachable from there, and also moves
+       some objects from 'objects_with_finalizers' to 'run_finalizers'. */
+    deal_with_objects_with_finalizers();
+
+    /* weakrefs and old light finalizers */
     stm_visit_old_weakrefs();
+    deal_with_old_objects_with_finalizers();
 
     /* cleanup */
     clean_up_segment_lists();
diff --git a/c7/stm/hashtable.c b/c7/stm/hashtable.c
new file mode 100644
--- /dev/null
+++ b/c7/stm/hashtable.c
@@ -0,0 +1,514 @@
+/*
+Design of stmgc's "hashtable" objects
+=====================================
+
+A "hashtable" is theoretically a lazily-filled array of objects of
+length 2**64.  Initially it is full of NULLs.  It's obviously
+implemented as a dictionary in which NULL objects are not needed.
+
+A real dictionary can be implemented on top of it, by using the index
+`hash(key)` in the hashtable, and storing a list of `(key, value)`
+pairs at that index (usually only one, unless there is a hash
+collision).
+
+The main operations on a hashtable are reading or writing an object at a
+given index.  It also supports fetching the list of non-NULL entries.
+
+There are two markers for every index (a read and a write marker).
+This is unlike regular arrays, which have only two markers in total.
+
+Additionally, we use the read marker for the hashtable object itself
+to mean "we have read the complete list of keys".  This plays the role
+of a "global" read marker: when any thread adds a new key/value object
+to the hashtable, this new object's read marker is initialized with a
+copy of the "global" read marker --- in all segments.
+
+
+Implementation
+--------------
+
+First idea: have the hashtable in raw memory, pointing to "entry"
+objects (which are regular, GC- and STM-managed objects).  The entry
+objects themselves point to the user-specified objects.  The entry
+objects hold the read/write markers.  Every entry object, once
+created, stays around.  It is only removed by the next major GC if it
+points to NULL and its read/write markers are not set in any
+currently-running transaction.
+
+References
+----------
+
+Inspired by: http://ppl.stanford.edu/papers/podc011-bronson.pdf
+*/
+
+
+uint32_t stm_hashtable_entry_userdata;
+
+
+#define INITIAL_HASHTABLE_SIZE   8
+#define PERTURB_SHIFT            5
+#define RESIZING_LOCK            0
+
+typedef struct {
+    uintptr_t mask;
+
+    /* 'resize_counter' start at an odd value, and is decremented (by
+       6) for every new item put in 'items'.  When it crosses 0, we
+       instead allocate a bigger table and change 'resize_counter' to
+       be a regular pointer to it (which is then even).  The whole
+       structure is immutable then.
+
+       The field 'resize_counter' also works as a write lock: changes
+       go via the intermediate value RESIZING_LOCK (0).
+    */
+    uintptr_t resize_counter;
+
+    stm_hashtable_entry_t *items[INITIAL_HASHTABLE_SIZE];

From noreply at buildbot.pypy.org  Tue Feb 24 13:42:00 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 24 Feb 2015 13:42:00 +0100 (CET)
Subject: [pypy-commit] stmgc gc-small-uniform: fix
Message-ID: <20150224124200.089381C014F@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: gc-small-uniform
Changeset: r1649:e21654d9b939
Date: 2015-02-24 12:49 +0100
http://bitbucket.org/pypy/stmgc/changeset/e21654d9b939/

Log:	fix

diff --git a/c7/stmgc.h b/c7/stmgc.h
--- a/c7/stmgc.h
+++ b/c7/stmgc.h
@@ -124,7 +124,7 @@
 #endif
 
 #define _STM_GCFLAG_WRITE_BARRIER      0x01
-#define _STM_GCFLAG_CARDS_SET          0x04
+#define _STM_GCFLAG_CARDS_SET          0x08
 #define _STM_CARD_SIZE                 32     /* must be >= 32 */
 #define _STM_MIN_CARD_COUNT            17
 #define _STM_MIN_CARD_OBJ_SIZE         (_STM_CARD_SIZE * _STM_MIN_CARD_COUNT)

From noreply at buildbot.pypy.org  Tue Feb 24 13:42:01 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 24 Feb 2015 13:42:01 +0100 (CET)
Subject: [pypy-commit] stmgc gc-small-uniform: fix forksupport for small
 objs (copy&paste mistake)
Message-ID: <20150224124201.08DA31C014F@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: gc-small-uniform
Changeset: r1650:ffeae4a203e7
Date: 2015-02-24 13:22 +0100
http://bitbucket.org/pypy/stmgc/changeset/ffeae4a203e7/

Log:	fix forksupport for small objs (copy&paste mistake)

diff --git a/c7/stm/forksupport.c b/c7/stm/forksupport.c
--- a/c7/stm/forksupport.c
+++ b/c7/stm/forksupport.c
@@ -221,8 +221,6 @@
                 break;   /* done */
             pagenum = (uninitialized_page_stop - stm_object_pages) / 4096UL;
             endpagenum = NB_PAGES;
-            if (endpagenum == NB_PAGES)
-                break;   /* done */
         }
 
         struct page_shared_s ps = pages_privatized[pagenum - PAGE_FLAG_START];

From noreply at buildbot.pypy.org  Tue Feb 24 13:42:02 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 24 Feb 2015 13:42:02 +0100 (CET)
Subject: [pypy-commit] stmgc default: fix (it seemed wrong)
Message-ID: <20150224124202.081F11C014F@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: 
Changeset: r1651:4e1c0df136dc
Date: 2015-02-24 13:40 +0100
http://bitbucket.org/pypy/stmgc/changeset/4e1c0df136dc/

Log:	fix (it seemed wrong)

diff --git a/c8/stm/smallmalloc.c b/c8/stm/smallmalloc.c
--- a/c8/stm/smallmalloc.c
+++ b/c8/stm/smallmalloc.c
@@ -64,7 +64,8 @@
         uninitialized_page_stop -= decrease_by;
         first_small_uniform_loc = uninitialized_page_stop - stm_object_pages;
 
-        if (!_stm_largemalloc_resize_arena(uninitialized_page_stop - uninitialized_page_start))
+        char *base = stm_object_pages + END_NURSERY_PAGE * 4096UL;
+        if (!_stm_largemalloc_resize_arena(uninitialized_page_stop - base))
             goto out_of_memory;
 
         /* make writable in sharing seg */

From noreply at buildbot.pypy.org  Tue Feb 24 13:42:03 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 24 Feb 2015 13:42:03 +0100 (CET)
Subject: [pypy-commit] stmgc default: fix forksupport for small objs
	(copy&paste mistake)
Message-ID: <20150224124203.024131C014F@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: 
Changeset: r1652:07f882f975d6
Date: 2015-02-24 13:22 +0100
http://bitbucket.org/pypy/stmgc/changeset/07f882f975d6/

Log:	fix forksupport for small objs (copy&paste mistake)

diff --git a/c7/stm/forksupport.c b/c7/stm/forksupport.c
--- a/c7/stm/forksupport.c
+++ b/c7/stm/forksupport.c
@@ -221,8 +221,6 @@
                 break;   /* done */
             pagenum = (uninitialized_page_stop - stm_object_pages) / 4096UL;
             endpagenum = NB_PAGES;
-            if (endpagenum == NB_PAGES)
-                break;   /* done */
         }
 
         struct page_shared_s ps = pages_privatized[pagenum - PAGE_FLAG_START];

From noreply at buildbot.pypy.org  Tue Feb 24 14:06:54 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 24 Feb 2015 14:06:54 +0100 (CET)
Subject: [pypy-commit] pypy optresult: fix merge
Message-ID: <20150224130654.A581E1C03D1@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r76097:a5e66cb352ac
Date: 2015-02-24 15:06 +0200
http://bitbucket.org/pypy/pypy/changeset/a5e66cb352ac/

Log:	fix merge

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -866,21 +866,21 @@
             return
         optpure = self.optpure
         if op.getopnum() == rop.INT_ADD:
-            optpure.pure(rop.INT_ADD, [op.getarg(1), op.getarg(0)], op.result)
+            optpure.pure(rop.INT_ADD, [op.getarg(1), op.getarg(0)], op)
             # Synthesize the reverse op for optimize_default to reuse
-            optpure.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
-            optpure.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
+            optpure.pure(rop.INT_SUB, [op, op.getarg(1)], op.getarg(0))
+            optpure.pure(rop.INT_SUB, [op, op.getarg(0)], op.getarg(1))
         elif op.getopnum() == rop.INT_SUB:
-            optpure.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
-            optpure.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
+            optpure.pure(rop.INT_ADD, [op, op.getarg(1)], op.getarg(0))
+            optpure.pure(rop.INT_SUB, [op.getarg(0), op], op.getarg(1))
         elif op.getopnum() == rop.FLOAT_MUL:
-            optpure.pure(rop.FLOAT_MUL, [op.getarg(1), op.getarg(0)], op.result)
+            optpure.pure(rop.FLOAT_MUL, [op.getarg(1), op.getarg(0)], op)
         elif op.getopnum() == rop.FLOAT_NEG:
-            optpure.pure(rop.FLOAT_NEG, [op.result], op.getarg(0))
+            optpure.pure(rop.FLOAT_NEG, [op], op.getarg(0))
         elif op.getopnum() == rop.CAST_INT_TO_PTR:
-            optpure.pure(rop.CAST_PTR_TO_INT, [op.result], op.getarg(0))
+            optpure.pure(rop.CAST_PTR_TO_INT, [op], op.getarg(0))
         elif op.getopnum() == rop.CAST_PTR_TO_INT:
-            optpure.pure(rop.CAST_INT_TO_PTR, [op.result], op.getarg(0))
+            optpure.pure(rop.CAST_INT_TO_PTR, [op], op.getarg(0))
 
     #def optimize_GUARD_NO_OVERFLOW(self, op):
     #    # otherwise the default optimizer will clear fields, which is unwanted
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -82,7 +82,6 @@
         args = op.getarglist()
         opnum = OpHelpers.call_for_descr(op.getdescr())
         newop = self.optimizer.replace_op_with(op, opnum)
-        self.remember_emitting_pure(op)
         self.emit_operation(newop)
         self.call_pure_positions.append(len(self.optimizer._newoperations) - 1)
     optimize_CALL_PURE_R = optimize_CALL_PURE_I
diff --git a/rpython/jit/metainterp/optimizeopt/util.py b/rpython/jit/metainterp/optimizeopt/util.py
--- a/rpython/jit/metainterp/optimizeopt/util.py
+++ b/rpython/jit/metainterp/optimizeopt/util.py
@@ -168,7 +168,7 @@
             assert op1.same_box(remap[op2])
         else:
             remap[op2] = op1
-        if op1.getopnum() not in [rop.JUMP, rop.LABEL] and not op1.is_guard():
+        if op1.getopnum() not in [rop.JUMP, rop.LABEL, rop.FINISH] and not op1.is_guard():
             assert op1.getdescr() == op2.getdescr()
         if op1.getfailargs() or op2.getfailargs():
             assert len(op1.getfailargs()) == len(op2.getfailargs())

From noreply at buildbot.pypy.org  Tue Feb 24 14:48:08 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 24 Feb 2015 14:48:08 +0100 (CET)
Subject: [pypy-commit] pypy default: Add a __repr__ for debugging
Message-ID: <20150224134808.B01341C0334@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76098:2e95ba978d82
Date: 2015-02-24 14:46 +0100
http://bitbucket.org/pypy/pypy/changeset/2e95ba978d82/

Log:	Add a __repr__ for debugging

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -60,6 +60,17 @@
             self.make_constant(box)
         # invariant: box is a Const if and only if level == LEVEL_CONSTANT
 
+    def __repr__(self):
+        level = {LEVEL_UNKNOWN: 'UNKNOWN',
+                 LEVEL_NONNULL: 'NONNULL',
+                 LEVEL_KNOWNCLASS: 'KNOWNCLASS',
+                 LEVEL_CONSTANT: 'CONSTANT'}.get(self.getlevel(),
+                                                 self.getlevel())
+        return '<%s %s %s>' % (
+            self.__class__.__name__,
+            level,
+            self.box)
+
     def getlevel(self):
         return self._tag & 0x3
 

From noreply at buildbot.pypy.org  Tue Feb 24 15:05:31 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 24 Feb 2015 15:05:31 +0100 (CET)
Subject: [pypy-commit] pypy optresult: (fijal, arigo) completely in-progress
Message-ID: <20150224140531.180461C03B3@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r76099:9e2e51f4f8a5
Date: 2015-02-24 16:03 +0200
http://bitbucket.org/pypy/pypy/changeset/9e2e51f4f8a5/

Log:	(fijal, arigo) completely in-progress

diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -177,6 +177,8 @@
 class Const(AbstractValue):
     __slots__ = ()
 
+    forwarded = None
+
     @staticmethod
     def _new(x):
         "NOT_RPYTHON"
@@ -207,6 +209,9 @@
     def repr(self, memo):
         return self.repr_rpython()
 
+    def is_constant(self):
+        return True
+
     def __repr__(self):
         return 'Const(%s)' % self._getrepr_()
 
diff --git a/rpython/jit/metainterp/optimizeopt/generalize.py b/rpython/jit/metainterp/optimizeopt/generalize.py
--- a/rpython/jit/metainterp/optimizeopt/generalize.py
+++ b/rpython/jit/metainterp/optimizeopt/generalize.py
@@ -1,5 +1,5 @@
 from rpython.jit.metainterp.optimizeopt.optimizer import MININT, MAXINT,\
-     IntOptValue
+     IntOptInfo
 
 
 class GeneralizationStrategy(object):
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -4,7 +4,7 @@
 from rpython.jit.metainterp.optimizeopt.intutils import (IntBound, IntLowerBound,
     IntUpperBound)
 from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, CONST_1,
-    CONST_0, MODE_ARRAY, MODE_STR, MODE_UNICODE, IntOptValue)
+    CONST_0, MODE_ARRAY, MODE_STR, MODE_UNICODE, IntOptInfo)
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from rpython.jit.metainterp.resoperation import rop, AbstractResOp
 from rpython.jit.backend.llsupport import symbolic
@@ -111,9 +111,10 @@
             r.getintbound().intersect(IntBound(0, next_pow2_m1(lesser)))
 
     def optimize_INT_SUB(self, op):
-        v1 = self.getvalue(op.getarg(0))
-        v2 = self.getvalue(op.getarg(1))
+        v1 = self.getinfo(op.getarg(0))
+        v2 = self.getinfo(op.getarg(1))
         self.emit_operation(op)
+        xxx
         r = self.getvalue(op)
         b = v1.getintbound().sub_bound(v2.getintbound())
         if b.bounded():
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -2,13 +2,11 @@
 from rpython.jit.metainterp.executor import execute_nonspec_const
 from rpython.jit.metainterp.logger import LogOperations
 from rpython.jit.metainterp.history import Const, ConstInt, REF
-from rpython.jit.metainterp.optimizeopt.intutils import IntBound, IntUnbounded, \
-                                                     ImmutableIntUnbounded, \
-                                                     IntLowerBound, MININT,\
-                                                     MAXINT
+from rpython.jit.metainterp.optimizeopt.intutils import IntBound,\
+     ImmutableIntUnbounded, IntLowerBound, MININT, MAXINT
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from rpython.jit.metainterp.resoperation import rop, ResOperation,\
-     AbstractResOp, AbstractInputArg, DONT_CHANGE, GuardResOp
+     AbstractResOp, AbstractInputArg, GuardResOp
 from rpython.jit.metainterp.typesystem import llhelper
 from rpython.tool.pairtype import extendabletype
 from rpython.rlib.debug import debug_print
@@ -45,21 +43,17 @@
                 self.descr == other.descr and
                 self.bound.contains_bound(other.bound))
 
-class OptValue(object):
-    __metaclass__ = extendabletype
-    _attrs_ = ('box', '_tag')
-
+class OptInfo(object):
+    _attrs_ = ('_tag',)
     _tag = 0
 
-    def __init__(self, box, level=None, known_class=None, intbound=None):
-        self.box = box
+    forwarded = None
+
+    def __init__(self, level=LEVEL_UNKNOWN, known_class=None, intbound=None):
+        assert isinstance(level, int)
         if level is not None:
             self._tag = level
 
-        if isinstance(box, Const):
-            self.make_constant(box)
-        # invariant: box is a Const if and only if level == LEVEL_CONSTANT
-
     def getlevel(self):
         return self._tag & 0x3
 
@@ -92,9 +86,7 @@
         self._tag = other_value._tag
 
     def force_box(self, optforce):
-        return self.box
-
-    def get_key_box(self):
+        xxx
         return self.box
 
     def force_at_end_of_preamble(self, already_forced, optforce):
@@ -155,10 +147,7 @@
         return box.getint()
 
     def is_virtual(self):
-        # Don't check this with 'isinstance(_, VirtualValue)'!
-        # Even if it is a VirtualValue, the 'box' can be non-None,
-        # meaning it has been forced.
-        return self.box is None
+        return False # overwridden in VirtualInfo
 
     def is_forced_virtual(self):
         return False
@@ -214,7 +203,7 @@
     def get_constant_class(self, cpu):
         return None
 
-class PtrOptValue(OptValue):
+class PtrOptInfo(OptInfo):
     _attrs_ = ('known_class', 'last_guard_pos', 'lenbound')
 
     known_class = None
@@ -320,15 +309,13 @@
     def get_known_class(self):
         return self.known_class
 
-class IntOptValue(OptValue):
+class IntOptInfo(OptInfo):
     _attrs_ = ('intbound',)
 
     intbound = ImmutableIntUnbounded()
 
-    def __init__(self, box, level=None, known_class=None, intbound=None):
-        OptValue.__init__(self, box, level, None, None)
-        if isinstance(box, Const):
-            return
+    def __init__(self, level=LEVEL_UNKNOWN, known_class=None, intbound=None):
+        OptInfo.__init__(self, level, None, None)
         if intbound:
             self.intbound = intbound
         else:
@@ -395,32 +382,11 @@
     def getlenbound(self):
         return None
 
-class ConstantFloatValue(OptValue):
-    def __init__(self, box):
-        self.make_constant(box)
-
-    def __repr__(self):
-        return 'Constant(%r)' % (self.box,)
-
-class ConstantIntValue(IntOptValue):
-    def __init__(self, box):
-        self.make_constant(box)
-
-    def __repr__(self):
-        return 'Constant(%r)' % (self.box,)
-
-class ConstantPtrValue(PtrOptValue):
-    def __init__(self, box):
-        self.make_constant(box)
-
-    def __repr__(self):
-        return 'Constant(%r)' % (self.box,)
 
 CONST_0      = ConstInt(0)
 CONST_1      = ConstInt(1)
-CVAL_ZERO    = ConstantIntValue(CONST_0)
-CVAL_ZERO_FLOAT = ConstantFloatValue(Const._new(0.0))
-llhelper.CVAL_NULLREF = ConstantPtrValue(llhelper.CONST_NULL)
+CONST_ZERO_FLOAT = Const._new(0.0)
+llhelper.CONST_NULLREF = llhelper.CONST_NULL
 REMOVED = AbstractResOp()
 
 
@@ -438,8 +404,11 @@
         self.next_optimization.propagate_forward(op)
 
     # FIXME: Move some of these here?
-    def getvalue(self, box):
-        return self.optimizer.getvalue(box)
+    def getinfo(self, op, create=True):
+        return self.optimizer.getinfo(op, create=create)
+
+    def get_box_replacement(self, op):
+        return self.optimizer.get_box_replacement(op)
 
     def getlastop(self):
         return self.optimizer._last_emitted_op
@@ -603,8 +572,25 @@
         else:
             return box
 
-    @specialize.argtype(0)
-    def getvalue(self, box):
+    def getinfo(self, op, create=False):
+        while op.forwarded is not None:
+            op = op.forwarded
+        if isinstance(op, OptInfo) or isinstance(op, Const):
+            return op
+        if not create:
+            return None
+        if op.type == 'r':
+            optinfo = PtrOptInfo()
+        elif op.type == 'i':
+            optinfo = IntOptInfo()
+        else:
+            optinfo = OptInfo()
+        op.forwarded = optinfo
+        return optinfo
+        xxx
+        yyy
+
+        XXX
         box = self.getinterned(box)
         try:
             value = self.values[box]
@@ -619,12 +605,14 @@
         self.ensure_imported(value)
         return value
 
-    def get_box_replacement(self, box):
-        try:
-            v = self.values[box]
-        except KeyError:
-            return box
-        return v.get_key_box()
+    def get_box_replacement(self, op):
+        orig_op = op
+        while (op.forwarded is not None and
+               not isinstance(op.forwarded, OptInfo)):
+            op = op.forwarded
+        if op is not orig_op:
+            orig_op.forwarded = op
+        return op
 
     def ensure_imported(self, value):
         pass
@@ -667,11 +655,8 @@
     def replace_op_with(self, op, newopnum, args=None, descr=None):
         newop = op.copy_and_change(newopnum, args, descr)
         if newop.type != 'v':
-            val = self.getvalue(op)
-            if val.box is not None:
-                assert val.box is op
-                val.box = newop
-            self.values[newop] = val
+            assert op.forwarded is None
+            op.forwarded = newop
         return newop
 
     def make_constant(self, box, constbox):
@@ -775,6 +760,7 @@
 
     def get_op_replacement(self, op):
         # XXX this is wrong
+        xxx
         changed = False
         for i, arg in enumerate(op.getarglist()):
             try:
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -39,12 +39,13 @@
                 return
 
             # did we do the exact same operation already?
-            args = self.optimizer.make_args_key(op.getopnum(),
-                                                op.getarglist(), op.getdescr())
-            oldval = self.pure_operations.get(args, None)
-            if oldval is not None:
-                self.optimizer.make_equal_to(op, oldval)
-                return
+            if 0:
+                args = self.optimizer.make_args_key(op.getopnum(),
+                                                    op.getarglist(), op.getdescr())
+                oldval = self.pure_operations.get(args, None)
+                if oldval is not None:
+                    self.optimizer.make_equal_to(op, oldval)
+                    return
 
         # otherwise, the operation remains
         self.emit_operation(op)
@@ -52,8 +53,8 @@
             self.optimizer.bool_boxes[self.getvalue(op)] = None
         if nextop:
             self.emit_operation(nextop)
-        if args is not None:
-            self.pure_operations[args] = self.getvalue(op)
+        #if args is not None:
+        #    self.pure_operations[args] = self.getvalue(op)
 
     def optimize_CALL_PURE_I(self, op):
         # Step 1: check if all arguments are constant
@@ -102,6 +103,7 @@
         self.optimizer.optpure = self
 
     def pure(self, opnum, args, result):
+        return # XXX
         key = self.optimizer.make_args_key(opnum, args, None)
         if key not in self.pure_operations:
             self.pure_operations[key] = self.getvalue(result)
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -6,7 +6,7 @@
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound
 from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, REMOVED,
-    CONST_0, CONST_1, PtrOptValue)
+    CONST_0, CONST_1, PtrOptInfo)
 from rpython.jit.metainterp.optimizeopt.util import _findall, make_dispatcher_method
 from rpython.jit.metainterp.resoperation import rop, ResOperation, opclasses,\
      OpHelpers
@@ -108,14 +108,16 @@
             self.emit_operation(op)
 
     def optimize_INT_SUB(self, op):
-        v1 = self.getvalue(op.getarg(0))
-        v2 = self.getvalue(op.getarg(1))
-        if v2.is_constant() and v2.box.getint() == 0:
+        arg1 = self.get_box_replacement(op.getarg(0))
+        arg2 = self.get_box_replacement(op.getarg(1))
+        if arg2.is_constant() and arg2.getint() == 0:
+            xxx
             self.make_equal_to(op, v1)
-        elif v1.is_constant() and v1.box.getint() == 0:
+        elif arg1.is_constant() and arg1.getint() == 0:
+            xxx
             op = self.replace_op_with(op, rop.INT_NEG, args=[v2.box])
             self.emit_operation(op)
-        elif v1 is v2:
+        elif arg1 is arg2:
             self.make_constant_int(op, 0)
         else:
             self.emit_operation(op)
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -4,7 +4,7 @@
 from rpython.jit.metainterp.history import Const, ConstInt, BoxInt
 from rpython.jit.metainterp.history import CONST_NULL, BoxPtr
 from rpython.jit.metainterp.optimizeopt import optimizer
-from rpython.jit.metainterp.optimizeopt.optimizer import OptValue, REMOVED
+from rpython.jit.metainterp.optimizeopt.optimizer import OptInfo, REMOVED
 from rpython.jit.metainterp.optimizeopt.util import (make_dispatcher_method,
                                                      descrlist_dict, sort_descrs)
 
@@ -13,7 +13,7 @@
 from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.jit.metainterp.optimizeopt.intutils import IntUnbounded
 
-class AbstractVirtualValue(optimizer.PtrOptValue):
+class AbstractVirtualValue(optimizer.PtrOptInfo):
     _attrs_ = ('keybox', 'source_op', '_cached_vinfo')
     box = None
     _tag = optimizer.LEVEL_NONNULL
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -4,7 +4,7 @@
 from rpython.jit.metainterp.optimizeopt import virtualize
 from rpython.jit.metainterp.optimizeopt.intutils import IntUnbounded
 from rpython.jit.metainterp.optimizeopt.optimizer import (LEVEL_CONSTANT,
-    LEVEL_KNOWNCLASS, LEVEL_NONNULL, LEVEL_UNKNOWN, OptValue)
+    LEVEL_KNOWNCLASS, LEVEL_NONNULL, LEVEL_UNKNOWN, OptInfo)
 from rpython.jit.metainterp.resoperation import rop, ResOperation,\
      AbstractInputArg
 from rpython.jit.metainterp.compile import Memo
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -45,46 +45,46 @@
 # ____________________________________________________________
 
 
-class __extend__(optimizer.OptValue):
-    """New methods added to the base class OptValue for this file."""
+# class __extend__(optimizer.OptValue):
+#     """New methods added to the base class OptValue for this file."""
 
-    def getstrlen(self, string_optimizer, mode, lengthop):
-        if mode is mode_string:
-            s = self.get_constant_string_spec(mode_string)
-            if s is not None:
-                return ConstInt(len(s))
-        else:
-            s = self.get_constant_string_spec(mode_unicode)
-            if s is not None:
-                return ConstInt(len(s))
-        if string_optimizer is None:
-            return None
-        self.ensure_nonnull()
-        box = self.force_box(string_optimizer)
-        if lengthop is not None:
-            lengthop = string_optimizer.optimizer.replace_op_with(lengthop,
-                mode.STRLEN, [box])
-        else:
-            lengthop = ResOperation(mode.STRLEN, [box])
-        string_optimizer.emit_operation(lengthop)
-        return lengthop
+#     def getstrlen(self, string_optimizer, mode, lengthop):
+#         if mode is mode_string:
+#             s = self.get_constant_string_spec(mode_string)
+#             if s is not None:
+#                 return ConstInt(len(s))
+#         else:
+#             s = self.get_constant_string_spec(mode_unicode)
+#             if s is not None:
+#                 return ConstInt(len(s))
+#         if string_optimizer is None:
+#             return None
+#         self.ensure_nonnull()
+#         box = self.force_box(string_optimizer)
+#         if lengthop is not None:
+#             lengthop = string_optimizer.optimizer.replace_op_with(lengthop,
+#                 mode.STRLEN, [box])
+#         else:
+#             lengthop = ResOperation(mode.STRLEN, [box])
+#         string_optimizer.emit_operation(lengthop)
+#         return lengthop
 
-    @specialize.arg(1)
-    def get_constant_string_spec(self, mode):
-        if self.is_constant():
-            s = self.box.getref(lltype.Ptr(mode.LLTYPE))
-            return mode.hlstr(s)
-        else:
-            return None
+#     @specialize.arg(1)
+#     def get_constant_string_spec(self, mode):
+#         if self.is_constant():
+#             s = self.box.getref(lltype.Ptr(mode.LLTYPE))
+#             return mode.hlstr(s)
+#         else:
+#             return None
 
-    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
-        # Copies the pointer-to-string 'self' into the target string
-        # given by 'targetbox', at the specified offset.  Returns the offset
-        # at the end of the copy.
-        lengthbox = self.getstrlen(string_optimizer, mode, None)
-        srcbox = self.force_box(string_optimizer)
-        return copy_str_content(string_optimizer, srcbox, targetbox,
-                                CONST_0, offsetbox, lengthbox, mode)
+#     def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
+#         # Copies the pointer-to-string 'self' into the target string
+#         # given by 'targetbox', at the specified offset.  Returns the offset
+#         # at the end of the copy.
+#         lengthbox = self.getstrlen(string_optimizer, mode, None)
+#         srcbox = self.force_box(string_optimizer)
+#         return copy_str_content(string_optimizer, srcbox, targetbox,
+#                                 CONST_0, offsetbox, lengthbox, mode)
 
 
 class VAbstractStringValue(virtualize.AbstractVirtualValue):
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -15,6 +15,9 @@
     def repr_short(self, memo):
         return self.repr(memo)
 
+    def is_constant(self):
+        return False
+
 DONT_CHANGE = AbstractValue()
 
 def ResOperation(opnum, args, descr=None):
@@ -42,8 +45,9 @@
     type = 'v'
     boolreflex = -1
     boolinverse = -1
+    forwarded = None # either another resop or OptInfo
 
-    _attrs_ = ()
+    _attrs_ = ('forwarded',)
 
     def getopnum(self):
         return self.opnum
@@ -394,6 +398,7 @@
         return InputArgRef()
 
 class AbstractInputArg(AbstractValue):
+    forwarded = None
     
     def repr(self, memo):
         try:
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -213,8 +213,8 @@
         numb = lltype.malloc(NUMBERING, length)
         for i in range(length):
             box = boxes[i]
-            value = optimizer.getvalue(box)
-            box = value.get_key_box()
+            box = optimizer.get_box_replacement(box)
+            info = optimizer.getinfo(box, create=False)
 
             if isinstance(box, Const):
                 tagged = self.getconst(box)

From noreply at buildbot.pypy.org  Tue Feb 24 15:08:57 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 24 Feb 2015 15:08:57 +0100 (CET)
Subject: [pypy-commit] pypy optresult: hg merge default
Message-ID: <20150224140857.02A321C0334@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: optresult
Changeset: r76100:a33561b74625
Date: 2015-02-24 15:07 +0100
http://bitbucket.org/pypy/pypy/changeset/a33561b74625/

Log:	hg merge default

diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py
--- a/pypy/module/_rawffi/array.py
+++ b/pypy/module/_rawffi/array.py
@@ -15,7 +15,7 @@
 from pypy.module._rawffi.interp_rawffi import unpack_shape_with_length
 from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr
 from rpython.rlib.rarithmetic import r_uint
-from rpython.rlib import rgc
+from rpython.rlib import rgc, clibffi
 
 
 class W_Array(W_DataShape):
@@ -84,14 +84,11 @@
 
 class W_ArrayInstance(W_DataInstance):
     def __init__(self, space, shape, length, address=r_uint(0)):
-        # Workaround for a strange behavior of libffi: make sure that
-        # we always have at least 8 bytes.  For W_ArrayInstances that are
-        # used as the result value of a function call, ffi_call() writes
-        # 8 bytes into it even if the function's result type asks for less.
-        # This strange behavior is documented.
         memsize = shape.size * length
-        if memsize < 8:
-            memsize = 8
+        # For W_ArrayInstances that are used as the result value of a
+        # function call, ffi_call() writes 8 bytes into it even if the
+        # function's result type asks for less.
+        memsize = clibffi.adjust_return_size(memsize)
         W_DataInstance.__init__(self, space, memsize, address)
         self.length = length
         self.shape = shape
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -495,6 +495,7 @@
         try:
             if self.resshape is not None:
                 result = self.resshape.allocate(space, 1, autofree=True)
+                # adjust_return_size() was used here on result.ll_buffer
                 self.ptr.call(args_ll, result.ll_buffer)
                 return space.wrap(result)
             else:
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -204,10 +204,6 @@
                 self._clear_caches_arraycopy(opnum, descr, argboxes, effectinfo)
                 return
             else:
-                # first escape arguments:
-                for argbox in argboxes:
-                    self._escape_box(argbox)
-
                 # Only invalidate things that are escaped
                 # XXX can do better, only do it for the descrs in the effectinfo
                 for descr, cache in self.heap_cache.iteritems():
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -54,6 +54,17 @@
         if level is not None:
             self._tag = level
 
+    def __repr__(self):
+        level = {LEVEL_UNKNOWN: 'UNKNOWN',
+                 LEVEL_NONNULL: 'NONNULL',
+                 LEVEL_KNOWNCLASS: 'KNOWNCLASS',
+                 LEVEL_CONSTANT: 'CONSTANT'}.get(self.getlevel(),
+                                                 self.getlevel())
+        return '<%s %s %s>' % (
+            self.__class__.__name__,
+            level,
+            self.box)
+
     def getlevel(self):
         return self._tag & 0x3
 
diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py
--- a/rpython/rlib/clibffi.py
+++ b/rpython/rlib/clibffi.py
@@ -564,6 +564,7 @@
         self.funcsym = funcsym
 
     def call(self, args_ll, ll_result):
+        # adjust_return_size() should always be used here on ll_result
         assert len(args_ll) == len(self.argtypes), (
             "wrong number of arguments in call to %s(): "
             "%d instead of %d" % (self.name, len(args_ll), len(self.argtypes)))
@@ -594,8 +595,8 @@
                                             intmask(argtypes[i].c_size),
                                             flavor='raw')
         if restype != ffi_type_void:
-            self.ll_result = lltype.malloc(rffi.VOIDP.TO,
-                                           intmask(restype.c_size),
+            size = adjust_return_size(intmask(restype.c_size))
+            self.ll_result = lltype.malloc(rffi.VOIDP.TO, size,
                                            flavor='raw')
 
     def push_arg(self, value):
@@ -693,3 +694,12 @@
             dlclose(self.lib)
             self.lib = rffi.cast(DLLHANDLE, -1)
 
+
+def adjust_return_size(memsize):
+    # Workaround for a strange behavior of libffi: make sure that
+    # we always have at least 8 bytes.  ffi_call() writes 8 bytes
+    # into the buffer even if the function's result type asks for
+    # less.  This strange behavior is documented.
+    if memsize < 8:
+        memsize = 8
+    return memsize
diff --git a/rpython/rlib/libffi.py b/rpython/rlib/libffi.py
--- a/rpython/rlib/libffi.py
+++ b/rpython/rlib/libffi.py
@@ -9,7 +9,8 @@
 from rpython.rlib import jit
 from rpython.rlib import clibffi
 from rpython.rlib.clibffi import FUNCFLAG_CDECL, FUNCFLAG_STDCALL, \
-        AbstractFuncPtr, push_arg_as_ffiptr, c_ffi_call, FFI_TYPE_STRUCT
+        AbstractFuncPtr, push_arg_as_ffiptr, c_ffi_call, FFI_TYPE_STRUCT, \
+        adjust_return_size
 from rpython.rlib.rdynload import dlopen, dlclose, dlsym, dlsym_byordinal
 from rpython.rlib.rdynload import DLLHANDLE
 
@@ -369,8 +370,8 @@
         # XXX: check len(args)?
         ll_result = lltype.nullptr(rffi.CCHARP.TO)
         if self.restype != types.void:
-            ll_result = lltype.malloc(rffi.CCHARP.TO,
-                                      intmask(self.restype.c_size),
+            size = adjust_return_size(intmask(self.restype.c_size))
+            ll_result = lltype.malloc(rffi.CCHARP.TO, size,
                                       flavor='raw')
         ffires = c_ffi_call(self.ll_cif,
                             self.funcsym,

From noreply at buildbot.pypy.org  Tue Feb 24 15:20:04 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 24 Feb 2015 15:20:04 +0100 (CET)
Subject: [pypy-commit] stmgc default: apply fixes from last few commit to C8
Message-ID: <20150224142004.045031C04A7@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: 
Changeset: r1653:d0ca59d95e84
Date: 2015-02-24 15:20 +0100
http://bitbucket.org/pypy/stmgc/changeset/d0ca59d95e84/

Log:	apply fixes from last few commit to C8

diff --git a/c8/stm/extra.c b/c8/stm/extra.c
--- a/c8/stm/extra.c
+++ b/c8/stm/extra.c
@@ -6,15 +6,24 @@
 static long register_callbacks(stm_thread_local_t *tl,
                                void *key, void callback(void *), long index)
 {
-    if (!_stm_in_transaction(tl)) {
-        /* check that the current thread-local is really running a
+    dprintf(("register_callbacks: tl=%p key=%p callback=%p index=%ld\n",
+             tl, key, callback, index));
+    if (tl->associated_segment_num == -1) {
+        /* check that the provided thread-local is really running a
            transaction, and do nothing otherwise. */
+        dprintf(("  NOT IN TRANSACTION\n"));
         return -1;
     }
-
-    if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {
+    /* The tl was only here to check that.  We're really using
+       STM_PSEGMENT below, which is often but not always the
+       segment corresponding to the tl.  One case where it's not
+       the case is if this gets called from stmcb_light_finalizer()
+       from abort_finalizers() from major collections or contention.
+    */
+    if (STM_PSEGMENT->transaction_state != TS_REGULAR) {
         /* ignore callbacks if we're in an inevitable transaction
-           (which cannot abort) */
+           (which cannot abort) or no transaction at all in this segment */
+        dprintf(("  STATE = %d\n", (int)STM_PSEGMENT->transaction_state));
         return -1;
     }
 
@@ -23,15 +32,19 @@
 
     if (callback == NULL) {
         /* double-unregistering works, but return 0 */
-        return tree_delete_item(callbacks, (uintptr_t)key);
+        long res = tree_delete_item(callbacks, (uintptr_t)key);
+        dprintf(("  DELETED %ld\n", res));
+        return res;
     }
     else {
         /* double-registering the same key will crash */
+        dprintf(("  INSERTING\n"));
         tree_insert(callbacks, (uintptr_t)key, (uintptr_t)callback);
         return 1;
     }
 }
 
+
 long stm_call_on_commit(stm_thread_local_t *tl,
                        void *key, void callback(void *))
 {
@@ -39,6 +52,7 @@
     if (result < 0 && callback != NULL) {
         /* no regular transaction running, invoke the callback
            immediately */
+        dprintf(("stm_call_on_commit calls now: %p(%p)\n", callback, key));
         callback(key);
     }
     return result;
@@ -72,8 +86,11 @@
         assert(key != NULL);
         assert(callback != NULL);
 
-        /* The callback may call stm_call_on_abort(key, NULL).  It is ignored,
-           because 'callbacks_on_commit_and_abort' was cleared already. */
+        /* The callback may call stm_call_on_abort(key, NULL)
+           (so with callback==NULL).  It is ignored, because
+           'callbacks_on_commit_and_abort' was cleared already. */
+        dprintf(("invoke_and_clear_user_callbacks(%ld): %p(%p)\n",
+                 index, callback, key));
         callback(key);
 
     } TREE_LOOP_END;
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -388,10 +388,10 @@
         */
 
         /* only for new, uncommitted objects:
-           If 'tl' is currently running, its 'associated_segment_num'
+           If 'tl' is currently running, its 'last_associated_segment_num'
            field is the segment number that contains the correct
            version of its overflowed objects. */
-        char *segment_base = get_segment_base(tl->associated_segment_num);
+        char *segment_base = get_segment_base(tl->last_associated_segment_num);
 
         struct stm_shadowentry_s *current = tl->shadowstack;
         struct stm_shadowentry_s *base = tl->shadowstack_base;
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -223,14 +223,15 @@
         tl->prev = stm_all_thread_locals->prev;
         stm_all_thread_locals->prev->next = tl;
         stm_all_thread_locals->prev = tl;
-        num = (tl->prev->associated_segment_num) % (NB_SEGMENTS-1);
+        num = (tl->prev->last_associated_segment_num) % (NB_SEGMENTS-1);
     }
     tl->thread_local_obj = NULL;
 
     /* assign numbers consecutively, but that's for tests; we could also
        assign the same number to all of them and they would get their own
        numbers automatically. */
-    tl->associated_segment_num = num + 1;
+    tl->associated_segment_num = -1;
+    tl->last_associated_segment_num = num + 1;
     *_get_cpth(tl) = pthread_self();
     _init_shadow_stack(tl);
     set_gs_register(get_segment_base(num + 1));
diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -110,7 +110,7 @@
     assert(_has_mutex());
     assert(_is_tl_registered(tl));
 
-    int num = tl->associated_segment_num - 1; // 0..NB_SEG-1
+    int num = tl->last_associated_segment_num - 1; // 0..NB_SEG-1
     OPT_ASSERT(num >= 0);
     if (sync_ctl.in_use1[num+1] == 0) {
         /* fast-path: we can get the same segment number than the one
@@ -130,8 +130,9 @@
         num = (num+1) % (NB_SEGMENTS-1);
         if (sync_ctl.in_use1[num+1] == 0) {
             /* we're getting 'num', a different number. */
-            dprintf(("acquired different segment: %d->%d\n", tl->associated_segment_num, num+1));
-            tl->associated_segment_num = num+1;
+            dprintf(("acquired different segment: %d->%d\n",
+                     tl->last_associated_segment_num, num+1));
+            tl->last_associated_segment_num = num+1;
             set_gs_register(get_segment_base(num+1));
             goto got_num;
         }
@@ -147,6 +148,7 @@
     sync_ctl.in_use1[num+1] = 1;
     assert(STM_SEGMENT->segment_num == num+1);
     assert(STM_SEGMENT->running_thread == NULL);
+    tl->associated_segment_num = tl->last_associated_segment_num;
     STM_SEGMENT->running_thread = tl;
     return true;
 }
@@ -158,10 +160,12 @@
     cond_signal(C_SEGMENT_FREE);
 
     assert(STM_SEGMENT->running_thread == tl);
+    assert(tl->associated_segment_num == tl->last_associated_segment_num);
+    tl->associated_segment_num = -1;
     STM_SEGMENT->running_thread = NULL;
 
-    assert(sync_ctl.in_use1[tl->associated_segment_num] == 1);
-    sync_ctl.in_use1[tl->associated_segment_num] = 0;
+    assert(sync_ctl.in_use1[tl->last_associated_segment_num] == 1);
+    sync_ctl.in_use1[tl->last_associated_segment_num] = 0;
 }
 
 __attribute__((unused))
@@ -172,9 +176,16 @@
 
 bool _stm_in_transaction(stm_thread_local_t *tl)
 {
-    int num = tl->associated_segment_num;
-    assert(1 <= num && num < NB_SEGMENTS);
-    return get_segment(num)->running_thread == tl;
+    if (tl->associated_segment_num == -1) {
+        return false;
+    }
+    else {
+        int num = tl->associated_segment_num;
+        OPT_ASSERT(1 <= num && num < NB_SEGMENTS);
+        OPT_ASSERT(num == tl->last_associated_segment_num);
+        OPT_ASSERT(get_segment(num)->running_thread == tl);
+        return true;
+    }
 }
 
 void _stm_test_switch(stm_thread_local_t *tl)
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -67,6 +67,7 @@
     long last_abort__bytes_in_nursery;
     /* the next fields are handled internally by the library */
     int associated_segment_num;
+    int last_associated_segment_num;
     struct stm_thread_local_s *prev, *next;
     void *creating_pthread[2];
 } stm_thread_local_t;

From noreply at buildbot.pypy.org  Tue Feb 24 15:57:30 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 24 Feb 2015 15:57:30 +0100 (CET)
Subject: [pypy-commit] pypy default: (fijal,
	arigo)  Kill random stupid subclasses
Message-ID: <20150224145730.A801C1C04BE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76101:32ed1868d4fa
Date: 2015-02-24 15:20 +0100
http://bitbucket.org/pypy/pypy/changeset/32ed1868d4fa/

Log:	(fijal, arigo) Kill random stupid subclasses

diff --git a/rpython/jit/metainterp/optimizeopt/intutils.py b/rpython/jit/metainterp/optimizeopt/intutils.py
--- a/rpython/jit/metainterp/optimizeopt/intutils.py
+++ b/rpython/jit/metainterp/optimizeopt/intutils.py
@@ -252,42 +252,21 @@
             guards.append(op)
 
 
-class IntUpperBound(IntBound):
-    def __init__(self, upper):
-        self.has_upper = True
-        self.has_lower = False
-        self.upper = upper
-        self.lower = 0
+def IntUpperBound(upper):
+    b = IntBound(lower=0, upper=upper)
+    b.has_lower = False
+    return b
 
-class IntLowerBound(IntBound):
-    def __init__(self, lower):
-        self.has_upper = False
-        self.has_lower = True
-        self.upper = 0
-        self.lower = lower
+def IntLowerBound(lower):
+    b = IntBound(upper=0, lower=lower)
+    b.has_upper = False
+    return b
 
-class IntUnbounded(IntBound):
-    def __init__(self):
-        self.has_upper = False
-        self.has_lower = False
-        self.upper = 0
-        self.lower = 0
-
-class ImmutableIntUnbounded(IntUnbounded):
-    def _raise(self):
-        raise TypeError('ImmutableIntUnbounded is immutable')
-    def make_le(self, other):
-        self._raise()
-    def make_lt(self, other):
-        self._raise()
-    def make_ge(self, other):
-        self._raise()
-    def make_gt(self, other):
-        self._raise()
-    def make_constant(self, value):
-        self._raise()
-    def intersect(self, other):
-        self._raise()
+def IntUnbounded():
+    b = IntBound(upper=0, lower=0)
+    b.has_lower = False
+    b.has_upper = False
+    return b
 
 def min4(t):
     return min(min(t[0], t[1]), min(t[2], t[3]))
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -3,7 +3,6 @@
 from rpython.jit.metainterp.history import BoxInt, BoxFloat, Const, ConstInt,\
      REF, BoxPtr, ConstPtr, ConstFloat
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound, IntUnbounded,\
-                                                     ImmutableIntUnbounded, \
                                                      IntLowerBound, MININT,\
                                                      MAXINT
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
@@ -323,19 +322,17 @@
 class IntOptValue(OptValue):
     _attrs_ = ('intbound',)
 
-    intbound = ImmutableIntUnbounded()
-
     def __init__(self, box, level=None, known_class=None, intbound=None):
         OptValue.__init__(self, box, level, None, None)
         if isinstance(box, Const):
+            value = box.getint()
+            self.intbound = IntBound(value, value)
             return
         if intbound:
             self.intbound = intbound
         else:
-            if isinstance(box, BoxInt):
-                self.intbound = IntBound(MININT, MAXINT)
-            else:
-                self.intbound = IntUnbounded()
+            assert isinstance(box, BoxInt)
+            self.intbound = IntBound(MININT, MAXINT)
 
     def copy_from(self, other_value):
         assert isinstance(other_value, IntOptValue)

From noreply at buildbot.pypy.org  Tue Feb 24 15:57:31 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 24 Feb 2015 15:57:31 +0100 (CET)
Subject: [pypy-commit] pypy optresult: hg merge default
Message-ID: <20150224145731.E5AA41C04BE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: optresult
Changeset: r76102:78beb9af0d46
Date: 2015-02-24 15:22 +0100
http://bitbucket.org/pypy/pypy/changeset/78beb9af0d46/

Log:	hg merge default

diff --git a/rpython/jit/metainterp/optimizeopt/intutils.py b/rpython/jit/metainterp/optimizeopt/intutils.py
--- a/rpython/jit/metainterp/optimizeopt/intutils.py
+++ b/rpython/jit/metainterp/optimizeopt/intutils.py
@@ -250,42 +250,21 @@
             guards.append(op)
 
 
-class IntUpperBound(IntBound):
-    def __init__(self, upper):
-        self.has_upper = True
-        self.has_lower = False
-        self.upper = upper
-        self.lower = 0
+def IntUpperBound(upper):
+    b = IntBound(lower=0, upper=upper)
+    b.has_lower = False
+    return b
 
-class IntLowerBound(IntBound):
-    def __init__(self, lower):
-        self.has_upper = False
-        self.has_lower = True
-        self.upper = 0
-        self.lower = lower
+def IntLowerBound(lower):
+    b = IntBound(upper=0, lower=lower)
+    b.has_upper = False
+    return b
 
-class IntUnbounded(IntBound):
-    def __init__(self):
-        self.has_upper = False
-        self.has_lower = False
-        self.upper = 0
-        self.lower = 0
-
-class ImmutableIntUnbounded(IntUnbounded):
-    def _raise(self):
-        raise TypeError('ImmutableIntUnbounded is immutable')
-    def make_le(self, other):
-        self._raise()
-    def make_lt(self, other):
-        self._raise()
-    def make_ge(self, other):
-        self._raise()
-    def make_gt(self, other):
-        self._raise()
-    def make_constant(self, value):
-        self._raise()
-    def intersect(self, other):
-        self._raise()
+def IntUnbounded():
+    b = IntBound(upper=0, lower=0)
+    b.has_lower = False
+    b.has_upper = False
+    return b
 
 def min4(t):
     return min(min(t[0], t[1]), min(t[2], t[3]))
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -3,7 +3,7 @@
 from rpython.jit.metainterp.logger import LogOperations
 from rpython.jit.metainterp.history import Const, ConstInt, REF
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound,\
-     ImmutableIntUnbounded, IntLowerBound, MININT, MAXINT
+     IntLowerBound, MININT, MAXINT
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from rpython.jit.metainterp.resoperation import rop, ResOperation,\
      AbstractResOp, AbstractInputArg, GuardResOp
@@ -323,8 +323,6 @@
 class IntOptInfo(OptInfo):
     _attrs_ = ('intbound',)
 
-    intbound = ImmutableIntUnbounded()
-
     def __init__(self, level=LEVEL_UNKNOWN, known_class=None, intbound=None):
         OptInfo.__init__(self, level, None, None)
         if intbound:

From noreply at buildbot.pypy.org  Tue Feb 24 15:57:33 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 24 Feb 2015 15:57:33 +0100 (CET)
Subject: [pypy-commit] pypy optresult: (fijal, arigo)  fix optimize_INT_SUB,
	really.
Message-ID: <20150224145733.1B7441C04BE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: optresult
Changeset: r76103:8f2d559186d6
Date: 2015-02-24 15:54 +0100
http://bitbucket.org/pypy/pypy/changeset/8f2d559186d6/

Log:	(fijal, arigo) fix optimize_INT_SUB, really.

diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -177,8 +177,6 @@
 class Const(AbstractValue):
     __slots__ = ()
 
-    forwarded = None
-
     @staticmethod
     def _new(x):
         "NOT_RPYTHON"
diff --git a/rpython/jit/metainterp/optimizeopt/generalize.py b/rpython/jit/metainterp/optimizeopt/generalize.py
--- a/rpython/jit/metainterp/optimizeopt/generalize.py
+++ b/rpython/jit/metainterp/optimizeopt/generalize.py
@@ -1,5 +1,4 @@
-from rpython.jit.metainterp.optimizeopt.optimizer import MININT, MAXINT,\
-     IntOptInfo
+from rpython.jit.metainterp.optimizeopt.optimizer import MININT, MAXINT
 
 
 class GeneralizationStrategy(object):
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -4,7 +4,7 @@
 from rpython.jit.metainterp.optimizeopt.intutils import (IntBound, IntLowerBound,
     IntUpperBound)
 from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, CONST_1,
-    CONST_0, MODE_ARRAY, MODE_STR, MODE_UNICODE, IntOptInfo)
+    CONST_0, MODE_ARRAY, MODE_STR, MODE_UNICODE)
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from rpython.jit.metainterp.resoperation import rop, AbstractResOp
 from rpython.jit.backend.llsupport import symbolic
@@ -111,14 +111,12 @@
             r.getintbound().intersect(IntBound(0, next_pow2_m1(lesser)))
 
     def optimize_INT_SUB(self, op):
-        v1 = self.getinfo(op.getarg(0))
-        v2 = self.getinfo(op.getarg(1))
         self.emit_operation(op)
-        xxx
-        r = self.getvalue(op)
-        b = v1.getintbound().sub_bound(v2.getintbound())
+        v1 = self.getintbound(op.getarg(0))
+        v2 = self.getintbound(op.getarg(1))
+        b = v1.sub_bound(v2)
         if b.bounded():
-            r.getintbound().intersect(b)
+            self.getintbound(op).intersect(b)
 
     def optimize_INT_ADD(self, op):
         v1 = self.getvalue(op.getarg(0))
diff --git a/rpython/jit/metainterp/optimizeopt/intutils.py b/rpython/jit/metainterp/optimizeopt/intutils.py
--- a/rpython/jit/metainterp/optimizeopt/intutils.py
+++ b/rpython/jit/metainterp/optimizeopt/intutils.py
@@ -1,14 +1,15 @@
 from rpython.rlib.rarithmetic import ovfcheck, LONG_BIT, maxint, is_valid_int
 from rpython.rlib.objectmodel import we_are_translated
-from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.jit.metainterp.resoperation import rop, ResOperation, AbstractValue
 from rpython.jit.metainterp.history import ConstInt
 
 MAXINT = maxint
 MININT = -maxint - 1
 
 
-class IntBound(object):
+class IntBound(AbstractValue):
     _attrs_ = ('has_upper', 'has_lower', 'upper', 'lower')
+    is_info_class = True
 
     def __init__(self, lower, upper):
         self.has_upper = True
@@ -266,6 +267,9 @@
     b.has_upper = False
     return b
 
+def ConstIntBound(value):
+    return IntBound(value, value)
+
 def min4(t):
     return min(min(t[0], t[1]), min(t[2], t[3]))
 
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -3,10 +3,10 @@
 from rpython.jit.metainterp.logger import LogOperations
 from rpython.jit.metainterp.history import Const, ConstInt, REF
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound,\
-     IntLowerBound, MININT, MAXINT
+     IntLowerBound, MININT, MAXINT, IntUnbounded, ConstIntBound
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from rpython.jit.metainterp.resoperation import rop, ResOperation,\
-     AbstractResOp, AbstractInputArg, GuardResOp
+     AbstractResOp, AbstractInputArg, GuardResOp, AbstractValue
 from rpython.jit.metainterp.typesystem import llhelper
 from rpython.tool.pairtype import extendabletype
 from rpython.rlib.debug import debug_print
@@ -43,16 +43,172 @@
                 self.descr == other.descr and
                 self.bound.contains_bound(other.bound))
 
-class OptInfo(object):
-    _attrs_ = ('_tag',)
+
+## class OptInfo(object):
+
+##     def getlevel(self):
+##         return self._tag & 0x3
+
+##     def setlevel(self, level):
+##         self._tag = (self._tag & (~0x3)) | level
+
+##     def import_from(self, other, optimizer):
+##         if self.getlevel() == LEVEL_CONSTANT:
+##             assert other.getlevel() == LEVEL_CONSTANT
+##             assert other.box.same_constant(self.box)
+##             return
+##         assert self.getlevel() <= LEVEL_NONNULL
+##         if other.getlevel() == LEVEL_CONSTANT:
+##             self.make_constant(other.get_key_box())
+##         elif other.getlevel() == LEVEL_KNOWNCLASS:
+##             self.make_constant_class(None, other.get_known_class())
+##         else:
+##             if other.getlevel() == LEVEL_NONNULL:
+##                 self.ensure_nonnull()
+
+##     def make_guards(self, box):
+##         if self.getlevel() == LEVEL_CONSTANT:
+##             op = ResOperation(rop.GUARD_VALUE, [box, self.box], None)
+##             return [op]
+##         return []
+
+##     def copy_from(self, other_value):
+##         assert isinstance(other_value, OptValue)
+##         self.box = other_value.box
+##         self._tag = other_value._tag
+
+##     def force_box(self, optforce):
+##         xxx
+##         return self.box
+
+##     def force_at_end_of_preamble(self, already_forced, optforce):
+##         return self
+
+##     # visitor API
+
+##     def visitor_walk_recursive(self, visitor):
+##         pass
+
+##     @specialize.argtype(1)
+##     def visitor_dispatch_virtual_type(self, visitor):
+##         if self.is_virtual():
+##             return self._visitor_dispatch_virtual_type(visitor)
+##         else:
+##             return visitor.visit_not_virtual(self)
+
+##     @specialize.argtype(1)
+##     def _visitor_dispatch_virtual_type(self, visitor):
+##         assert 0, "unreachable"
+
+##     def is_constant(self):
+##         return self.getlevel() == LEVEL_CONSTANT
+
+##     def is_null(self):
+##         if self.is_constant():
+##             box = self.box
+##             assert isinstance(box, Const)
+##             return not box.nonnull()
+##         return False
+
+##     def same_value(self, other):
+##         if not other:
+##             return False
+##         if self.is_constant() and other.is_constant():
+##             return self.box.same_constant(other.box)
+##         return self is other
+
+##     def is_nonnull(self):
+##         level = self.getlevel()
+##         if level == LEVEL_NONNULL or level == LEVEL_KNOWNCLASS:
+##             return True
+##         elif level == LEVEL_CONSTANT:
+##             box = self.box
+##             assert isinstance(box, Const)
+##             return box.nonnull()
+##         else:
+##             return False
+
+##     def ensure_nonnull(self):
+##         if self.getlevel() < LEVEL_NONNULL:
+##             self.setlevel(LEVEL_NONNULL)
+
+##     def get_constant_int(self):
+##         assert self.is_constant()
+##         box = self.box
+##         assert isinstance(box, ConstInt)
+##         return box.getint()
+
+##     def is_virtual(self):
+##         return False # overwridden in VirtualInfo
+
+##     def is_forced_virtual(self):
+##         return False
+
+##     def getfield(self, ofs, default):
+##         raise NotImplementedError
+
+##     def setfield(self, ofs, value):
+##         raise NotImplementedError
+
+##     def getlength(self):
+##         raise NotImplementedError
+
+##     def getitem(self, index):
+##         raise NotImplementedError
+
+##     def setitem(self, index, value):
+##         raise NotImplementedError
+
+##     def getitem_raw(self, offset, length, descr):
+##         raise NotImplementedError
+
+##     def setitem_raw(self, offset, length, descr, value):
+##         raise NotImplementedError
+
+##     def getinteriorfield(self, index, ofs, default):
+##         raise NotImplementedError
+
+##     def setinteriorfield(self, index, ofs, value):
+##         raise NotImplementedError
+
+##     def get_missing_null_value(self):
+##         raise NotImplementedError    # only for VArrayValue
+
+##     def make_constant(self, constbox):
+##         """Replace 'self.box' with a Const box."""
+##         assert isinstance(constbox, Const)
+##         self.box = constbox
+##         self.setlevel(LEVEL_CONSTANT)
+
+##     def get_last_guard(self, optimizer):
+##         return None
+
+##     def get_known_class(self):
+##         return None
+
+##     def getlenbound(self):
+##         return None
+
+##     def getintbound(self):
+##         return None
+
+##     def get_constant_class(self, cpu):
+##         return None
+
+
+class PtrOptInfo(AbstractValue):
+    _attrs_ = ('_tag', 'known_class', 'last_guard_pos', 'lenbound')
+    is_info_class = True
+
     _tag = 0
+    known_class = None
+    last_guard_pos = -1
+    lenbound = None
 
-    forwarded = None
-
-    def __init__(self, level=LEVEL_UNKNOWN, known_class=None, intbound=None):
-        assert isinstance(level, int)
-        if level is not None:
-            self._tag = level
+    def __init__(self, box, level=None, known_class=None, intbound=None):
+        OptValue.__init__(self, box, level, None, intbound)
+        if not isinstance(box, Const):
+            self.known_class = known_class
 
     def __repr__(self):
         level = {LEVEL_UNKNOWN: 'UNKNOWN',
@@ -65,167 +221,6 @@
             level,
             self.box)
 
-    def getlevel(self):
-        return self._tag & 0x3
-
-    def setlevel(self, level):
-        self._tag = (self._tag & (~0x3)) | level
-
-    def import_from(self, other, optimizer):
-        if self.getlevel() == LEVEL_CONSTANT:
-            assert other.getlevel() == LEVEL_CONSTANT
-            assert other.box.same_constant(self.box)
-            return
-        assert self.getlevel() <= LEVEL_NONNULL
-        if other.getlevel() == LEVEL_CONSTANT:
-            self.make_constant(other.get_key_box())
-        elif other.getlevel() == LEVEL_KNOWNCLASS:
-            self.make_constant_class(None, other.get_known_class())
-        else:
-            if other.getlevel() == LEVEL_NONNULL:
-                self.ensure_nonnull()
-
-    def make_guards(self, box):
-        if self.getlevel() == LEVEL_CONSTANT:
-            op = ResOperation(rop.GUARD_VALUE, [box, self.box], None)
-            return [op]
-        return []
-
-    def copy_from(self, other_value):
-        assert isinstance(other_value, OptValue)
-        self.box = other_value.box
-        self._tag = other_value._tag
-
-    def force_box(self, optforce):
-        xxx
-        return self.box
-
-    def force_at_end_of_preamble(self, already_forced, optforce):
-        return self
-
-    # visitor API
-
-    def visitor_walk_recursive(self, visitor):
-        pass
-
-    @specialize.argtype(1)
-    def visitor_dispatch_virtual_type(self, visitor):
-        if self.is_virtual():
-            return self._visitor_dispatch_virtual_type(visitor)
-        else:
-            return visitor.visit_not_virtual(self)
-
-    @specialize.argtype(1)
-    def _visitor_dispatch_virtual_type(self, visitor):
-        assert 0, "unreachable"
-
-    def is_constant(self):
-        return self.getlevel() == LEVEL_CONSTANT
-
-    def is_null(self):
-        if self.is_constant():
-            box = self.box
-            assert isinstance(box, Const)
-            return not box.nonnull()
-        return False
-
-    def same_value(self, other):
-        if not other:
-            return False
-        if self.is_constant() and other.is_constant():
-            return self.box.same_constant(other.box)
-        return self is other
-
-    def is_nonnull(self):
-        level = self.getlevel()
-        if level == LEVEL_NONNULL or level == LEVEL_KNOWNCLASS:
-            return True
-        elif level == LEVEL_CONSTANT:
-            box = self.box
-            assert isinstance(box, Const)
-            return box.nonnull()
-        else:
-            return False
-
-    def ensure_nonnull(self):
-        if self.getlevel() < LEVEL_NONNULL:
-            self.setlevel(LEVEL_NONNULL)
-
-    def get_constant_int(self):
-        assert self.is_constant()
-        box = self.box
-        assert isinstance(box, ConstInt)
-        return box.getint()
-
-    def is_virtual(self):
-        return False # overwridden in VirtualInfo
-
-    def is_forced_virtual(self):
-        return False
-
-    def getfield(self, ofs, default):
-        raise NotImplementedError
-
-    def setfield(self, ofs, value):
-        raise NotImplementedError
-
-    def getlength(self):
-        raise NotImplementedError
-
-    def getitem(self, index):
-        raise NotImplementedError
-
-    def setitem(self, index, value):
-        raise NotImplementedError
-
-    def getitem_raw(self, offset, length, descr):
-        raise NotImplementedError
-
-    def setitem_raw(self, offset, length, descr, value):
-        raise NotImplementedError
-
-    def getinteriorfield(self, index, ofs, default):
-        raise NotImplementedError
-
-    def setinteriorfield(self, index, ofs, value):
-        raise NotImplementedError
-
-    def get_missing_null_value(self):
-        raise NotImplementedError    # only for VArrayValue
-
-    def make_constant(self, constbox):
-        """Replace 'self.box' with a Const box."""
-        assert isinstance(constbox, Const)
-        self.box = constbox
-        self.setlevel(LEVEL_CONSTANT)
-
-    def get_last_guard(self, optimizer):
-        return None
-
-    def get_known_class(self):
-        return None
-
-    def getlenbound(self):
-        return None
-
-    def getintbound(self):
-        return None
-
-    def get_constant_class(self, cpu):
-        return None
-
-class PtrOptInfo(OptInfo):
-    _attrs_ = ('known_class', 'last_guard_pos', 'lenbound')
-
-    known_class = None
-    last_guard_pos = -1
-    lenbound = None
-
-    def __init__(self, box, level=None, known_class=None, intbound=None):
-        OptValue.__init__(self, box, level, None, intbound)
-        if not isinstance(box, Const):
-            self.known_class = known_class
-
     def copy_from(self, other_value):
         assert isinstance(other_value, PtrOptValue)
         self.box = other_value.box
@@ -320,76 +315,76 @@
     def get_known_class(self):
         return self.known_class
 
-class IntOptInfo(OptInfo):
-    _attrs_ = ('intbound',)
+## class IntOptInfo(OptInfo):
+##     _attrs_ = ('intbound',)
 
-    def __init__(self, level=LEVEL_UNKNOWN, known_class=None, intbound=None):
-        OptInfo.__init__(self, level, None, None)
-        if intbound:
-            self.intbound = intbound
-        else:
-            self.intbound = IntBound(MININT, MAXINT)
+##     def __init__(self, level=LEVEL_UNKNOWN, known_class=None, intbound=None):
+##         OptInfo.__init__(self, level, None, None)
+##         if intbound:
+##             self.intbound = intbound
+##         else:
+##             self.intbound = IntBound(MININT, MAXINT)
 
-    def copy_from(self, other_value):
-        assert isinstance(other_value, IntOptValue)
-        self.box = other_value.box
-        self.intbound = other_value.intbound
-        self._tag = other_value._tag
+##     def copy_from(self, other_value):
+##         assert isinstance(other_value, IntOptValue)
+##         self.box = other_value.box
+##         self.intbound = other_value.intbound
+##         self._tag = other_value._tag
 
-    def make_constant(self, constbox):
-        """Replace 'self.box' with a Const box."""
-        assert isinstance(constbox, ConstInt)
-        self.box = constbox
-        self.setlevel(LEVEL_CONSTANT)
-        val = constbox.getint()
-        self.intbound = IntBound(val, val)
+##     def make_constant(self, constbox):
+##         """Replace 'self.box' with a Const box."""
+##         assert isinstance(constbox, ConstInt)
+##         self.box = constbox
+##         self.setlevel(LEVEL_CONSTANT)
+##         val = constbox.getint()
+##         self.intbound = IntBound(val, val)
 
-    def is_nonnull(self):
-        if OptValue.is_nonnull(self):
-            return True
-        if self.intbound:
-            if self.intbound.known_gt(IntBound(0, 0)) or \
-               self.intbound.known_lt(IntBound(0, 0)):
-                return True
-        return False
+##     def is_nonnull(self):
+##         if OptValue.is_nonnull(self):
+##             return True
+##         if self.intbound:
+##             if self.intbound.known_gt(IntBound(0, 0)) or \
+##                self.intbound.known_lt(IntBound(0, 0)):
+##                 return True
+##         return False
 
-    def make_nonnull(self, optimizer):
-        assert self.getlevel() < LEVEL_NONNULL
-        self.setlevel(LEVEL_NONNULL)
+##     def make_nonnull(self, optimizer):
+##         assert self.getlevel() < LEVEL_NONNULL
+##         self.setlevel(LEVEL_NONNULL)
 
-    def import_from(self, other, optimizer):
-        OptValue.import_from(self, other, optimizer)
-        if self.getlevel() != LEVEL_CONSTANT:
-            if other.getintbound() is not None: # VRawBufferValue
-                self.intbound.intersect(other.getintbound())
+##     def import_from(self, other, optimizer):
+##         OptValue.import_from(self, other, optimizer)
+##         if self.getlevel() != LEVEL_CONSTANT:
+##             if other.getintbound() is not None: # VRawBufferValue
+##                 self.intbound.intersect(other.getintbound())
 
-    def make_guards(self, box):
-        guards = []
-        level = self.getlevel()
-        if level == LEVEL_CONSTANT:
-            op = ResOperation(rop.GUARD_VALUE, [box, self.box], None)
-            guards.append(op)
-        elif level == LEVEL_KNOWNCLASS:
-            op = ResOperation(rop.GUARD_NONNULL, [box], None)
-            guards.append(op)
-        else:
-            if level == LEVEL_NONNULL:
-                op = ResOperation(rop.GUARD_NONNULL, [box], None)
-                guards.append(op)
-            self.intbound.make_guards(box, guards)
-        return guards
+##     def make_guards(self, box):
+##         guards = []
+##         level = self.getlevel()
+##         if level == LEVEL_CONSTANT:
+##             op = ResOperation(rop.GUARD_VALUE, [box, self.box], None)
+##             guards.append(op)
+##         elif level == LEVEL_KNOWNCLASS:
+##             op = ResOperation(rop.GUARD_NONNULL, [box], None)
+##             guards.append(op)
+##         else:
+##             if level == LEVEL_NONNULL:
+##                 op = ResOperation(rop.GUARD_NONNULL, [box], None)
+##                 guards.append(op)
+##             self.intbound.make_guards(box, guards)
+##         return guards
 
-    def getintbound(self):
-        return self.intbound
+##     def getintbound(self):
+##         return self.intbound
 
-    def get_last_guard(self, optimizer):
-        return None
+##     def get_last_guard(self, optimizer):
+##         return None
 
-    def get_known_class(self):
-        return None
+##     def get_known_class(self):
+##         return None
 
-    def getlenbound(self):
-        return None
+##     def getlenbound(self):
+##         return None
 
 
 CONST_0      = ConstInt(0)
@@ -412,9 +407,18 @@
         self.last_emitted_operation = op
         self.next_optimization.propagate_forward(op)
 
-    # FIXME: Move some of these here?
-    def getinfo(self, op, create=True):
-        return self.optimizer.getinfo(op, create=create)
+    def getintbound(self, op):#, create=True):
+        assert op.type == 'i'
+        while op.get_forwarded() is not None:
+            op = op.get_forwarded()
+        if isinstance(op, IntBound):
+            return op
+        if isinstance(op, ConstInt):
+            return ConstIntBound(op.getint())
+        assert op.type == 'i'
+        intbound = IntUnbounded()
+        op.set_forwarded(intbound)
+        return intbound
 
     def get_box_replacement(self, op):
         return self.optimizer.get_box_replacement(op)
@@ -581,46 +585,32 @@
         else:
             return box
 
-    def getinfo(self, op, create=False):
-        while op.forwarded is not None:
-            op = op.forwarded
-        if isinstance(op, OptInfo) or isinstance(op, Const):
-            return op
-        if not create:
-            return None
-        if op.type == 'r':
-            optinfo = PtrOptInfo()
-        elif op.type == 'i':
-            optinfo = IntOptInfo()
-        else:
-            optinfo = OptInfo()
-        op.forwarded = optinfo
-        return optinfo
-        xxx
-        yyy
+    ## def getinfo(self, op, create=False):
+    ##     xxx
+    ##     yyy
 
-        XXX
-        box = self.getinterned(box)
-        try:
-            value = self.values[box]
-        except KeyError:
-            if box.type == "r":
-                value = self.values[box] = PtrOptValue(box)
-            elif box.type == "i":
-                value = self.values[box] = IntOptValue(box)
-            else:
-                assert box.type == "f"
-                value = self.values[box] = OptValue(box)
-        self.ensure_imported(value)
-        return value
+    ##     XXX
+    ##     box = self.getinterned(box)
+    ##     try:
+    ##         value = self.values[box]
+    ##     except KeyError:
+    ##         if box.type == "r":
+    ##             value = self.values[box] = PtrOptValue(box)
+    ##         elif box.type == "i":
+    ##             value = self.values[box] = IntOptValue(box)
+    ##         else:
+    ##             assert box.type == "f"
+    ##             value = self.values[box] = OptValue(box)
+    ##     self.ensure_imported(value)
+    ##     return value
 
     def get_box_replacement(self, op):
         orig_op = op
-        while (op.forwarded is not None and
-               not isinstance(op.forwarded, OptInfo)):
-            op = op.forwarded
+        while (op.get_forwarded() is not None and
+               not op.get_forwarded().is_info_class):
+            op = op.get_forwarded()
         if op is not orig_op:
-            orig_op.forwarded = op
+            orig_op.set_forwarded(op)
         return op
 
     def ensure_imported(self, value):
@@ -664,8 +654,8 @@
     def replace_op_with(self, op, newopnum, args=None, descr=None):
         newop = op.copy_and_change(newopnum, args, descr)
         if newop.type != 'v':
-            assert op.forwarded is None
-            op.forwarded = newop
+            assert op.get_forwarded() is None  #XXX
+            op.set_forwarded(newop)
         return newop
 
     def make_constant(self, box, constbox):
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -4,7 +4,7 @@
 from rpython.jit.metainterp.history import Const, ConstInt, BoxInt
 from rpython.jit.metainterp.history import CONST_NULL, BoxPtr
 from rpython.jit.metainterp.optimizeopt import optimizer
-from rpython.jit.metainterp.optimizeopt.optimizer import OptInfo, REMOVED
+from rpython.jit.metainterp.optimizeopt.optimizer import REMOVED
 from rpython.jit.metainterp.optimizeopt.util import (make_dispatcher_method,
                                                      descrlist_dict, sort_descrs)
 
@@ -13,7 +13,7 @@
 from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.jit.metainterp.optimizeopt.intutils import IntUnbounded
 
-class AbstractVirtualValue(optimizer.PtrOptInfo):
+class AbstractVirtualValue:  #XXX (optimizer.PtrOptInfo):
     _attrs_ = ('keybox', 'source_op', '_cached_vinfo')
     box = None
     _tag = optimizer.LEVEL_NONNULL
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -4,7 +4,7 @@
 from rpython.jit.metainterp.optimizeopt import virtualize
 from rpython.jit.metainterp.optimizeopt.intutils import IntUnbounded
 from rpython.jit.metainterp.optimizeopt.optimizer import (LEVEL_CONSTANT,
-    LEVEL_KNOWNCLASS, LEVEL_NONNULL, LEVEL_UNKNOWN, OptInfo)
+    LEVEL_KNOWNCLASS, LEVEL_NONNULL, LEVEL_UNKNOWN)
 from rpython.jit.metainterp.resoperation import rop, ResOperation,\
      AbstractInputArg
 from rpython.jit.metainterp.compile import Memo
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -5,7 +5,8 @@
 
 class AbstractValue(object):
     _repr_memo = {}
-    
+    is_info_class = False
+
     def _get_hash_(self):
         return compute_identity_hash(self)
 
@@ -18,6 +19,12 @@
     def is_constant(self):
         return False
 
+    def get_forwarded(self):
+        return None
+
+    def set_forwarded(self, forwarded_to):
+        raise Exception("oups")
+
 DONT_CHANGE = AbstractValue()
 
 def ResOperation(opnum, args, descr=None):
@@ -37,6 +44,8 @@
 class AbstractResOp(AbstractValue):
     """The central ResOperation class, representing one operation."""
 
+    _attrs_ = ('_forwarded',)
+
     # debug
     name = ""
     pc = 0
@@ -45,13 +54,17 @@
     type = 'v'
     boolreflex = -1
     boolinverse = -1
-    forwarded = None # either another resop or OptInfo
-
-    _attrs_ = ('forwarded',)
+    _forwarded = None # either another resop or OptInfo
 
     def getopnum(self):
         return self.opnum
 
+    def get_forwarded(self):
+        return self._forwarded
+
+    def set_forwarded(self, forwarded_to):
+        self._forwarded = forwarded_to
+
     # methods implemented by the arity mixins
     # ---------------------------------------
 
@@ -398,8 +411,14 @@
         return InputArgRef()
 
 class AbstractInputArg(AbstractValue):
-    forwarded = None
-    
+    _forwarded = None
+
+    def get_forwarded(self):
+        return self._forwarded
+
+    def set_forwarded(self, forwarded_to):
+        self._forwarded = forwarded_to
+
     def repr(self, memo):
         try:
             return memo[self]
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -221,7 +221,7 @@
             elif box in liveboxes:
                 tagged = liveboxes[box]
             else:
-                if value is not None and value.is_virtual():
+                if info is not None and info.is_virtual():
                     tagged = tag(v, TAGVIRTUAL)
                     v += 1
                 else:

From noreply at buildbot.pypy.org  Tue Feb 24 15:57:34 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 24 Feb 2015 15:57:34 +0100 (CET)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20150224145734.4252F1C04BE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76104:8919872440c0
Date: 2015-02-24 15:55 +0100
http://bitbucket.org/pypy/pypy/changeset/8919872440c0/

Log:	merge heads

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -59,6 +59,17 @@
             self.make_constant(box)
         # invariant: box is a Const if and only if level == LEVEL_CONSTANT
 
+    def __repr__(self):
+        level = {LEVEL_UNKNOWN: 'UNKNOWN',
+                 LEVEL_NONNULL: 'NONNULL',
+                 LEVEL_KNOWNCLASS: 'KNOWNCLASS',
+                 LEVEL_CONSTANT: 'CONSTANT'}.get(self.getlevel(),
+                                                 self.getlevel())
+        return '<%s %s %s>' % (
+            self.__class__.__name__,
+            level,
+            self.box)
+
     def getlevel(self):
         return self._tag & 0x3
 

From noreply at buildbot.pypy.org  Tue Feb 24 16:43:54 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 24 Feb 2015 16:43:54 +0100 (CET)
Subject: [pypy-commit] pypy optresult: (fijal, arigo)  push stuff
Message-ID: <20150224154354.3C6B21C014F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: optresult
Changeset: r76105:5240488a3374
Date: 2015-02-24 16:42 +0100
http://bitbucket.org/pypy/pypy/changeset/5240488a3374/

Log:	(fijal, arigo) push stuff

diff --git a/rpython/jit/metainterp/optimizeopt/earlyforce.py b/rpython/jit/metainterp/optimizeopt/earlyforce.py
--- a/rpython/jit/metainterp/optimizeopt/earlyforce.py
+++ b/rpython/jit/metainterp/optimizeopt/earlyforce.py
@@ -14,7 +14,8 @@
     def propagate_forward(self, op):
         opnum = op.getopnum()
 
-        if (opnum != rop.SETFIELD_GC and
+        if 0:   # XXX
+          if (opnum != rop.SETFIELD_GC and
             opnum != rop.SETARRAYITEM_GC and
             opnum != rop.SETARRAYITEM_RAW and
             opnum != rop.QUASIIMMUT_FIELD and
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -63,9 +63,9 @@
             dispatch_bounds_ops(self, box)
 
     def optimize_GUARD_TRUE(self, op):
-        v = self.getvalue(op.getarg(0))
         self.emit_operation(op)
-        self.propagate_bounds_backward(op.getarg(0), v)
+        #v = self.getintbound(op.getarg(0))
+        #self.propagate_bounds_backward(op.getarg(0), v) XXX
 
     optimize_GUARD_FALSE = optimize_GUARD_TRUE
     optimize_GUARD_VALUE = optimize_GUARD_TRUE
diff --git a/rpython/jit/metainterp/optimizeopt/intutils.py b/rpython/jit/metainterp/optimizeopt/intutils.py
--- a/rpython/jit/metainterp/optimizeopt/intutils.py
+++ b/rpython/jit/metainterp/optimizeopt/intutils.py
@@ -45,12 +45,14 @@
         return self.make_ge(other.add(1))
 
     def make_constant(self, value):
+        XXXX # don't call me
         self.has_lower = True
         self.has_upper = True
         self.lower = value
         self.upper = value
 
     def make_unbounded(self):
+        XXX  # hum
         self.has_lower = False
         self.has_upper = False
 
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -10,7 +10,7 @@
 from rpython.jit.metainterp.typesystem import llhelper
 from rpython.tool.pairtype import extendabletype
 from rpython.rlib.debug import debug_print
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, we_are_translated
 
 """ The tag field on OptValue has a following meaning:
 
@@ -509,7 +509,6 @@
         self.cpu = metainterp_sd.cpu
         self.loop = loop
         self.logops = LogOperations(metainterp_sd, False)
-        self.values = {}
         self.interned_refs = self.cpu.ts.new_ref_dict()
         self.interned_ints = {}
         self.resumedata_memo = resume.ResumeDataLoopMemo(metainterp_sd)
@@ -659,7 +658,12 @@
         return newop
 
     def make_constant(self, box, constbox):
-        self.getvalue(box).make_constant(constbox)
+        assert isinstance(constbox, ConstInt)
+        box = self.get_box_replacement(box)
+        if not we_are_translated():    # safety-check
+            if box.get_forwarded() is not None:
+                assert box.get_forwarded().contains(constbox.getint())
+        box.set_forwarded(constbox)
 
     def make_constant_int(self, box, intvalue):
         self.make_constant(box, ConstInt(intvalue))
@@ -804,7 +808,7 @@
             raise compile.giveup()
         descr.store_final_boxes(op, newboxes, self.metainterp_sd)
         #
-        if op.getopnum() == rop.GUARD_VALUE:
+        if op.getopnum() == rop.GUARD_VALUE and 0: # XXX
             val = self.getvalue(op.getarg(0))
             if val in self.bool_boxes:
                 # Hack: turn guard_value(bool) into guard_true/guard_false.
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -248,9 +248,8 @@
         self.optimizer.pure_reverse(op)
 
     def optimize_guard(self, op, constbox, emit_operation=True):
-        value = self.getvalue(op.getarg(0))
-        if value.is_constant():
-            box = value.box
+        box = self.get_box_replacement(op.getarg(0))
+        if box.is_constant():
             assert isinstance(box, Const)
             if not box.same_constant(constbox):
                 r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op)
@@ -259,9 +258,9 @@
             return
         if emit_operation:
             self.emit_operation(op)
-        value.make_constant(constbox)
-        if self.optimizer.optheap:
-            self.optimizer.optheap.value_updated(value, self.getvalue(constbox))
+        self.make_constant(box, constbox)
+        #if self.optimizer.optheap:  XXX
+        #    self.optimizer.optheap.value_updated(value, self.getvalue(constbox))
 
     def optimize_GUARD_ISNULL(self, op):
         value = self.getvalue(op.getarg(0))
@@ -286,46 +285,46 @@
         value.make_nonnull(self.optimizer)
 
     def optimize_GUARD_VALUE(self, op):
-        value = self.getvalue(op.getarg(0))
-        if value.is_virtual():
-            arg = value.get_constant_class(self.optimizer.cpu)
-            if arg:
-                addr = arg.getaddr()
-                name = self.optimizer.metainterp_sd.get_name_from_address(addr)
-            else:
-                name = ""
-            raise InvalidLoop('A promote of a virtual %s (a recently allocated object) never makes sense!' % name)
-        old_guard_op = value.get_last_guard(self.optimizer)
-        if old_guard_op and not isinstance(old_guard_op.getdescr(),
-                                           compile.ResumeAtPositionDescr):
-            # there already has been a guard_nonnull or guard_class or
-            # guard_nonnull_class on this value, which is rather silly.
-            # replace the original guard with a guard_value
-            if old_guard_op.getopnum() != rop.GUARD_NONNULL:
-                # This is only safe if the class of the guard_value matches the
-                # class of the guard_*_class, otherwise the intermediate ops might
-                # be executed with wrong classes.
-                previous_classbox = value.get_constant_class(self.optimizer.cpu)
-                expected_classbox = self.optimizer.cpu.ts.cls_of_box(op.getarg(1))
-                assert previous_classbox is not None
-                assert expected_classbox is not None
-                if not previous_classbox.same_constant(expected_classbox):
-                    r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op)
-                    raise InvalidLoop('A GUARD_VALUE (%s) was proven to always fail' % r)
-            descr = compile.ResumeGuardValueDescr()
-            op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
-                        args = [old_guard_op.getarg(0), op.getarg(1)],
-                        descr = descr)
-            # Note: we give explicitly a new descr for 'op'; this is why the
-            # old descr must not be ResumeAtPositionDescr (checked above).
-            # Better-safe-than-sorry but it should never occur: we should
-            # not put in short preambles guard_xxx and guard_value
-            # on the same box.
-            self.optimizer.replace_guard(op, value)
-            descr.make_a_counter_per_value(op)
-            # to be safe
-            if isinstance(value, PtrOptValue):
-                value.last_guard_pos = -1
+        ## value = self.getvalue(op.getarg(0))
+        ## if value.is_virtual():
+        ##     arg = value.get_constant_class(self.optimizer.cpu)
+        ##     if arg:
+        ##         addr = arg.getaddr()
+        ##         name = self.optimizer.metainterp_sd.get_name_from_address(addr)
+        ##     else:
+        ##         name = ""
+        ##     raise InvalidLoop('A promote of a virtual %s (a recently allocated object) never makes sense!' % name)
+        ## old_guard_op = value.get_last_guard(self.optimizer)
+        ## if old_guard_op and not isinstance(old_guard_op.getdescr(),
+        ##                                    compile.ResumeAtPositionDescr):
+        ##     # there already has been a guard_nonnull or guard_class or
+        ##     # guard_nonnull_class on this value, which is rather silly.
+        ##     # replace the original guard with a guard_value
+        ##     if old_guard_op.getopnum() != rop.GUARD_NONNULL:
+        ##         # This is only safe if the class of the guard_value matches the
+        ##         # class of the guard_*_class, otherwise the intermediate ops might
+        ##         # be executed with wrong classes.
+        ##         previous_classbox = value.get_constant_class(self.optimizer.cpu)
+        ##         expected_classbox = self.optimizer.cpu.ts.cls_of_box(op.getarg(1))
+        ##         assert previous_classbox is not None
+        ##         assert expected_classbox is not None
+        ##         if not previous_classbox.same_constant(expected_classbox):
+        ##             r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op)
+        ##             raise InvalidLoop('A GUARD_VALUE (%s) was proven to always fail' % r)
+        ##     descr = compile.ResumeGuardValueDescr()
+        ##     op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
+        ##                 args = [old_guard_op.getarg(0), op.getarg(1)],
+        ##                 descr = descr)
+        ##     # Note: we give explicitly a new descr for 'op'; this is why the
+        ##     # old descr must not be ResumeAtPositionDescr (checked above).
+        ##     # Better-safe-than-sorry but it should never occur: we should
+        ##     # not put in short preambles guard_xxx and guard_value
+        ##     # on the same box.
+        ##     self.optimizer.replace_guard(op, value)
+        ##     descr.make_a_counter_per_value(op)
+        ##     # to be safe
+        ##     if isinstance(value, PtrOptValue):
+        ##         value.last_guard_pos = -1
         constbox = op.getarg(1)
         assert isinstance(constbox, Const)
         self.optimize_guard(op, constbox)
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -214,14 +214,19 @@
         for i in range(length):
             box = boxes[i]
             box = optimizer.get_box_replacement(box)
-            info = optimizer.getinfo(box, create=False)
 
             if isinstance(box, Const):
                 tagged = self.getconst(box)
             elif box in liveboxes:
                 tagged = liveboxes[box]
             else:
-                if info is not None and info.is_virtual():
+                if box.type == 'r':
+                    xxx
+                    info = optimizer.getinfo(box, create=False)
+                    is_virtual = (info is not None and info.is_virtual())
+                else:
+                    is_virtual = False
+                if is_virtual:
                     tagged = tag(v, TAGVIRTUAL)
                     v += 1
                 else:

From noreply at buildbot.pypy.org  Tue Feb 24 16:55:52 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 24 Feb 2015 16:55:52 +0100 (CET)
Subject: [pypy-commit] stmgc default: Bug fix: the test removed by
 07f882f975d6 was wrong but the
Message-ID: <20150224155552.863561C014F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1654:783f8ec3b46f
Date: 2015-02-24 16:50 +0100
http://bitbucket.org/pypy/stmgc/changeset/783f8ec3b46f/

Log:	Bug fix: the test removed by 07f882f975d6 was wrong but the point is
	that it should have broken out of the loop. Fix it differently.

diff --git a/c7/stm/forksupport.c b/c7/stm/forksupport.c
--- a/c7/stm/forksupport.c
+++ b/c7/stm/forksupport.c
@@ -220,6 +220,7 @@
             if (endpagenum == NB_PAGES)
                 break;   /* done */
             pagenum = (uninitialized_page_stop - stm_object_pages) / 4096UL;
+            pagenum--;    /* contains data from largemalloc */
             endpagenum = NB_PAGES;
         }
 

From noreply at buildbot.pypy.org  Tue Feb 24 16:55:53 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 24 Feb 2015 16:55:53 +0100 (CET)
Subject: [pypy-commit] stmgc default: Kill this logic,
 which might be dangerously close to being broken, 
Message-ID: <20150224155553.A8D101C014F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1655:c6c18f188568
Date: 2015-02-24 16:52 +0100
http://bitbucket.org/pypy/stmgc/changeset/c6c18f188568/

Log:	Kill this logic, which might be dangerously close to being broken,
	notably because "minor_collection(/*commit=*/ true)" can then be
	called more than once.

diff --git a/c7/stm/core.c b/c7/stm/core.c
--- a/c7/stm/core.c
+++ b/c7/stm/core.c
@@ -833,7 +833,6 @@
 
 void stm_commit_transaction(void)
 {
- restart_all:
     exec_local_finalizers();
 
     assert(!_has_mutex());
@@ -853,11 +852,6 @@
        Important: we should not call cond_wait() in the meantime. */
     synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK);
 
-    if (any_local_finalizers()) {
-        s_mutex_unlock();
-        goto restart_all;
-    }
-
     /* detect conflicts */
     if (detect_write_read_conflicts())
         goto restart;

From noreply at buildbot.pypy.org  Tue Feb 24 16:55:54 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 24 Feb 2015 16:55:54 +0100 (CET)
Subject: [pypy-commit] stmgc default: merge heads
Message-ID: <20150224155554.A773F1C014F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1656:a4584a403dd7
Date: 2015-02-24 16:52 +0100
http://bitbucket.org/pypy/stmgc/changeset/a4584a403dd7/

Log:	merge heads

diff --git a/c8/stm/extra.c b/c8/stm/extra.c
--- a/c8/stm/extra.c
+++ b/c8/stm/extra.c
@@ -6,15 +6,24 @@
 static long register_callbacks(stm_thread_local_t *tl,
                                void *key, void callback(void *), long index)
 {
-    if (!_stm_in_transaction(tl)) {
-        /* check that the current thread-local is really running a
+    dprintf(("register_callbacks: tl=%p key=%p callback=%p index=%ld\n",
+             tl, key, callback, index));
+    if (tl->associated_segment_num == -1) {
+        /* check that the provided thread-local is really running a
            transaction, and do nothing otherwise. */
+        dprintf(("  NOT IN TRANSACTION\n"));
         return -1;
     }
-
-    if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {
+    /* The tl was only here to check that.  We're really using
+       STM_PSEGMENT below, which is often but not always the
+       segment corresponding to the tl.  One case where it's not
+       the case is if this gets called from stmcb_light_finalizer()
+       from abort_finalizers() from major collections or contention.
+    */
+    if (STM_PSEGMENT->transaction_state != TS_REGULAR) {
         /* ignore callbacks if we're in an inevitable transaction
-           (which cannot abort) */
+           (which cannot abort) or no transaction at all in this segment */
+        dprintf(("  STATE = %d\n", (int)STM_PSEGMENT->transaction_state));
         return -1;
     }
 
@@ -23,15 +32,19 @@
 
     if (callback == NULL) {
         /* double-unregistering works, but return 0 */
-        return tree_delete_item(callbacks, (uintptr_t)key);
+        long res = tree_delete_item(callbacks, (uintptr_t)key);
+        dprintf(("  DELETED %ld\n", res));
+        return res;
     }
     else {
         /* double-registering the same key will crash */
+        dprintf(("  INSERTING\n"));
         tree_insert(callbacks, (uintptr_t)key, (uintptr_t)callback);
         return 1;
     }
 }
 
+
 long stm_call_on_commit(stm_thread_local_t *tl,
                        void *key, void callback(void *))
 {
@@ -39,6 +52,7 @@
     if (result < 0 && callback != NULL) {
         /* no regular transaction running, invoke the callback
            immediately */
+        dprintf(("stm_call_on_commit calls now: %p(%p)\n", callback, key));
         callback(key);
     }
     return result;
@@ -72,8 +86,11 @@
         assert(key != NULL);
         assert(callback != NULL);
 
-        /* The callback may call stm_call_on_abort(key, NULL).  It is ignored,
-           because 'callbacks_on_commit_and_abort' was cleared already. */
+        /* The callback may call stm_call_on_abort(key, NULL)
+           (so with callback==NULL).  It is ignored, because
+           'callbacks_on_commit_and_abort' was cleared already. */
+        dprintf(("invoke_and_clear_user_callbacks(%ld): %p(%p)\n",
+                 index, callback, key));
         callback(key);
 
     } TREE_LOOP_END;
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -388,10 +388,10 @@
         */
 
         /* only for new, uncommitted objects:
-           If 'tl' is currently running, its 'associated_segment_num'
+           If 'tl' is currently running, its 'last_associated_segment_num'
            field is the segment number that contains the correct
            version of its overflowed objects. */
-        char *segment_base = get_segment_base(tl->associated_segment_num);
+        char *segment_base = get_segment_base(tl->last_associated_segment_num);
 
         struct stm_shadowentry_s *current = tl->shadowstack;
         struct stm_shadowentry_s *base = tl->shadowstack_base;
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -223,14 +223,15 @@
         tl->prev = stm_all_thread_locals->prev;
         stm_all_thread_locals->prev->next = tl;
         stm_all_thread_locals->prev = tl;
-        num = (tl->prev->associated_segment_num) % (NB_SEGMENTS-1);
+        num = (tl->prev->last_associated_segment_num) % (NB_SEGMENTS-1);
     }
     tl->thread_local_obj = NULL;
 
     /* assign numbers consecutively, but that's for tests; we could also
        assign the same number to all of them and they would get their own
        numbers automatically. */
-    tl->associated_segment_num = num + 1;
+    tl->associated_segment_num = -1;
+    tl->last_associated_segment_num = num + 1;
     *_get_cpth(tl) = pthread_self();
     _init_shadow_stack(tl);
     set_gs_register(get_segment_base(num + 1));
diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -110,7 +110,7 @@
     assert(_has_mutex());
     assert(_is_tl_registered(tl));
 
-    int num = tl->associated_segment_num - 1; // 0..NB_SEG-1
+    int num = tl->last_associated_segment_num - 1; // 0..NB_SEG-1
     OPT_ASSERT(num >= 0);
     if (sync_ctl.in_use1[num+1] == 0) {
         /* fast-path: we can get the same segment number than the one
@@ -130,8 +130,9 @@
         num = (num+1) % (NB_SEGMENTS-1);
         if (sync_ctl.in_use1[num+1] == 0) {
             /* we're getting 'num', a different number. */
-            dprintf(("acquired different segment: %d->%d\n", tl->associated_segment_num, num+1));
-            tl->associated_segment_num = num+1;
+            dprintf(("acquired different segment: %d->%d\n",
+                     tl->last_associated_segment_num, num+1));
+            tl->last_associated_segment_num = num+1;
             set_gs_register(get_segment_base(num+1));
             goto got_num;
         }
@@ -147,6 +148,7 @@
     sync_ctl.in_use1[num+1] = 1;
     assert(STM_SEGMENT->segment_num == num+1);
     assert(STM_SEGMENT->running_thread == NULL);
+    tl->associated_segment_num = tl->last_associated_segment_num;
     STM_SEGMENT->running_thread = tl;
     return true;
 }
@@ -158,10 +160,12 @@
     cond_signal(C_SEGMENT_FREE);
 
     assert(STM_SEGMENT->running_thread == tl);
+    assert(tl->associated_segment_num == tl->last_associated_segment_num);
+    tl->associated_segment_num = -1;
     STM_SEGMENT->running_thread = NULL;
 
-    assert(sync_ctl.in_use1[tl->associated_segment_num] == 1);
-    sync_ctl.in_use1[tl->associated_segment_num] = 0;
+    assert(sync_ctl.in_use1[tl->last_associated_segment_num] == 1);
+    sync_ctl.in_use1[tl->last_associated_segment_num] = 0;
 }
 
 __attribute__((unused))
@@ -172,9 +176,16 @@
 
 bool _stm_in_transaction(stm_thread_local_t *tl)
 {
-    int num = tl->associated_segment_num;
-    assert(1 <= num && num < NB_SEGMENTS);
-    return get_segment(num)->running_thread == tl;
+    if (tl->associated_segment_num == -1) {
+        return false;
+    }
+    else {
+        int num = tl->associated_segment_num;
+        OPT_ASSERT(1 <= num && num < NB_SEGMENTS);
+        OPT_ASSERT(num == tl->last_associated_segment_num);
+        OPT_ASSERT(get_segment(num)->running_thread == tl);
+        return true;
+    }
 }
 
 void _stm_test_switch(stm_thread_local_t *tl)
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -67,6 +67,7 @@
     long last_abort__bytes_in_nursery;
     /* the next fields are handled internally by the library */
     int associated_segment_num;
+    int last_associated_segment_num;
     struct stm_thread_local_s *prev, *next;
     void *creating_pthread[2];
 } stm_thread_local_t;

From noreply at buildbot.pypy.org  Tue Feb 24 16:57:38 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 24 Feb 2015 16:57:38 +0100 (CET)
Subject: [pypy-commit] pypy optresult: write a very simple test and fix it
	until it passes
Message-ID: <20150224155738.725A21C0334@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r76106:9c7babe1f395
Date: 2015-02-24 17:55 +0200
http://bitbucket.org/pypy/pypy/changeset/9c7babe1f395/

Log:	write a very simple test and fix it until it passes

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -612,23 +612,17 @@
             orig_op.set_forwarded(op)
         return op
 
+    def force_box(self, op):
+        return self.get_box_replacement(op)
+
     def ensure_imported(self, value):
         pass
 
-    @specialize.argtype(0)
     def get_constant_box(self, box):
+        box = self.get_box_replacement(box)
         if isinstance(box, Const):
             return box
-        try:
-            value = self.values[box]
-            self.ensure_imported(value)
-        except KeyError:
-            return None
-        if value.is_constant():
-            constbox = value.box
-            assert isinstance(constbox, Const)
-            return constbox
-        return None
+        #self.ensure_imported(value)
 
     def get_newoperations(self):
         self.flush()
@@ -734,16 +728,10 @@
         orig_op = op
         op = self.replace_op_with(op, op.getopnum())
         for i in range(op.numargs()):
-            arg = op.getarg(i)
-            try:
-                value = self.values[arg]
-            except KeyError:
-                pass
-            else:
-                self.ensure_imported(value)
-                newbox = value.force_box(self)
-                if arg is not newbox:
-                    op.setarg(i, newbox)
+            arg = self.force_box(op.getarg(i))
+            #self.ensure_imported(value)
+            #    newbox = value.force_box(self)
+            op.setarg(i, arg)
         self.metainterp_sd.profiler.count(jitprof.Counters.OPT_OPS)
         if op.is_guard():
             self.metainterp_sd.profiler.count(jitprof.Counters.OPT_GUARDS)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -115,6 +115,21 @@
 
 class BaseTestOptimizeBasic(BaseTestBasic):
 
+    def test_very_simple(self):
+        ops = """
+        [i]
+        i0 = int_sub(i, 1)
+        guard_value(i0, 0) [i0]
+        jump(i0)
+        """
+        expected = """
+        [i]
+        i0 = int_sub(i, 1)
+        guard_value(i0, 0) [i0]
+        jump(0)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_simple(self):
         ops = """
         [i]

From noreply at buildbot.pypy.org  Tue Feb 24 17:37:06 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 24 Feb 2015 17:37:06 +0100 (CET)
Subject: [pypy-commit] pypy optresult: pass the test_simple
Message-ID: <20150224163706.C86D11C04A7@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r76107:f73c8eedd105
Date: 2015-02-24 18:05 +0200
http://bitbucket.org/pypy/pypy/changeset/f73c8eedd105/

Log:	pass the test_simple

diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -49,26 +49,24 @@
         assert not op.is_ovf()
         self.emit_operation(op)
 
-    def propagate_bounds_backward(self, box, v):
+    def propagate_bounds_backward(self, box):
         # FIXME: This takes care of the instruction where box is the reuslt
         #        but the bounds produced by all instructions where box is
         #        an argument might also be tighten
-        b = v.getintbound()
-        if b is None:
-            return # pointer
+        b = self.getintbound(box)
         if b.has_lower and b.has_upper and b.lower == b.upper:
-            v.make_constant(ConstInt(b.lower))
+            self.make_constant_int(box, b.lower)
 
         if isinstance(box, AbstractResOp):
             dispatch_bounds_ops(self, box)
 
-    def optimize_GUARD_TRUE(self, op):
+    def _optimize_guard_true_false_value(self, op):
         self.emit_operation(op)
-        #v = self.getintbound(op.getarg(0))
-        #self.propagate_bounds_backward(op.getarg(0), v) XXX
+        self.propagate_bounds_backward(op.getarg(0))
 
-    optimize_GUARD_FALSE = optimize_GUARD_TRUE
-    optimize_GUARD_VALUE = optimize_GUARD_TRUE
+    optimize_GUARD_TRUE = _optimize_guard_true_false_value
+    optimize_GUARD_FALSE = _optimize_guard_true_false_value
+    optimize_GUARD_VALUE = _optimize_guard_true_false_value
 
     def optimize_INT_OR_or_XOR(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -533,15 +531,15 @@
             self.propagate_bounds_backward(op.getarg(1), v2)
 
     def propagate_bounds_INT_SUB(self, op):
-        v1 = self.getvalue(op.getarg(0))
-        v2 = self.getvalue(op.getarg(1))
-        r = self.getvalue(op)
-        b = r.getintbound().add_bound(v2.getintbound())
-        if v1.getintbound().intersect(b):
-            self.propagate_bounds_backward(op.getarg(0), v1)
-        b = r.getintbound().sub_bound(v1.getintbound()).mul(-1)
-        if v2.getintbound().intersect(b):
-            self.propagate_bounds_backward(op.getarg(1), v2)
+        b1 = self.getintbound(op.getarg(0))
+        b2 = self.getintbound(op.getarg(1))
+        r = self.getintbound(op)
+        b = r.add_bound(b2)
+        if b1.intersect(b):
+            self.propagate_bounds_backward(op.getarg(0))
+        b = r.sub_bound(b1).mul(-1)
+        if b2.intersect(b):
+            self.propagate_bounds_backward(op.getarg(1))
 
     def propagate_bounds_INT_MUL(self, op):
         v1 = self.getvalue(op.getarg(0))
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -657,6 +657,8 @@
         if not we_are_translated():    # safety-check
             if box.get_forwarded() is not None:
                 assert box.get_forwarded().contains(constbox.getint())
+        if box.is_constant():
+            return
         box.set_forwarded(constbox)
 
     def make_constant_int(self, box, intvalue):

From noreply at buildbot.pypy.org  Tue Feb 24 17:37:07 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 24 Feb 2015 17:37:07 +0100 (CET)
Subject: [pypy-commit] pypy optresult: (arigo,
 fijal) hack until we pass test_constant_propagate
Message-ID: <20150224163707.F2E2F1C04A7@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r76108:67f401e16660
Date: 2015-02-24 18:33 +0200
http://bitbucket.org/pypy/pypy/changeset/67f401e16660/

Log:	(arigo, fijal) hack until we pass test_constant_propagate

diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -110,20 +110,20 @@
 
     def optimize_INT_SUB(self, op):
         self.emit_operation(op)
-        v1 = self.getintbound(op.getarg(0))
-        v2 = self.getintbound(op.getarg(1))
-        b = v1.sub_bound(v2)
+        b1 = self.getintbound(op.getarg(0))
+        b2 = self.getintbound(op.getarg(1))
+        b = b1.sub_bound(b2)
         if b.bounded():
             self.getintbound(op).intersect(b)
 
     def optimize_INT_ADD(self, op):
-        v1 = self.getvalue(op.getarg(0))
-        v2 = self.getvalue(op.getarg(1))
         self.emit_operation(op)
-        r = self.getvalue(op)
-        b = v1.getintbound().add_bound(v2.getintbound())
+        b1 = self.getintbound(op.getarg(0))
+        b2 = self.getintbound(op.getarg(1))
+        r = self.getintbound(op)
+        b = b1.add_bound(b2)
         if b.bounded():
-            r.getintbound().intersect(b)
+            r.intersect(b)
 
     def optimize_INT_MUL(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -499,18 +499,19 @@
                     self.propagate_bounds_backward(op.getarg(1), v2)
 
     def propagate_bounds_INT_IS_TRUE(self, op):
-        r = self.getvalue(op)
+        r = self.get_box_replacement(op)
         if r.is_constant():
-            if r.box.same_constant(CONST_1):
-                v1 = self.getvalue(op.getarg(0))
-                if v1.getintbound().known_ge(IntBound(0, 0)):
-                    v1.getintbound().make_gt(IntBound(0, 0))
-                    self.propagate_bounds_backward(op.getarg(0), v1)
+            if r.getint() == 1:
+                b1 = self.getintbound(op.getarg(0))
+                if b1.known_ge(IntBound(0, 0)):
+                    b1.make_gt(IntBound(0, 0))
+                    self.propagate_bounds_backward(op.getarg(0))
 
     def propagate_bounds_INT_IS_ZERO(self, op):
-        r = self.getvalue(op)
+        r = self.get_box_replacement(op)
         if r.is_constant():
-            if r.box.same_constant(CONST_1):
+            if r.getint() == 1:
+                xxx
                 v1 = self.getvalue(op.getarg(0))
                 # Clever hack, we can't use self.make_constant_int yet because
                 # the args aren't in the values dictionary yet so it runs into
@@ -520,15 +521,15 @@
                 self.propagate_bounds_backward(op.getarg(0), v1)
 
     def propagate_bounds_INT_ADD(self, op):
-        v1 = self.getvalue(op.getarg(0))
-        v2 = self.getvalue(op.getarg(1))
-        r = self.getvalue(op)
-        b = r.getintbound().sub_bound(v2.getintbound())
-        if v1.getintbound().intersect(b):
-            self.propagate_bounds_backward(op.getarg(0), v1)
-        b = r.getintbound().sub_bound(v1.getintbound())
-        if v2.getintbound().intersect(b):
-            self.propagate_bounds_backward(op.getarg(1), v2)
+        b1 = self.getintbound(op.getarg(0))
+        b2 = self.getintbound(op.getarg(1))
+        r = self.getintbound(op)
+        b = r.sub_bound(b2)
+        if b1.intersect(b):
+            self.propagate_bounds_backward(op.getarg(0))
+        b = r.sub_bound(b1)
+        if b2.intersect(b):
+            self.propagate_bounds_backward(op.getarg(1))
 
     def propagate_bounds_INT_SUB(self, op):
         b1 = self.getintbound(op.getarg(0))
diff --git a/rpython/jit/metainterp/optimizeopt/intutils.py b/rpython/jit/metainterp/optimizeopt/intutils.py
--- a/rpython/jit/metainterp/optimizeopt/intutils.py
+++ b/rpython/jit/metainterp/optimizeopt/intutils.py
@@ -252,6 +252,20 @@
             op = ResOperation(rop.GUARD_TRUE, [op])
             guards.append(op)
 
+    def is_bool(self):
+        return (self.bounded() and self.known_ge(ConstIntBound(0)) and
+                self.known_le(ConstIntBound(1)))
+
+    def getnullness(self):
+        from rpython.jit.metainterp.optimizeopt import optimizer
+
+        if self.known_gt(IntBound(0, 0)) or \
+           self.known_lt(IntBound(0, 0)):
+            return optimizer.INFO_NONNULL
+        if self.known_ge(IntBound(0, 0)) and \
+           self.known_le(IntBound(0, 0)):
+            return optimizer.INFO_NULL
+        return optimizer.INFO_UNKNOWN
 
 def IntUpperBound(upper):
     b = IntBound(lower=0, upper=upper)
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -393,6 +393,9 @@
 llhelper.CONST_NULLREF = llhelper.CONST_NULL
 REMOVED = AbstractResOp()
 
+INFO_NULL = 0
+INFO_NONNULL = 1
+INFO_UNKNOWN = 2
 
 class Optimization(object):
     next_optimization = None
@@ -420,6 +423,11 @@
         op.set_forwarded(intbound)
         return intbound
 
+    def getnullness(self, op):
+        if op.type == 'i':
+            return self.getintbound(op).getnullness()
+        xxxx
+
     def get_box_replacement(self, op):
         return self.optimizer.get_box_replacement(op)
 
@@ -512,7 +520,6 @@
         self.interned_refs = self.cpu.ts.new_ref_dict()
         self.interned_ints = {}
         self.resumedata_memo = resume.ResumeDataLoopMemo(metainterp_sd)
-        self.bool_boxes = {}
         self.pendingfields = None # set temporarily to a list, normally by
                                   # heap.py, as we're about to generate a guard
         self.quasi_immutable_deps = None
@@ -721,7 +728,7 @@
 
     def emit_operation(self, op):
         if op.returns_bool_result():
-            self.bool_boxes[self.getvalue(op)] = None
+            self.getintbound(op).make_bool()
         self._emit_operation(op)
 
     @specialize.argtype(0)
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -6,7 +6,7 @@
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound
 from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, REMOVED,
-    CONST_0, CONST_1, PtrOptInfo)
+    CONST_0, CONST_1, PtrOptInfo, INFO_NONNULL, INFO_NULL)
 from rpython.jit.metainterp.optimizeopt.util import _findall, make_dispatcher_method
 from rpython.jit.metainterp.resoperation import rop, ResOperation, opclasses,\
      OpHelpers
@@ -124,13 +124,13 @@
             self.optimizer.pure_reverse(op)
 
     def optimize_INT_ADD(self, op):
-        v1 = self.getvalue(op.getarg(0))
-        v2 = self.getvalue(op.getarg(1))
+        arg1 = self.get_box_replacement(op.getarg(0))
+        arg2 = self.get_box_replacement(op.getarg(1))
 
         # If one side of the op is 0 the result is the other side.
-        if v1.is_constant() and v1.box.getint() == 0:
+        if arg1.is_constant() and arg1.getint() == 0:
             self.make_equal_to(op, v2)
-        elif v2.is_constant() and v2.box.getint() == 0:
+        elif arg2.is_constant() and arg2.getint() == 0:
             self.make_equal_to(op, v1)
         else:
             self.emit_operation(op)
@@ -427,16 +427,17 @@
         self.emit_operation(op)
 
     def _optimize_nullness(self, op, box, expect_nonnull):
-        value = self.getvalue(box)
-        if value.is_nonnull():
+        info = self.getnullness(box)
+        if info == INFO_NONNULL:
             self.make_constant_int(op, expect_nonnull)
-        elif value.is_null():
+        elif info == INFO_NULL:
             self.make_constant_int(op, not expect_nonnull)
         else:
             self.emit_operation(op)
 
     def optimize_INT_IS_TRUE(self, op):
-        if self.getvalue(op.getarg(0)) in self.optimizer.bool_boxes:
+        if self.getintbound(op.getarg(0)).is_bool():
+            xxx
             self.make_equal_to(op, self.getvalue(op.getarg(0)))
             return
         self._optimize_nullness(op, op.getarg(0), True)
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -759,21 +759,23 @@
         self.emit_operation(op)
 
     def optimize_INT_ADD(self, op):
-        value = self.getvalue(op.getarg(0))
-        offsetbox = self.get_constant_box(op.getarg(1))
-        if value.is_virtual() and offsetbox is not None:
-            offset = offsetbox.getint()
-            # the following check is constant-folded to False if the
-            # translation occurs without any VRawXxxValue instance around
-            if value.is_about_raw:
-                if isinstance(value, VRawBufferValue):
-                    self.make_virtual_raw_slice(value, offset, op)
-                    return
-                elif isinstance(value, VRawSliceValue):
-                    offset = offset + value.offset
-                    self.make_virtual_raw_slice(value.rawbuffer_value, offset,
-                                                op)
-                    return
+        if 0:
+            XXX
+            value = self.getvalue(op.getarg(0))
+            offsetbox = self.get_constant_box(op.getarg(1))
+            if value.is_virtual() and offsetbox is not None:
+                offset = offsetbox.getint()
+                # the following check is constant-folded to False if the
+                # translation occurs without any VRawXxxValue instance around
+                if value.is_about_raw:
+                    if isinstance(value, VRawBufferValue):
+                        self.make_virtual_raw_slice(value, offset, op)
+                        return
+                    elif isinstance(value, VRawSliceValue):
+                        offset = offset + value.offset
+                        self.make_virtual_raw_slice(value.rawbuffer_value, offset,
+                                                    op)
+                        return
         self.emit_operation(op)
 
     def optimize_ARRAYLEN_GC(self, op):

From noreply at buildbot.pypy.org  Tue Feb 24 17:37:09 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 24 Feb 2015 17:37:09 +0100 (CET)
Subject: [pypy-commit] pypy optresult: whack until we pass the next test
Message-ID: <20150224163709.1EBB81C04A7@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r76109:bca22d1493a6
Date: 2015-02-24 18:34 +0200
http://bitbucket.org/pypy/pypy/changeset/bca22d1493a6/

Log:	whack until we pass the next test

diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -235,17 +235,17 @@
         self.emit_operation(op)
 
     def optimize_INT_ADD_OVF(self, op):
-        v1 = self.getvalue(op.getarg(0))
-        v2 = self.getvalue(op.getarg(1))
-        resbound = v1.getintbound().add_bound(v2.getintbound())
+        b1 = self.getintbound(op.getarg(0))
+        b2 = self.getintbound(op.getarg(1))
+        resbound = b1.add_bound(b2)
         if resbound.bounded():
             # Transform into INT_ADD.  The following guard will be killed
             # by optimize_GUARD_NO_OVERFLOW; if we see instead an
             # optimize_GUARD_OVERFLOW, then InvalidLoop.
             op = self.replace_op_with(op, rop.INT_ADD)
         self.emit_operation(op) # emit the op
-        r = self.getvalue(op)
-        r.getintbound().intersect(resbound)
+        r = self.getintbound(op)
+        r.intersect(resbound)
 
     def optimize_INT_SUB_OVF(self, op):
         v1 = self.getvalue(op.getarg(0))

From noreply at buildbot.pypy.org  Tue Feb 24 17:40:58 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 24 Feb 2015 17:40:58 +0100 (CET)
Subject: [pypy-commit] pypy default: Unify INT_IS_TRUE and INT_IS_ZERO here
Message-ID: <20150224164058.02A1A1C0107@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76110:36e8e37db37a
Date: 2015-02-24 17:39 +0100
http://bitbucket.org/pypy/pypy/changeset/36e8e37db37a/

Log:	Unify INT_IS_TRUE and INT_IS_ZERO here

diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -495,19 +495,15 @@
                 if v2.getintbound().intersect(v1.getintbound()):
                     self.propagate_bounds_backward(op.getarg(1))
 
-    def propagate_bounds_INT_IS_TRUE(self, op):
+    def _propagate_int_is_true_or_zero(self, op, constnonzero, constzero):
         r = self.getvalue(op.result)
         if r.is_constant():
-            if r.box.same_constant(CONST_1):
+            if r.box.same_constant(constnonzero):
                 v1 = self.getvalue(op.getarg(0))
                 if v1.getintbound().known_ge(IntBound(0, 0)):
                     v1.getintbound().make_gt(IntBound(0, 0))
                     self.propagate_bounds_backward(op.getarg(0))
-
-    def propagate_bounds_INT_IS_ZERO(self, op):
-        r = self.getvalue(op.result)
-        if r.is_constant():
-            if r.box.same_constant(CONST_1):
+            elif r.box.same_constant(constzero):
                 v1 = self.getvalue(op.getarg(0))
                 # Clever hack, we can't use self.make_constant_int yet because
                 # the args aren't in the values dictionary yet so it runs into
@@ -516,6 +512,12 @@
                 v1.getintbound().make_lt(IntBound(1, 1))
                 self.propagate_bounds_backward(op.getarg(0))
 
+    def propagate_bounds_INT_IS_TRUE(self, op):
+        self._propagate_int_is_true_or_zero(op, CONST_1, CONST_0)
+
+    def propagate_bounds_INT_IS_ZERO(self, op):
+        self._propagate_int_is_true_or_zero(op, CONST_0, CONST_1)
+
     def propagate_bounds_INT_ADD(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -512,7 +512,7 @@
         [i]
         i1 = int_is_true(i)
         guard_false(i1) [i]
-        jump(i)
+        jump(0)
         """
         self.optimize_loop(ops, expected)
 
@@ -4774,6 +4774,25 @@
         """
         self.optimize_strunicode_loop(ops, expected)
 
+    def test_int_is_zero_bounds(self):
+        ops = """
+        [p0]
+        i0 = strlen(p0)
+        i1 = int_is_zero(i0)
+        guard_false(i1) []
+        i2 = int_ge(0, i0)
+        guard_false(i2) []
+        jump(p0)
+        """
+        expected = """
+        [p0]
+        i0 = strlen(p0)
+        i1 = int_is_zero(i0)
+        guard_false(i1) []
+        jump(p0)
+        """
+        self.optimize_strunicode_loop(ops, expected)
+
     def test_strslice_subtraction_folds(self):
         ops = """
         [p0, i0]
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -669,11 +669,11 @@
         [i]
         i1 = int_is_true(i)
         guard_false(i1) [i]
-        jump(i)
-        """
-        expected = """
-        [i]
-        jump(i)
+        jump()
+        """
+        expected = """
+        []
+        jump()
         """
         self.optimize_loop(ops, expected, preamble)
 

From noreply at buildbot.pypy.org  Tue Feb 24 17:49:53 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 24 Feb 2015 17:49:53 +0100 (CET)
Subject: [pypy-commit] pypy optresult: hg merge default
Message-ID: <20150224164953.8DC491C029A@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: optresult
Changeset: r76111:f66ed19fc711
Date: 2015-02-24 17:47 +0100
http://bitbucket.org/pypy/pypy/changeset/f66ed19fc711/

Log:	hg merge default

diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -498,20 +498,16 @@
                 if v2.getintbound().intersect(v1.getintbound()):
                     self.propagate_bounds_backward(op.getarg(1), v2)
 
-    def propagate_bounds_INT_IS_TRUE(self, op):
+    def _propagate_int_is_true_or_zero(self, op, valnonzero, valzero):
         r = self.get_box_replacement(op)
         if r.is_constant():
-            if r.getint() == 1:
+            if r.getint() == valnonzero:
                 b1 = self.getintbound(op.getarg(0))
                 if b1.known_ge(IntBound(0, 0)):
                     b1.make_gt(IntBound(0, 0))
                     self.propagate_bounds_backward(op.getarg(0))
-
-    def propagate_bounds_INT_IS_ZERO(self, op):
-        r = self.get_box_replacement(op)
-        if r.is_constant():
-            if r.getint() == 1:
-                xxx
+            elif r.getint() == valzero:
+                XXX
                 v1 = self.getvalue(op.getarg(0))
                 # Clever hack, we can't use self.make_constant_int yet because
                 # the args aren't in the values dictionary yet so it runs into
@@ -520,6 +516,12 @@
                 v1.getintbound().make_lt(IntBound(1, 1))
                 self.propagate_bounds_backward(op.getarg(0), v1)
 
+    def propagate_bounds_INT_IS_TRUE(self, op):
+        self._propagate_int_is_true_or_zero(op, 1, 0)
+
+    def propagate_bounds_INT_IS_ZERO(self, op):
+        self._propagate_int_is_true_or_zero(op, 0, 1)
+
     def propagate_bounds_INT_ADD(self, op):
         b1 = self.getintbound(op.getarg(0))
         b2 = self.getintbound(op.getarg(1))
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -530,7 +530,7 @@
         [i]
         i1 = int_is_true(i)
         guard_false(i1) [i]
-        jump(i)
+        jump(0)
         """
         self.optimize_loop(ops, expected)
 
@@ -4797,6 +4797,25 @@
         """
         self.optimize_strunicode_loop(ops, expected)
 
+    def test_int_is_zero_bounds(self):
+        ops = """
+        [p0]
+        i0 = strlen(p0)
+        i1 = int_is_zero(i0)
+        guard_false(i1) []
+        i2 = int_ge(0, i0)
+        guard_false(i2) []
+        jump(p0)
+        """
+        expected = """
+        [p0]
+        i0 = strlen(p0)
+        i1 = int_is_zero(i0)
+        guard_false(i1) []
+        jump(p0)
+        """
+        self.optimize_strunicode_loop(ops, expected)
+
     def test_strslice_subtraction_folds(self):
         ops = """
         [p0, i0]
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -669,11 +669,11 @@
         [i]
         i1 = int_is_true(i)
         guard_false(i1) [i]
-        jump(i)
-        """
-        expected = """
-        [i]
-        jump(i)
+        jump()
+        """
+        expected = """
+        []
+        jump()
         """
         self.optimize_loop(ops, expected, preamble)
 

From noreply at buildbot.pypy.org  Tue Feb 24 18:08:19 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 24 Feb 2015 18:08:19 +0100 (CET)
Subject: [pypy-commit] pypy optresult: pass the first test about ptr
Message-ID: <20150224170819.6CF451C03D1@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r76112:b39c993ea8ec
Date: 2015-02-24 19:06 +0200
http://bitbucket.org/pypy/pypy/changeset/b39c993ea8ec/

Log:	pass the first test about ptr

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -1,7 +1,7 @@
 from rpython.jit.metainterp import jitprof, resume, compile
 from rpython.jit.metainterp.executor import execute_nonspec_const
 from rpython.jit.metainterp.logger import LogOperations
-from rpython.jit.metainterp.history import Const, ConstInt, REF
+from rpython.jit.metainterp.history import Const, ConstInt, REF, ConstPtr
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound,\
      IntLowerBound, MININT, MAXINT, IntUnbounded, ConstIntBound
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
@@ -205,10 +205,16 @@
     last_guard_pos = -1
     lenbound = None
 
-    def __init__(self, box, level=None, known_class=None, intbound=None):
-        OptValue.__init__(self, box, level, None, intbound)
-        if not isinstance(box, Const):
-            self.known_class = known_class
+    #def __init__(self, level=None, known_class=None, intbound=None):
+    #    OptValue.__init__(self, box, level, None, intbound)
+    #    if not isinstance(box, Const):
+    #        self.known_class = known_class
+
+    def getlevel(self):
+        return self._tag & 0x3
+
+    def setlevel(self, level):
+        self._tag = (self._tag & (~0x3)) | level
 
     def __repr__(self):
         level = {LEVEL_UNKNOWN: 'UNKNOWN',
@@ -412,12 +418,13 @@
 
     def getintbound(self, op):#, create=True):
         assert op.type == 'i'
-        while op.get_forwarded() is not None:
-            op = op.get_forwarded()
-        if isinstance(op, IntBound):
-            return op
+        op = self.get_box_replacement(op)
         if isinstance(op, ConstInt):
             return ConstIntBound(op.getint())
+        fw = op.get_forwarded()
+        if isinstance(fw, IntBound):
+            return fw
+        assert fw is None
         assert op.type == 'i'
         intbound = IntUnbounded()
         op.set_forwarded(intbound)
@@ -428,6 +435,20 @@
             return self.getintbound(op).getnullness()
         xxxx
 
+    def getptrinfo(self, op):
+        assert op.type == 'r'
+        op = self.get_box_replacement(op)
+        assert op.type == 'r'
+        if isinstance(op, ConstPtr):
+            xxx
+        fw = op.get_forwarded()
+        if fw is not None:
+            assert isinstance(fw, PtrOptInfo)
+            return fw
+        ptrinfo = PtrOptInfo()
+        op.set_forwarded(ptrinfo)
+        return ptrinfo
+
     def get_box_replacement(self, op):
         return self.optimizer.get_box_replacement(op)
 
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -346,7 +346,7 @@
         value.make_constant_class(None, expectedclassbox)
 
     def optimize_GUARD_CLASS(self, op):
-        value = self.getvalue(op.getarg(0))
+        value = self.getptrinfo(op.getarg(0))
         expectedclassbox = op.getarg(1)
         assert isinstance(expectedclassbox, Const)
         realclassbox = value.get_constant_class(self.optimizer.cpu)
@@ -356,7 +356,6 @@
             r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op)
             raise InvalidLoop('A GUARD_CLASS (%s) was proven to always fail'
                               % r)
-        assert isinstance(value, PtrOptValue)
         old_guard_op = value.get_last_guard(self.optimizer)
         if old_guard_op and not isinstance(old_guard_op.getdescr(),
                                            compile.ResumeAtPositionDescr):

From noreply at buildbot.pypy.org  Tue Feb 24 19:16:02 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 24 Feb 2015 19:16:02 +0100 (CET)
Subject: [pypy-commit] pypy default: Don't compile bridges out of
 guard_not_invalidated. Additionally for it to make
Message-ID: <20150224181602.6B8881C014F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r76113:e36875ce12ab
Date: 2015-02-24 20:14 +0200
http://bitbucket.org/pypy/pypy/changeset/e36875ce12ab/

Log:	Don't compile bridges out of guard_not_invalidated. Additionally for
	it to make sense, we can double-patch stuff (for a bridge and for
	invalidation)

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -682,6 +682,9 @@
 class ResumeGuardNotInvalidated(ResumeGuardDescr):
     guard_opnum = rop.GUARD_NOT_INVALIDATED
 
+    def must_compile(self, deadframe, metainterp_sd, jitdriver_sd):
+        return False
+
 class ResumeAtPositionDescr(ResumeGuardDescr):
     guard_opnum = rop.GUARD_FUTURE_CONDITION
 

From noreply at buildbot.pypy.org  Tue Feb 24 19:16:03 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 24 Feb 2015 19:16:03 +0100 (CET)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20150224181603.AC9EF1C014F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r76114:6d8fc302e70c
Date: 2015-02-24 20:14 +0200
http://bitbucket.org/pypy/pypy/changeset/6d8fc302e70c/

Log:	merge

diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py
--- a/pypy/module/_rawffi/array.py
+++ b/pypy/module/_rawffi/array.py
@@ -15,7 +15,7 @@
 from pypy.module._rawffi.interp_rawffi import unpack_shape_with_length
 from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr
 from rpython.rlib.rarithmetic import r_uint
-from rpython.rlib import rgc
+from rpython.rlib import rgc, clibffi
 
 
 class W_Array(W_DataShape):
@@ -84,14 +84,11 @@
 
 class W_ArrayInstance(W_DataInstance):
     def __init__(self, space, shape, length, address=r_uint(0)):
-        # Workaround for a strange behavior of libffi: make sure that
-        # we always have at least 8 bytes.  For W_ArrayInstances that are
-        # used as the result value of a function call, ffi_call() writes
-        # 8 bytes into it even if the function's result type asks for less.
-        # This strange behavior is documented.
         memsize = shape.size * length
-        if memsize < 8:
-            memsize = 8
+        # For W_ArrayInstances that are used as the result value of a
+        # function call, ffi_call() writes 8 bytes into it even if the
+        # function's result type asks for less.
+        memsize = clibffi.adjust_return_size(memsize)
         W_DataInstance.__init__(self, space, memsize, address)
         self.length = length
         self.shape = shape
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -495,6 +495,7 @@
         try:
             if self.resshape is not None:
                 result = self.resshape.allocate(space, 1, autofree=True)
+                # adjust_return_size() was used here on result.ll_buffer
                 self.ptr.call(args_ll, result.ll_buffer)
                 return space.wrap(result)
             else:
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -199,10 +199,6 @@
                 self._clear_caches_arraycopy(opnum, descr, argboxes, effectinfo)
                 return
             else:
-                # first escape arguments:
-                for argbox in argboxes:
-                    self._escape_box(argbox)
-
                 # Only invalidate things that are escaped
                 # XXX can do better, only do it for the descrs in the effectinfo
                 for descr, cache in self.heap_cache.iteritems():
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -495,19 +495,15 @@
                 if v2.getintbound().intersect(v1.getintbound()):
                     self.propagate_bounds_backward(op.getarg(1))
 
-    def propagate_bounds_INT_IS_TRUE(self, op):
+    def _propagate_int_is_true_or_zero(self, op, constnonzero, constzero):
         r = self.getvalue(op.result)
         if r.is_constant():
-            if r.box.same_constant(CONST_1):
+            if r.box.same_constant(constnonzero):
                 v1 = self.getvalue(op.getarg(0))
                 if v1.getintbound().known_ge(IntBound(0, 0)):
                     v1.getintbound().make_gt(IntBound(0, 0))
                     self.propagate_bounds_backward(op.getarg(0))
-
-    def propagate_bounds_INT_IS_ZERO(self, op):
-        r = self.getvalue(op.result)
-        if r.is_constant():
-            if r.box.same_constant(CONST_1):
+            elif r.box.same_constant(constzero):
                 v1 = self.getvalue(op.getarg(0))
                 # Clever hack, we can't use self.make_constant_int yet because
                 # the args aren't in the values dictionary yet so it runs into
@@ -516,6 +512,12 @@
                 v1.getintbound().make_lt(IntBound(1, 1))
                 self.propagate_bounds_backward(op.getarg(0))
 
+    def propagate_bounds_INT_IS_TRUE(self, op):
+        self._propagate_int_is_true_or_zero(op, CONST_1, CONST_0)
+
+    def propagate_bounds_INT_IS_ZERO(self, op):
+        self._propagate_int_is_true_or_zero(op, CONST_0, CONST_1)
+
     def propagate_bounds_INT_ADD(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
diff --git a/rpython/jit/metainterp/optimizeopt/intutils.py b/rpython/jit/metainterp/optimizeopt/intutils.py
--- a/rpython/jit/metainterp/optimizeopt/intutils.py
+++ b/rpython/jit/metainterp/optimizeopt/intutils.py
@@ -252,42 +252,21 @@
             guards.append(op)
 
 
-class IntUpperBound(IntBound):
-    def __init__(self, upper):
-        self.has_upper = True
-        self.has_lower = False
-        self.upper = upper
-        self.lower = 0
+def IntUpperBound(upper):
+    b = IntBound(lower=0, upper=upper)
+    b.has_lower = False
+    return b
 
-class IntLowerBound(IntBound):
-    def __init__(self, lower):
-        self.has_upper = False
-        self.has_lower = True
-        self.upper = 0
-        self.lower = lower
+def IntLowerBound(lower):
+    b = IntBound(upper=0, lower=lower)
+    b.has_upper = False
+    return b
 
-class IntUnbounded(IntBound):
-    def __init__(self):
-        self.has_upper = False
-        self.has_lower = False
-        self.upper = 0
-        self.lower = 0
-
-class ImmutableIntUnbounded(IntUnbounded):
-    def _raise(self):
-        raise TypeError('ImmutableIntUnbounded is immutable')
-    def make_le(self, other):
-        self._raise()
-    def make_lt(self, other):
-        self._raise()
-    def make_ge(self, other):
-        self._raise()
-    def make_gt(self, other):
-        self._raise()
-    def make_constant(self, value):
-        self._raise()
-    def intersect(self, other):
-        self._raise()
+def IntUnbounded():
+    b = IntBound(upper=0, lower=0)
+    b.has_lower = False
+    b.has_upper = False
+    return b
 
 def min4(t):
     return min(min(t[0], t[1]), min(t[2], t[3]))
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -3,7 +3,6 @@
 from rpython.jit.metainterp.history import BoxInt, BoxFloat, Const, ConstInt,\
      REF, BoxPtr, ConstPtr, ConstFloat
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound, IntUnbounded,\
-                                                     ImmutableIntUnbounded, \
                                                      IntLowerBound, MININT,\
                                                      MAXINT
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
@@ -60,6 +59,17 @@
             self.make_constant(box)
         # invariant: box is a Const if and only if level == LEVEL_CONSTANT
 
+    def __repr__(self):
+        level = {LEVEL_UNKNOWN: 'UNKNOWN',
+                 LEVEL_NONNULL: 'NONNULL',
+                 LEVEL_KNOWNCLASS: 'KNOWNCLASS',
+                 LEVEL_CONSTANT: 'CONSTANT'}.get(self.getlevel(),
+                                                 self.getlevel())
+        return '<%s %s %s>' % (
+            self.__class__.__name__,
+            level,
+            self.box)
+
     def getlevel(self):
         return self._tag & 0x3
 
@@ -323,19 +333,17 @@
 class IntOptValue(OptValue):
     _attrs_ = ('intbound',)
 
-    intbound = ImmutableIntUnbounded()
-
     def __init__(self, box, level=None, known_class=None, intbound=None):
         OptValue.__init__(self, box, level, None, None)
         if isinstance(box, Const):
+            value = box.getint()
+            self.intbound = IntBound(value, value)
             return
         if intbound:
             self.intbound = intbound
         else:
-            if isinstance(box, BoxInt):
-                self.intbound = IntBound(MININT, MAXINT)
-            else:
-                self.intbound = IntUnbounded()
+            assert isinstance(box, BoxInt)
+            self.intbound = IntBound(MININT, MAXINT)
 
     def copy_from(self, other_value):
         assert isinstance(other_value, IntOptValue)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -512,7 +512,7 @@
         [i]
         i1 = int_is_true(i)
         guard_false(i1) [i]
-        jump(i)
+        jump(0)
         """
         self.optimize_loop(ops, expected)
 
@@ -4774,6 +4774,25 @@
         """
         self.optimize_strunicode_loop(ops, expected)
 
+    def test_int_is_zero_bounds(self):
+        ops = """
+        [p0]
+        i0 = strlen(p0)
+        i1 = int_is_zero(i0)
+        guard_false(i1) []
+        i2 = int_ge(0, i0)
+        guard_false(i2) []
+        jump(p0)
+        """
+        expected = """
+        [p0]
+        i0 = strlen(p0)
+        i1 = int_is_zero(i0)
+        guard_false(i1) []
+        jump(p0)
+        """
+        self.optimize_strunicode_loop(ops, expected)
+
     def test_strslice_subtraction_folds(self):
         ops = """
         [p0, i0]
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -669,11 +669,11 @@
         [i]
         i1 = int_is_true(i)
         guard_false(i1) [i]
-        jump(i)
-        """
-        expected = """
-        [i]
-        jump(i)
+        jump()
+        """
+        expected = """
+        []
+        jump()
         """
         self.optimize_loop(ops, expected, preamble)
 
diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py
--- a/rpython/rlib/clibffi.py
+++ b/rpython/rlib/clibffi.py
@@ -564,6 +564,7 @@
         self.funcsym = funcsym
 
     def call(self, args_ll, ll_result):
+        # adjust_return_size() should always be used here on ll_result
         assert len(args_ll) == len(self.argtypes), (
             "wrong number of arguments in call to %s(): "
             "%d instead of %d" % (self.name, len(args_ll), len(self.argtypes)))
@@ -594,8 +595,8 @@
                                             intmask(argtypes[i].c_size),
                                             flavor='raw')
         if restype != ffi_type_void:
-            self.ll_result = lltype.malloc(rffi.VOIDP.TO,
-                                           intmask(restype.c_size),
+            size = adjust_return_size(intmask(restype.c_size))
+            self.ll_result = lltype.malloc(rffi.VOIDP.TO, size,
                                            flavor='raw')
 
     def push_arg(self, value):
@@ -693,3 +694,12 @@
             dlclose(self.lib)
             self.lib = rffi.cast(DLLHANDLE, -1)
 
+
+def adjust_return_size(memsize):
+    # Workaround for a strange behavior of libffi: make sure that
+    # we always have at least 8 bytes.  ffi_call() writes 8 bytes
+    # into the buffer even if the function's result type asks for
+    # less.  This strange behavior is documented.
+    if memsize < 8:
+        memsize = 8
+    return memsize
diff --git a/rpython/rlib/libffi.py b/rpython/rlib/libffi.py
--- a/rpython/rlib/libffi.py
+++ b/rpython/rlib/libffi.py
@@ -9,7 +9,8 @@
 from rpython.rlib import jit
 from rpython.rlib import clibffi
 from rpython.rlib.clibffi import FUNCFLAG_CDECL, FUNCFLAG_STDCALL, \
-        AbstractFuncPtr, push_arg_as_ffiptr, c_ffi_call, FFI_TYPE_STRUCT
+        AbstractFuncPtr, push_arg_as_ffiptr, c_ffi_call, FFI_TYPE_STRUCT, \
+        adjust_return_size
 from rpython.rlib.rdynload import dlopen, dlclose, dlsym, dlsym_byordinal
 from rpython.rlib.rdynload import DLLHANDLE
 
@@ -369,8 +370,8 @@
         # XXX: check len(args)?
         ll_result = lltype.nullptr(rffi.CCHARP.TO)
         if self.restype != types.void:
-            ll_result = lltype.malloc(rffi.CCHARP.TO,
-                                      intmask(self.restype.c_size),
+            size = adjust_return_size(intmask(self.restype.c_size))
+            ll_result = lltype.malloc(rffi.CCHARP.TO, size,
                                       flavor='raw')
         ffires = c_ffi_call(self.ll_cif,
                             self.funcsym,

From noreply at buildbot.pypy.org  Tue Feb 24 23:50:35 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 24 Feb 2015 23:50:35 +0100 (CET)
Subject: [pypy-commit] stmgc default: Merge
Message-ID: <20150224225035.3B79F1C0107@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: 
Changeset: r1658:028d5446e347
Date: 2015-02-24 16:58 +0100
http://bitbucket.org/pypy/stmgc/changeset/028d5446e347/

Log:	Merge

diff --git a/c7/stm/core.c b/c7/stm/core.c
--- a/c7/stm/core.c
+++ b/c7/stm/core.c
@@ -833,7 +833,6 @@
 
 void stm_commit_transaction(void)
 {
- restart_all:
     exec_local_finalizers();
 
     assert(!_has_mutex());
@@ -853,11 +852,6 @@
        Important: we should not call cond_wait() in the meantime. */
     synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK);
 
-    if (any_local_finalizers()) {
-        s_mutex_unlock();
-        goto restart_all;
-    }
-
     /* detect conflicts */
     if (detect_write_read_conflicts())
         goto restart;
diff --git a/c7/stm/forksupport.c b/c7/stm/forksupport.c
--- a/c7/stm/forksupport.c
+++ b/c7/stm/forksupport.c
@@ -220,6 +220,7 @@
             if (endpagenum == NB_PAGES)
                 break;   /* done */
             pagenum = (uninitialized_page_stop - stm_object_pages) / 4096UL;
+            pagenum--;    /* contains data from largemalloc */
             endpagenum = NB_PAGES;
         }
 

From noreply at buildbot.pypy.org  Tue Feb 24 23:50:37 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 24 Feb 2015 23:50:37 +0100 (CET)
Subject: [pypy-commit] stmgc default: c8: fix for stmcb overwriting the
 jmpbuf (one of the leftovers from when we aborted in the signal handler)
Message-ID: <20150224225037.5F48B1C0107@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: 
Changeset: r1660:e2ab71a70a13
Date: 2015-02-24 18:23 +0100
http://bitbucket.org/pypy/stmgc/changeset/e2ab71a70a13/

Log:	c8: fix for stmcb overwriting the jmpbuf (one of the leftovers from
	when we aborted in the signal handler)

diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -28,6 +28,7 @@
     size_t mem_bytes_to_clear_on_abort;
     long last_abort__bytes_in_nursery;
     int associated_segment_num;
+    int last_associated_segment_num;
     struct stm_thread_local_s *prev, *next;
     void *creating_pthread[2];
     ...;
@@ -55,7 +56,7 @@
 void _stm_start_safe_point(void);
 bool _check_stop_safe_point(void);
 
-ssize_t _checked_stmcb_size_rounded_up(struct object_s *obj);
+ssize_t stmcb_size_rounded_up(struct object_s *obj);
 
 bool _checked_stm_write(object_t *obj);
 bool _stm_was_read(object_t *obj);
@@ -123,8 +124,6 @@
 
 void *memset(void *s, int c, size_t n);
 
-ssize_t stmcb_size_rounded_up(struct object_s *obj);
-
 
 object_t *_stm_allocate_old_small(ssize_t size_rounded_up);
 bool (*_stm_smallmalloc_keep)(char *data);
@@ -209,19 +208,6 @@
     return 1;
 }
 
-ssize_t _checked_stmcb_size_rounded_up(struct object_s *obj)
-{
-    stm_thread_local_t *_tl = STM_SEGMENT->running_thread;      \
-    void **jmpbuf = _tl->rjthread.jmpbuf;                       \
-    if (__builtin_setjmp(jmpbuf) == 0) { /* returned directly */\
-        ssize_t res = stmcb_size_rounded_up(obj);
-        clear_jmpbuf(_tl);
-        return res;
-    }
-    clear_jmpbuf(_tl);
-    return 1;
-}
-
 
 bool _check_stop_safe_point(void) {
     CHECKED(_stm_stop_safe_point());
@@ -492,10 +478,7 @@
     return lib._stm_is_accessible_page(pagenum)
 
 def stm_get_obj_size(o):
-    res = lib._checked_stmcb_size_rounded_up(stm_get_real_address(o))
-    if res == 1:
-        raise Conflict()
-    return res
+    return lib.stmcb_size_rounded_up(stm_get_real_address(o))
 
 def stm_get_obj_pages(o):
     start = int(ffi.cast('uintptr_t', o))

From noreply at buildbot.pypy.org  Tue Feb 24 23:50:33 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 24 Feb 2015 23:50:33 +0100 (CET)
Subject: [pypy-commit] stmgc default: c8: add files for finalizers
Message-ID: <20150224225033.BF6581C0107@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: 
Changeset: r1657:88b09f52d77b
Date: 2015-02-24 15:25 +0100
http://bitbucket.org/pypy/stmgc/changeset/88b09f52d77b/

Log:	c8: add files for finalizers

diff --git a/c8/stm/finalizer.c b/c8/stm/finalizer.c
new file mode 100644
--- /dev/null
+++ b/c8/stm/finalizer.c
@@ -0,0 +1,473 @@
+
+
+/* callbacks */
+void (*stmcb_light_finalizer)(object_t *);
+void (*stmcb_finalizer)(object_t *);
+
+
+static void init_finalizers(struct finalizers_s *f)
+{
+    f->objects_with_finalizers = list_create();
+    f->count_non_young = 0;
+    f->run_finalizers = NULL;
+    f->running_next = NULL;
+}
+
+static void setup_finalizer(void)
+{
+    init_finalizers(&g_finalizers);
+}
+
+static void teardown_finalizer(void)
+{
+    if (g_finalizers.run_finalizers != NULL)
+        list_free(g_finalizers.run_finalizers);
+    list_free(g_finalizers.objects_with_finalizers);
+    memset(&g_finalizers, 0, sizeof(g_finalizers));
+}
+
+static void _commit_finalizers(void)
+{
+    if (STM_PSEGMENT->finalizers->run_finalizers != NULL) {
+        /* copy 'STM_PSEGMENT->finalizers->run_finalizers' into
+           'g_finalizers.run_finalizers', dropping any initial NULLs
+           (finalizers already called) */
+        struct list_s *src = STM_PSEGMENT->finalizers->run_finalizers;
+        uintptr_t frm = 0;
+        if (STM_PSEGMENT->finalizers->running_next != NULL) {
+            frm = *STM_PSEGMENT->finalizers->running_next;
+            assert(frm <= list_count(src));
+            *STM_PSEGMENT->finalizers->running_next = (uintptr_t)-1;
+        }
+        if (frm < list_count(src)) {
+            if (g_finalizers.run_finalizers == NULL)
+                g_finalizers.run_finalizers = list_create();
+            g_finalizers.run_finalizers = list_extend(
+                g_finalizers.run_finalizers,
+                src, frm);
+        }
+        list_free(src);
+    }
+
+    /* copy the whole 'STM_PSEGMENT->finalizers->objects_with_finalizers'
+       into 'g_finalizers.objects_with_finalizers' */
+    g_finalizers.objects_with_finalizers = list_extend(
+        g_finalizers.objects_with_finalizers,
+        STM_PSEGMENT->finalizers->objects_with_finalizers, 0);
+    list_free(STM_PSEGMENT->finalizers->objects_with_finalizers);
+
+    free(STM_PSEGMENT->finalizers);
+    STM_PSEGMENT->finalizers = NULL;
+}
+
+static void abort_finalizers(struct stm_priv_segment_info_s *pseg)
+{
+    /* like _commit_finalizers(), but forget everything from the
+       current transaction */
+    if (pseg->finalizers != NULL) {
+        if (pseg->finalizers->run_finalizers != NULL) {
+            if (pseg->finalizers->running_next != NULL) {
+                *pseg->finalizers->running_next = (uintptr_t)-1;
+            }
+            list_free(pseg->finalizers->run_finalizers);
+        }
+        list_free(pseg->finalizers->objects_with_finalizers);
+        free(pseg->finalizers);
+        pseg->finalizers = NULL;
+    }
+
+    /* call the light finalizers for objects that are about to
+       be forgotten from the current transaction */
+    char *old_gs_register = STM_SEGMENT->segment_base;
+    bool must_fix_gs = old_gs_register != pseg->pub.segment_base;
+
+    struct list_s *lst = pseg->young_objects_with_light_finalizers;
+    long i, count = list_count(lst);
+    if (lst > 0) {
+        for (i = 0; i < count; i++) {
+            object_t *obj = (object_t *)list_item(lst, i);
+            assert(_is_young(obj));
+            if (must_fix_gs) {
+                set_gs_register(pseg->pub.segment_base);
+                must_fix_gs = false;
+            }
+            stmcb_light_finalizer(obj);
+        }
+        list_clear(lst);
+    }
+
+    /* also deals with overflow objects: they are at the tail of
+       old_objects_with_light_finalizers (this list is kept in order
+       and we cannot add any already-committed object) */
+    lst = pseg->old_objects_with_light_finalizers;
+    count = list_count(lst);
+    while (count > 0) {
+        object_t *obj = (object_t *)list_item(lst, --count);
+        if (!IS_OVERFLOW_OBJ(pseg, obj))
+            break;
+        lst->count = count;
+        if (must_fix_gs) {
+            set_gs_register(pseg->pub.segment_base);
+            must_fix_gs = false;
+        }
+        stmcb_light_finalizer(obj);
+    }
+
+    if (STM_SEGMENT->segment_base != old_gs_register)
+        set_gs_register(old_gs_register);
+}
+
+
+void stm_enable_light_finalizer(object_t *obj)
+{
+    if (_is_young(obj)) {
+        LIST_APPEND(STM_PSEGMENT->young_objects_with_light_finalizers, obj);
+    }
+    else {
+        assert(_is_from_same_transaction(obj));
+        LIST_APPEND(STM_PSEGMENT->old_objects_with_light_finalizers, obj);
+    }
+}
+
+object_t *stm_allocate_with_finalizer(ssize_t size_rounded_up)
+{
+    object_t *obj = _stm_allocate_external(size_rounded_up);
+
+    if (STM_PSEGMENT->finalizers == NULL) {
+        struct finalizers_s *f = malloc(sizeof(struct finalizers_s));
+        if (f == NULL)
+            stm_fatalerror("out of memory in create_finalizers");   /* XXX */
+        init_finalizers(f);
+        STM_PSEGMENT->finalizers = f;
+    }
+    assert(STM_PSEGMENT->finalizers->count_non_young
+           <= list_count(STM_PSEGMENT->finalizers->objects_with_finalizers));
+    LIST_APPEND(STM_PSEGMENT->finalizers->objects_with_finalizers, obj);
+    return obj;
+}
+
+
+/************************************************************/
+/*  Light finalizers
+*/
+
+static void deal_with_young_objects_with_finalizers(void)
+{
+    /* for light finalizers */
+    struct list_s *lst = STM_PSEGMENT->young_objects_with_light_finalizers;
+    long i, count = list_count(lst);
+    for (i = 0; i < count; i++) {
+        object_t *obj = (object_t *)list_item(lst, i);
+        assert(_is_young(obj));
+
+        object_t *TLPREFIX *pforwarded_array = (object_t *TLPREFIX *)obj;
+        if (pforwarded_array[0] != GCWORD_MOVED) {
+            /* not moved: the object dies */
+            stmcb_light_finalizer(obj);
+        }
+        else {
+            obj = pforwarded_array[1]; /* moved location */
+            assert(!_is_young(obj));
+            LIST_APPEND(STM_PSEGMENT->old_objects_with_light_finalizers, obj);
+        }
+    }
+    list_clear(lst);
+}
+
+static void deal_with_old_objects_with_finalizers(void)
+{
+    /* for light finalizers */
+    int old_gs_register = STM_SEGMENT->segment_num;
+    int current_gs_register = old_gs_register;
+    long j;
+    for (j = 1; j <= NB_SEGMENTS; j++) {
+        struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
+
+        struct list_s *lst = pseg->old_objects_with_light_finalizers;
+        long i, count = list_count(lst);
+        lst->count = 0;
+        for (i = 0; i < count; i++) {
+            object_t *obj = (object_t *)list_item(lst, i);
+            if (!mark_visited_test(obj)) {
+                /* not marked: object dies */
+                /* we're calling the light finalizer in the same
+                   segment as where it was originally registered.  For
+                   objects that existed since a long time, it doesn't
+                   change anything: any thread should see the same old
+                   content (because if it wasn't the case, the object
+                   would be in a 'modified_old_objects' list
+                   somewhere, and so it wouldn't be dead).  But it's
+                   important if the object was created by the same
+                   transaction: then only that segment sees valid
+                   content.
+                */
+                if (j != current_gs_register) {
+                    set_gs_register(get_segment_base(j));
+                    current_gs_register = j;
+                }
+                stmcb_light_finalizer(obj);
+            }
+            else {
+                /* object survives */
+                list_set_item(lst, lst->count++, (uintptr_t)obj);
+            }
+        }
+    }
+    if (old_gs_register != current_gs_register)
+        set_gs_register(get_segment_base(old_gs_register));
+}
+
+
+/************************************************************/
+/*  Algorithm for regular (non-light) finalizers.
+    Follows closely pypy/doc/discussion/finalizer-order.rst
+    as well as rpython/memory/gc/minimark.py.
+*/
+
+static inline int _finalization_state(object_t *obj)
+{
+    /* Returns the state, "0", 1, 2 or 3, as per finalizer-order.rst.
+       One difference is that the official state 0 is returned here
+       as a number that is <= 0. */
+    uintptr_t lock_idx = mark_loc(obj);
+    return write_locks[lock_idx] - (WL_FINALIZ_ORDER_1 - 1);
+}
+
+static void _bump_finalization_state_from_0_to_1(object_t *obj)
+{
+    uintptr_t lock_idx = mark_loc(obj);
+    assert(write_locks[lock_idx] < WL_FINALIZ_ORDER_1);
+    write_locks[lock_idx] = WL_FINALIZ_ORDER_1;
+}
+
+static struct list_s *_finalizer_tmpstack;
+static struct list_s *_finalizer_emptystack;
+static struct list_s *_finalizer_pending;
+
+static inline void _append_to_finalizer_tmpstack(object_t **pobj)
+{
+    object_t *obj = *pobj;
+    if (obj != NULL)
+        LIST_APPEND(_finalizer_tmpstack, obj);
+}
+
+static inline struct list_s *finalizer_trace(char *base, object_t *obj,
+                                             struct list_s *lst)
+{
+    struct object_s *realobj = (struct object_s *)REAL_ADDRESS(base, obj);
+    _finalizer_tmpstack = lst;
+    stmcb_trace(realobj, &_append_to_finalizer_tmpstack);
+    return _finalizer_tmpstack;
+}
+
+static void _recursively_bump_finalization_state(char *base, object_t *obj,
+                                                 int to_state)
+{
+    struct list_s *tmpstack = _finalizer_emptystack;
+    assert(list_is_empty(tmpstack));
+
+    while (1) {
+        if (_finalization_state(obj) == to_state - 1) {
+            /* bump to the next state */
+            write_locks[mark_loc(obj)]++;
+
+            /* trace */
+            tmpstack = finalizer_trace(base, obj, tmpstack);
+        }
+
+        if (list_is_empty(tmpstack))
+            break;
+
+        obj = (object_t *)list_pop_item(tmpstack);
+    }
+    _finalizer_emptystack = tmpstack;
+}
+
+static struct list_s *mark_finalize_step1(char *base, struct finalizers_s *f)
+{
+    if (f == NULL)
+        return NULL;
+
+    struct list_s *marked = list_create();
+
+    struct list_s *lst = f->objects_with_finalizers;
+    long i, count = list_count(lst);
+    lst->count = 0;
+    f->count_non_young = 0;
+
+    for (i = 0; i < count; i++) {
+        object_t *x = (object_t *)list_item(lst, i);
+
+        assert(_finalization_state(x) != 1);
+        if (_finalization_state(x) >= 2) {
+            list_set_item(lst, lst->count++, (uintptr_t)x);
+            continue;
+        }
+        LIST_APPEND(marked, x);
+
+        struct list_s *pending = _finalizer_pending;
+        LIST_APPEND(pending, x);
+        while (!list_is_empty(pending)) {
+            object_t *y = (object_t *)list_pop_item(pending);
+            int state = _finalization_state(y);
+            if (state <= 0) {
+                _bump_finalization_state_from_0_to_1(y);
+                pending = finalizer_trace(base, y, pending);
+            }
+            else if (state == 2) {
+                _recursively_bump_finalization_state(base, y, 3);
+            }
+        }
+        _finalizer_pending = pending;
+        assert(_finalization_state(x) == 1);
+        _recursively_bump_finalization_state(base, x, 2);
+    }
+    return marked;
+}
+
+static void mark_finalize_step2(char *base, struct finalizers_s *f,
+                                struct list_s *marked)
+{
+    if (f == NULL)
+        return;
+
+    struct list_s *run_finalizers = f->run_finalizers;
+
+    long i, count = list_count(marked);
+    for (i = 0; i < count; i++) {
+        object_t *x = (object_t *)list_item(marked, i);
+
+        int state = _finalization_state(x);
+        assert(state >= 2);
+        if (state == 2) {
+            if (run_finalizers == NULL)
+                run_finalizers = list_create();
+            LIST_APPEND(run_finalizers, x);
+            _recursively_bump_finalization_state(base, x, 3);
+        }
+        else {
+            struct list_s *lst = f->objects_with_finalizers;
+            list_set_item(lst, lst->count++, (uintptr_t)x);
+        }
+    }
+    list_free(marked);
+
+    f->run_finalizers = run_finalizers;
+}
+
+static void deal_with_objects_with_finalizers(void)
+{
+    /* for non-light finalizers */
+
+    /* there is one 'objects_with_finalizers' list per segment.
+       Objects that die at a major collection running in the same
+       transaction as they were created will be put in the
+       'run_finalizers' list of that segment.  Objects that survive at
+       least one commit move to the global g_objects_with_finalizers,
+       and when they die they go to g_run_finalizers.  The former kind
+       of dying object must have its finalizer called in the correct
+       thread; the latter kind can be called in any thread, through
+       any segment, because they all should see the same old content
+       anyway.  (If the content was different between segments at this
+       point, the object would be in a 'modified_old_objects' list
+       somewhere, and so it wouldn't be dead).
+    */
+    struct list_s *marked_seg[NB_SEGMENTS + 1];
+    LIST_CREATE(_finalizer_emptystack);
+    LIST_CREATE(_finalizer_pending);
+
+    long j;
+    for (j = 1; j <= NB_SEGMENTS; j++) {
+        struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
+        marked_seg[j] = mark_finalize_step1(pseg->pub.segment_base,
+                                            pseg->finalizers);
+    }
+    marked_seg[0] = mark_finalize_step1(stm_object_pages, &g_finalizers);
+
+    LIST_FREE(_finalizer_pending);
+
+    for (j = 1; j <= NB_SEGMENTS; j++) {
+        struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
+        mark_finalize_step2(pseg->pub.segment_base, pseg->finalizers,
+                            marked_seg[j]);
+    }
+    mark_finalize_step2(stm_object_pages, &g_finalizers, marked_seg[0]);
+
+    LIST_FREE(_finalizer_emptystack);
+}
+
+static void mark_visit_from_finalizer1(char *base, struct finalizers_s *f)
+{
+    if (f != NULL && f->run_finalizers != NULL) {
+        LIST_FOREACH_R(f->run_finalizers, object_t * /*item*/,
+                       mark_visit_object(item, base));
+    }
+}
+
+static void mark_visit_from_finalizer_pending(void)
+{
+    long j;
+    for (j = 1; j <= NB_SEGMENTS; j++) {
+        struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
+        mark_visit_from_finalizer1(pseg->pub.segment_base, pseg->finalizers);
+    }
+    mark_visit_from_finalizer1(stm_object_pages, &g_finalizers);
+}
+
+static void _execute_finalizers(struct finalizers_s *f)
+{
+    if (f->run_finalizers == NULL)
+        return;   /* nothing to do */
+
+ restart:
+    if (f->running_next != NULL)
+        return;   /* in a nested invocation of execute_finalizers() */
+
+    uintptr_t next = 0, total = list_count(f->run_finalizers);
+    f->running_next = &next;
+
+    while (next < total) {
+        object_t *obj = (object_t *)list_item(f->run_finalizers, next);
+        list_set_item(f->run_finalizers, next, 0);
+        next++;
+
+        stmcb_finalizer(obj);
+    }
+    if (next == (uintptr_t)-1) {
+        /* transaction committed: the whole 'f' was freed */
+        return;
+    }
+    f->running_next = NULL;
+
+    if (f->run_finalizers->count > total) {
+        memmove(f->run_finalizers->items,
+                f->run_finalizers->items + total,
+                (f->run_finalizers->count - total) * sizeof(uintptr_t));
+        goto restart;
+    }
+
+    LIST_FREE(f->run_finalizers);
+}
+
+static void _invoke_general_finalizers(stm_thread_local_t *tl)
+{
+    /* called between transactions */
+    static int lock = 0;
+
+    if (__sync_lock_test_and_set(&lock, 1) != 0) {
+        /* can't acquire the lock: someone else is likely already
+           running this function, so don't wait. */
+        return;
+    }
+
+    rewind_jmp_buf rjbuf;
+    stm_rewind_jmp_enterframe(tl, &rjbuf);
+    stm_start_transaction(tl);
+
+    _execute_finalizers(&g_finalizers);
+
+    stm_commit_transaction();
+    stm_rewind_jmp_leaveframe(tl, &rjbuf);
+
+    __sync_lock_release(&lock);
+}
diff --git a/c8/stm/finalizer.h b/c8/stm/finalizer.h
new file mode 100644
--- /dev/null
+++ b/c8/stm/finalizer.h
@@ -0,0 +1,43 @@
+
+struct finalizers_s {
+    struct list_s *objects_with_finalizers;
+    uintptr_t count_non_young;
+    struct list_s *run_finalizers;
+    uintptr_t *running_next;
+};
+
+static void mark_visit_from_finalizer_pending(void);
+static void deal_with_young_objects_with_finalizers(void);
+static void deal_with_old_objects_with_finalizers(void);
+static void deal_with_objects_with_finalizers(void);
+
+static void setup_finalizer(void);
+static void teardown_finalizer(void);
+
+static void _commit_finalizers(void);
+static void abort_finalizers(struct stm_priv_segment_info_s *);
+
+#define commit_finalizers()   do {              \
+    if (STM_PSEGMENT->finalizers != NULL)       \
+        _commit_finalizers();                   \
+} while (0)
+
+
+/* regular finalizers (objs from already-committed transactions) */
+static struct finalizers_s g_finalizers;
+
+static void _invoke_general_finalizers(stm_thread_local_t *tl);
+
+#define invoke_general_finalizers(tl)    do {   \
+    if (g_finalizers.run_finalizers != NULL)    \
+        _invoke_general_finalizers(tl);         \
+} while (0)
+
+static void _execute_finalizers(struct finalizers_s *f);
+
+#define any_local_finalizers() (STM_PSEGMENT->finalizers != NULL &&         \
+                               STM_PSEGMENT->finalizers->run_finalizers != NULL)
+#define exec_local_finalizers()  do {                   \
+    if (any_local_finalizers())                         \
+        _execute_finalizers(STM_PSEGMENT->finalizers);  \
+} while (0)
diff --git a/c8/stmgc.c b/c8/stmgc.c
--- a/c8/stmgc.c
+++ b/c8/stmgc.c
@@ -15,6 +15,7 @@
 #include "stm/extra.h"
 #include "stm/fprintcolor.h"
 #include "stm/rewind_setjmp.h"
+#include "stm/finalizer.h"
 
 #include "stm/misc.c"
 #include "stm/list.c"
@@ -34,3 +35,4 @@
 #include "stm/extra.c"
 #include "stm/fprintcolor.c"
 #include "stm/rewind_setjmp.c"
+#include "stm/finalizer.c"
diff --git a/c8/test/test_finalizer.py b/c8/test/test_finalizer.py
new file mode 100644
--- /dev/null
+++ b/c8/test/test_finalizer.py
@@ -0,0 +1,256 @@
+from support import *
+import py
+
+
+class TestLightFinalizer(BaseTest):
+
+    def setup_method(self, meth):
+        BaseTest.setup_method(self, meth)
+        #
+        @ffi.callback("void(object_t *)")
+        def light_finalizer(obj):
+            assert stm_get_obj_size(obj) == 48
+            segnum = lib.current_segment_num()
+            tlnum = '?'
+            for n, tl in enumerate(self.tls):
+                if tl.associated_segment_num == segnum:
+                    tlnum = n
+                    break
+            self.light_finalizers_called.append((obj, tlnum))
+        self.light_finalizers_called = []
+        lib.stmcb_light_finalizer = light_finalizer
+        self._light_finalizer_keepalive = light_finalizer
+
+    def teardown_method(self, meth):
+        lib.stmcb_light_finalizer = ffi.NULL
+        BaseTest.teardown_method(self, meth)
+
+    def expect_finalized(self, objs, from_tlnum=None):
+        assert [obj for (obj, tlnum) in self.light_finalizers_called] == objs
+        if from_tlnum is not None:
+            for obj, tlnum in self.light_finalizers_called:
+                assert tlnum == from_tlnum
+        self.light_finalizers_called = []
+
+    def test_no_finalizer(self):
+        self.start_transaction()
+        lp1 = stm_allocate(48)
+        self.commit_transaction()
+        self.expect_finalized([])
+
+    def test_young_light_finalizer(self):
+        self.start_transaction()
+        lp1 = stm_allocate(48)
+        lib.stm_enable_light_finalizer(lp1)
+        self.expect_finalized([])
+        self.commit_transaction()
+        self.expect_finalized([lp1], from_tlnum=0)
+
+    def test_young_light_finalizer_survives(self):
+        self.start_transaction()
+        lp1 = stm_allocate(48)
+        lib.stm_enable_light_finalizer(lp1)
+        self.push_root(lp1)       # stays alive
+        self.commit_transaction()
+        self.expect_finalized([])
+
+    def test_young_light_finalizer_aborts(self):
+        self.start_transaction()
+        lp1 = stm_allocate(48)
+        lib.stm_enable_light_finalizer(lp1)
+        self.expect_finalized([])
+        self.abort_transaction()
+        self.start_transaction()
+        self.expect_finalized([lp1], from_tlnum=0)
+
+    def test_old_light_finalizer(self):
+        self.start_transaction()
+        lp1 = stm_allocate(48)
+        self.push_root(lp1)
+        stm_minor_collect()
+        lp1 = self.pop_root()
+        lib.stm_enable_light_finalizer(lp1)
+        self.commit_transaction()
+        self.expect_finalized([])
+
+    def test_old_light_finalizer_2(self):
+        self.start_transaction()
+        lp1 = stm_allocate(48)
+        lib.stm_enable_light_finalizer(lp1)
+        self.push_root(lp1)
+        stm_minor_collect()
+        lp1 = self.pop_root()
+        self.expect_finalized([])
+        stm_major_collect()
+        self.expect_finalized([lp1])
+        self.commit_transaction()
+
+    def test_old_light_finalizer_survives(self):
+        self.start_transaction()
+        lp1 = stm_allocate(48)
+        lib.stm_enable_light_finalizer(lp1)
+        self.push_root(lp1)
+        stm_minor_collect()
+        lp1 = self.pop_root()
+        self.push_root(lp1)
+        stm_major_collect()
+        self.commit_transaction()
+        self.expect_finalized([])
+
+    def test_old_light_finalizer_segment(self):
+        self.start_transaction()
+        #
+        self.switch(1)
+        self.start_transaction()
+        lp1 = stm_allocate(48)
+        lib.stm_enable_light_finalizer(lp1)
+        self.push_root(lp1)
+        stm_minor_collect()
+        lp1 = self.pop_root()
+        #
+        self.switch(0)
+        self.expect_finalized([])
+        stm_major_collect()
+        self.expect_finalized([lp1], from_tlnum=1)
+
+    def test_old_light_finalizer_aborts(self):
+        self.start_transaction()
+        lp1 = stm_allocate(48)
+        lib.stm_enable_light_finalizer(lp1)
+        self.push_root(lp1)
+        self.commit_transaction()
+        #
+        self.start_transaction()
+        self.expect_finalized([])
+        self.abort_transaction()
+        self.expect_finalized([])
+
+    def test_overflow_light_finalizer_aborts(self):
+        self.start_transaction()
+        lp1 = stm_allocate(48)
+        lib.stm_enable_light_finalizer(lp1)
+        self.push_root(lp1)
+        stm_minor_collect()
+        lp1 = self.pop_root()
+        self.push_root(lp1)
+        self.expect_finalized([])
+        self.abort_transaction()
+        self.expect_finalized([lp1], from_tlnum=0)
+
+
+class TestRegularFinalizer(BaseTest):
+    expect_content_character = None
+    run_major_collect_in_finalizer = False
+
+    def setup_method(self, meth):
+        BaseTest.setup_method(self, meth)
+        #
+        @ffi.callback("void(object_t *)")
+        def finalizer(obj):
+            print "finalizing!", obj
+            assert stm_get_obj_size(obj) in [16, 32, 48, 56]
+            if self.expect_content_character is not None:
+                assert stm_get_char(obj) == self.expect_content_character
+            self.finalizers_called.append(obj)
+            if self.run_major_collect_in_finalizer:
+                stm_major_collect()
+        self.finalizers_called = []
+        lib.stmcb_finalizer = finalizer
+        self._finalizer_keepalive = finalizer
+
+    def expect_finalized(self, objs):
+        assert self.finalizers_called == objs
+        self.finalizers_called = []
+
+    def test_no_finalizer(self):
+        self.start_transaction()
+        lp1 = stm_allocate(48)
+        stm_major_collect()
+        self.expect_finalized([])
+
+    def test_no_finalizer_in_minor_collection(self):
+        self.start_transaction()
+        lp1 = stm_allocate_with_finalizer(48)
+        stm_minor_collect()
+        self.expect_finalized([])
+
+    def test_finalizer_in_major_collection(self):
+        self.start_transaction()
+        for repeat in range(2):
+            lp1 = stm_allocate_with_finalizer(48)
+            lp2 = stm_allocate_with_finalizer(48)
+            lp3 = stm_allocate_with_finalizer(48)
+            print repeat, lp1, lp2, lp3
+            self.expect_finalized([])
+            stm_major_collect()
+            self.expect_finalized([lp1, lp2, lp3])
+
+    def test_finalizer_from_other_thread(self):
+        self.start_transaction()
+        lp1 = stm_allocate_with_finalizer(48)
+        stm_set_char(lp1, 'H')
+        self.expect_content_character = 'H'
+        print lp1
+        #
+        self.switch(1)
+        self.start_transaction()
+        stm_major_collect()
+        self.expect_finalized([])      # marked as dead, but wrong thread
+        #
+        self.switch(0)
+        self.expect_finalized([lp1])   # now it has been finalized
+
+    def test_finalizer_ordering(self):
+        self.start_transaction()
+        lp1 = stm_allocate_with_finalizer_refs(1)
+        lp2 = stm_allocate_with_finalizer_refs(1)
+        lp3 = stm_allocate_with_finalizer_refs(1)
+        print lp1, lp2, lp3
+        stm_set_ref(lp3, 0, lp1)
+        stm_set_ref(lp1, 0, lp2)
+        stm_major_collect()
+        self.expect_finalized([lp3])
+
+    def test_finalizer_extra_transaction(self):
+        self.start_transaction()
+        lp1 = stm_allocate_with_finalizer(32)
+        print lp1
+        self.push_root(lp1)
+        self.commit_transaction()
+
+        self.start_transaction()
+        lp1b = self.pop_root()
+        assert lp1b == lp1
+        self.expect_finalized([])
+        self.commit_transaction()
+        self.expect_finalized([])
+
+        self.start_transaction()
+        stm_major_collect()
+        self.expect_finalized([])
+        self.commit_transaction()
+        self.expect_finalized([lp1])
+
+    def test_run_cb_for_all_threads(self):
+        self.start_transaction()
+        lp1 = stm_allocate_with_finalizer(48)
+        print lp1
+        #
+        self.switch(1)
+        self.start_transaction()
+        lp2 = stm_allocate_with_finalizer(56)
+        print lp2
+
+        self.expect_finalized([])
+        stm_major_collect()
+        self.switch(0)
+        self.expect_finalized([lp2, lp1])
+
+    def test_run_major_collect_in_finalizer(self):
+        self.run_major_collect_in_finalizer = True
+        self.start_transaction()
+        lp1 = stm_allocate_with_finalizer(32)
+        lp2 = stm_allocate_with_finalizer(32)
+        lp3 = stm_allocate_with_finalizer(32)
+        print lp1, lp2, lp3
+        stm_major_collect()

From noreply at buildbot.pypy.org  Tue Feb 24 23:50:36 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 24 Feb 2015 23:50:36 +0100 (CET)
Subject: [pypy-commit] stmgc default: c8: pass some first finalizer tests
Message-ID: <20150224225036.54C841C0107@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: 
Changeset: r1659:7aceea4cc94e
Date: 2015-02-24 17:12 +0100
http://bitbucket.org/pypy/stmgc/changeset/7aceea4cc94e/

Log:	c8: pass some first finalizer tests

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -773,6 +773,8 @@
     assert(tree_is_cleared(STM_PSEGMENT->nursery_objects_shadows));
     assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[0]));
     assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[1]));
+    assert(list_is_empty(STM_PSEGMENT->young_objects_with_light_finalizers));
+    assert(STM_PSEGMENT->finalizers == NULL);
 
     check_nursery_at_transaction_start();
 
@@ -871,6 +873,8 @@
 
 void stm_commit_transaction(void)
 {
+    exec_local_finalizers();
+
     assert(!_has_mutex());
     assert(STM_PSEGMENT->safe_point == SP_RUNNING);
     assert(STM_PSEGMENT->running_pthread == pthread_self());
@@ -884,7 +888,6 @@
     _validate_and_add_to_commit_log();
 
 
-
     /* XXX do we still need a s_mutex_lock() section here? */
     s_mutex_lock();
 
@@ -893,6 +896,8 @@
 
     stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
 
+    commit_finalizers();
+
     invoke_and_clear_user_callbacks(0);   /* for commit */
 
     if (globally_unique_transaction && was_inev) {
@@ -900,10 +905,15 @@
     }
 
     /* done */
+    stm_thread_local_t *tl = STM_SEGMENT->running_thread;
     _finish_transaction();
     /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */
 
     s_mutex_unlock();
+
+    /* between transactions, call finalizers. this will execute
+       a transaction itself */
+    invoke_general_finalizers(tl);
 }
 
 static void reset_modified_from_backup_copies(int segment_num)
@@ -959,6 +969,8 @@
                        (int)pseg->transaction_state);
     }
 
+    abort_finalizers(pseg);
+
     long bytes_in_nursery = throw_away_nursery(pseg);
 
     acquire_modification_lock(segment_num);
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -110,6 +110,14 @@
     pthread_t running_pthread;
 #endif
 
+    /* light finalizers */
+    struct list_s *young_objects_with_light_finalizers;
+    struct list_s *old_objects_with_light_finalizers;
+
+    /* regular finalizers (objs from the current transaction only) */
+    struct finalizers_s *finalizers;
+
+
     /* This is for smallmalloc.c */
     struct small_malloc_data_s small_malloc_data;
 
diff --git a/c8/stm/finalizer.c b/c8/stm/finalizer.c
--- a/c8/stm/finalizer.c
+++ b/c8/stm/finalizer.c
@@ -28,6 +28,8 @@
 
 static void _commit_finalizers(void)
 {
+    /* move finalizer lists to g_finalizers for major collections */
+
     if (STM_PSEGMENT->finalizers->run_finalizers != NULL) {
         /* copy 'STM_PSEGMENT->finalizers->run_finalizers' into
            'g_finalizers.run_finalizers', dropping any initial NULLs
@@ -96,14 +98,14 @@
         list_clear(lst);
     }
 
-    /* also deals with overflow objects: they are at the tail of
+    /* also deals with newly created objects: they are at the tail of
        old_objects_with_light_finalizers (this list is kept in order
        and we cannot add any already-committed object) */
     lst = pseg->old_objects_with_light_finalizers;
     count = list_count(lst);
     while (count > 0) {
         object_t *obj = (object_t *)list_item(lst, --count);
-        if (!IS_OVERFLOW_OBJ(pseg, obj))
+        if (!(obj->stm_flags & GCFLAG_WB_EXECUTED))
             break;
         lst->count = count;
         if (must_fix_gs) {
@@ -153,7 +155,8 @@
 
 static void deal_with_young_objects_with_finalizers(void)
 {
-    /* for light finalizers */
+    /* for light finalizers: executes finalizers for objs that don't survive
+       this minor gc */
     struct list_s *lst = STM_PSEGMENT->young_objects_with_light_finalizers;
     long i, count = list_count(lst);
     for (i = 0; i < count; i++) {
@@ -174,48 +177,48 @@
     list_clear(lst);
 }
 
-static void deal_with_old_objects_with_finalizers(void)
-{
-    /* for light finalizers */
-    int old_gs_register = STM_SEGMENT->segment_num;
-    int current_gs_register = old_gs_register;
-    long j;
-    for (j = 1; j <= NB_SEGMENTS; j++) {
-        struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
+/* static void deal_with_old_objects_with_finalizers(void) */
+/* { */
+/*     /\* for light finalizers *\/ */
+/*     int old_gs_register = STM_SEGMENT->segment_num; */
+/*     int current_gs_register = old_gs_register; */
+/*     long j; */
+/*     for (j = 1; j <= NB_SEGMENTS; j++) { */
+/*         struct stm_priv_segment_info_s *pseg = get_priv_segment(j); */
 
-        struct list_s *lst = pseg->old_objects_with_light_finalizers;
-        long i, count = list_count(lst);
-        lst->count = 0;
-        for (i = 0; i < count; i++) {
-            object_t *obj = (object_t *)list_item(lst, i);
-            if (!mark_visited_test(obj)) {
-                /* not marked: object dies */
-                /* we're calling the light finalizer in the same
-                   segment as where it was originally registered.  For
-                   objects that existed since a long time, it doesn't
-                   change anything: any thread should see the same old
-                   content (because if it wasn't the case, the object
-                   would be in a 'modified_old_objects' list
-                   somewhere, and so it wouldn't be dead).  But it's
-                   important if the object was created by the same
-                   transaction: then only that segment sees valid
-                   content.
-                */
-                if (j != current_gs_register) {
-                    set_gs_register(get_segment_base(j));
-                    current_gs_register = j;
-                }
-                stmcb_light_finalizer(obj);
-            }
-            else {
-                /* object survives */
-                list_set_item(lst, lst->count++, (uintptr_t)obj);
-            }
-        }
-    }
-    if (old_gs_register != current_gs_register)
-        set_gs_register(get_segment_base(old_gs_register));
-}
+/*         struct list_s *lst = pseg->old_objects_with_light_finalizers; */
+/*         long i, count = list_count(lst); */
+/*         lst->count = 0; */
+/*         for (i = 0; i < count; i++) { */
+/*             object_t *obj = (object_t *)list_item(lst, i); */
+/*             if (!mark_visited_test(obj)) { */
+/*                 /\* not marked: object dies *\/ */
+/*                 /\* we're calling the light finalizer in the same */
+/*                    segment as where it was originally registered.  For */
+/*                    objects that existed since a long time, it doesn't */
+/*                    change anything: any thread should see the same old */
+/*                    content (because if it wasn't the case, the object */
+/*                    would be in a 'modified_old_objects' list */
+/*                    somewhere, and so it wouldn't be dead).  But it's */
+/*                    important if the object was created by the same */
+/*                    transaction: then only that segment sees valid */
+/*                    content. */
+/*                 *\/ */
+/*                 if (j != current_gs_register) { */
+/*                     set_gs_register(get_segment_base(j)); */
+/*                     current_gs_register = j; */
+/*                 } */
+/*                 stmcb_light_finalizer(obj); */
+/*             } */
+/*             else { */
+/*                 /\* object survives *\/ */
+/*                 list_set_item(lst, lst->count++, (uintptr_t)obj); */
+/*             } */
+/*         } */
+/*     } */
+/*     if (old_gs_register != current_gs_register) */
+/*         set_gs_register(get_segment_base(old_gs_register)); */
+/* } */
 
 
 /************************************************************/
@@ -224,195 +227,195 @@
     as well as rpython/memory/gc/minimark.py.
 */
 
-static inline int _finalization_state(object_t *obj)
-{
-    /* Returns the state, "0", 1, 2 or 3, as per finalizer-order.rst.
-       One difference is that the official state 0 is returned here
-       as a number that is <= 0. */
-    uintptr_t lock_idx = mark_loc(obj);
-    return write_locks[lock_idx] - (WL_FINALIZ_ORDER_1 - 1);
-}
+/* static inline int _finalization_state(object_t *obj) */
+/* { */
+/*     /\* Returns the state, "0", 1, 2 or 3, as per finalizer-order.rst. */
+/*        One difference is that the official state 0 is returned here */
+/*        as a number that is <= 0. *\/ */
+/*     uintptr_t lock_idx = mark_loc(obj); */
+/*     return write_locks[lock_idx] - (WL_FINALIZ_ORDER_1 - 1); */
+/* } */
 
-static void _bump_finalization_state_from_0_to_1(object_t *obj)
-{
-    uintptr_t lock_idx = mark_loc(obj);
-    assert(write_locks[lock_idx] < WL_FINALIZ_ORDER_1);
-    write_locks[lock_idx] = WL_FINALIZ_ORDER_1;
-}
+/* static void _bump_finalization_state_from_0_to_1(object_t *obj) */
+/* { */
+/*     uintptr_t lock_idx = mark_loc(obj); */
+/*     assert(write_locks[lock_idx] < WL_FINALIZ_ORDER_1); */
+/*     write_locks[lock_idx] = WL_FINALIZ_ORDER_1; */
+/* } */
 
-static struct list_s *_finalizer_tmpstack;
-static struct list_s *_finalizer_emptystack;
-static struct list_s *_finalizer_pending;
+/* static struct list_s *_finalizer_tmpstack; */
+/* static struct list_s *_finalizer_emptystack; */
+/* static struct list_s *_finalizer_pending; */
 
-static inline void _append_to_finalizer_tmpstack(object_t **pobj)
-{
-    object_t *obj = *pobj;
-    if (obj != NULL)
-        LIST_APPEND(_finalizer_tmpstack, obj);
-}
+/* static inline void _append_to_finalizer_tmpstack(object_t **pobj) */
+/* { */
+/*     object_t *obj = *pobj; */
+/*     if (obj != NULL) */
+/*         LIST_APPEND(_finalizer_tmpstack, obj); */
+/* } */
 
-static inline struct list_s *finalizer_trace(char *base, object_t *obj,
-                                             struct list_s *lst)
-{
-    struct object_s *realobj = (struct object_s *)REAL_ADDRESS(base, obj);
-    _finalizer_tmpstack = lst;
-    stmcb_trace(realobj, &_append_to_finalizer_tmpstack);
-    return _finalizer_tmpstack;
-}
+/* static inline struct list_s *finalizer_trace(char *base, object_t *obj, */
+/*                                              struct list_s *lst) */
+/* { */
+/*     struct object_s *realobj = (struct object_s *)REAL_ADDRESS(base, obj); */
+/*     _finalizer_tmpstack = lst; */
+/*     stmcb_trace(realobj, &_append_to_finalizer_tmpstack); */
+/*     return _finalizer_tmpstack; */
+/* } */
 
-static void _recursively_bump_finalization_state(char *base, object_t *obj,
-                                                 int to_state)
-{
-    struct list_s *tmpstack = _finalizer_emptystack;
-    assert(list_is_empty(tmpstack));
+/* static void _recursively_bump_finalization_state(char *base, object_t *obj, */
+/*                                                  int to_state) */
+/* { */
+/*     struct list_s *tmpstack = _finalizer_emptystack; */
+/*     assert(list_is_empty(tmpstack)); */
 
-    while (1) {
-        if (_finalization_state(obj) == to_state - 1) {
-            /* bump to the next state */
-            write_locks[mark_loc(obj)]++;
+/*     while (1) { */
+/*         if (_finalization_state(obj) == to_state - 1) { */
+/*             /\* bump to the next state *\/ */
+/*             write_locks[mark_loc(obj)]++; */
 
-            /* trace */
-            tmpstack = finalizer_trace(base, obj, tmpstack);
-        }
+/*             /\* trace *\/ */
+/*             tmpstack = finalizer_trace(base, obj, tmpstack); */
+/*         } */
 
-        if (list_is_empty(tmpstack))
-            break;
+/*         if (list_is_empty(tmpstack)) */
+/*             break; */
 
-        obj = (object_t *)list_pop_item(tmpstack);
-    }
-    _finalizer_emptystack = tmpstack;
-}
+/*         obj = (object_t *)list_pop_item(tmpstack); */
+/*     } */
+/*     _finalizer_emptystack = tmpstack; */
+/* } */
 
-static struct list_s *mark_finalize_step1(char *base, struct finalizers_s *f)
-{
-    if (f == NULL)
-        return NULL;
+/* static struct list_s *mark_finalize_step1(char *base, struct finalizers_s *f) */
+/* { */
+/*     if (f == NULL) */
+/*         return NULL; */
 
-    struct list_s *marked = list_create();
+/*     struct list_s *marked = list_create(); */
 
-    struct list_s *lst = f->objects_with_finalizers;
-    long i, count = list_count(lst);
-    lst->count = 0;
-    f->count_non_young = 0;
+/*     struct list_s *lst = f->objects_with_finalizers; */
+/*     long i, count = list_count(lst); */
+/*     lst->count = 0; */
+/*     f->count_non_young = 0; */
 
-    for (i = 0; i < count; i++) {
-        object_t *x = (object_t *)list_item(lst, i);
+/*     for (i = 0; i < count; i++) { */
+/*         object_t *x = (object_t *)list_item(lst, i); */
 
-        assert(_finalization_state(x) != 1);
-        if (_finalization_state(x) >= 2) {
-            list_set_item(lst, lst->count++, (uintptr_t)x);
-            continue;
-        }
-        LIST_APPEND(marked, x);
+/*         assert(_finalization_state(x) != 1); */
+/*         if (_finalization_state(x) >= 2) { */
+/*             list_set_item(lst, lst->count++, (uintptr_t)x); */
+/*             continue; */
+/*         } */
+/*         LIST_APPEND(marked, x); */
 
-        struct list_s *pending = _finalizer_pending;
-        LIST_APPEND(pending, x);
-        while (!list_is_empty(pending)) {
-            object_t *y = (object_t *)list_pop_item(pending);
-            int state = _finalization_state(y);
-            if (state <= 0) {
-                _bump_finalization_state_from_0_to_1(y);
-                pending = finalizer_trace(base, y, pending);
-            }
-            else if (state == 2) {
-                _recursively_bump_finalization_state(base, y, 3);
-            }
-        }
-        _finalizer_pending = pending;
-        assert(_finalization_state(x) == 1);
-        _recursively_bump_finalization_state(base, x, 2);
-    }
-    return marked;
-}
+/*         struct list_s *pending = _finalizer_pending; */
+/*         LIST_APPEND(pending, x); */
+/*         while (!list_is_empty(pending)) { */
+/*             object_t *y = (object_t *)list_pop_item(pending); */
+/*             int state = _finalization_state(y); */
+/*             if (state <= 0) { */
+/*                 _bump_finalization_state_from_0_to_1(y); */
+/*                 pending = finalizer_trace(base, y, pending); */
+/*             } */
+/*             else if (state == 2) { */
+/*                 _recursively_bump_finalization_state(base, y, 3); */
+/*             } */
+/*         } */
+/*         _finalizer_pending = pending; */
+/*         assert(_finalization_state(x) == 1); */
+/*         _recursively_bump_finalization_state(base, x, 2); */
+/*     } */
+/*     return marked; */
+/* } */
 
-static void mark_finalize_step2(char *base, struct finalizers_s *f,
-                                struct list_s *marked)
-{
-    if (f == NULL)
-        return;
+/* static void mark_finalize_step2(char *base, struct finalizers_s *f, */
+/*                                 struct list_s *marked) */
+/* { */
+/*     if (f == NULL) */
+/*         return; */
 
-    struct list_s *run_finalizers = f->run_finalizers;
+/*     struct list_s *run_finalizers = f->run_finalizers; */
 
-    long i, count = list_count(marked);
-    for (i = 0; i < count; i++) {
-        object_t *x = (object_t *)list_item(marked, i);
+/*     long i, count = list_count(marked); */
+/*     for (i = 0; i < count; i++) { */
+/*         object_t *x = (object_t *)list_item(marked, i); */
 
-        int state = _finalization_state(x);
-        assert(state >= 2);
-        if (state == 2) {
-            if (run_finalizers == NULL)
-                run_finalizers = list_create();
-            LIST_APPEND(run_finalizers, x);
-            _recursively_bump_finalization_state(base, x, 3);
-        }
-        else {
-            struct list_s *lst = f->objects_with_finalizers;
-            list_set_item(lst, lst->count++, (uintptr_t)x);
-        }
-    }
-    list_free(marked);
+/*         int state = _finalization_state(x); */
+/*         assert(state >= 2); */
+/*         if (state == 2) { */
+/*             if (run_finalizers == NULL) */
+/*                 run_finalizers = list_create(); */
+/*             LIST_APPEND(run_finalizers, x); */
+/*             _recursively_bump_finalization_state(base, x, 3); */
+/*         } */
+/*         else { */
+/*             struct list_s *lst = f->objects_with_finalizers; */
+/*             list_set_item(lst, lst->count++, (uintptr_t)x); */
+/*         } */
+/*     } */
+/*     list_free(marked); */
 
-    f->run_finalizers = run_finalizers;
-}
+/*     f->run_finalizers = run_finalizers; */
+/* } */
 
-static void deal_with_objects_with_finalizers(void)
-{
-    /* for non-light finalizers */
+/* static void deal_with_objects_with_finalizers(void) */
+/* { */
+/*     /\* for non-light finalizers *\/ */
 
-    /* there is one 'objects_with_finalizers' list per segment.
-       Objects that die at a major collection running in the same
-       transaction as they were created will be put in the
-       'run_finalizers' list of that segment.  Objects that survive at
-       least one commit move to the global g_objects_with_finalizers,
-       and when they die they go to g_run_finalizers.  The former kind
-       of dying object must have its finalizer called in the correct
-       thread; the latter kind can be called in any thread, through
-       any segment, because they all should see the same old content
-       anyway.  (If the content was different between segments at this
-       point, the object would be in a 'modified_old_objects' list
-       somewhere, and so it wouldn't be dead).
-    */
-    struct list_s *marked_seg[NB_SEGMENTS + 1];
-    LIST_CREATE(_finalizer_emptystack);
-    LIST_CREATE(_finalizer_pending);
+/*     /\* there is one 'objects_with_finalizers' list per segment. */
+/*        Objects that die at a major collection running in the same */
+/*        transaction as they were created will be put in the */
+/*        'run_finalizers' list of that segment.  Objects that survive at */
+/*        least one commit move to the global g_objects_with_finalizers, */
+/*        and when they die they go to g_run_finalizers.  The former kind */
+/*        of dying object must have its finalizer called in the correct */
+/*        thread; the latter kind can be called in any thread, through */
+/*        any segment, because they all should see the same old content */
+/*        anyway.  (If the content was different between segments at this */
+/*        point, the object would be in a 'modified_old_objects' list */
+/*        somewhere, and so it wouldn't be dead). */
+/*     *\/ */
+/*     struct list_s *marked_seg[NB_SEGMENTS + 1]; */
+/*     LIST_CREATE(_finalizer_emptystack); */
+/*     LIST_CREATE(_finalizer_pending); */
 
-    long j;
-    for (j = 1; j <= NB_SEGMENTS; j++) {
-        struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
-        marked_seg[j] = mark_finalize_step1(pseg->pub.segment_base,
-                                            pseg->finalizers);
-    }
-    marked_seg[0] = mark_finalize_step1(stm_object_pages, &g_finalizers);
+/*     long j; */
+/*     for (j = 1; j <= NB_SEGMENTS; j++) { */
+/*         struct stm_priv_segment_info_s *pseg = get_priv_segment(j); */
+/*         marked_seg[j] = mark_finalize_step1(pseg->pub.segment_base, */
+/*                                             pseg->finalizers); */
+/*     } */
+/*     marked_seg[0] = mark_finalize_step1(stm_object_pages, &g_finalizers); */
 
-    LIST_FREE(_finalizer_pending);
+/*     LIST_FREE(_finalizer_pending); */
 
-    for (j = 1; j <= NB_SEGMENTS; j++) {
-        struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
-        mark_finalize_step2(pseg->pub.segment_base, pseg->finalizers,
-                            marked_seg[j]);
-    }
-    mark_finalize_step2(stm_object_pages, &g_finalizers, marked_seg[0]);
+/*     for (j = 1; j <= NB_SEGMENTS; j++) { */
+/*         struct stm_priv_segment_info_s *pseg = get_priv_segment(j); */
+/*         mark_finalize_step2(pseg->pub.segment_base, pseg->finalizers, */
+/*                             marked_seg[j]); */
+/*     } */
+/*     mark_finalize_step2(stm_object_pages, &g_finalizers, marked_seg[0]); */
 
-    LIST_FREE(_finalizer_emptystack);
-}
+/*     LIST_FREE(_finalizer_emptystack); */
+/* } */
 
-static void mark_visit_from_finalizer1(char *base, struct finalizers_s *f)
-{
-    if (f != NULL && f->run_finalizers != NULL) {
-        LIST_FOREACH_R(f->run_finalizers, object_t * /*item*/,
-                       mark_visit_object(item, base));
-    }
-}
+/* static void mark_visit_from_finalizer1(char *base, struct finalizers_s *f) */
+/* { */
+/*     if (f != NULL && f->run_finalizers != NULL) { */
+/*         LIST_FOREACH_R(f->run_finalizers, object_t * /\*item*\/, */
+/*                        mark_visit_object(item, base)); */
+/*     } */
+/* } */
 
-static void mark_visit_from_finalizer_pending(void)
-{
-    long j;
-    for (j = 1; j <= NB_SEGMENTS; j++) {
-        struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
-        mark_visit_from_finalizer1(pseg->pub.segment_base, pseg->finalizers);
-    }
-    mark_visit_from_finalizer1(stm_object_pages, &g_finalizers);
-}
+/* static void mark_visit_from_finalizer_pending(void) */
+/* { */
+/*     long j; */
+/*     for (j = 1; j <= NB_SEGMENTS; j++) { */
+/*         struct stm_priv_segment_info_s *pseg = get_priv_segment(j); */
+/*         mark_visit_from_finalizer1(pseg->pub.segment_base, pseg->finalizers); */
+/*     } */
+/*     mark_visit_from_finalizer1(stm_object_pages, &g_finalizers); */
+/* } */
 
 static void _execute_finalizers(struct finalizers_s *f)
 {
diff --git a/c8/stm/list.c b/c8/stm/list.c
--- a/c8/stm/list.c
+++ b/c8/stm/list.c
@@ -30,6 +30,21 @@
     return lst;
 }
 
+static struct list_s *list_extend(struct list_s *lst, struct list_s *lst2,
+                                  uintptr_t slicestart)
+{
+    if (lst2->count <= slicestart)
+        return lst;
+    uintptr_t baseindex = lst->count;
+    lst->count = baseindex + lst2->count - slicestart;
+    uintptr_t lastindex = lst->count - 1;
+    if (lastindex > lst->last_allocated)
+        lst = _list_grow(lst, lastindex);
+    memcpy(lst->items + baseindex, lst2->items + slicestart,
+           (lst2->count - slicestart) * sizeof(uintptr_t));
+    return lst;
+}
+
 
 /************************************************************/
 
diff --git a/c8/stm/list.h b/c8/stm/list.h
--- a/c8/stm/list.h
+++ b/c8/stm/list.h
@@ -96,6 +96,10 @@
     return &lst->items[index];
 }
 
+static struct list_s *list_extend(struct list_s *lst, struct list_s *lst2,
+                                  uintptr_t slicestart);
+
+
 #define LIST_FOREACH_R(lst, TYPE, CODE)         \
     do {                                        \
         struct list_s *_lst = (lst);            \
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -37,6 +37,10 @@
         tree_contains(STM_PSEGMENT->young_outside_nursery, (uintptr_t)obj));
 }
 
+static inline bool _is_from_same_transaction(object_t *obj) {
+    return _is_young(obj) || (obj->stm_flags & GCFLAG_WB_EXECUTED);
+}
+
 long stm_can_move(object_t *obj)
 {
     /* 'long' return value to avoid using 'bool' in the public interface */
@@ -226,6 +230,22 @@
     }
 }
 
+static void collect_objs_still_young_but_with_finalizers(void)
+{
+    struct list_s *lst = STM_PSEGMENT->finalizers->objects_with_finalizers;
+    uintptr_t i, total = list_count(lst);
+
+    for (i = STM_PSEGMENT->finalizers->count_non_young; i < total; i++) {
+
+        object_t *o = (object_t *)list_item(lst, i);
+        minor_trace_if_young(&o);
+
+        /* was not actually movable */
+        assert(o == (object_t *)list_item(lst, i));
+    }
+    STM_PSEGMENT->finalizers->count_non_young = total;
+}
+
 
 static size_t throw_away_nursery(struct stm_priv_segment_info_s *pseg)
 {
@@ -296,12 +316,16 @@
 
     collect_roots_in_nursery();
 
+    if (STM_PSEGMENT->finalizers != NULL)
+        collect_objs_still_young_but_with_finalizers();
+
     collect_oldrefs_to_nursery();
 
     /* now all surviving nursery objects have been moved out */
     acquire_privatization_lock(STM_SEGMENT->segment_num);
     stm_move_young_weakrefs();
     release_privatization_lock(STM_SEGMENT->segment_num);
+    deal_with_young_objects_with_finalizers();
 
     assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery));
 
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -108,6 +108,8 @@
         pr->nursery_objects_shadows = tree_create();
         pr->callbacks_on_commit_and_abort[0] = tree_create();
         pr->callbacks_on_commit_and_abort[1] = tree_create();
+        pr->young_objects_with_light_finalizers = list_create();
+        pr->old_objects_with_light_finalizers = list_create();
 
         pr->last_commit_log_entry = &commit_log_root;
         pr->pub.transaction_read_version = 0xff;
@@ -127,6 +129,7 @@
     setup_gcpage();
     setup_pages();
     setup_forksupport();
+    setup_finalizer();
 
     set_gs_register(get_segment_base(0));
 }
@@ -153,6 +156,8 @@
         tree_free(pr->nursery_objects_shadows);
         tree_free(pr->callbacks_on_commit_and_abort[0]);
         tree_free(pr->callbacks_on_commit_and_abort[1]);
+        list_free(pr->young_objects_with_light_finalizers);
+        list_free(pr->old_objects_with_light_finalizers);
     }
 
     munmap(stm_object_pages, TOTAL_MEMORY);
@@ -160,6 +165,7 @@
     commit_log_root.next = NULL; /* xxx:free them */
     commit_log_root.segment_num = -1;
 
+    teardown_finalizer();
     teardown_sync();
     teardown_gcpage();
     teardown_smallmalloc();
diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -193,6 +193,7 @@
     assert(_stm_in_transaction(tl));
     set_gs_register(get_segment_base(tl->associated_segment_num));
     assert(STM_SEGMENT->running_thread == tl);
+    exec_local_finalizers();
 }
 
 void _stm_test_switch_segment(int segnum)
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -270,6 +270,31 @@
 void stm_validate(void);
 
 
+
+/* Support for light finalizers.  This is a simple version of
+   finalizers that guarantees not to do anything fancy, like not
+   resurrecting objects. */
+extern void (*stmcb_light_finalizer)(object_t *);
+void stm_enable_light_finalizer(object_t *);
+
+/* Support for regular finalizers.  Unreachable objects with
+   finalizers are kept alive, as well as everything they point to, and
+   stmcb_finalizer() is called after the major GC.  If there are
+   several objects with finalizers that reference each other in a
+   well-defined order (i.e. there are no cycles), then they are
+   finalized in order from outermost to innermost (i.e. starting with
+   the ones that are unreachable even from others).
+
+   For objects that have been created by the current transaction, if a
+   major GC runs while that transaction is alive and finds the object
+   unreachable, the finalizer is called immediately in the same
+   transaction.  For older objects, the finalizer is called from a
+   random thread between regular transactions, in a new custom
+   transaction. */
+extern void (*stmcb_finalizer)(object_t *);
+object_t *stm_allocate_with_finalizer(ssize_t size_rounded_up);
+
+
 /* dummies for now: */
 __attribute__((always_inline))
 static inline void stm_write_card(object_t *obj, uintptr_t index)
diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -40,6 +40,7 @@
 /*void stm_write(object_t *obj); use _checked_stm_write() instead */
 object_t *stm_allocate(ssize_t size_rounded_up);
 object_t *stm_allocate_weakref(ssize_t size_rounded_up);
+object_t *stm_allocate_with_finalizer(ssize_t size_rounded_up);
 
 void stm_setup(void);
 void stm_teardown(void);
@@ -77,6 +78,7 @@
 bool _check_become_inevitable(stm_thread_local_t *tl);
 bool _check_become_globally_unique_transaction(stm_thread_local_t *tl);
 int stm_is_inevitable(void);
+long current_segment_num(void);
 
 void _set_type_id(object_t *obj, uint32_t h);
 uint32_t _get_type_id(object_t *obj);
@@ -128,6 +130,12 @@
 bool (*_stm_smallmalloc_keep)(char *data);
 void _stm_smallmalloc_sweep_test(void);
 
+
+void (*stmcb_light_finalizer)(object_t *);
+void stm_enable_light_finalizer(object_t *);
+
+void (*stmcb_finalizer)(object_t *);
+
 """)
 
 
@@ -322,6 +330,10 @@
     }
 }
 
+long current_segment_num(void)
+{
+    return STM_SEGMENT->segment_num;
+}
 ''', sources=source_files,
      define_macros=[('STM_TESTS', '1'),
                     ('STM_NO_AUTOMATIC_SETJMP', '1'),
@@ -332,7 +344,7 @@
                     ],
      undef_macros=['NDEBUG'],
      include_dirs=[parent_dir],
-     extra_compile_args=['-g', '-O0', '-Wall', '-Werror', '-ferror-limit=5'],
+     extra_compile_args=['-g', '-O0', '-Wall', '-ferror-limit=5'],
      extra_link_args=['-g', '-lrt'],
      force_generic_engine=True)
 
@@ -386,15 +398,27 @@
     lib._set_weakref(o, point_to_obj)
     return o
 
-def stm_get_weakref(o):
-    return lib._get_weakref(o)
-
 def stm_allocate_refs(n):
     o = lib.stm_allocate(HDR + n * WORD)
     tid = 421420 + n
     lib._set_type_id(o, tid)
     return o
 
+def stm_allocate_with_finalizer(size):
+    o = lib.stm_allocate_with_finalizer(size)
+    tid = 42 + size
+    lib._set_type_id(o, tid)
+    return o
+
+def stm_allocate_with_finalizer_refs(n):
+    o = lib.stm_allocate_with_finalizer(HDR + n * WORD)
+    tid = 421420 + n
+    lib._set_type_id(o, tid)
+    return o
+
+def stm_get_weakref(o):
+    return lib._get_weakref(o)
+
 def stm_set_ref(obj, idx, ref, use_cards=False):
     if use_cards:
         stm_write_card(obj, idx)
diff --git a/c8/test/test_list.py b/c8/test/test_list.py
--- a/c8/test/test_list.py
+++ b/c8/test/test_list.py
@@ -6,6 +6,12 @@
 ffi = cffi.FFI()
 ffi.cdef("""
 struct list_s *list_create(void);
+void list_free(struct list_s *lst);
+struct list_s *list_append(struct list_s *lst, uintptr_t item);
+uintptr_t list_count(struct list_s *lst);
+uintptr_t list_item(struct list_s *lst, uintptr_t index);
+struct list_s *list_extend(struct list_s *lst, struct list_s *lst2,
+                           uintptr_t slicestart);
 
 struct tree_s *tree_create(void);
 void tree_free(struct tree_s *tree);
@@ -127,3 +133,16 @@
 def test_hash_permutation():
     hashes = [((n ^ (n << 4)) & 0xFF0) for n in range(256)]
     assert set(hashes) == set(range(0, 4096, 16))
+
+def test_list_extend():
+    a = lib.list_create()
+    b = lib.list_create()
+    for i in range(100, 120):
+        b = lib.list_append(b, i)
+    a = lib.list_extend(a, b, 3)
+    a = lib.list_extend(a, b, 13)
+    assert lib.list_count(a) == 17 + 7
+    for i, expected in enumerate(range(103, 120) + range(113, 120)):
+        assert lib.list_item(a, i) == expected
+    lib.list_free(b)
+    lib.list_free(a)

From noreply at buildbot.pypy.org  Tue Feb 24 23:50:38 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 24 Feb 2015 23:50:38 +0100 (CET)
Subject: [pypy-commit] stmgc default: c8: pass some more tests
Message-ID: <20150224225038.5BF1B1C0107@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: 
Changeset: r1661:ba1ea6a6ff54
Date: 2015-02-24 18:32 +0100
http://bitbucket.org/pypy/stmgc/changeset/ba1ea6a6ff54/

Log:	c8: pass some more tests

diff --git a/c8/stm/finalizer.c b/c8/stm/finalizer.c
--- a/c8/stm/finalizer.c
+++ b/c8/stm/finalizer.c
@@ -177,48 +177,49 @@
     list_clear(lst);
 }
 
-/* static void deal_with_old_objects_with_finalizers(void) */
-/* { */
-/*     /\* for light finalizers *\/ */
-/*     int old_gs_register = STM_SEGMENT->segment_num; */
-/*     int current_gs_register = old_gs_register; */
-/*     long j; */
-/*     for (j = 1; j <= NB_SEGMENTS; j++) { */
-/*         struct stm_priv_segment_info_s *pseg = get_priv_segment(j); */
+static void deal_with_old_objects_with_finalizers(void)
+{
+    /* for light finalizers */
+    int old_gs_register = STM_SEGMENT->segment_num;
+    int current_gs_register = old_gs_register;
+    long j;
+    assert(list_is_empty(get_priv_segment(0)->old_objects_with_light_finalizers));
+    for (j = 1; j < NB_SEGMENTS; j++) {
+        struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
 
-/*         struct list_s *lst = pseg->old_objects_with_light_finalizers; */
-/*         long i, count = list_count(lst); */
-/*         lst->count = 0; */
-/*         for (i = 0; i < count; i++) { */
-/*             object_t *obj = (object_t *)list_item(lst, i); */
-/*             if (!mark_visited_test(obj)) { */
-/*                 /\* not marked: object dies *\/ */
-/*                 /\* we're calling the light finalizer in the same */
-/*                    segment as where it was originally registered.  For */
-/*                    objects that existed since a long time, it doesn't */
-/*                    change anything: any thread should see the same old */
-/*                    content (because if it wasn't the case, the object */
-/*                    would be in a 'modified_old_objects' list */
-/*                    somewhere, and so it wouldn't be dead).  But it's */
-/*                    important if the object was created by the same */
-/*                    transaction: then only that segment sees valid */
-/*                    content. */
-/*                 *\/ */
-/*                 if (j != current_gs_register) { */
-/*                     set_gs_register(get_segment_base(j)); */
-/*                     current_gs_register = j; */
-/*                 } */
-/*                 stmcb_light_finalizer(obj); */
-/*             } */
-/*             else { */
-/*                 /\* object survives *\/ */
-/*                 list_set_item(lst, lst->count++, (uintptr_t)obj); */
-/*             } */
-/*         } */
-/*     } */
-/*     if (old_gs_register != current_gs_register) */
-/*         set_gs_register(get_segment_base(old_gs_register)); */
-/* } */
+        struct list_s *lst = pseg->old_objects_with_light_finalizers;
+        long i, count = list_count(lst);
+        lst->count = 0;
+        for (i = 0; i < count; i++) {
+            object_t *obj = (object_t *)list_item(lst, i);
+            if (!mark_visited_test(obj)) {
+                /* not marked: object dies */
+                /* we're calling the light finalizer in the same
+                   segment as where it was originally registered.  For
+                   objects that existed since a long time, it doesn't
+                   change anything: any thread should see the same old
+                   content (because if it wasn't the case, the object
+                   would be in a 'modified_old_objects' list
+                   somewhere, and so it wouldn't be dead).  But it's
+                   important if the object was created by the same
+                   transaction: then only that segment sees valid
+                   content.
+                */
+                if (j != current_gs_register) {
+                    set_gs_register(get_segment_base(j));
+                    current_gs_register = j;
+                }
+                stmcb_light_finalizer(obj);
+            }
+            else {
+                /* object survives */
+                list_set_item(lst, lst->count++, (uintptr_t)obj);
+            }
+        }
+    }
+    if (old_gs_register != current_gs_register)
+        set_gs_register(get_segment_base(old_gs_register));
+}
 
 
 /************************************************************/
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -661,6 +661,7 @@
 
     /* weakrefs */
     stm_visit_old_weakrefs();
+    deal_with_old_objects_with_finalizers();
 
     /* cleanup */
     clean_up_segment_lists();

From noreply at buildbot.pypy.org  Tue Feb 24 23:50:39 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 24 Feb 2015 23:50:39 +0100 (CET)
Subject: [pypy-commit] stmgc default: c8: port major gc finalizer handling
Message-ID: <20150224225039.5830C1C0107@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: 
Changeset: r1662:df47b9331468
Date: 2015-02-24 23:20 +0100
http://bitbucket.org/pypy/stmgc/changeset/df47b9331468/

Log:	c8: port major gc finalizer handling

diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -39,6 +39,7 @@
     GCFLAG_HAS_SHADOW = 0x02,
     GCFLAG_WB_EXECUTED = 0x04,
     GCFLAG_VISITED = 0x08,
+    GCFLAG_FINALIZATION_ORDERING = 0x10,
 };
 
 
diff --git a/c8/stm/finalizer.c b/c8/stm/finalizer.c
--- a/c8/stm/finalizer.c
+++ b/c8/stm/finalizer.c
@@ -228,195 +228,212 @@
     as well as rpython/memory/gc/minimark.py.
 */
 
-/* static inline int _finalization_state(object_t *obj) */
-/* { */
-/*     /\* Returns the state, "0", 1, 2 or 3, as per finalizer-order.rst. */
-/*        One difference is that the official state 0 is returned here */
-/*        as a number that is <= 0. *\/ */
-/*     uintptr_t lock_idx = mark_loc(obj); */
-/*     return write_locks[lock_idx] - (WL_FINALIZ_ORDER_1 - 1); */
-/* } */
+static inline int _finalization_state(object_t *obj)
+{
+    /* Returns the state, "0", 1, 2 or 3, as per finalizer-order.rst.
+       One difference is that the official state 0 is returned here
+       as a number that is <= 0. */
+    struct object_s *realobj = mark_loc(obj);
+    if (realobj->stm_flags & GCFLAG_VISITED) {
+        if (realobj->stm_flags & GCFLAG_FINALIZATION_ORDERING)
+            return 2;
+        return 3;
+    }
 
-/* static void _bump_finalization_state_from_0_to_1(object_t *obj) */
-/* { */
-/*     uintptr_t lock_idx = mark_loc(obj); */
-/*     assert(write_locks[lock_idx] < WL_FINALIZ_ORDER_1); */
-/*     write_locks[lock_idx] = WL_FINALIZ_ORDER_1; */
-/* } */
+    if (realobj->stm_flags & GCFLAG_FINALIZATION_ORDERING)
+        return 1;
 
-/* static struct list_s *_finalizer_tmpstack; */
-/* static struct list_s *_finalizer_emptystack; */
-/* static struct list_s *_finalizer_pending; */
+    return 0;
+}
 
-/* static inline void _append_to_finalizer_tmpstack(object_t **pobj) */
-/* { */
-/*     object_t *obj = *pobj; */
-/*     if (obj != NULL) */
-/*         LIST_APPEND(_finalizer_tmpstack, obj); */
-/* } */
+static void _bump_finalization_state_from_0_to_1(object_t *obj)
+{
+    assert(_finalization_state(obj) == 0);
+    struct object_s *realobj = mark_loc(obj);
+    realobj->stm_flags |= GCFLAG_FINALIZATION_ORDERING;
+}
 
-/* static inline struct list_s *finalizer_trace(char *base, object_t *obj, */
-/*                                              struct list_s *lst) */
-/* { */
-/*     struct object_s *realobj = (struct object_s *)REAL_ADDRESS(base, obj); */
-/*     _finalizer_tmpstack = lst; */
-/*     stmcb_trace(realobj, &_append_to_finalizer_tmpstack); */
-/*     return _finalizer_tmpstack; */
-/* } */
+static struct list_s *_finalizer_tmpstack;
+static struct list_s *_finalizer_emptystack;
+static struct list_s *_finalizer_pending;
 
-/* static void _recursively_bump_finalization_state(char *base, object_t *obj, */
-/*                                                  int to_state) */
-/* { */
-/*     struct list_s *tmpstack = _finalizer_emptystack; */
-/*     assert(list_is_empty(tmpstack)); */
+static inline void _append_to_finalizer_tmpstack(object_t **pobj)
+{
+    object_t *obj = *pobj;
+    if (obj != NULL)
+        LIST_APPEND(_finalizer_tmpstack, obj);
+}
 
-/*     while (1) { */
-/*         if (_finalization_state(obj) == to_state - 1) { */
-/*             /\* bump to the next state *\/ */
-/*             write_locks[mark_loc(obj)]++; */
+static inline struct list_s *finalizer_trace(char *base, object_t *obj,
+                                             struct list_s *lst)
+{
+    struct object_s *realobj = (struct object_s *)REAL_ADDRESS(base, obj);
+    _finalizer_tmpstack = lst;
+    stmcb_trace(realobj, &_append_to_finalizer_tmpstack);
+    return _finalizer_tmpstack;
+}
 
-/*             /\* trace *\/ */
-/*             tmpstack = finalizer_trace(base, obj, tmpstack); */
-/*         } */
 
-/*         if (list_is_empty(tmpstack)) */
-/*             break; */
+static void _recursively_bump_finalization_state_from_2_to_3(char *base, object_t *obj)
+{
+    assert(_finalization_state(obj) == 2);
+    struct list_s *tmpstack = _finalizer_emptystack;
+    assert(list_is_empty(tmpstack));
 
-/*         obj = (object_t *)list_pop_item(tmpstack); */
-/*     } */
-/*     _finalizer_emptystack = tmpstack; */
-/* } */
+    while (1) {
+        struct object_s *realobj = mark_loc(obj);
+        if (realobj->stm_flags & GCFLAG_FINALIZATION_ORDERING) {
+            realobj->stm_flags &= ~GCFLAG_FINALIZATION_ORDERING;
 
-/* static struct list_s *mark_finalize_step1(char *base, struct finalizers_s *f) */
-/* { */
-/*     if (f == NULL) */
-/*         return NULL; */
+            /* trace */
+            tmpstack = finalizer_trace(base, obj, tmpstack);
+        }
 
-/*     struct list_s *marked = list_create(); */
+        if (list_is_empty(tmpstack))
+            break;
 
-/*     struct list_s *lst = f->objects_with_finalizers; */
-/*     long i, count = list_count(lst); */
-/*     lst->count = 0; */
-/*     f->count_non_young = 0; */
+        obj = (object_t *)list_pop_item(tmpstack);
+    }
+    _finalizer_emptystack = tmpstack;
+}
 
-/*     for (i = 0; i < count; i++) { */
-/*         object_t *x = (object_t *)list_item(lst, i); */
+static void _recursively_bump_finalization_state_from_1_to_2(char *base, object_t *obj)
+{
+    assert(_finalization_state(obj) == 1);
+    /* The call will add GCFLAG_VISITED recursively, thus bump state 1->2 */
+    mark_visit_possibly_new_object(base, obj);
+}
 
-/*         assert(_finalization_state(x) != 1); */
-/*         if (_finalization_state(x) >= 2) { */
-/*             list_set_item(lst, lst->count++, (uintptr_t)x); */
-/*             continue; */
-/*         } */
-/*         LIST_APPEND(marked, x); */
+static struct list_s *mark_finalize_step1(char *base, struct finalizers_s *f)
+{
+    if (f == NULL)
+        return NULL;
 
-/*         struct list_s *pending = _finalizer_pending; */
-/*         LIST_APPEND(pending, x); */
-/*         while (!list_is_empty(pending)) { */
-/*             object_t *y = (object_t *)list_pop_item(pending); */
-/*             int state = _finalization_state(y); */
-/*             if (state <= 0) { */
-/*                 _bump_finalization_state_from_0_to_1(y); */
-/*                 pending = finalizer_trace(base, y, pending); */
-/*             } */
-/*             else if (state == 2) { */
-/*                 _recursively_bump_finalization_state(base, y, 3); */
-/*             } */
-/*         } */
-/*         _finalizer_pending = pending; */
-/*         assert(_finalization_state(x) == 1); */
-/*         _recursively_bump_finalization_state(base, x, 2); */
-/*     } */
-/*     return marked; */
-/* } */
+    struct list_s *marked = list_create();
 
-/* static void mark_finalize_step2(char *base, struct finalizers_s *f, */
-/*                                 struct list_s *marked) */
-/* { */
-/*     if (f == NULL) */
-/*         return; */
+    struct list_s *lst = f->objects_with_finalizers;
+    long i, count = list_count(lst);
+    lst->count = 0;
+    f->count_non_young = 0;
 
-/*     struct list_s *run_finalizers = f->run_finalizers; */
+    for (i = 0; i < count; i++) {
+        object_t *x = (object_t *)list_item(lst, i);
 
-/*     long i, count = list_count(marked); */
-/*     for (i = 0; i < count; i++) { */
-/*         object_t *x = (object_t *)list_item(marked, i); */
+        assert(_finalization_state(x) != 1);
+        if (_finalization_state(x) >= 2) {
+            list_set_item(lst, lst->count++, (uintptr_t)x);
+            continue;
+        }
+        LIST_APPEND(marked, x);
 
-/*         int state = _finalization_state(x); */
-/*         assert(state >= 2); */
-/*         if (state == 2) { */
-/*             if (run_finalizers == NULL) */
-/*                 run_finalizers = list_create(); */
-/*             LIST_APPEND(run_finalizers, x); */
-/*             _recursively_bump_finalization_state(base, x, 3); */
-/*         } */
-/*         else { */
-/*             struct list_s *lst = f->objects_with_finalizers; */
-/*             list_set_item(lst, lst->count++, (uintptr_t)x); */
-/*         } */
-/*     } */
-/*     list_free(marked); */
+        struct list_s *pending = _finalizer_pending;
+        LIST_APPEND(pending, x);
+        while (!list_is_empty(pending)) {
+            object_t *y = (object_t *)list_pop_item(pending);
+            int state = _finalization_state(y);
+            if (state <= 0) {
+                _bump_finalization_state_from_0_to_1(y);
+                pending = finalizer_trace(base, y, pending);
+            }
+            else if (state == 2) {
+                _recursively_bump_finalization_state_from_2_to_3(base, y);
+            }
+        }
+        _finalizer_pending = pending;
+        assert(_finalization_state(x) == 1);
+        _recursively_bump_finalization_state_from_1_to_2(base, x);
+    }
+    return marked;
+}
 
-/*     f->run_finalizers = run_finalizers; */
-/* } */
+static void mark_finalize_step2(char *base, struct finalizers_s *f,
+                                struct list_s *marked)
+{
+    if (f == NULL)
+        return;
 
-/* static void deal_with_objects_with_finalizers(void) */
-/* { */
-/*     /\* for non-light finalizers *\/ */
+    struct list_s *run_finalizers = f->run_finalizers;
 
-/*     /\* there is one 'objects_with_finalizers' list per segment. */
-/*        Objects that die at a major collection running in the same */
-/*        transaction as they were created will be put in the */
-/*        'run_finalizers' list of that segment.  Objects that survive at */
-/*        least one commit move to the global g_objects_with_finalizers, */
-/*        and when they die they go to g_run_finalizers.  The former kind */
-/*        of dying object must have its finalizer called in the correct */
-/*        thread; the latter kind can be called in any thread, through */
-/*        any segment, because they all should see the same old content */
-/*        anyway.  (If the content was different between segments at this */
-/*        point, the object would be in a 'modified_old_objects' list */
-/*        somewhere, and so it wouldn't be dead). */
-/*     *\/ */
-/*     struct list_s *marked_seg[NB_SEGMENTS + 1]; */
-/*     LIST_CREATE(_finalizer_emptystack); */
-/*     LIST_CREATE(_finalizer_pending); */
+    long i, count = list_count(marked);
+    for (i = 0; i < count; i++) {
+        object_t *x = (object_t *)list_item(marked, i);
 
-/*     long j; */
-/*     for (j = 1; j <= NB_SEGMENTS; j++) { */
-/*         struct stm_priv_segment_info_s *pseg = get_priv_segment(j); */
-/*         marked_seg[j] = mark_finalize_step1(pseg->pub.segment_base, */
-/*                                             pseg->finalizers); */
-/*     } */
-/*     marked_seg[0] = mark_finalize_step1(stm_object_pages, &g_finalizers); */
+        int state = _finalization_state(x);
+        assert(state >= 2);
+        if (state == 2) {
+            if (run_finalizers == NULL)
+                run_finalizers = list_create();
+            LIST_APPEND(run_finalizers, x);
+            _recursively_bump_finalization_state_from_2_to_3(base, x);
+        }
+        else {
+            struct list_s *lst = f->objects_with_finalizers;
+            list_set_item(lst, lst->count++, (uintptr_t)x);
+        }
+    }
+    list_free(marked);
 
-/*     LIST_FREE(_finalizer_pending); */
+    f->run_finalizers = run_finalizers;
+}
 
-/*     for (j = 1; j <= NB_SEGMENTS; j++) { */
-/*         struct stm_priv_segment_info_s *pseg = get_priv_segment(j); */
-/*         mark_finalize_step2(pseg->pub.segment_base, pseg->finalizers, */
-/*                             marked_seg[j]); */
-/*     } */
-/*     mark_finalize_step2(stm_object_pages, &g_finalizers, marked_seg[0]); */
+static void deal_with_objects_with_finalizers(void)
+{
+    /* for non-light finalizers */
 
-/*     LIST_FREE(_finalizer_emptystack); */
-/* } */
+    /* there is one 'objects_with_finalizers' list per segment.
+       Objects that die at a major collection running in the same
+       transaction as they were created will be put in the
+       'run_finalizers' list of that segment.  Objects that survive at
+       least one commit move to the global g_objects_with_finalizers,
+       and when they die they go to g_run_finalizers.  The former kind
+       of dying object must have its finalizer called in the correct
+       thread; the latter kind can be called in any thread, through
+       any segment, because they all should see the same old content
+       anyway.  (If the content was different between segments at this
+       point, the object would be in a 'modified_old_objects' list
+       somewhere, and so it wouldn't be dead).
+    */
+    struct list_s *marked_seg[NB_SEGMENTS];
+    LIST_CREATE(_finalizer_emptystack);
+    LIST_CREATE(_finalizer_pending);
 
-/* static void mark_visit_from_finalizer1(char *base, struct finalizers_s *f) */
-/* { */
-/*     if (f != NULL && f->run_finalizers != NULL) { */
-/*         LIST_FOREACH_R(f->run_finalizers, object_t * /\*item*\/, */
-/*                        mark_visit_object(item, base)); */
-/*     } */
-/* } */
+    long j;
+    for (j = 1; j < NB_SEGMENTS; j++) {
+        struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
+        marked_seg[j] = mark_finalize_step1(pseg->pub.segment_base,
+                                            pseg->finalizers);
+    }
+    marked_seg[0] = mark_finalize_step1(stm_object_pages, &g_finalizers);
 
-/* static void mark_visit_from_finalizer_pending(void) */
-/* { */
-/*     long j; */
-/*     for (j = 1; j <= NB_SEGMENTS; j++) { */
-/*         struct stm_priv_segment_info_s *pseg = get_priv_segment(j); */
-/*         mark_visit_from_finalizer1(pseg->pub.segment_base, pseg->finalizers); */
-/*     } */
-/*     mark_visit_from_finalizer1(stm_object_pages, &g_finalizers); */
-/* } */
+    LIST_FREE(_finalizer_pending);
+
+    for (j = 1; j < NB_SEGMENTS; j++) {
+        struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
+        mark_finalize_step2(pseg->pub.segment_base, pseg->finalizers,
+                            marked_seg[j]);
+    }
+    mark_finalize_step2(stm_object_pages, &g_finalizers, marked_seg[0]);
+
+    LIST_FREE(_finalizer_emptystack);
+}
+
+static void mark_visit_from_finalizer1(char *base, struct finalizers_s *f)
+{
+    if (f != NULL && f->run_finalizers != NULL) {
+        LIST_FOREACH_R(f->run_finalizers, object_t * /*item*/,
+                       mark_visit_object(item, base));
+    }
+}
+
+static void mark_visit_from_finalizer_pending(void)
+{
+    long j;
+    for (j = 1; j < NB_SEGMENTS; j++) {
+        struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
+        mark_visit_from_finalizer1(pseg->pub.segment_base, pseg->finalizers);
+    }
+    mark_visit_from_finalizer1(stm_object_pages, &g_finalizers);
+}
 
 static void _execute_finalizers(struct finalizers_s *f)
 {
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -146,6 +146,7 @@
     }
 
     s_mutex_unlock();
+    exec_local_finalizers();
 }
 
 
@@ -456,6 +457,9 @@
                 mark_visited_test_and_clear(item);
                 realobj = (struct object_s*)REAL_ADDRESS(stm_object_pages, item);
                 realobj->stm_flags |= GCFLAG_WB_EXECUTED;
+
+                /* make sure this flag is cleared as well */
+                realobj->stm_flags &= ~GCFLAG_FINALIZATION_ORDERING;
             }));
     }
 #pragma pop_macro("STM_SEGMENT")
@@ -657,9 +661,16 @@
     LIST_CREATE(marked_objects_to_trace);
     mark_visit_from_modified_objects();
     mark_visit_from_roots();
+    mark_visit_from_finalizer_pending();
+
+    /* finalizer support: will mark as visited all objects with a
+       finalizer and all objects reachable from there, and also moves
+       some objects from 'objects_with_finalizers' to 'run_finalizers'. */
+    deal_with_objects_with_finalizers();
+
     LIST_FREE(marked_objects_to_trace);
 
-    /* weakrefs */
+    /* weakrefs and execute old light finalizers */
     stm_visit_old_weakrefs();
     deal_with_old_objects_with_finalizers();
 

From noreply at buildbot.pypy.org  Tue Feb 24 23:50:40 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 24 Feb 2015 23:50:40 +0100 (CET)
Subject: [pypy-commit] stmgc default: c8: possibly safer by tracing in the
	right segment
Message-ID: <20150224225040.5B2E81C0107@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: 
Changeset: r1663:8c547a55af94
Date: 2015-02-24 23:50 +0100
http://bitbucket.org/pypy/stmgc/changeset/8c547a55af94/

Log:	c8: possibly safer by tracing in the right segment

diff --git a/c8/stm/finalizer.c b/c8/stm/finalizer.c
--- a/c8/stm/finalizer.c
+++ b/c8/stm/finalizer.c
@@ -267,6 +267,9 @@
 static inline struct list_s *finalizer_trace(char *base, object_t *obj,
                                              struct list_s *lst)
 {
+    if (!is_new_object(obj))
+        base = stm_object_pages;
+
     struct object_s *realobj = (struct object_s *)REAL_ADDRESS(base, obj);
     _finalizer_tmpstack = lst;
     stmcb_trace(realobj, &_append_to_finalizer_tmpstack);
@@ -421,7 +424,9 @@
 {
     if (f != NULL && f->run_finalizers != NULL) {
         LIST_FOREACH_R(f->run_finalizers, object_t * /*item*/,
-                       mark_visit_object(item, base));
+                       ({
+                           mark_visit_possibly_new_object(base, item);
+                       }));
     }
 }
 
diff --git a/c8/test/test_finalizer.py b/c8/test/test_finalizer.py
--- a/c8/test/test_finalizer.py
+++ b/c8/test/test_finalizer.py
@@ -254,3 +254,20 @@
         lp3 = stm_allocate_with_finalizer(32)
         print lp1, lp2, lp3
         stm_major_collect()
+
+    def test_new_objects_w_finalizers(self):
+        self.switch(2)
+        self.start_transaction()
+        lp1 = stm_allocate_with_finalizer_refs(3)
+        lp2 = stm_allocate_with_finalizer_refs(3)
+        stm_set_ref(lp1, 0, lp2)
+
+        self.push_root(lp1)
+        stm_minor_collect()
+        lp1 = self.pop_root()
+        lp2 = stm_get_ref(lp1, 0)
+        # lp1, lp2 have WB_EXECUTED objs
+
+        self.expect_finalized([])
+        stm_major_collect()
+        self.expect_finalized([lp1])

From noreply at buildbot.pypy.org  Wed Feb 25 09:22:27 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 25 Feb 2015 09:22:27 +0100 (CET)
Subject: [pypy-commit] pypy default: fix test_rx86_*_encoding
Message-ID: <20150225082227.D907B1C02BD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76115:a09b79ad2d51
Date: 2015-02-25 09:21 +0100
http://bitbucket.org/pypy/pypy/changeset/a09b79ad2d51/

Log:	fix test_rx86_*_encoding

diff --git a/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py b/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
--- a/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
+++ b/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
@@ -317,7 +317,9 @@
                 # CALL_j is actually relative, so tricky to test
                 (instrname == 'CALL' and argmodes == 'j') or
                 # SET_ir must be tested manually
-                (instrname == 'SET' and argmodes == 'ir')
+                (instrname == 'SET' and argmodes == 'ir') or
+                # MULTIBYTE_NOPs can't easily be tested the same way
+                (instrname == 'MULTIBYTE')
         )
 
 

From noreply at buildbot.pypy.org  Wed Feb 25 10:54:39 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 25 Feb 2015 10:54:39 +0100 (CET)
Subject: [pypy-commit] pypy default: issue #1991: fixed by removing
 sys.executable from the list of
Message-ID: <20150225095439.9BAE41C02BB@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76116:19e305e27e67
Date: 2015-02-25 10:54 +0100
http://bitbucket.org/pypy/pypy/changeset/19e305e27e67/

Log:	issue #1991: fixed by removing sys.executable from the list of
	prebuilt attributes. Related fixes of other places that would end
	up with sys.executable == 'py.py'.

diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -36,7 +36,8 @@
    "PyPy home directory".  The arguments are:
 
    * ``home``: NULL terminated path to an executable inside the pypy directory
-     (can be a .so name, can be made up)
+     (can be a .so name, can be made up).  Used to look up the standard
+     library, and is also set as ``sys.executable``.
 
    * ``verbose``: if non-zero, it will print error messages to stderr
 
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -106,6 +106,9 @@
         space.call_function(w_pathsetter, w_path)
         # import site
         try:
+            space.setattr(space.getbuiltinmodule('sys'),
+                          space.wrap('executable'),
+                          space.wrap(home))
             import_ = space.getattr(space.getbuiltinmodule('__builtin__'),
                                     space.wrap('__import__'))
             space.call_function(import_, space.wrap('site'))
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -1,5 +1,5 @@
 #! /usr/bin/env python
-# App-level version of py.py.
+# This is pure Python code that handles the main entry point into "pypy".
 # See test/test_app_main.
 
 # Missing vs CPython: -d, -t, -v, -x, -3
@@ -157,10 +157,13 @@
             current = group
     raise SystemExit
 
+def get_sys_executable():
+    return getattr(sys, 'executable', 'pypy')
+
 def print_help(*args):
     import os
     print 'usage: %s [option] ... [-c cmd | -m mod | file | -] [arg] ...' % (
-        sys.executable,)
+        get_sys_executable(),)
     print USAGE1,
     if 'pypyjit' in sys.builtin_module_names:
         print "--jit options: advanced JIT options: try 'off' or 'help'"
@@ -171,7 +174,7 @@
     try:
         import pypyjit
     except ImportError:
-        print >> sys.stderr, "No jit support in %s" % (sys.executable,)
+        print >> sys.stderr, "No jit support in %s" % (get_sys_executable(),)
         return
     items = sorted(pypyjit.defaults.items())
     print 'Advanced JIT options: a comma-separated list of OPTION=VALUE:'
@@ -209,7 +212,7 @@
         raise SystemExit
     if 'pypyjit' not in sys.builtin_module_names:
         print >> sys.stderr, ("Warning: No jit support in %s" %
-                              (sys.executable,))
+                              (get_sys_executable(),))
     else:
         import pypyjit
         pypyjit.set_param(jitparam)
@@ -219,8 +222,8 @@
 
 def print_error(msg):
     print >> sys.stderr, msg
-    print >> sys.stderr, 'usage: %s [options]' % (sys.executable,)
-    print >> sys.stderr, 'Try `%s -h` for more information.' % (sys.executable,)
+    print >> sys.stderr, 'usage: %s [options]' % (get_sys_executable(),)
+    print >> sys.stderr, 'Try `%s -h` for more information.' % (get_sys_executable(),)
 
 def fdopen(fd, mode, bufsize=-1):
     try:
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -373,7 +373,7 @@
         config = make_config(None)
         space = make_objspace(config)
         w_executable = space.wrap('executable')
-        assert space.str_w(space.getattr(space.sys, w_executable)) == 'py.py'
+        assert space.findattr(space.sys, w_executable) is None
         space.setattr(space.sys, w_executable, space.wrap('foobar'))
         assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
         space.startup()
diff --git a/pypy/interpreter/test/test_targetpypy.py b/pypy/interpreter/test/test_targetpypy.py
--- a/pypy/interpreter/test/test_targetpypy.py
+++ b/pypy/interpreter/test/test_targetpypy.py
@@ -8,7 +8,7 @@
         entry_point = get_entry_point(config)[0]
         entry_point(['pypy-c' , '-S', '-c', 'print 3'])
 
-def test_exeucte_source(space):
+def test_execute_source(space):
     _, d = create_entry_point(space, None)
     execute_source = d['pypy_execute_source']
     lls = rffi.str2charp("import sys; sys.modules['xyz'] = 3")
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -64,7 +64,6 @@
         'call_tracing'          : 'vm.call_tracing',
         'getsizeof'             : 'vm.getsizeof',
 
-        'executable'            : 'space.wrap("py.py")',
         'api_version'           : 'version.get_api_version(space)',
         'version_info'          : 'version.get_version_info(space)',
         'version'               : 'version.get_version(space)',
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -500,7 +500,7 @@
         assert isinstance(sys.builtin_module_names, tuple)
         assert isinstance(sys.copyright, basestring)
         #assert isinstance(sys.exec_prefix, basestring) -- not present!
-        assert isinstance(sys.executable, basestring)
+        #assert isinstance(sys.executable, basestring) -- not present!
         assert isinstance(sys.hexversion, int)
         assert isinstance(sys.maxint, int)
         assert isinstance(sys.maxsize, int)
@@ -519,6 +519,12 @@
         assert vi[3] in ("alpha", "beta", "candidate", "final")
         assert isinstance(vi[4], int)
 
+    def test_reload_doesnt_override_sys_executable(self):
+        import sys
+        sys.executable = 'from_test_sysmodule'
+        reload(sys)
+        assert sys.executable == 'from_test_sysmodule'
+
     def test_settrace(self):
         import sys
         counts = []

From noreply at buildbot.pypy.org  Wed Feb 25 11:53:13 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 25 Feb 2015 11:53:13 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c7: add a TODO
Message-ID: <20150225105313.28D0B1C13B0@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: stmgc-c7
Changeset: r76117:a47e96925062
Date: 2015-02-25 10:19 +0100
http://bitbucket.org/pypy/pypy/changeset/a47e96925062/

Log:	add a TODO

diff --git a/TODO b/TODO
--- a/TODO
+++ b/TODO
@@ -1,3 +1,10 @@
+------------------------------------------------------------
+
+maybe statically optimize away some stm_become_inevitable(), there
+are some loops that call it repeatedly (may be not relevant
+for performance) 
+
+------------------------------------------------------------
 
 1b664888133d (March 15, 2014): the overhead of a non-JIT STM, when
 compared with a non-JIT plain PyPy, is measured to be 54% in a

From noreply at buildbot.pypy.org  Wed Feb 25 11:53:23 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 25 Feb 2015 11:53:23 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c8: Merge with stmgc-c7
Message-ID: <20150225105323.A2AA81C13B0@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: stmgc-c8
Changeset: r76118:f9544d0d6739
Date: 2015-02-25 10:23 +0100
http://bitbucket.org/pypy/pypy/changeset/f9544d0d6739/

Log:	Merge with stmgc-c7

diff too long, truncating to 2000 out of 40925 lines

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -28,7 +28,7 @@
     DEALINGS IN THE SOFTWARE.
 
 
-PyPy Copyright holders 2003-2014
+PyPy Copyright holders 2003-2015
 ----------------------------------- 
 
 Except when otherwise stated (look for LICENSE files or information at
@@ -42,19 +42,19 @@
   Amaury Forgeot d'Arc
   Samuele Pedroni
   Alex Gaynor
+  Brian Kearns
+  Matti Picus
+  Philip Jenvey
   Michael Hudson
   David Schneider
-  Matti Picus
-  Brian Kearns
-  Philip Jenvey
   Holger Krekel
   Christian Tismer
   Hakan Ardo
   Benjamin Peterson
   Manuel Jacob
+  Ronan Lamy
   Anders Chrigstrom
   Eric van Riet Paap
-  Ronan Lamy
   Wim Lavrijsen
   Richard Emslie
   Alexander Schremmer
@@ -68,9 +68,9 @@
   Camillo Bruni
   Laura Creighton
   Toon Verwaest
+  Romain Guillebert
   Leonardo Santagada
   Seo Sanghyeon
-  Romain Guillebert
   Justin Peel
   Ronny Pfannschmidt
   David Edelsohn
@@ -91,15 +91,16 @@
   Michal Bendowski
   Jan de Mooij
   stian
+  Tyler Wade
   Michael Foord
   Stephan Diehl
-  Tyler Wade
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
   Patrick Maupin
   Bob Ippolito
   Bruno Gola
+  David Malcolm
   Jean-Paul Calderone
   Timo Paulssen
   Squeaky
@@ -108,18 +109,19 @@
   Marius Gedminas
   Martin Matusiak
   Konstantin Lopuhin
+  Wenzhu Man
   John Witulski
-  Wenzhu Man
+  Laurence Tratt
+  Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
   Mark Pearse
   Simon Cross
-  Ivan Sichmann Freitas
   Andreas Stührk
+  Stefano Rivera
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Stefano Rivera
   Paweł Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -129,7 +131,6 @@
   tav
   Taavi Burns
   Georg Brandl
-  Laurence Tratt
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -141,13 +142,12 @@
   Jeremy Thurgood
   Rami Chowdhury
   Tobias Pape
-  David Malcolm
   Eugene Oden
   Henry Mason
   Vasily Kuznetsov
   Preston Timmons
+  David Ripton
   Jeff Terrace
-  David Ripton
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
@@ -166,13 +166,16 @@
   Gintautas Miliauskas
   Michael Twomey
   Lucian Branescu Mihaila
+  Yichao Yu
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
   Karl Bartel
+  Wouter van Heyst
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  anatoly techtonik
   Stuart Williams
   Jasper Schulz
   Christian Hudon
@@ -182,12 +185,11 @@
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
-  anatoly techtonik
   Neil Shepperd
+  Stanislaw Halik
   Mikael Schönenberg
   Elmo M?ntynen
   Jonathan David Riehl
-  Stanislaw Halik
   Anders Qvist
   Corbin Simpson
   Chirag Jadwani
@@ -196,10 +198,13 @@
   Vincent Legoll
   Alan McIntyre
   Alexander Sedov
+  Attila Gobi
   Christopher Pope
   Christian Tismer 
   Marc Abramowitz
   Dan Stromberg
+  Arjun Naik
+  Valentina Mukhamedzhanova
   Stefano Parmesan
   Alexis Daboville
   Jens-Uwe Mager
@@ -213,8 +218,6 @@
   Sylvain Thenault
   Nathan Taylor
   Vladimir Kryachko
-  Arjun Naik
-  Attila Gobi
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -222,22 +225,23 @@
   Ryan Gonzalez
   Ian Foote
   Kristjan Valur Jonsson
+  David Lievens
   Neil Blakey-Milner
   Lutz Paelike
   Lucio Torre
   Lars Wassermann
-  Valentina Mukhamedzhanova
   Henrik Vendelbo
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
   Sergey Kishchenko
-  Yichao Yu
   Ignas Mikalajunas
   Christoph Gerum
   Martin Blais
   Lene Wagner
   Tomo Cocoa
+  Toni Mattis
+  Lucas Stadler
   roberto at goyle
   Yury V. Zaytsev
   Anna Katrina Dominguez
@@ -265,23 +269,30 @@
   Stephan Busemann
   Rafał Gałczyński
   Christian Muirhead
+  Berker Peksag
   James Lan
   shoma hosaka
-  Daniel Neuh?user
-  Matthew Miller
+  Daniel Neuhäuser
+  Ben Mather
+  halgari
+  Boglarka Vezer
+  Chris Pressey
   Buck Golemon
   Konrad Delong
   Dinu Gherman
   Chris Lambacher
   coolbutuseless at gmail.com
+  Jim Baker
   Rodrigo Araújo
-  Jim Baker
+  Nikolaos-Digenis Karagiannis
   James Robert
   Armin Ronacher
   Brett Cannon
+  Donald Stufft
   yrttyr
   aliceinwire
   OlivierBlanvillain
+  Dan Sanders
   Zooko Wilcox-O Hearn
   Tomer Chachamu
   Christopher Groskopf
@@ -295,6 +306,7 @@
   Markus Unterwaditzer
   Even Wiik Thomassen
   jbs
+  squeaky
   soareschen
   Kurt Griffiths
   Mike Bayer
@@ -306,6 +318,7 @@
   Anna Ravencroft
   Dan Crosta
   Julien Phalip
+  Roman Podoliaka
   Dan Loewenherz
 
   Heinrich-Heine University, Germany 
diff --git a/TODO b/TODO
--- a/TODO
+++ b/TODO
@@ -1,3 +1,10 @@
+------------------------------------------------------------
+
+maybe statically optimize away some stm_become_inevitable(), there
+are some loops that call it repeatedly (may be not relevant
+for performance) 
+
+------------------------------------------------------------
 
 1b664888133d (March 15, 2014): the overhead of a non-JIT STM, when
 compared with a non-JIT plain PyPy, is measured to be 54% in a
@@ -146,6 +153,12 @@
 stm_read(p125)
 cond_call_gc_wb_array(p125...)    # don't need the stm_read maybe?
 
+------------------------------------------------------------
+
+we should fake the stm_location inside jit/metainterp, so that it
+is reported correctly even if we're (1) tracing, (2) blackholing,
+or (3) in ResumeDataDirectReader.change_stm_location
+
 
 
 ===============================================================================
diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -17,6 +17,10 @@
 except ImportError:
     assert '__pypy__' not in _sys.builtin_module_names
     newdict = lambda _ : {}
+try:
+    from __pypy__ import reversed_dict
+except ImportError:
+    reversed_dict = lambda d: reversed(d.keys())
 
 try:
     from thread import get_ident as _get_ident
@@ -29,142 +33,35 @@
 ################################################################################
 
 class OrderedDict(dict):
-    'Dictionary that remembers insertion order'
-    # An inherited dict maps keys to values.
-    # The inherited dict provides __getitem__, __len__, __contains__, and get.
-    # The remaining methods are order-aware.
-    # Big-O running times for all methods are the same as regular dictionaries.
+    '''Dictionary that remembers insertion order.
 
-    # The internal self.__map dict maps keys to links in a doubly linked list.
-    # The circular doubly linked list starts and ends with a sentinel element.
-    # The sentinel element never gets deleted (this simplifies the algorithm).
-    # Each link is stored as a list of length three:  [PREV, NEXT, KEY].
+    In PyPy all dicts are ordered anyway.  This is mostly useful as a
+    placeholder to mean "this dict must be ordered even on CPython".
 
-    def __init__(self, *args, **kwds):
-        '''Initialize an ordered dictionary.  The signature is the same as
-        regular dictionaries, but keyword arguments are not recommended because
-        their insertion order is arbitrary.
-
-        '''
-        if len(args) > 1:
-            raise TypeError('expected at most 1 arguments, got %d' % len(args))
-        try:
-            self.__root
-        except AttributeError:
-            self.__root = root = []                     # sentinel node
-            root[:] = [root, root, None]
-            self.__map = {}
-        self.__update(*args, **kwds)
-
-    def __setitem__(self, key, value, dict_setitem=dict.__setitem__):
-        'od.__setitem__(i, y) <==> od[i]=y'
-        # Setting a new item creates a new link at the end of the linked list,
-        # and the inherited dictionary is updated with the new key/value pair.
-        if key not in self:
-            root = self.__root
-            last = root[0]
-            last[1] = root[0] = self.__map[key] = [last, root, key]
-        return dict_setitem(self, key, value)
-
-    def __delitem__(self, key, dict_delitem=dict.__delitem__):
-        'od.__delitem__(y) <==> del od[y]'
-        # Deleting an existing item uses self.__map to find the link which gets
-        # removed by updating the links in the predecessor and successor nodes.
-        dict_delitem(self, key)
-        link_prev, link_next, _ = self.__map.pop(key)
-        link_prev[1] = link_next                        # update link_prev[NEXT]
-        link_next[0] = link_prev                        # update link_next[PREV]
-
-    def __iter__(self):
-        'od.__iter__() <==> iter(od)'
-        # Traverse the linked list in order.
-        root = self.__root
-        curr = root[1]                                  # start at the first node
-        while curr is not root:
-            yield curr[2]                               # yield the curr[KEY]
-            curr = curr[1]                              # move to next node
+    Known difference: iterating over an OrderedDict which is being
+    concurrently modified raises RuntimeError in PyPy.  In CPython
+    instead we get some behavior that appears reasonable in some
+    cases but is nonsensical in other cases.  This is officially
+    forbidden by the CPython docs, so we forbid it explicitly for now.
+    '''
 
     def __reversed__(self):
-        'od.__reversed__() <==> reversed(od)'
-        # Traverse the linked list in reverse order.
-        root = self.__root
-        curr = root[0]                                  # start at the last node
-        while curr is not root:
-            yield curr[2]                               # yield the curr[KEY]
-            curr = curr[0]                              # move to previous node
-
-    def clear(self):
-        'od.clear() -> None.  Remove all items from od.'
-        root = self.__root
-        root[:] = [root, root, None]
-        self.__map.clear()
-        dict.clear(self)
-
-    # -- the following methods do not depend on the internal structure --
-
-    def keys(self):
-        'od.keys() -> list of keys in od'
-        return list(self)
-
-    def values(self):
-        'od.values() -> list of values in od'
-        return [self[key] for key in self]
-
-    def items(self):
-        'od.items() -> list of (key, value) pairs in od'
-        return [(key, self[key]) for key in self]
-
-    def iterkeys(self):
-        'od.iterkeys() -> an iterator over the keys in od'
-        return iter(self)
-
-    def itervalues(self):
-        'od.itervalues -> an iterator over the values in od'
-        for k in self:
-            yield self[k]
-
-    def iteritems(self):
-        'od.iteritems -> an iterator over the (key, value) pairs in od'
-        for k in self:
-            yield (k, self[k])
-
-    update = MutableMapping.update
-
-    __update = update # let subclasses override update without breaking __init__
-
-    __marker = object()
-
-    def pop(self, key, default=__marker):
-        '''od.pop(k[,d]) -> v, remove specified key and return the corresponding
-        value.  If key is not found, d is returned if given, otherwise KeyError
-        is raised.
-
-        '''
-        if key in self:
-            result = self[key]
-            del self[key]
-            return result
-        if default is self.__marker:
-            raise KeyError(key)
-        return default
-
-    def setdefault(self, key, default=None):
-        'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
-        if key in self:
-            return self[key]
-        self[key] = default
-        return default
+        return reversed_dict(self)
 
     def popitem(self, last=True):
         '''od.popitem() -> (k, v), return and remove a (key, value) pair.
         Pairs are returned in LIFO order if last is true or FIFO order if false.
 
         '''
-        if not self:
-            raise KeyError('dictionary is empty')
-        key = next(reversed(self) if last else iter(self))
-        value = self.pop(key)
-        return key, value
+        if last:
+            return dict.popitem(self)
+        else:
+            it = dict.__iter__(self)
+            try:
+                k = it.next()
+            except StopIteration:
+                raise KeyError('dictionary is empty')
+            return (k, self.pop(k))
 
     def __repr__(self, _repr_running={}):
         'od.__repr__() <==> repr(od)'
@@ -183,8 +80,6 @@
         'Return state information for pickling'
         items = [[k, self[k]] for k in self]
         inst_dict = vars(self).copy()
-        for k in vars(OrderedDict()):
-            inst_dict.pop(k, None)
         if inst_dict:
             return (self.__class__, (items,), inst_dict)
         return self.__class__, (items,)
@@ -193,17 +88,6 @@
         'od.copy() -> a shallow copy of od'
         return self.__class__(self)
 
-    @classmethod
-    def fromkeys(cls, iterable, value=None):
-        '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S.
-        If not specified, the value defaults to None.
-
-        '''
-        self = cls()
-        for key in iterable:
-            self[key] = value
-        return self
-
     def __eq__(self, other):
         '''od.__eq__(y) <==> od==y.  Comparison to another OD is order-sensitive
         while comparison to a regular mapping is order-insensitive.
diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py
@@ -2,7 +2,6 @@
 import array
 import gc
 import unittest
-from ctypes.test import xfail
 
 class X(Structure):
     _fields_ = [("c_int", c_int)]
@@ -11,7 +10,6 @@
         self._init_called = True
 
 class Test(unittest.TestCase):
-    @xfail
     def test_fom_buffer(self):
         a = array.array("i", range(16))
         x = (c_int * 16).from_buffer(a)
@@ -34,10 +32,9 @@
         del a; gc.collect(); gc.collect(); gc.collect()
         self.assertEqual(x[:], expected)
 
-        self.assertRaises(TypeError,
+        self.assertRaises((TypeError, ValueError),
                           (c_char * 16).from_buffer, "a" * 16)
 
-    @xfail
     def test_fom_buffer_with_offset(self):
         a = array.array("i", range(16))
         x = (c_int * 15).from_buffer(a, sizeof(c_int))
@@ -46,7 +43,6 @@
         self.assertRaises(ValueError, lambda: (c_int * 16).from_buffer(a, sizeof(c_int)))
         self.assertRaises(ValueError, lambda: (c_int * 1).from_buffer(a, 16 * sizeof(c_int)))
 
-    @xfail
     def test_from_buffer_copy(self):
         a = array.array("i", range(16))
         x = (c_int * 16).from_buffer_copy(a)
@@ -71,7 +67,6 @@
         x = (c_char * 16).from_buffer_copy("a" * 16)
         self.assertEqual(x[:], "a" * 16)
 
-    @xfail
     def test_fom_buffer_copy_with_offset(self):
         a = array.array("i", range(16))
         x = (c_int * 15).from_buffer_copy(a, sizeof(c_int))
diff --git a/lib-python/2.7/distutils/unixccompiler.py b/lib-python/2.7/distutils/unixccompiler.py
--- a/lib-python/2.7/distutils/unixccompiler.py
+++ b/lib-python/2.7/distutils/unixccompiler.py
@@ -58,7 +58,7 @@
     executables = {'preprocessor' : None,
                    'compiler'     : ["cc"],
                    'compiler_so'  : ["cc"],
-                   'compiler_cxx' : ["cc"],
+                   'compiler_cxx' : ["c++"],  # pypy: changed, 'cc' is bogus
                    'linker_so'    : ["cc", "-shared"],
                    'linker_exe'   : ["cc"],
                    'archiver'     : ["ar", "-cr"],
diff --git a/lib-python/2.7/platform.py b/lib-python/2.7/platform.py
--- a/lib-python/2.7/platform.py
+++ b/lib-python/2.7/platform.py
@@ -1382,7 +1382,7 @@
 _pypy_sys_version_parser = re.compile(
     r'([\w.+]+)\s*'
     '\(#?([^,]+),\s*([\w ]+),\s*([\w :]+)\)\s*'
-    '\[PyPy [^\]]+\]?')
+    '\[PyPy[^\]]+\]?')   # this also covers 'PyPy-STM x.y'
 
 _sys_version_cache = {}
 
diff --git a/lib-python/2.7/sqlite3/test/dbapi.py b/lib-python/2.7/sqlite3/test/dbapi.py
--- a/lib-python/2.7/sqlite3/test/dbapi.py
+++ b/lib-python/2.7/sqlite3/test/dbapi.py
@@ -478,6 +478,29 @@
         except TypeError:
             pass
 
+    def CheckCurDescription(self):
+        self.cu.execute("select * from test")
+
+        actual = self.cu.description
+        expected = [
+            ('id', None, None, None, None, None, None),
+            ('name', None, None, None, None, None, None),
+            ('income', None, None, None, None, None, None),
+        ]
+        self.assertEqual(expected, actual)
+
+    def CheckCurDescriptionVoidStatement(self):
+        self.cu.execute("insert into test(name) values (?)", ("foo",))
+        self.assertIsNone(self.cu.description)
+
+    def CheckCurDescriptionWithoutStatement(self):
+        cu = self.cx.cursor()
+        try:
+            self.assertIsNone(cu.description)
+        finally:
+            cu.close()
+
+
 @unittest.skipUnless(threading, 'This test requires threading.')
 class ThreadTests(unittest.TestCase):
     def setUp(self):
diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -1589,7 +1589,7 @@
                   'copyfile' in caller.f_globals):
         dest_dir = sys.pypy_resolvedirof(target_executable)
         src_dir = sys.pypy_resolvedirof(sys.executable)
-        for libname in ['libpypy-c.so']:
+        for libname in ['libpypy-c.so', 'libpypy-c.dylib']:
             dest_library = os.path.join(dest_dir, libname)
             src_library = os.path.join(src_dir, libname)
             if os.path.exists(src_library):
diff --git a/lib-python/2.7/test/test_collections.py b/lib-python/2.7/test/test_collections.py
--- a/lib-python/2.7/test/test_collections.py
+++ b/lib-python/2.7/test/test_collections.py
@@ -578,7 +578,12 @@
             def __repr__(self):
                 return "MySet(%s)" % repr(list(self))
         s = MySet([5,43,2,1])
-        self.assertEqual(s.pop(), 1)
+        # changed from CPython 2.7: it was "s.pop() == 1" but I see
+        # nothing that guarantees a particular order here.  In the
+        # 'all_ordered_dicts' branch of PyPy (or with OrderedDict
+        # instead of sets), it consistently returns 5, but this test
+        # should not rely on this or any other order.
+        self.assert_(s.pop() in [5,43,2,1])
 
     def test_issue8750(self):
         empty = WithSet()
@@ -1010,8 +1015,9 @@
                                           c=3, e=5).items()), pairs)                # mixed input
 
         # make sure no positional args conflict with possible kwdargs
-        self.assertEqual(inspect.getargspec(OrderedDict.__dict__['__init__']).args,
-                         ['self'])
+        if '__init__' in OrderedDict.__dict__:   # absent in PyPy
+            self.assertEqual(inspect.getargspec(OrderedDict.__dict__['__init__']).args,
+                             ['self'])
 
         # Make sure that direct calls to __init__ do not clear previous contents
         d = OrderedDict([('a', 1), ('b', 2), ('c', 3), ('d', 44), ('e', 55)])
@@ -1108,6 +1114,16 @@
             od.popitem()
         self.assertEqual(len(od), 0)
 
+    def test_popitem_first(self):
+        pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
+        shuffle(pairs)
+        od = OrderedDict(pairs)
+        while pairs:
+            self.assertEqual(od.popitem(last=False), pairs.pop(0))
+        with self.assertRaises(KeyError):
+            od.popitem(last=False)
+        self.assertEqual(len(od), 0)
+
     def test_pop(self):
         pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
         shuffle(pairs)
@@ -1179,7 +1195,11 @@
         od = OrderedDict(pairs)
         # yaml.dump(od) -->
         # '!!python/object/apply:__main__.OrderedDict\n- - [a, 1]\n  - [b, 2]\n'
-        self.assertTrue(all(type(pair)==list for pair in od.__reduce__()[1]))
+
+        # PyPy bug fix: added [0] at the end of this line, because the
+        # test is really about the 2-tuples that need to be 2-lists
+        # inside the list of 6 of them
+        self.assertTrue(all(type(pair)==list for pair in od.__reduce__()[1][0]))
 
     def test_reduce_not_too_fat(self):
         # do not save instance dictionary if not needed
@@ -1189,6 +1209,16 @@
         od.x = 10
         self.assertEqual(len(od.__reduce__()), 3)
 
+    def test_reduce_exact_output(self):
+        # PyPy: test that __reduce__() produces the exact same answer as
+        # CPython does, even though in the 'all_ordered_dicts' branch we
+        # have to emulate it.
+        pairs = [['c', 1], ['b', 2], ['d', 4]]
+        od = OrderedDict(pairs)
+        self.assertEqual(od.__reduce__(), (OrderedDict, (pairs,)))
+        od.x = 10
+        self.assertEqual(od.__reduce__(), (OrderedDict, (pairs,), {'x': 10}))
+
     def test_repr(self):
         od = OrderedDict([('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)])
         self.assertEqual(repr(od),
diff --git a/lib-python/2.7/test/test_xml_etree.py b/lib-python/2.7/test/test_xml_etree.py
--- a/lib-python/2.7/test/test_xml_etree.py
+++ b/lib-python/2.7/test/test_xml_etree.py
@@ -225,9 +225,9 @@
     >>> element.remove(subelement)
     >>> serialize(element) # 5
     ''
-    >>> element.remove(subelement)
+    >>> element.remove(subelement)    # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    ValueError: list.remove(x): x not in list
+    ValueError: list.remove(...
     >>> serialize(element) # 6
     ''
     >>> element[0:0] = [subelement, subelement, subelement]
diff --git a/lib-python/stdlib-upgrade.txt b/lib-python/stdlib-upgrade.txt
--- a/lib-python/stdlib-upgrade.txt
+++ b/lib-python/stdlib-upgrade.txt
@@ -7,7 +7,7 @@
 
 1. check out the branch vendor/stdlib
 2. upgrade the files there
-3. update stdlib-versions.txt with the output of hg -id from the cpython repo
+3. update stdlib-version.txt with the output of hg -id from the cpython repo
 4. commit
 5. update to default/py3k
 6. create a integration branch for the new stdlib
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -83,6 +83,37 @@
     def in_dll(self, dll, name):
         return self.from_address(dll._handle.getaddressindll(name))
 
+    def from_buffer(self, obj, offset=0):
+        size = self._sizeofinstances()
+        buf = buffer(obj, offset, size)
+        if len(buf) < size:
+            raise ValueError(
+                "Buffer size too small (%d instead of at least %d bytes)"
+                % (len(buf) + offset, size + offset))
+        raw_addr = buf._pypy_raw_address()
+        result = self.from_address(raw_addr)
+        result._ensure_objects()['ffffffff'] = obj
+        return result
+
+    def from_buffer_copy(self, obj, offset=0):
+        size = self._sizeofinstances()
+        buf = buffer(obj, offset, size)
+        if len(buf) < size:
+            raise ValueError(
+                "Buffer size too small (%d instead of at least %d bytes)"
+                % (len(buf) + offset, size + offset))
+        result = self()
+        dest = result._buffer.buffer
+        try:
+            raw_addr = buf._pypy_raw_address()
+        except ValueError:
+            _rawffi.rawstring2charp(dest, buf)
+        else:
+            from ctypes import memmove
+            memmove(dest, raw_addr, size)
+        return result
+
+
 class CArgObject(object):
     """ simple wrapper around buffer, just for the case of freeing
     it afterwards
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -1,3 +1,4 @@
+import sys
 import _rawffi
 from _ctypes.basics import _CData, _CDataMeta, keepalive_key,\
      store_reference, ensure_objects, CArgObject
@@ -178,6 +179,8 @@
         instance = StructOrUnion.__new__(self)
         if isinstance(address, _rawffi.StructureInstance):
             address = address.buffer
+        # fix the address: turn it into as unsigned, in case it is negative
+        address = address & (sys.maxint * 2 + 1)
         instance.__dict__['_buffer'] = self._ffistruct.fromaddress(address)
         return instance
 
diff --git a/lib_pypy/_functools.py b/lib_pypy/_functools.py
--- a/lib_pypy/_functools.py
+++ b/lib_pypy/_functools.py
@@ -9,7 +9,10 @@
     of the given arguments and keywords.
     """
 
-    def __init__(self, func, *args, **keywords):
+    def __init__(self, *args, **keywords):
+        if not args:
+            raise TypeError('__init__() takes at least 2 arguments (1 given)')
+        func, args = args[0], args[1:]
         if not callable(func):
             raise TypeError("the first argument must be callable")
         self._func = func
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -1175,8 +1175,9 @@
         try:
             return self.__description
         except AttributeError:
-            self.__description = self.__statement._get_description()
-            return self.__description
+            if self.__statement:
+                self.__description = self.__statement._get_description()
+                return self.__description
     description = property(__get_description)
 
     def __get_lastrowid(self):
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -6,3 +6,8 @@
 
 __version__ = "0.8.6"
 __version_info__ = (0, 8, 6)
+
+# The verifier module file names are based on the CRC32 of a string that
+# contains the following version number.  It may be older than __version__
+# if nothing is clearly incompatible.
+__version_verifier_modules__ = "0.8.6"
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -69,6 +69,7 @@
         self._function_caches = []
         self._libraries = []
         self._cdefsources = []
+        self._windows_unicode = None
         if hasattr(backend, 'set_ffi'):
             backend.set_ffi(self)
         for name in backend.__dict__:
@@ -77,6 +78,7 @@
         #
         with self._lock:
             self.BVoidP = self._get_cached_btype(model.voidp_type)
+            self.BCharA = self._get_cached_btype(model.char_array_type)
         if isinstance(backend, types.ModuleType):
             # _cffi_backend: attach these constants to the class
             if not hasattr(FFI, 'NULL'):
@@ -189,13 +191,16 @@
             cdecl = self._typeof(cdecl)
         return self._backend.alignof(cdecl)
 
-    def offsetof(self, cdecl, fieldname):
+    def offsetof(self, cdecl, *fields_or_indexes):
         """Return the offset of the named field inside the given
-        structure, which must be given as a C type name.
+        structure or array, which must be given as a C type name.  
+        You can give several field names in case of nested structures.
+        You can also give numeric values which correspond to array
+        items, in case of an array type.
         """
         if isinstance(cdecl, basestring):
             cdecl = self._typeof(cdecl)
-        return self._backend.typeoffsetof(cdecl, fieldname)[1]
+        return self._typeoffsetof(cdecl, *fields_or_indexes)[1]
 
     def new(self, cdecl, init=None):
         """Allocate an instance according to the specified C type and
@@ -264,6 +269,16 @@
         """
         return self._backend.buffer(cdata, size)
 
+    def from_buffer(self, python_buffer):
+        """Return a  that points to the data of the
+        given Python object, which must support the buffer interface.
+        Note that this is not meant to be used on the built-in types str,
+        unicode, or bytearray (you can build 'char[]' arrays explicitly)
+        but only on objects containing large quantities of raw data
+        in some other format, like 'array.array' or numpy arrays.
+        """
+        return self._backend.from_buffer(self.BCharA, python_buffer)
+
     def callback(self, cdecl, python_callable=None, error=None):
         """Return a callback object or a decorator making such a
         callback object.  'cdecl' must name a C function pointer type.
@@ -335,9 +350,23 @@
         which requires binary compatibility in the signatures.
         """
         from .verifier import Verifier, _caller_dir_pycache
+        #
+        # If set_unicode(True) was called, insert the UNICODE and
+        # _UNICODE macro declarations
+        if self._windows_unicode:
+            self._apply_windows_unicode(kwargs)
+        #
+        # Set the tmpdir here, and not in Verifier.__init__: it picks
+        # up the caller's directory, which we want to be the caller of
+        # ffi.verify(), as opposed to the caller of Veritier().
         tmpdir = tmpdir or _caller_dir_pycache()
+        #
+        # Make a Verifier() and use it to load the library.
         self.verifier = Verifier(self, source, tmpdir, **kwargs)
         lib = self.verifier.load_library()
+        #
+        # Save the loaded library for keep-alive purposes, even
+        # if the caller doesn't keep it alive itself (it should).
         self._libraries.append(lib)
         return lib
 
@@ -356,15 +385,29 @@
         with self._lock:
             return model.pointer_cache(self, ctype)
 
-    def addressof(self, cdata, field=None):
+    def addressof(self, cdata, *fields_or_indexes):
         """Return the address of a .
-        If 'field' is specified, return the address of this field.
+        If 'fields_or_indexes' are given, returns the address of that
+        field or array item in the structure or array, recursively in
+        case of nested structures.
         """
         ctype = self._backend.typeof(cdata)
-        ctype, offset = self._backend.typeoffsetof(ctype, field)
+        if fields_or_indexes:
+            ctype, offset = self._typeoffsetof(ctype, *fields_or_indexes)
+        else:
+            if ctype.kind == "pointer":
+                raise TypeError("addressof(pointer)")
+            offset = 0
         ctypeptr = self._pointer_to(ctype)
         return self._backend.rawaddressof(ctypeptr, cdata, offset)
 
+    def _typeoffsetof(self, ctype, field_or_index, *fields_or_indexes):
+        ctype, offset = self._backend.typeoffsetof(ctype, field_or_index)
+        for field1 in fields_or_indexes:
+            ctype, offset1 = self._backend.typeoffsetof(ctype, field1, 1)
+            offset += offset1
+        return ctype, offset
+
     def include(self, ffi_to_include):
         """Includes the typedefs, structs, unions and enums defined
         in another FFI instance.  Usage is similar to a #include in C,
@@ -387,6 +430,44 @@
     def from_handle(self, x):
         return self._backend.from_handle(x)
 
+    def set_unicode(self, enabled_flag):
+        """Windows: if 'enabled_flag' is True, enable the UNICODE and
+        _UNICODE defines in C, and declare the types like TCHAR and LPTCSTR
+        to be (pointers to) wchar_t.  If 'enabled_flag' is False,
+        declare these types to be (pointers to) plain 8-bit characters.
+        This is mostly for backward compatibility; you usually want True.
+        """
+        if self._windows_unicode is not None:
+            raise ValueError("set_unicode() can only be called once")
+        enabled_flag = bool(enabled_flag)
+        if enabled_flag:
+            self.cdef("typedef wchar_t TBYTE;"
+                      "typedef wchar_t TCHAR;"
+                      "typedef const wchar_t *LPCTSTR;"
+                      "typedef const wchar_t *PCTSTR;"
+                      "typedef wchar_t *LPTSTR;"
+                      "typedef wchar_t *PTSTR;"
+                      "typedef TBYTE *PTBYTE;"
+                      "typedef TCHAR *PTCHAR;")
+        else:
+            self.cdef("typedef char TBYTE;"
+                      "typedef char TCHAR;"
+                      "typedef const char *LPCTSTR;"
+                      "typedef const char *PCTSTR;"
+                      "typedef char *LPTSTR;"
+                      "typedef char *PTSTR;"
+                      "typedef TBYTE *PTBYTE;"
+                      "typedef TCHAR *PTCHAR;")
+        self._windows_unicode = enabled_flag
+
+    def _apply_windows_unicode(self, kwds):
+        defmacros = kwds.get('define_macros', ())
+        if not isinstance(defmacros, (list, tuple)):
+            raise TypeError("'define_macros' must be a list or tuple")
+        defmacros = list(defmacros) + [('UNICODE', '1'),
+                                       ('_UNICODE', '1')]
+        kwds['define_macros'] = defmacros
+
 
 def _load_backend_lib(backend, name, flags):
     if name is None:
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -169,6 +169,7 @@
 class CTypesGenericPtr(CTypesData):
     __slots__ = ['_address', '_as_ctype_ptr']
     _automatic_casts = False
+    kind = "pointer"
 
     @classmethod
     def _newp(cls, init):
@@ -370,10 +371,12 @@
                                 (CTypesPrimitive, type(source).__name__))
             return source
         #
+        kind1 = kind
         class CTypesPrimitive(CTypesGenericPrimitive):
             __slots__ = ['_value']
             _ctype = ctype
             _reftypename = '%s &' % name
+            kind = kind1
 
             def __init__(self, value):
                 self._value = value
@@ -703,12 +706,13 @@
         class struct_or_union(base_ctypes_class):
             pass
         struct_or_union.__name__ = '%s_%s' % (kind, name)
+        kind1 = kind
         #
         class CTypesStructOrUnion(CTypesBaseStructOrUnion):
             __slots__ = ['_blob']
             _ctype = struct_or_union
             _reftypename = '%s &' % (name,)
-            _kind = kind
+            _kind = kind = kind1
         #
         CTypesStructOrUnion._fix_class()
         return CTypesStructOrUnion
@@ -994,27 +998,42 @@
     def getcname(self, BType, replace_with):
         return BType._get_c_name(replace_with)
 
-    def typeoffsetof(self, BType, fieldname):
-        if fieldname is not None and issubclass(BType, CTypesGenericPtr):
-            BType = BType._BItem
-        if not issubclass(BType, CTypesBaseStructOrUnion):
-            raise TypeError("expected a struct or union ctype")
-        if fieldname is None:
-            return (BType, 0)
-        else:
+    def typeoffsetof(self, BType, fieldname, num=0):
+        if isinstance(fieldname, str):
+            if num == 0 and issubclass(BType, CTypesGenericPtr):
+                BType = BType._BItem
+            if not issubclass(BType, CTypesBaseStructOrUnion):
+                raise TypeError("expected a struct or union ctype")
             BField = BType._bfield_types[fieldname]
             if BField is Ellipsis:
                 raise TypeError("not supported for bitfields")
             return (BField, BType._offsetof(fieldname))
+        elif isinstance(fieldname, (int, long)):
+            if issubclass(BType, CTypesGenericArray):
+                BType = BType._CTPtr
+            if not issubclass(BType, CTypesGenericPtr):
+                raise TypeError("expected an array or ptr ctype")
+            BItem = BType._BItem
+            offset = BItem._get_size() * fieldname
+            if offset > sys.maxsize:
+                raise OverflowError
+            return (BItem, offset)
+        else:
+            raise TypeError(type(fieldname))
 
-    def rawaddressof(self, BTypePtr, cdata, offset):
+    def rawaddressof(self, BTypePtr, cdata, offset=None):
         if isinstance(cdata, CTypesBaseStructOrUnion):
             ptr = ctypes.pointer(type(cdata)._to_ctypes(cdata))
         elif isinstance(cdata, CTypesGenericPtr):
+            if offset is None or not issubclass(type(cdata)._BItem,
+                                                CTypesBaseStructOrUnion):
+                raise TypeError("unexpected cdata type")
+            ptr = type(cdata)._to_ctypes(cdata)
+        elif isinstance(cdata, CTypesGenericArray):
             ptr = type(cdata)._to_ctypes(cdata)
         else:
             raise TypeError("expected a ")
-        if offset != 0:
+        if offset:
             ptr = ctypes.cast(
                 ctypes.c_void_p(
                     ctypes.cast(ptr, ctypes.c_void_p).value + offset),
diff --git a/lib_pypy/cffi/commontypes.py b/lib_pypy/cffi/commontypes.py
--- a/lib_pypy/cffi/commontypes.py
+++ b/lib_pypy/cffi/commontypes.py
@@ -29,6 +29,9 @@
                 result = model.PointerType(resolve_common_type(result[:-2]))
         elif result in model.PrimitiveType.ALL_PRIMITIVE_TYPES:
             result = model.PrimitiveType(result)
+        elif result == 'set-unicode-needed':
+            raise api.FFIError("The Windows type %r is only available after "
+                               "you call ffi.set_unicode()" % (commontype,))
         else:
             if commontype == result:
                 raise api.FFIError("Unsupported type: %r.  Please file a bug "
@@ -86,8 +89,6 @@
         "ULONGLONG": "unsigned long long",
         "WCHAR": "wchar_t",
         "SHORT": "short",
-        "TBYTE": "WCHAR",
-        "TCHAR": "WCHAR",
         "UCHAR": "unsigned char",
         "UINT": "unsigned int",
         "UINT8": "unsigned char",
@@ -157,14 +158,12 @@
 
         "LPCVOID": model.const_voidp_type,
         "LPCWSTR": "const WCHAR *",
-        "LPCTSTR": "LPCWSTR",
         "LPDWORD": "DWORD *",
         "LPHANDLE": "HANDLE *",
         "LPINT": "int *",
         "LPLONG": "long *",
         "LPSTR": "CHAR *",
         "LPWSTR": "WCHAR *",
-        "LPTSTR": "LPWSTR",
         "LPVOID": model.voidp_type,
         "LPWORD": "WORD *",
         "LRESULT": "LONG_PTR",
@@ -173,7 +172,6 @@
         "PBYTE": "BYTE *",
         "PCHAR": "CHAR *",
         "PCSTR": "const CHAR *",
-        "PCTSTR": "LPCWSTR",
         "PCWSTR": "const WCHAR *",
         "PDWORD": "DWORD *",
         "PDWORDLONG": "DWORDLONG *",
@@ -200,9 +198,6 @@
         "PSIZE_T": "SIZE_T *",
         "PSSIZE_T": "SSIZE_T *",
         "PSTR": "CHAR *",
-        "PTBYTE": "TBYTE *",
-        "PTCHAR": "TCHAR *",
-        "PTSTR": "LPWSTR",
         "PUCHAR": "UCHAR *",
         "PUHALF_PTR": "UHALF_PTR *",
         "PUINT": "UINT *",
@@ -240,6 +235,15 @@
         "USN": "LONGLONG",
         "VOID": model.void_type,
         "WPARAM": "UINT_PTR",
+
+        "TBYTE": "set-unicode-needed",
+        "TCHAR": "set-unicode-needed",
+        "LPCTSTR": "set-unicode-needed",
+        "PCTSTR": "set-unicode-needed",
+        "LPTSTR": "set-unicode-needed",
+        "PTSTR": "set-unicode-needed",
+        "PTBYTE": "set-unicode-needed",
+        "PTCHAR": "set-unicode-needed",
         })
     return result
 
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -1,4 +1,3 @@
-
 from . import api, model
 from .commontypes import COMMON_TYPES, resolve_common_type
 try:
@@ -209,6 +208,8 @@
 
     def _add_constants(self, key, val):
         if key in self._int_constants:
+            if self._int_constants[key] == val:
+                return     # ignore identical double declarations
             raise api.FFIError(
                 "multiple declarations of constant: %s" % (key,))
         self._int_constants[key] = val
@@ -228,12 +229,18 @@
 
                 pyvalue = int(int_str, 0)
                 self._add_constants(key, pyvalue)
+                self._declare('macro ' + key, pyvalue)
             elif value == '...':
                 self._declare('macro ' + key, value)
             else:
-                raise api.CDefError('only supports the syntax "#define '
-                                    '%s ..." (literally) or "#define '
-                                    '%s 0x1FF" for now' % (key, key))
+                raise api.CDefError(
+                    'only supports one of the following syntax:\n'
+                    '  #define %s ...     (literally dot-dot-dot)\n'
+                    '  #define %s NUMBER  (with NUMBER an integer'
+                                    ' constant, decimal/hex/octal)\n'
+                    'got:\n'
+                    '  #define %s %s'
+                    % (key, key, key, value))
 
     def _parse_decl(self, decl):
         node = decl.type
@@ -460,6 +467,8 @@
             elif kind == 'union':
                 tp = model.UnionType(explicit_name, None, None, None)
             elif kind == 'enum':
+                if explicit_name == '__dotdotdot__':
+                    raise CDefError("Enums cannot be declared with ...")
                 tp = self._build_enum_type(explicit_name, type.values)
             else:
                 raise AssertionError("kind = %r" % (kind,))
@@ -532,9 +541,24 @@
 
     def _parse_constant(self, exprnode, partial_length_ok=False):
         # for now, limited to expressions that are an immediate number
-        # or negative number
+        # or positive/negative number
         if isinstance(exprnode, pycparser.c_ast.Constant):
-            return int(exprnode.value, 0)
+            s = exprnode.value
+            if s.startswith('0'):
+                if s.startswith('0x') or s.startswith('0X'):
+                    return int(s, 16)
+                return int(s, 8)
+            elif '1' <= s[0] <= '9':
+                return int(s, 10)
+            elif s[0] == "'" and s[-1] == "'" and (
+                    len(s) == 3 or (len(s) == 4 and s[1] == "\\")):
+                return ord(s[-2])
+            else:
+                raise api.CDefError("invalid constant %r" % (s,))
+        #
+        if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and
+                exprnode.op == '+'):
+            return self._parse_constant(exprnode.expr)
         #
         if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and
                 exprnode.op == '-'):
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -11,6 +11,9 @@
     """
 
 
+LIST_OF_FILE_NAMES = ['sources', 'include_dirs', 'library_dirs',
+                      'extra_objects', 'depends']
+
 def get_extension(srcfilename, modname, sources=(), **kwds):
     from distutils.core import Extension
     allsources = [srcfilename]
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -235,6 +235,8 @@
         BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist)
         return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length)
 
+char_array_type = ArrayType(PrimitiveType('char'), None)
+
 
 class StructOrUnionOrEnum(BaseTypeByIdentity):
     _attrs_ = ('name',)
@@ -478,7 +480,7 @@
     try:
         res = getattr(ffi._backend, funcname)(*args)
     except NotImplementedError as e:
-        raise NotImplementedError("%r: %s" % (srctype, e))
+        raise NotImplementedError("%s: %r: %s" % (funcname, srctype, e))
     # note that setdefault() on WeakValueDictionary is not atomic
     # and contains a rare bug (http://bugs.python.org/issue19542);
     # we have to use a lock and do it ourselves
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -65,7 +65,7 @@
         # The following two 'chained_list_constants' items contains
         # the head of these two chained lists, as a string that gives the
         # call to do, if any.
-        self._chained_list_constants = ['0', '0']
+        self._chained_list_constants = ['((void)lib,0)', '((void)lib,0)']
         #
         prnt = self._prnt
         # first paste some standard set of lines that are mostly '#define'
@@ -138,15 +138,22 @@
         prnt()
         prnt('#endif')
 
-    def load_library(self):
+    def load_library(self, flags=None):
         # XXX review all usages of 'self' here!
         # import it as a new extension module
+        if hasattr(sys, "getdlopenflags"):
+            previous_flags = sys.getdlopenflags()
         try:
+            if hasattr(sys, "setdlopenflags") and flags is not None:
+                sys.setdlopenflags(flags)
             module = imp.load_dynamic(self.verifier.get_module_name(),
                                       self.verifier.modulefilename)
         except ImportError as e:
             error = "importing %r: %s" % (self.verifier.modulefilename, e)
             raise ffiplatform.VerificationError(error)
+        finally:
+            if hasattr(sys, "setdlopenflags"):
+                sys.setdlopenflags(previous_flags)
         #
         # call loading_cpy_struct() to get the struct layout inferred by
         # the C compiler
@@ -228,7 +235,8 @@
                 converter = '_cffi_to_c_int'
                 extraarg = ', %s' % tp.name
             else:
-                converter = '_cffi_to_c_%s' % (tp.name.replace(' ', '_'),)
+                converter = '(%s)_cffi_to_c_%s' % (tp.get_c_name(''),
+                                                   tp.name.replace(' ', '_'))
             errvalue = '-1'
         #
         elif isinstance(tp, model.PointerType):
@@ -267,8 +275,8 @@
         self._prnt('  if (datasize != 0) {')
         self._prnt('    if (datasize < 0)')
         self._prnt('      %s;' % errcode)
-        self._prnt('    %s = alloca(datasize);' % (tovar,))
-        self._prnt('    memset((void *)%s, 0, datasize);' % (tovar,))
+        self._prnt('    %s = alloca((size_t)datasize);' % (tovar,))
+        self._prnt('    memset((void *)%s, 0, (size_t)datasize);' % (tovar,))
         self._prnt('    if (_cffi_convert_array_from_object('
                    '(char *)%s, _cffi_type(%d), %s) < 0)' % (
             tovar, self._gettypenum(tp), fromvar))
@@ -336,7 +344,7 @@
         prnt = self._prnt
         numargs = len(tp.args)
         if numargs == 0:
-            argname = 'no_arg'
+            argname = 'noarg'
         elif numargs == 1:
             argname = 'arg0'
         else:
@@ -386,6 +394,9 @@
         prnt('  Py_END_ALLOW_THREADS')
         prnt()
         #
+        prnt('  (void)self; /* unused */')
+        if numargs == 0:
+            prnt('  (void)noarg; /* unused */')
         if result_code:
             prnt('  return %s;' %
                  self._convert_expr_from_c(tp.result, 'result', 'result type'))
@@ -452,6 +463,7 @@
         prnt('static void %s(%s *p)' % (checkfuncname, cname))
         prnt('{')
         prnt('  /* only to generate compile-time warnings or errors */')
+        prnt('  (void)p;')
         for fname, ftype, fbitsize in tp.enumfields():
             if (isinstance(ftype, model.PrimitiveType)
                 and ftype.is_integer_type()) or fbitsize >= 0:
@@ -482,6 +494,8 @@
                 prnt('    sizeof(((%s *)0)->%s),' % (cname, fname))
         prnt('    -1')
         prnt('  };')
+        prnt('  (void)self; /* unused */')
+        prnt('  (void)noarg; /* unused */')
         prnt('  return _cffi_get_struct_layout(nums);')
         prnt('  /* the next line is not executed, but compiled */')
         prnt('  %s(0);' % (checkfuncname,))
@@ -578,7 +592,8 @@
     # constants, likely declared with '#define'
 
     def _generate_cpy_const(self, is_int, name, tp=None, category='const',
-                            vartp=None, delayed=True, size_too=False):
+                            vartp=None, delayed=True, size_too=False,
+                            check_value=None):
         prnt = self._prnt
         funcname = '_cffi_%s_%s' % (category, name)
         prnt('static int %s(PyObject *lib)' % funcname)
@@ -590,6 +605,9 @@
         else:
             assert category == 'const'
         #
+        if check_value is not None:
+            self._check_int_constant_value(name, check_value)
+        #
         if not is_int:
             if category == 'var':
                 realexpr = '&' + name
@@ -637,6 +655,27 @@
     # ----------
     # enums
 
+    def _check_int_constant_value(self, name, value, err_prefix=''):
+        prnt = self._prnt
+        if value <= 0:
+            prnt('  if ((%s) > 0 || (long)(%s) != %dL) {' % (
+                name, name, value))
+        else:
+            prnt('  if ((%s) <= 0 || (unsigned long)(%s) != %dUL) {' % (
+                name, name, value))
+        prnt('    char buf[64];')
+        prnt('    if ((%s) <= 0)' % name)
+        prnt('        snprintf(buf, 63, "%%ld", (long)(%s));' % name)
+        prnt('    else')
+        prnt('        snprintf(buf, 63, "%%lu", (unsigned long)(%s));' %
+             name)
+        prnt('    PyErr_Format(_cffi_VerificationError,')
+        prnt('                 "%s%s has the real value %s, not %s",')
+        prnt('                 "%s", "%s", buf, "%d");' % (
+            err_prefix, name, value))
+        prnt('    return -1;')
+        prnt('  }')
+
     def _enum_funcname(self, prefix, name):
         # "$enum_$1" => "___D_enum____D_1"
         name = name.replace('$', '___D_')
@@ -653,25 +692,8 @@
         prnt('static int %s(PyObject *lib)' % funcname)
         prnt('{')
         for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
-            if enumvalue < 0:
-                prnt('  if ((%s) >= 0 || (long)(%s) != %dL) {' % (
-                    enumerator, enumerator, enumvalue))
-            else:
-                prnt('  if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % (
-                    enumerator, enumerator, enumvalue))
-            prnt('    char buf[64];')
-            prnt('    if ((%s) < 0)' % enumerator)
-            prnt('        snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator)
-            prnt('    else')
-            prnt('        snprintf(buf, 63, "%%lu", (unsigned long)(%s));' %
-                 enumerator)
-            prnt('    PyErr_Format(_cffi_VerificationError,')
-            prnt('                 "enum %s: %s has the real value %s, '
-                 'not %s",')
-            prnt('                 "%s", "%s", buf, "%d");' % (
-                name, enumerator, enumvalue))
-            prnt('    return -1;')
-            prnt('  }')
+            self._check_int_constant_value(enumerator, enumvalue,
+                                           "enum %s: " % name)
         prnt('  return %s;' % self._chained_list_constants[True])
         self._chained_list_constants[True] = funcname + '(lib)'
         prnt('}')
@@ -695,8 +717,11 @@
     # macros: for now only for integers
 
     def _generate_cpy_macro_decl(self, tp, name):
-        assert tp == '...'
-        self._generate_cpy_const(True, name)
+        if tp == '...':
+            check_value = None
+        else:
+            check_value = tp     # an integer
+        self._generate_cpy_const(True, name, check_value=check_value)
 
     _generate_cpy_macro_collecttype = _generate_nothing
     _generate_cpy_macro_method = _generate_nothing
@@ -783,6 +808,24 @@
    typedef unsigned __int16 uint16_t;
    typedef unsigned __int32 uint32_t;
    typedef unsigned __int64 uint64_t;
+   typedef __int8 int_least8_t;
+   typedef __int16 int_least16_t;
+   typedef __int32 int_least32_t;
+   typedef __int64 int_least64_t;
+   typedef unsigned __int8 uint_least8_t;
+   typedef unsigned __int16 uint_least16_t;
+   typedef unsigned __int32 uint_least32_t;
+   typedef unsigned __int64 uint_least64_t;
+   typedef __int8 int_fast8_t;
+   typedef __int16 int_fast16_t;
+   typedef __int32 int_fast32_t;
+   typedef __int64 int_fast64_t;
+   typedef unsigned __int8 uint_fast8_t;
+   typedef unsigned __int16 uint_fast16_t;
+   typedef unsigned __int32 uint_fast32_t;
+   typedef unsigned __int64 uint_fast64_t;
+   typedef __int64 intmax_t;
+   typedef unsigned __int64 uintmax_t;
 # else
 #  include 
 # endif
@@ -828,12 +871,15 @@
             PyLong_FromLongLong((long long)(x)))
 
 #define _cffi_from_c_int(x, type)                                        \
-    (((type)-1) > 0 ?   /* unsigned */                                   \
-        (sizeof(type) < sizeof(long) ? PyInt_FromLong(x) :               \
-         sizeof(type) == sizeof(long) ? PyLong_FromUnsignedLong(x) :     \
-                                        PyLong_FromUnsignedLongLong(x))  \
-      : (sizeof(type) <= sizeof(long) ? PyInt_FromLong(x) :              \
-                                        PyLong_FromLongLong(x)))
+    (((type)-1) > 0 ? /* unsigned */                                     \
+        (sizeof(type) < sizeof(long) ?                                   \
+            PyInt_FromLong((long)x) :                                    \
+         sizeof(type) == sizeof(long) ?                                  \
+            PyLong_FromUnsignedLong((unsigned long)x) :                  \
+            PyLong_FromUnsignedLongLong((unsigned long long)x)) :        \
+        (sizeof(type) <= sizeof(long) ?                                  \
+            PyInt_FromLong((long)x) :                                    \
+            PyLong_FromLongLong((long long)x)))
 
 #define _cffi_to_c_int(o, type)                                          \
     (sizeof(type) == 1 ? (((type)-1) > 0 ? (type)_cffi_to_c_u8(o)        \
@@ -844,7 +890,7 @@
                                          : (type)_cffi_to_c_i32(o)) :    \
      sizeof(type) == 8 ? (((type)-1) > 0 ? (type)_cffi_to_c_u64(o)       \
                                          : (type)_cffi_to_c_i64(o)) :    \
-     (Py_FatalError("unsupported size for type " #type), 0))
+     (Py_FatalError("unsupported size for type " #type), (type)0))
 
 #define _cffi_to_c_i8                                                    \
                  ((int(*)(PyObject *))_cffi_exports[1])
@@ -907,6 +953,7 @@
 {
     PyObject *library;
     int was_alive = (_cffi_types != NULL);
+    (void)self; /* unused */
     if (!PyArg_ParseTuple(args, "OOO", &_cffi_types, &_cffi_VerificationError,
                                        &library))
         return NULL;
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -58,12 +58,12 @@
             modname = self.verifier.get_module_name()
             prnt("void %s%s(void) { }\n" % (prefix, modname))
 
-    def load_library(self):
+    def load_library(self, flags=0):
         # import it with the CFFI backend
         backend = self.ffi._backend
         # needs to make a path that contains '/', on Posix
         filename = os.path.join(os.curdir, self.verifier.modulefilename)
-        module = backend.load_library(filename)
+        module = backend.load_library(filename, flags)
         #
         # call loading_gen_struct() to get the struct layout inferred by
         # the C compiler
@@ -235,6 +235,7 @@
         prnt('static void %s(%s *p)' % (checkfuncname, cname))
         prnt('{')
         prnt('  /* only to generate compile-time warnings or errors */')
+        prnt('  (void)p;')
         for fname, ftype, fbitsize in tp.enumfields():
             if (isinstance(ftype, model.PrimitiveType)
                 and ftype.is_integer_type()) or fbitsize >= 0:
@@ -354,11 +355,20 @@
     # ----------
     # constants, likely declared with '#define'
 
-    def _generate_gen_const(self, is_int, name, tp=None, category='const'):
+    def _generate_gen_const(self, is_int, name, tp=None, category='const',
+                            check_value=None):
         prnt = self._prnt
         funcname = '_cffi_%s_%s' % (category, name)
         self.export_symbols.append(funcname)
-        if is_int:
+        if check_value is not None:
+            assert is_int
+            assert category == 'const'
+            prnt('int %s(char *out_error)' % funcname)
+            prnt('{')
+            self._check_int_constant_value(name, check_value)
+            prnt('  return 0;')
+            prnt('}')
+        elif is_int:
             assert category == 'const'
             prnt('int %s(long long *out_value)' % funcname)
             prnt('{')
@@ -367,6 +377,7 @@
             prnt('}')
         else:
             assert tp is not None
+            assert check_value is None
             prnt(tp.get_c_name(' %s(void)' % funcname, name),)
             prnt('{')
             if category == 'var':
@@ -383,9 +394,13 @@
 
     _loading_gen_constant = _loaded_noop
 
-    def _load_constant(self, is_int, tp, name, module):
+    def _load_constant(self, is_int, tp, name, module, check_value=None):
         funcname = '_cffi_const_%s' % name
-        if is_int:
+        if check_value is not None:
+            assert is_int
+            self._load_known_int_constant(module, funcname)
+            value = check_value
+        elif is_int:
             BType = self.ffi._typeof_locked("long long*")[0]
             BFunc = self.ffi._typeof_locked("int(*)(long long*)")[0]
             function = module.load_function(BFunc, funcname)
@@ -396,6 +411,7 @@
                 BLongLong = self.ffi._typeof_locked("long long")[0]
                 value += (1 << (8*self.ffi.sizeof(BLongLong)))
         else:
+            assert check_value is None
             BFunc = self.ffi._typeof_locked(tp.get_c_name('(*)(void)', name))[0]
             function = module.load_function(BFunc, funcname)
             value = function()
@@ -410,6 +426,36 @@
     # ----------
     # enums
 
+    def _check_int_constant_value(self, name, value):
+        prnt = self._prnt
+        if value <= 0:
+            prnt('  if ((%s) > 0 || (long)(%s) != %dL) {' % (
+                name, name, value))
+        else:
+            prnt('  if ((%s) <= 0 || (unsigned long)(%s) != %dUL) {' % (
+                name, name, value))
+        prnt('    char buf[64];')
+        prnt('    if ((%s) <= 0)' % name)
+        prnt('        sprintf(buf, "%%ld", (long)(%s));' % name)
+        prnt('    else')
+        prnt('        sprintf(buf, "%%lu", (unsigned long)(%s));' %
+             name)
+        prnt('    sprintf(out_error, "%s has the real value %s, not %s",')
+        prnt('            "%s", buf, "%d");' % (name[:100], value))
+        prnt('    return -1;')
+        prnt('  }')
+
+    def _load_known_int_constant(self, module, funcname):
+        BType = self.ffi._typeof_locked("char[]")[0]
+        BFunc = self.ffi._typeof_locked("int(*)(char*)")[0]
+        function = module.load_function(BFunc, funcname)
+        p = self.ffi.new(BType, 256)
+        if function(p) < 0:
+            error = self.ffi.string(p)
+            if sys.version_info >= (3,):
+                error = str(error, 'utf-8')
+            raise ffiplatform.VerificationError(error)
+
     def _enum_funcname(self, prefix, name):
         # "$enum_$1" => "___D_enum____D_1"
         name = name.replace('$', '___D_')
@@ -427,24 +473,7 @@
         prnt('int %s(char *out_error)' % funcname)
         prnt('{')
         for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
-            if enumvalue < 0:
-                prnt('  if ((%s) >= 0 || (long)(%s) != %dL) {' % (
-                    enumerator, enumerator, enumvalue))
-            else:
-                prnt('  if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % (
-                    enumerator, enumerator, enumvalue))
-            prnt('    char buf[64];')
-            prnt('    if ((%s) < 0)' % enumerator)
-            prnt('        sprintf(buf, "%%ld", (long)(%s));' % enumerator)
-            prnt('    else')
-            prnt('        sprintf(buf, "%%lu", (unsigned long)(%s));' %
-                 enumerator)
-            prnt('    sprintf(out_error,'
-                             ' "%s has the real value %s, not %s",')
-            prnt('            "%s", buf, "%d");' % (
-                enumerator[:100], enumvalue))
-            prnt('    return -1;')
-            prnt('  }')
+            self._check_int_constant_value(enumerator, enumvalue)
         prnt('  return 0;')
         prnt('}')
         prnt()
@@ -456,16 +485,8 @@
             tp.enumvalues = tuple(enumvalues)
             tp.partial_resolved = True
         else:
-            BType = self.ffi._typeof_locked("char[]")[0]
-            BFunc = self.ffi._typeof_locked("int(*)(char*)")[0]
             funcname = self._enum_funcname(prefix, name)
-            function = module.load_function(BFunc, funcname)
-            p = self.ffi.new(BType, 256)
-            if function(p) < 0:
-                error = self.ffi.string(p)
-                if sys.version_info >= (3,):
-                    error = str(error, 'utf-8')
-                raise ffiplatform.VerificationError(error)
+            self._load_known_int_constant(module, funcname)
 
     def _loaded_gen_enum(self, tp, name, module, library):
         for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
@@ -476,13 +497,21 @@
     # macros: for now only for integers
 
     def _generate_gen_macro_decl(self, tp, name):
-        assert tp == '...'
-        self._generate_gen_const(True, name)
+        if tp == '...':
+            check_value = None
+        else:
+            check_value = tp     # an integer
+        self._generate_gen_const(True, name, check_value=check_value)
 
     _loading_gen_macro = _loaded_noop
 
     def _loaded_gen_macro(self, tp, name, module, library):
-        value = self._load_constant(True, tp, name, module)
+        if tp == '...':
+            check_value = None
+        else:
+            check_value = tp     # an integer
+        value = self._load_constant(True, tp, name, module,
+                                    check_value=check_value)
         setattr(library, name, value)
         type(library)._cffi_dir.append(name)
 
@@ -565,6 +594,24 @@
    typedef unsigned __int16 uint16_t;
    typedef unsigned __int32 uint32_t;
    typedef unsigned __int64 uint64_t;
+   typedef __int8 int_least8_t;
+   typedef __int16 int_least16_t;
+   typedef __int32 int_least32_t;
+   typedef __int64 int_least64_t;
+   typedef unsigned __int8 uint_least8_t;
+   typedef unsigned __int16 uint_least16_t;
+   typedef unsigned __int32 uint_least32_t;
+   typedef unsigned __int64 uint_least64_t;
+   typedef __int8 int_fast8_t;
+   typedef __int16 int_fast16_t;
+   typedef __int32 int_fast32_t;
+   typedef __int64 int_fast64_t;
+   typedef unsigned __int8 uint_fast8_t;
+   typedef unsigned __int16 uint_fast16_t;
+   typedef unsigned __int32 uint_fast32_t;
+   typedef unsigned __int64 uint_fast64_t;
+   typedef __int64 intmax_t;
+   typedef unsigned __int64 uintmax_t;
 # else
 #  include 
 # endif
diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py
--- a/lib_pypy/cffi/verifier.py
+++ b/lib_pypy/cffi/verifier.py
@@ -1,12 +1,23 @@
-import sys, os, binascii, imp, shutil
-from . import __version__
+import sys, os, binascii, shutil
+from . import __version_verifier_modules__
 from . import ffiplatform
 
+if sys.version_info >= (3, 3):
+    import importlib.machinery
+    def _extension_suffixes():
+        return importlib.machinery.EXTENSION_SUFFIXES[:]
+else:
+    import imp
+    def _extension_suffixes():
+        return [suffix for suffix, _, type in imp.get_suffixes()
+                if type == imp.C_EXTENSION]
+
 
 class Verifier(object):
 
     def __init__(self, ffi, preamble, tmpdir=None, modulename=None,
-                 ext_package=None, tag='', force_generic_engine=False, **kwds):
+                 ext_package=None, tag='', force_generic_engine=False,
+                 source_extension='.c', flags=None, relative_to=None, **kwds):
         self.ffi = ffi
         self.preamble = preamble
         if not modulename:
@@ -14,14 +25,15 @@
         vengine_class = _locate_engine_class(ffi, force_generic_engine)
         self._vengine = vengine_class(self)
         self._vengine.patch_extension_kwds(kwds)
-        self.kwds = kwds
+        self.flags = flags
+        self.kwds = self.make_relative_to(kwds, relative_to)
         #
         if modulename:
             if tag:
                 raise TypeError("can't specify both 'modulename' and 'tag'")
         else:
-            key = '\x00'.join([sys.version[:3], __version__, preamble,
-                               flattened_kwds] +
+            key = '\x00'.join([sys.version[:3], __version_verifier_modules__,
+                               preamble, flattened_kwds] +
                               ffi._cdefsources)
             if sys.version_info >= (3,):
                 key = key.encode('utf-8')
@@ -33,7 +45,7 @@
                                               k1, k2)
         suffix = _get_so_suffixes()[0]
         self.tmpdir = tmpdir or _caller_dir_pycache()
-        self.sourcefilename = os.path.join(self.tmpdir, modulename + '.c')
+        self.sourcefilename = os.path.join(self.tmpdir, modulename + source_extension)
         self.modulefilename = os.path.join(self.tmpdir, modulename + suffix)
         self.ext_package = ext_package
         self._has_source = False
@@ -97,6 +109,20 @@
     def generates_python_module(self):
         return self._vengine._gen_python_module
 
+    def make_relative_to(self, kwds, relative_to):
+        if relative_to and os.path.dirname(relative_to):
+            dirname = os.path.dirname(relative_to)
+            kwds = kwds.copy()
+            for key in ffiplatform.LIST_OF_FILE_NAMES:
+                if key in kwds:
+                    lst = kwds[key]
+                    if not isinstance(lst, (list, tuple)):
+                        raise TypeError("keyword '%s' should be a list or tuple"
+                                        % (key,))
+                    lst = [os.path.join(dirname, fn) for fn in lst]
+                    kwds[key] = lst
+        return kwds
+
     # ----------
 
     def _locate_module(self):
@@ -148,7 +174,10 @@
 
     def _load_library(self):
         assert self._has_module
-        return self._vengine.load_library()
+        if self.flags is not None:
+            return self._vengine.load_library(self.flags)
+        else:
+            return self._vengine.load_library()
 
 # ____________________________________________________________
 
@@ -181,6 +210,9 @@
 def _caller_dir_pycache():
     if _TMPDIR:
         return _TMPDIR
+    result = os.environ.get('CFFI_TMPDIR')
+    if result:
+        return result
     filename = sys._getframe(2).f_code.co_filename
     return os.path.abspath(os.path.join(os.path.dirname(filename),
                            '__pycache__'))
@@ -222,11 +254,7 @@
             pass
 
 def _get_so_suffixes():
-    suffixes = []
-    for suffix, mode, type in imp.get_suffixes():
-        if type == imp.C_EXTENSION:
-            suffixes.append(suffix)
-
+    suffixes = _extension_suffixes()
     if not suffixes:
         # bah, no C_EXTENSION available.  Occurs on pypy without cpyext
         if sys.platform == 'win32':
diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
--- a/lib_pypy/greenlet.egg-info
+++ b/lib_pypy/greenlet.egg-info
@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: greenlet
-Version: 0.4.0
+Version: 0.4.5
 Summary: Lightweight in-process concurrent programming
 Home-page: https://github.com/python-greenlet/greenlet
 Author: Ralf Schmitt (for CPython), PyPy team
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -1,7 +1,7 @@
 import sys
 import _continuation
 
-__version__ = "0.4.0"
+__version__ = "0.4.5"
 
 # ____________________________________________________________
 # Exceptions
diff --git a/lib_pypy/pypy_test/test_transaction.py b/lib_pypy/pypy_test/test_transaction.py
--- a/lib_pypy/pypy_test/test_transaction.py
+++ b/lib_pypy/pypy_test/test_transaction.py
@@ -8,9 +8,10 @@
 def test_simple_random_order():
     for x in range(N):
         lst = []
-        with transaction.TransactionQueue():
-            for i in range(10):
-                transaction.add(lst.append, i)
+        tq = transaction.TransactionQueue()
+        for i in range(10):
+            tq.add(lst.append, i)
+        tq.run()
         if VERBOSE:
             print lst
         assert sorted(lst) == range(10), lst
@@ -22,9 +23,10 @@
             lst.append(i)
             i += 1
             if i < 10:
-                transaction.add(do_stuff, i)
-        with transaction.TransactionQueue():
-            transaction.add(do_stuff, 0)
+                tq.add(do_stuff, i)
+        tq = transaction.TransactionQueue()
+        tq.add(do_stuff, 0)
+        tq.run()
         if VERBOSE:
             print lst
         assert lst == range(10), lst
@@ -36,10 +38,11 @@
             lsts[i].append(j)
             j += 1
             if j < 10:
-                transaction.add(do_stuff, i, j)
-        with transaction.TransactionQueue():
-            for i in range(5):
-                transaction.add(do_stuff, i, 0)
+                tq.add(do_stuff, i, j)
+        tq = transaction.TransactionQueue()
+        for i in range(5):
+            tq.add(do_stuff, i, 0)
+        tq.run()
         if VERBOSE:
             print lsts
         assert lsts == (range(10),) * 5, lsts
@@ -53,14 +56,15 @@
             lsts[i].append(j)
             j += 1
             if j < 5:
-                transaction.add(do_stuff, i, j)
+                tq.add(do_stuff, i, j)
             else:
                 lsts[i].append('foo')
                 raise FooError
+        tq = transaction.TransactionQueue()
+        for i in range(10):
+            tq.add(do_stuff, i, 0)
         try:
-            with transaction.TransactionQueue():
-                for i in range(10):
-                    transaction.add(do_stuff, i, 0)
+            tq.run()
         except FooError:
             pass
         else:
@@ -78,19 +82,74 @@
 
 
 def test_number_of_transactions_reported():
-    py.test.skip("not reimplemented")
-    with transaction.TransactionQueue():
-        transaction.add(lambda: None)
-    assert transaction.number_of_transactions_in_last_run() == 1
+    tq = transaction.TransactionQueue()
+    tq.add(lambda: None)
+    tq.add(lambda: None)
+    tq.run()
+    assert tq.number_of_transactions_executed() == 2
+
+    tq.run()
+    assert tq.number_of_transactions_executed() == 2
+
+    tq.add(lambda: None)
+    tq.run()
+    assert tq.number_of_transactions_executed() == 3
+
+    tq.add(lambda: some_name_that_is_not_defined)
+    try:
+        tq.run()
+    except NameError:
+        pass
+    else:
+        raise AssertionError("should have raised NameError")
+    assert tq.number_of_transactions_executed() == 4
+
+    tq.add(tq.number_of_transactions_executed)
+    try:
+        tq.run()
+    except transaction.TransactionError:
+        pass
+    else:
+        raise AssertionError("should have raised TransactionError")
 
     def add_transactions(l):
         if l:
             for x in range(l[0]):
-                transaction.add(add_transactions, l[1:])
+                tq.add(add_transactions, l[1:])
 
-    with transaction.TransactionQueue():
-        transaction.add(add_transactions, [10, 10, 10])
-    assert transaction.number_of_transactions_in_last_run() == 1111
+    tq = transaction.TransactionQueue()
+    tq.add(add_transactions, [10, 10, 10])
+    tq.run()
+    assert tq.number_of_transactions_executed() == 1111
+
+def test_unexecuted_transactions_after_exception():
+    class FooError(Exception):
+        pass
+    class BarError(Exception):
+        pass
+    def raiseme(exc):
+        raise exc
+    seen = []
+    tq = transaction.TransactionQueue()
+    tq.add(raiseme, FooError)
+    tq.add(raiseme, BarError)
+    tq.add(seen.append, 42)
+    tq.add(seen.append, 42)
+    try:
+        tq.run()
+    except (FooError, BarError), e:
+        seen_exc = e.__class__
+    else:
+        raise AssertionError("should have raised FooError or BarError")
+    try:
+        tq.run()
+    except (FooError, BarError), e:
+        assert e.__class__ != seen_exc
+    else:
+        raise AssertionError("unexecuted transactions have disappeared")
+    for i in range(2):
+        tq.run()
+        assert seen == [42, 42]
 
 
 def test_stmidset():
diff --git a/lib_pypy/readline.py b/lib_pypy/readline.py
--- a/lib_pypy/readline.py
+++ b/lib_pypy/readline.py
@@ -6,4 +6,11 @@
 are only stubs at the moment.
 """
 
-from pyrepl.readline import *
+try:
+    from pyrepl.readline import *
+except ImportError:
+    import sys
+    if sys.platform == 'win32':
+        raise ImportError("the 'readline' module is not available on Windows"
+                          " (on either PyPy or CPython)")
+    raise
diff --git a/lib_pypy/transaction.py b/lib_pypy/transaction.py
--- a/lib_pypy/transaction.py
+++ b/lib_pypy/transaction.py
@@ -108,106 +108,103 @@
     pass
 
 
-def add(f, *args, **kwds):
-    """Register a new transaction that will be done by 'f(*args, **kwds)'.
-    Must be called within the transaction in the "with TransactionQueue()"
-    block, or within a transaction started by this one, directly or
-    indirectly.
-    """
-    _thread_local.pending.append((f, args, kwds))
+class TransactionQueue(object):
+    """A queue of pending transactions.
 
-

From noreply at buildbot.pypy.org  Wed Feb 25 11:53:24 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 25 Feb 2015 11:53:24 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c8: import stmgc 8c547a55af94
Message-ID: <20150225105324.EAD721C13B0@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: stmgc-c8
Changeset: r76119:ee48741bae4a
Date: 2015-02-25 10:31 +0100
http://bitbucket.org/pypy/pypy/changeset/ee48741bae4a/

Log:	import stmgc 8c547a55af94

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-ee8dd5569dea
+8c547a55af94
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -2,7 +2,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
 #endif
-
 /* *** MISC *** */
 static void free_bk(struct stm_undo_s *undo)
 {
@@ -80,9 +79,9 @@
         assert((get_page_status_in(STM_SEGMENT->segment_num,
                                    current_page_num) != PAGE_NO_ACCESS));
 
-        dprintf(("import slice seg=%d obj=%p off=%lu sz=%d pg=%lu\n",
-                 from_segnum, obj, SLICE_OFFSET(undo->slice),
-                 SLICE_SIZE(undo->slice), current_page_num));
+        /* dprintf(("import slice seg=%d obj=%p off=%lu sz=%d pg=%lu\n", */
+        /*          from_segnum, obj, SLICE_OFFSET(undo->slice), */
+        /*          SLICE_SIZE(undo->slice), current_page_num)); */
         char *src, *dst;
         if (src_segment_base != NULL)
             src = REAL_ADDRESS(src_segment_base, oslice);
@@ -180,7 +179,7 @@
     /* increment_total_allocated(4096); */
 
     if (copy_from_segnum == -1) {
-        /* this page is only accessible in the sharing segment so far (new
+        /* this page is only accessible in the sharing segment seg0 so far (new
            allocation). We can thus simply mark it accessible here. */
         pagecopy(get_virtual_page(my_segnum, pagenum),
                  get_virtual_page(0, pagenum));
@@ -229,14 +228,15 @@
         addr >= stm_object_pages+TOTAL_MEMORY) {
         /* actual segfault, unrelated to stmgc */
         fprintf(stderr, "Segmentation fault: accessing %p\n", addr);
-        abort();
+        raise(SIGINT);
     }
 
     int segnum = get_segment_of_linear_address(addr);
+    OPT_ASSERT(segnum != 0);
     if (segnum != STM_SEGMENT->segment_num) {
         fprintf(stderr, "Segmentation fault: accessing %p (seg %d) from"
                 " seg %d\n", addr, segnum, STM_SEGMENT->segment_num);
-        abort();
+        raise(SIGINT);
     }
     dprintf(("-> segment: %d\n", segnum));
 
@@ -245,7 +245,7 @@
     if (pagenum < END_NURSERY_PAGE) {
         fprintf(stderr, "Segmentation fault: accessing %p (seg %d "
                         "page %lu)\n", addr, segnum, pagenum);
-        abort();
+        raise(SIGINT);
     }
 
     DEBUG_EXPECT_SEGFAULT(false);
@@ -264,11 +264,7 @@
     struct stm_commit_log_entry_s *cl = &commit_log_root;
 
     fprintf(stderr, "commit log:\n");
-    while ((cl = cl->next)) {
-        if (cl == INEV_RUNNING) {
-            fprintf(stderr, "  INEVITABLE\n");
-            return;
-        }
+    while (cl) {
         fprintf(stderr, "  entry at %p: seg %d, rev %lu\n", cl, cl->segment_num, cl->rev_num);
         struct stm_undo_s *undo = cl->written;
         struct stm_undo_s *end = undo + cl->written_count;
@@ -280,6 +276,12 @@
             /*     fprintf(stderr, " 0x%016lx", *(long *)(undo->backup + i)); */
             fprintf(stderr, "\n");
         }
+
+        cl = cl->next;
+        if (cl == INEV_RUNNING) {
+            fprintf(stderr, "  INEVITABLE\n");
+            return;
+        }
     }
 }
 
@@ -289,7 +291,8 @@
 {
     /* returns true if we reached a valid state, or false if
        we need to abort now */
-    dprintf(("_stm_validate()\n"));
+    dprintf(("_stm_validate() at cl=%p, rev=%lu\n", STM_PSEGMENT->last_commit_log_entry,
+             STM_PSEGMENT->last_commit_log_entry->rev_num));
     /* go from last known entry in commit log to the
        most current one and apply all changes done
        by other transactions. Abort if we have read one of
@@ -302,7 +305,8 @@
     if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {
         //assert(first_cl->next == INEV_RUNNING);
         /* the above assert may fail when running a major collection
-           while the commit of the inevitable transaction is in progress */
+           while the commit of the inevitable transaction is in progress
+           and the element is already attached */
         return true;
     }
 
@@ -389,6 +393,7 @@
                          !needs_abort);  /* if we abort, we still want to copy everything */
                 }
 
+                dprintf(("_stm_validate() to cl=%p, rev=%lu\n", cl, cl->rev_num));
                 /* last fully validated entry */
                 STM_PSEGMENT->last_commit_log_entry = cl;
                 if (cl == last_cl)
@@ -475,6 +480,12 @@
             reset_wb_executed_flags();
             check_all_write_barrier_flags(STM_SEGMENT->segment_base,
                                           STM_PSEGMENT->modified_old_objects);
+
+            /* need to remove the entries in modified_old_objects "at the same
+               time" as the attach to commit log. Otherwise, another thread may
+               see the new CL entry, import it, look for backup copies in this
+               segment and find the old backup copies! */
+            acquire_modification_lock(STM_SEGMENT->segment_num);
         }
 
         /* try to attach to commit log: */
@@ -491,6 +502,7 @@
         }
 
         if (is_commit) {
+            release_modification_lock(STM_SEGMENT->segment_num);
             /* XXX: unfortunately, if we failed to attach our CL entry,
                we have to re-add the WB_EXECUTED flags before we try to
                validate again because of said condition (s.a) */
@@ -501,8 +513,24 @@
 
         /* check for requested safe point. otherwise an INEV transaction
            may try to commit but cannot because of the busy-loop here. */
+        /* minor gc is fine here because we did one immediately before, so
+           there are no young objs anyway. major gc is fine because the
+           modified_old_objects list is still populated with the same
+           cl-entry objs */
+        /* XXXXXXXX: memory leak if we happen to do a major gc, we get aborted
+           in major_do_validation_and_minor_collections, and don't free 'new' */
         _stm_collectable_safe_point();
     }
+
+    if (is_commit) {
+        /* compare with _validate_and_add_to_commit_log */
+        STM_PSEGMENT->transaction_state = TS_NONE;
+        STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
+
+        list_clear(STM_PSEGMENT->modified_old_objects);
+        STM_PSEGMENT->last_commit_log_entry = new;
+        release_modification_lock(STM_SEGMENT->segment_num);
+    }
 }
 
 static void _validate_and_turn_inevitable(void)
@@ -525,17 +553,19 @@
         check_all_write_barrier_flags(STM_SEGMENT->segment_base,
                                       STM_PSEGMENT->modified_old_objects);
 
+        /* compare with _validate_and_attach: */
+        STM_PSEGMENT->transaction_state = TS_NONE;
+        STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
+        list_clear(STM_PSEGMENT->modified_old_objects);
+        STM_PSEGMENT->last_commit_log_entry = new;
+
+        /* do it: */
         bool yes = __sync_bool_compare_and_swap(&old->next, INEV_RUNNING, new);
         OPT_ASSERT(yes);
     }
     else {
         _validate_and_attach(new);
     }
-
-    acquire_modification_lock(STM_SEGMENT->segment_num);
-    list_clear(STM_PSEGMENT->modified_old_objects);
-    STM_PSEGMENT->last_commit_log_entry = new;
-    release_modification_lock(STM_SEGMENT->segment_num);
 }
 
 /* ############# STM ############# */
@@ -668,7 +698,7 @@
              (long)(NB_READMARKER_PAGES * 4096UL)));
     if (mmap(readmarkers, NB_READMARKER_PAGES * 4096UL,
              PROT_READ | PROT_WRITE,
-             MAP_FIXED | MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0) != readmarkers) {
+             MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0) != readmarkers) {
         /* fall-back */
 #if STM_TESTS
         stm_fatalerror("reset_transaction_read_version: %m");
@@ -743,6 +773,8 @@
     assert(tree_is_cleared(STM_PSEGMENT->nursery_objects_shadows));
     assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[0]));
     assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[1]));
+    assert(list_is_empty(STM_PSEGMENT->young_objects_with_light_finalizers));
+    assert(STM_PSEGMENT->finalizers == NULL);
 
     check_nursery_at_transaction_start();
 
@@ -763,9 +795,15 @@
 
 void stm_start_inevitable_transaction(stm_thread_local_t *tl)
 {
-    s_mutex_lock();
-    _stm_start_transaction(tl);
-    _stm_become_inevitable("stm_start_inevitable_transaction");
+    /* used to be more efficient, starting directly an inevitable transaction,
+       but there is no real point any more, I believe */
+    rewind_jmp_buf rjbuf;
+    stm_rewind_jmp_enterframe(tl, &rjbuf);
+
+    stm_start_transaction(tl);
+    stm_become_inevitable(tl, "start_inevitable_transaction");
+
+    stm_rewind_jmp_leaveframe(tl, &rjbuf);
 }
 
 #ifdef STM_NO_AUTOMATIC_SETJMP
@@ -835,6 +873,8 @@
 
 void stm_commit_transaction(void)
 {
+    exec_local_finalizers();
+
     assert(!_has_mutex());
     assert(STM_PSEGMENT->safe_point == SP_RUNNING);
     assert(STM_PSEGMENT->running_pthread == pthread_self());
@@ -844,26 +884,36 @@
 
     push_new_objects_to_other_segments();
     /* push before validate. otherwise they are reachable too early */
+    bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE;
     _validate_and_add_to_commit_log();
 
-    invoke_and_clear_user_callbacks(0);   /* for commit */
 
     /* XXX do we still need a s_mutex_lock() section here? */
     s_mutex_lock();
+
     enter_safe_point_if_requested();
     assert(STM_SEGMENT->nursery_end == NURSERY_END);
 
     stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
 
-    if (globally_unique_transaction && STM_PSEGMENT->transaction_state == TS_INEVITABLE) {
+    commit_finalizers();
+
+    invoke_and_clear_user_callbacks(0);   /* for commit */
+
+    if (globally_unique_transaction && was_inev) {
         committed_globally_unique_transaction();
     }
 
     /* done */
+    stm_thread_local_t *tl = STM_SEGMENT->running_thread;
     _finish_transaction();
     /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */
 
     s_mutex_unlock();
+
+    /* between transactions, call finalizers. this will execute
+       a transaction itself */
+    invoke_general_finalizers(tl);
 }
 
 static void reset_modified_from_backup_copies(int segment_num)
@@ -919,6 +969,8 @@
                        (int)pseg->transaction_state);
     }
 
+    abort_finalizers(pseg);
+
     long bytes_in_nursery = throw_away_nursery(pseg);
 
     acquire_modification_lock(segment_num);
@@ -1119,7 +1171,7 @@
             if (get_page_status_in(i, page) != PAGE_NO_ACCESS) {
                 /* shared or private, but never segfault */
                 char *dst = REAL_ADDRESS(get_segment_base(i), frag);
-                //dprintf(("-> flush %p to seg %lu, sz=%lu\n", frag, i, frag_size));
+                dprintf(("-> flush %p to seg %lu, sz=%lu\n", frag, i, frag_size));
                 memcpy(dst, src, frag_size);
             }
         }
diff --git a/rpython/translator/stm/src_stm/stm/core.h b/rpython/translator/stm/src_stm/stm/core.h
--- a/rpython/translator/stm/src_stm/stm/core.h
+++ b/rpython/translator/stm/src_stm/stm/core.h
@@ -1,6 +1,5 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
 #define _STM_CORE_H_
-
 #include 
 #include 
 #include 
@@ -40,6 +39,7 @@
     GCFLAG_HAS_SHADOW = 0x02,
     GCFLAG_WB_EXECUTED = 0x04,
     GCFLAG_VISITED = 0x08,
+    GCFLAG_FINALIZATION_ORDERING = 0x10,
 };
 
 
@@ -111,6 +111,14 @@
     pthread_t running_pthread;
 #endif
 
+    /* light finalizers */
+    struct list_s *young_objects_with_light_finalizers;
+    struct list_s *old_objects_with_light_finalizers;
+
+    /* regular finalizers (objs from the current transaction only) */
+    struct finalizers_s *finalizers;
+
+
     /* This is for smallmalloc.c */
     struct small_malloc_data_s small_malloc_data;
 
diff --git a/rpython/translator/stm/src_stm/stm/extra.c b/rpython/translator/stm/src_stm/stm/extra.c
--- a/rpython/translator/stm/src_stm/stm/extra.c
+++ b/rpython/translator/stm/src_stm/stm/extra.c
@@ -6,15 +6,24 @@
 static long register_callbacks(stm_thread_local_t *tl,
                                void *key, void callback(void *), long index)
 {
-    if (!_stm_in_transaction(tl)) {
-        /* check that the current thread-local is really running a
+    dprintf(("register_callbacks: tl=%p key=%p callback=%p index=%ld\n",
+             tl, key, callback, index));
+    if (tl->associated_segment_num == -1) {
+        /* check that the provided thread-local is really running a
            transaction, and do nothing otherwise. */
+        dprintf(("  NOT IN TRANSACTION\n"));
         return -1;
     }
-
-    if (STM_PSEGMENT->transaction_state == TS_INEVITABLE) {
+    /* The tl was only here to check that.  We're really using
+       STM_PSEGMENT below, which is often but not always the
+       segment corresponding to the tl.  One case where it's not
+       the case is if this gets called from stmcb_light_finalizer()
+       from abort_finalizers() from major collections or contention.
+    */
+    if (STM_PSEGMENT->transaction_state != TS_REGULAR) {
         /* ignore callbacks if we're in an inevitable transaction
-           (which cannot abort) */
+           (which cannot abort) or no transaction at all in this segment */
+        dprintf(("  STATE = %d\n", (int)STM_PSEGMENT->transaction_state));
         return -1;
     }
 
@@ -23,15 +32,19 @@
 
     if (callback == NULL) {
         /* double-unregistering works, but return 0 */
-        return tree_delete_item(callbacks, (uintptr_t)key);
+        long res = tree_delete_item(callbacks, (uintptr_t)key);
+        dprintf(("  DELETED %ld\n", res));
+        return res;
     }
     else {
         /* double-registering the same key will crash */
+        dprintf(("  INSERTING\n"));
         tree_insert(callbacks, (uintptr_t)key, (uintptr_t)callback);
         return 1;
     }
 }
 
+
 long stm_call_on_commit(stm_thread_local_t *tl,
                        void *key, void callback(void *))
 {
@@ -39,6 +52,7 @@
     if (result < 0 && callback != NULL) {
         /* no regular transaction running, invoke the callback
            immediately */
+        dprintf(("stm_call_on_commit calls now: %p(%p)\n", callback, key));
         callback(key);
     }
     return result;
@@ -72,8 +86,11 @@
         assert(key != NULL);
         assert(callback != NULL);
 
-        /* The callback may call stm_call_on_abort(key, NULL).  It is ignored,
-           because 'callbacks_on_commit_and_abort' was cleared already. */
+        /* The callback may call stm_call_on_abort(key, NULL)
+           (so with callback==NULL).  It is ignored, because
+           'callbacks_on_commit_and_abort' was cleared already. */
+        dprintf(("invoke_and_clear_user_callbacks(%ld): %p(%p)\n",
+                 index, callback, key));
         callback(key);
 
     } TREE_LOOP_END;
diff --git a/rpython/translator/stm/src_stm/stm/finalizer.c b/rpython/translator/stm/src_stm/stm/finalizer.c
new file mode 100644
--- /dev/null
+++ b/rpython/translator/stm/src_stm/stm/finalizer.c
@@ -0,0 +1,499 @@
+/* Imported by rpython/translator/stm/import_stmgc.py */
+
+/* callbacks */
+void (*stmcb_light_finalizer)(object_t *);
+void (*stmcb_finalizer)(object_t *);
+
+
+static void init_finalizers(struct finalizers_s *f)
+{
+    f->objects_with_finalizers = list_create();
+    f->count_non_young = 0;
+    f->run_finalizers = NULL;
+    f->running_next = NULL;
+}
+
+static void setup_finalizer(void)
+{
+    init_finalizers(&g_finalizers);
+}
+
+static void teardown_finalizer(void)
+{
+    if (g_finalizers.run_finalizers != NULL)
+        list_free(g_finalizers.run_finalizers);
+    list_free(g_finalizers.objects_with_finalizers);
+    memset(&g_finalizers, 0, sizeof(g_finalizers));
+}
+
+static void _commit_finalizers(void)
+{
+    /* move finalizer lists to g_finalizers for major collections */
+
+    if (STM_PSEGMENT->finalizers->run_finalizers != NULL) {
+        /* copy 'STM_PSEGMENT->finalizers->run_finalizers' into
+           'g_finalizers.run_finalizers', dropping any initial NULLs
+           (finalizers already called) */
+        struct list_s *src = STM_PSEGMENT->finalizers->run_finalizers;
+        uintptr_t frm = 0;
+        if (STM_PSEGMENT->finalizers->running_next != NULL) {
+            frm = *STM_PSEGMENT->finalizers->running_next;
+            assert(frm <= list_count(src));
+            *STM_PSEGMENT->finalizers->running_next = (uintptr_t)-1;
+        }
+        if (frm < list_count(src)) {
+            if (g_finalizers.run_finalizers == NULL)
+                g_finalizers.run_finalizers = list_create();
+            g_finalizers.run_finalizers = list_extend(
+                g_finalizers.run_finalizers,
+                src, frm);
+        }
+        list_free(src);
+    }
+
+    /* copy the whole 'STM_PSEGMENT->finalizers->objects_with_finalizers'
+       into 'g_finalizers.objects_with_finalizers' */
+    g_finalizers.objects_with_finalizers = list_extend(
+        g_finalizers.objects_with_finalizers,
+        STM_PSEGMENT->finalizers->objects_with_finalizers, 0);
+    list_free(STM_PSEGMENT->finalizers->objects_with_finalizers);
+
+    free(STM_PSEGMENT->finalizers);
+    STM_PSEGMENT->finalizers = NULL;
+}
+
+static void abort_finalizers(struct stm_priv_segment_info_s *pseg)
+{
+    /* like _commit_finalizers(), but forget everything from the
+       current transaction */
+    if (pseg->finalizers != NULL) {
+        if (pseg->finalizers->run_finalizers != NULL) {
+            if (pseg->finalizers->running_next != NULL) {
+                *pseg->finalizers->running_next = (uintptr_t)-1;
+            }
+            list_free(pseg->finalizers->run_finalizers);
+        }
+        list_free(pseg->finalizers->objects_with_finalizers);
+        free(pseg->finalizers);
+        pseg->finalizers = NULL;
+    }
+
+    /* call the light finalizers for objects that are about to
+       be forgotten from the current transaction */
+    char *old_gs_register = STM_SEGMENT->segment_base;
+    bool must_fix_gs = old_gs_register != pseg->pub.segment_base;
+
+    struct list_s *lst = pseg->young_objects_with_light_finalizers;
+    long i, count = list_count(lst);
+    if (lst > 0) {
+        for (i = 0; i < count; i++) {
+            object_t *obj = (object_t *)list_item(lst, i);
+            assert(_is_young(obj));
+            if (must_fix_gs) {
+                set_gs_register(pseg->pub.segment_base);
+                must_fix_gs = false;
+            }
+            stmcb_light_finalizer(obj);
+        }
+        list_clear(lst);
+    }
+
+    /* also deals with newly created objects: they are at the tail of
+       old_objects_with_light_finalizers (this list is kept in order
+       and we cannot add any already-committed object) */
+    lst = pseg->old_objects_with_light_finalizers;
+    count = list_count(lst);
+    while (count > 0) {
+        object_t *obj = (object_t *)list_item(lst, --count);
+        if (!(obj->stm_flags & GCFLAG_WB_EXECUTED))
+            break;
+        lst->count = count;
+        if (must_fix_gs) {
+            set_gs_register(pseg->pub.segment_base);
+            must_fix_gs = false;
+        }
+        stmcb_light_finalizer(obj);
+    }
+
+    if (STM_SEGMENT->segment_base != old_gs_register)
+        set_gs_register(old_gs_register);
+}
+
+
+void stm_enable_light_finalizer(object_t *obj)
+{
+    if (_is_young(obj)) {
+        LIST_APPEND(STM_PSEGMENT->young_objects_with_light_finalizers, obj);
+    }
+    else {
+        assert(_is_from_same_transaction(obj));
+        LIST_APPEND(STM_PSEGMENT->old_objects_with_light_finalizers, obj);
+    }
+}
+
+object_t *stm_allocate_with_finalizer(ssize_t size_rounded_up)
+{
+    object_t *obj = _stm_allocate_external(size_rounded_up);
+
+    if (STM_PSEGMENT->finalizers == NULL) {
+        struct finalizers_s *f = malloc(sizeof(struct finalizers_s));
+        if (f == NULL)
+            stm_fatalerror("out of memory in create_finalizers");   /* XXX */
+        init_finalizers(f);
+        STM_PSEGMENT->finalizers = f;
+    }
+    assert(STM_PSEGMENT->finalizers->count_non_young
+           <= list_count(STM_PSEGMENT->finalizers->objects_with_finalizers));
+    LIST_APPEND(STM_PSEGMENT->finalizers->objects_with_finalizers, obj);
+    return obj;
+}
+
+
+/************************************************************/
+/*  Light finalizers
+*/
+
+static void deal_with_young_objects_with_finalizers(void)
+{
+    /* for light finalizers: executes finalizers for objs that don't survive
+       this minor gc */
+    struct list_s *lst = STM_PSEGMENT->young_objects_with_light_finalizers;
+    long i, count = list_count(lst);
+    for (i = 0; i < count; i++) {
+        object_t *obj = (object_t *)list_item(lst, i);
+        assert(_is_young(obj));
+
+        object_t *TLPREFIX *pforwarded_array = (object_t *TLPREFIX *)obj;
+        if (pforwarded_array[0] != GCWORD_MOVED) {
+            /* not moved: the object dies */
+            stmcb_light_finalizer(obj);
+        }
+        else {
+            obj = pforwarded_array[1]; /* moved location */
+            assert(!_is_young(obj));
+            LIST_APPEND(STM_PSEGMENT->old_objects_with_light_finalizers, obj);
+        }
+    }
+    list_clear(lst);
+}
+
+static void deal_with_old_objects_with_finalizers(void)
+{
+    /* for light finalizers */
+    int old_gs_register = STM_SEGMENT->segment_num;
+    int current_gs_register = old_gs_register;
+    long j;
+    assert(list_is_empty(get_priv_segment(0)->old_objects_with_light_finalizers));
+    for (j = 1; j < NB_SEGMENTS; j++) {
+        struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
+
+        struct list_s *lst = pseg->old_objects_with_light_finalizers;
+        long i, count = list_count(lst);
+        lst->count = 0;
+        for (i = 0; i < count; i++) {
+            object_t *obj = (object_t *)list_item(lst, i);
+            if (!mark_visited_test(obj)) {
+                /* not marked: object dies */
+                /* we're calling the light finalizer in the same
+                   segment as where it was originally registered.  For
+                   objects that existed since a long time, it doesn't
+                   change anything: any thread should see the same old
+                   content (because if it wasn't the case, the object
+                   would be in a 'modified_old_objects' list
+                   somewhere, and so it wouldn't be dead).  But it's
+                   important if the object was created by the same
+                   transaction: then only that segment sees valid
+                   content.
+                */
+                if (j != current_gs_register) {
+                    set_gs_register(get_segment_base(j));
+                    current_gs_register = j;
+                }
+                stmcb_light_finalizer(obj);
+            }
+            else {
+                /* object survives */
+                list_set_item(lst, lst->count++, (uintptr_t)obj);
+            }
+        }
+    }
+    if (old_gs_register != current_gs_register)
+        set_gs_register(get_segment_base(old_gs_register));
+}
+
+
+/************************************************************/
+/*  Algorithm for regular (non-light) finalizers.
+    Follows closely pypy/doc/discussion/finalizer-order.rst
+    as well as rpython/memory/gc/minimark.py.
+*/
+
+static inline int _finalization_state(object_t *obj)
+{
+    /* Returns the state, "0", 1, 2 or 3, as per finalizer-order.rst.
+       One difference is that the official state 0 is returned here
+       as a number that is <= 0. */
+    struct object_s *realobj = mark_loc(obj);
+    if (realobj->stm_flags & GCFLAG_VISITED) {
+        if (realobj->stm_flags & GCFLAG_FINALIZATION_ORDERING)
+            return 2;
+        return 3;
+    }
+
+    if (realobj->stm_flags & GCFLAG_FINALIZATION_ORDERING)
+        return 1;
+
+    return 0;
+}
+
+static void _bump_finalization_state_from_0_to_1(object_t *obj)
+{
+    assert(_finalization_state(obj) == 0);
+    struct object_s *realobj = mark_loc(obj);
+    realobj->stm_flags |= GCFLAG_FINALIZATION_ORDERING;
+}
+
+static struct list_s *_finalizer_tmpstack;
+static struct list_s *_finalizer_emptystack;
+static struct list_s *_finalizer_pending;
+
+static inline void _append_to_finalizer_tmpstack(object_t **pobj)
+{
+    object_t *obj = *pobj;
+    if (obj != NULL)
+        LIST_APPEND(_finalizer_tmpstack, obj);
+}
+
+static inline struct list_s *finalizer_trace(char *base, object_t *obj,
+                                             struct list_s *lst)
+{
+    if (!is_new_object(obj))
+        base = stm_object_pages;
+
+    struct object_s *realobj = (struct object_s *)REAL_ADDRESS(base, obj);
+    _finalizer_tmpstack = lst;
+    stmcb_trace(realobj, &_append_to_finalizer_tmpstack);
+    return _finalizer_tmpstack;
+}
+
+
+static void _recursively_bump_finalization_state_from_2_to_3(char *base, object_t *obj)
+{
+    assert(_finalization_state(obj) == 2);
+    struct list_s *tmpstack = _finalizer_emptystack;
+    assert(list_is_empty(tmpstack));
+
+    while (1) {
+        struct object_s *realobj = mark_loc(obj);
+        if (realobj->stm_flags & GCFLAG_FINALIZATION_ORDERING) {
+            realobj->stm_flags &= ~GCFLAG_FINALIZATION_ORDERING;
+
+            /* trace */
+            tmpstack = finalizer_trace(base, obj, tmpstack);
+        }
+
+        if (list_is_empty(tmpstack))
+            break;
+
+        obj = (object_t *)list_pop_item(tmpstack);
+    }
+    _finalizer_emptystack = tmpstack;
+}
+
+static void _recursively_bump_finalization_state_from_1_to_2(char *base, object_t *obj)
+{
+    assert(_finalization_state(obj) == 1);
+    /* The call will add GCFLAG_VISITED recursively, thus bump state 1->2 */
+    mark_visit_possibly_new_object(base, obj);
+}
+
+static struct list_s *mark_finalize_step1(char *base, struct finalizers_s *f)
+{
+    if (f == NULL)
+        return NULL;
+
+    struct list_s *marked = list_create();
+
+    struct list_s *lst = f->objects_with_finalizers;
+    long i, count = list_count(lst);
+    lst->count = 0;
+    f->count_non_young = 0;
+
+    for (i = 0; i < count; i++) {
+        object_t *x = (object_t *)list_item(lst, i);
+
+        assert(_finalization_state(x) != 1);
+        if (_finalization_state(x) >= 2) {
+            list_set_item(lst, lst->count++, (uintptr_t)x);
+            continue;
+        }
+        LIST_APPEND(marked, x);
+
+        struct list_s *pending = _finalizer_pending;
+        LIST_APPEND(pending, x);
+        while (!list_is_empty(pending)) {
+            object_t *y = (object_t *)list_pop_item(pending);
+            int state = _finalization_state(y);
+            if (state <= 0) {
+                _bump_finalization_state_from_0_to_1(y);
+                pending = finalizer_trace(base, y, pending);
+            }
+            else if (state == 2) {
+                _recursively_bump_finalization_state_from_2_to_3(base, y);
+            }
+        }
+        _finalizer_pending = pending;
+        assert(_finalization_state(x) == 1);
+        _recursively_bump_finalization_state_from_1_to_2(base, x);
+    }
+    return marked;
+}
+
+static void mark_finalize_step2(char *base, struct finalizers_s *f,
+                                struct list_s *marked)
+{
+    if (f == NULL)
+        return;
+
+    struct list_s *run_finalizers = f->run_finalizers;
+
+    long i, count = list_count(marked);
+    for (i = 0; i < count; i++) {
+        object_t *x = (object_t *)list_item(marked, i);
+
+        int state = _finalization_state(x);
+        assert(state >= 2);
+        if (state == 2) {
+            if (run_finalizers == NULL)
+                run_finalizers = list_create();
+            LIST_APPEND(run_finalizers, x);
+            _recursively_bump_finalization_state_from_2_to_3(base, x);
+        }
+        else {
+            struct list_s *lst = f->objects_with_finalizers;
+            list_set_item(lst, lst->count++, (uintptr_t)x);
+        }
+    }
+    list_free(marked);
+
+    f->run_finalizers = run_finalizers;
+}
+
+static void deal_with_objects_with_finalizers(void)
+{
+    /* for non-light finalizers */
+
+    /* there is one 'objects_with_finalizers' list per segment.
+       Objects that die at a major collection running in the same
+       transaction as they were created will be put in the
+       'run_finalizers' list of that segment.  Objects that survive at
+       least one commit move to the global g_objects_with_finalizers,
+       and when they die they go to g_run_finalizers.  The former kind
+       of dying object must have its finalizer called in the correct
+       thread; the latter kind can be called in any thread, through
+       any segment, because they all should see the same old content
+       anyway.  (If the content was different between segments at this
+       point, the object would be in a 'modified_old_objects' list
+       somewhere, and so it wouldn't be dead).
+    */
+    struct list_s *marked_seg[NB_SEGMENTS];
+    LIST_CREATE(_finalizer_emptystack);
+    LIST_CREATE(_finalizer_pending);
+
+    long j;
+    for (j = 1; j < NB_SEGMENTS; j++) {
+        struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
+        marked_seg[j] = mark_finalize_step1(pseg->pub.segment_base,
+                                            pseg->finalizers);
+    }
+    marked_seg[0] = mark_finalize_step1(stm_object_pages, &g_finalizers);
+
+    LIST_FREE(_finalizer_pending);
+
+    for (j = 1; j < NB_SEGMENTS; j++) {
+        struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
+        mark_finalize_step2(pseg->pub.segment_base, pseg->finalizers,
+                            marked_seg[j]);
+    }
+    mark_finalize_step2(stm_object_pages, &g_finalizers, marked_seg[0]);
+
+    LIST_FREE(_finalizer_emptystack);
+}
+
+static void mark_visit_from_finalizer1(char *base, struct finalizers_s *f)
+{
+    if (f != NULL && f->run_finalizers != NULL) {
+        LIST_FOREACH_R(f->run_finalizers, object_t * /*item*/,
+                       ({
+                           mark_visit_possibly_new_object(base, item);
+                       }));
+    }
+}
+
+static void mark_visit_from_finalizer_pending(void)
+{
+    long j;
+    for (j = 1; j < NB_SEGMENTS; j++) {
+        struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
+        mark_visit_from_finalizer1(pseg->pub.segment_base, pseg->finalizers);
+    }
+    mark_visit_from_finalizer1(stm_object_pages, &g_finalizers);
+}
+
+static void _execute_finalizers(struct finalizers_s *f)
+{
+    if (f->run_finalizers == NULL)
+        return;   /* nothing to do */
+
+ restart:
+    if (f->running_next != NULL)
+        return;   /* in a nested invocation of execute_finalizers() */
+
+    uintptr_t next = 0, total = list_count(f->run_finalizers);
+    f->running_next = &next;
+
+    while (next < total) {
+        object_t *obj = (object_t *)list_item(f->run_finalizers, next);
+        list_set_item(f->run_finalizers, next, 0);
+        next++;
+
+        stmcb_finalizer(obj);
+    }
+    if (next == (uintptr_t)-1) {
+        /* transaction committed: the whole 'f' was freed */
+        return;
+    }
+    f->running_next = NULL;
+
+    if (f->run_finalizers->count > total) {
+        memmove(f->run_finalizers->items,
+                f->run_finalizers->items + total,
+                (f->run_finalizers->count - total) * sizeof(uintptr_t));
+        goto restart;
+    }
+
+    LIST_FREE(f->run_finalizers);
+}
+
+static void _invoke_general_finalizers(stm_thread_local_t *tl)
+{
+    /* called between transactions */
+    static int lock = 0;
+
+    if (__sync_lock_test_and_set(&lock, 1) != 0) {
+        /* can't acquire the lock: someone else is likely already
+           running this function, so don't wait. */
+        return;
+    }
+
+    rewind_jmp_buf rjbuf;
+    stm_rewind_jmp_enterframe(tl, &rjbuf);
+    stm_start_transaction(tl);
+
+    _execute_finalizers(&g_finalizers);
+
+    stm_commit_transaction();
+    stm_rewind_jmp_leaveframe(tl, &rjbuf);
+
+    __sync_lock_release(&lock);
+}
diff --git a/rpython/translator/stm/src_stm/stm/finalizer.h b/rpython/translator/stm/src_stm/stm/finalizer.h
new file mode 100644
--- /dev/null
+++ b/rpython/translator/stm/src_stm/stm/finalizer.h
@@ -0,0 +1,43 @@
+/* Imported by rpython/translator/stm/import_stmgc.py */
+struct finalizers_s {
+    struct list_s *objects_with_finalizers;
+    uintptr_t count_non_young;
+    struct list_s *run_finalizers;
+    uintptr_t *running_next;
+};
+
+static void mark_visit_from_finalizer_pending(void);
+static void deal_with_young_objects_with_finalizers(void);
+static void deal_with_old_objects_with_finalizers(void);
+static void deal_with_objects_with_finalizers(void);
+
+static void setup_finalizer(void);
+static void teardown_finalizer(void);
+
+static void _commit_finalizers(void);
+static void abort_finalizers(struct stm_priv_segment_info_s *);
+
+#define commit_finalizers()   do {              \
+    if (STM_PSEGMENT->finalizers != NULL)       \
+        _commit_finalizers();                   \
+} while (0)
+
+
+/* regular finalizers (objs from already-committed transactions) */
+static struct finalizers_s g_finalizers;
+
+static void _invoke_general_finalizers(stm_thread_local_t *tl);
+
+#define invoke_general_finalizers(tl)    do {   \
+    if (g_finalizers.run_finalizers != NULL)    \
+        _invoke_general_finalizers(tl);         \
+} while (0)
+
+static void _execute_finalizers(struct finalizers_s *f);
+
+#define any_local_finalizers() (STM_PSEGMENT->finalizers != NULL &&         \
+                               STM_PSEGMENT->finalizers->run_finalizers != NULL)
+#define exec_local_finalizers()  do {                   \
+    if (any_local_finalizers())                         \
+        _execute_finalizers(STM_PSEGMENT->finalizers);  \
+} while (0)
diff --git a/rpython/translator/stm/src_stm/stm/forksupport.c b/rpython/translator/stm/src_stm/stm/forksupport.c
new file mode 100644
--- /dev/null
+++ b/rpython/translator/stm/src_stm/stm/forksupport.c
@@ -0,0 +1,164 @@
+/* Imported by rpython/translator/stm/import_stmgc.py */
+#ifndef _STM_CORE_H_
+# error "must be compiled via stmgc.c"
+#endif
+
+static stm_thread_local_t *fork_this_tl;
+static bool fork_was_in_transaction;
+
+
+static void forksupport_prepare(void)
+{
+    if (stm_object_pages == NULL)
+        return;
+
+    /* So far we attempt to check this by walking all stm_thread_local_t,
+       marking the one from the current thread, and verifying that it's not
+       running a transaction.  This assumes that the stm_thread_local_t is just
+       a __thread variable, so never changes threads.
+    */
+    s_mutex_lock();
+
+    dprintf(("forksupport_prepare\n"));
+    fprintf(stderr, "[forking: for now, this operation can take some time]\n");
+
+    stm_thread_local_t *this_tl = NULL;
+    stm_thread_local_t *tl = stm_all_thread_locals;
+    do {
+        if (pthread_equal(*_get_cpth(tl), pthread_self())) {
+            if (this_tl != NULL)
+                stm_fatalerror("fork(): found several stm_thread_local_t"
+                               " from the same thread");
+            this_tl = tl;
+        }
+        tl = tl->next;
+    } while (tl != stm_all_thread_locals);
+
+    if (this_tl == NULL)
+        stm_fatalerror("fork(): found no stm_thread_local_t from this thread");
+    s_mutex_unlock();
+
+    bool was_in_transaction = _stm_in_transaction(this_tl);
+    if (!was_in_transaction)
+        stm_start_transaction(this_tl);
+
+    stm_become_inevitable(this_tl, "fork");
+    /* Note that the line above can still fail and abort, which should
+       be fine */
+
+    s_mutex_lock();
+    synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK);
+
+    fork_this_tl = this_tl;
+    fork_was_in_transaction = was_in_transaction;
+
+    assert(_has_mutex());
+    dprintf(("forksupport_prepare: from %p %p\n", fork_this_tl,
+             fork_this_tl->creating_pthread[0]));
+}
+
+static void forksupport_parent(void)
+{
+    if (stm_object_pages == NULL)
+        return;
+
+    dprintf(("forksupport_parent: continuing to run %p %p\n", fork_this_tl,
+             fork_this_tl->creating_pthread[0]));
+    assert(_has_mutex());
+    assert(_is_tl_registered(fork_this_tl));
+
+
+    bool was_in_transaction = fork_was_in_transaction;
+    s_mutex_unlock();
+
+    if (!was_in_transaction) {
+        stm_commit_transaction();
+    }
+
+    dprintf(("forksupport_parent: continuing to run\n"));
+}
+
+static void fork_abort_thread(long i)
+{
+    struct stm_priv_segment_info_s *pr = get_priv_segment(i);
+    stm_thread_local_t *tl = pr->pub.running_thread;
+    dprintf(("forksupport_child: abort in seg%ld\n", i));
+    assert(tl->associated_segment_num == i);
+    assert(pr->transaction_state == TS_REGULAR);
+    set_gs_register(get_segment_base(i));
+    assert(STM_SEGMENT->segment_num == i);
+
+    s_mutex_lock();
+#ifndef NDEBUG
+    pr->running_pthread = pthread_self();
+#endif
+    tl->shadowstack = NULL;
+    pr->shadowstack_at_start_of_transaction = NULL;
+    stm_rewind_jmp_forget(tl);
+    abort_with_mutex_no_longjmp();
+    s_mutex_unlock();
+}
+
+static void forksupport_child(void)
+{
+    if (stm_object_pages == NULL)
+        return;
+
+    /* this new process contains no other thread, so we can
+       just release these locks early */
+    s_mutex_unlock();
+
+
+    /* Unregister all other stm_thread_local_t, mostly as a way to free
+       the memory used by the shadowstacks
+     */
+    while (stm_all_thread_locals->next != stm_all_thread_locals) {
+        if (stm_all_thread_locals == fork_this_tl)
+            stm_unregister_thread_local(stm_all_thread_locals->next);
+        else
+            stm_unregister_thread_local(stm_all_thread_locals);
+    }
+    assert(stm_all_thread_locals == fork_this_tl);
+
+
+
+    /* Force the interruption of other running segments (seg0 never runs)
+     */
+    long i;
+    for (i = 1; i < NB_SEGMENTS; i++) {
+        struct stm_priv_segment_info_s *pr = get_priv_segment(i);
+        if (pr->pub.running_thread != NULL &&
+            pr->pub.running_thread != fork_this_tl) {
+            fork_abort_thread(i);
+        }
+    }
+
+    /* Restore a few things: the new pthread_self(), and the %gs
+       register */
+    int segnum = fork_this_tl->associated_segment_num;
+    assert(1 <= segnum && segnum < NB_SEGMENTS);
+    *_get_cpth(fork_this_tl) = pthread_self();
+    set_gs_register(get_segment_base(segnum));
+    assert(STM_SEGMENT->segment_num == segnum);
+
+    if (!fork_was_in_transaction) {
+        stm_commit_transaction();
+    }
+
+    /* Done */
+    dprintf(("forksupport_child: running one thread now\n"));
+}
+
+
+static void setup_forksupport(void)
+{
+    static bool fork_support_ready = false;
+
+    if (!fork_support_ready) {
+        int res = pthread_atfork(forksupport_prepare, forksupport_parent,
+                                 forksupport_child);
+        if (res != 0)
+            stm_fatalerror("pthread_atfork() failed: %m");
+        fork_support_ready = true;
+    }
+}
diff --git a/rpython/translator/stm/src_stm/stm/gcpage.c b/rpython/translator/stm/src_stm/stm/gcpage.c
--- a/rpython/translator/stm/src_stm/stm/gcpage.c
+++ b/rpython/translator/stm/src_stm/stm/gcpage.c
@@ -2,8 +2,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
 #endif
-
-static struct list_s *testing_prebuilt_objs = NULL;
 static struct tree_s *tree_prebuilt_objs = NULL;     /* XXX refactor */
 
 
@@ -64,6 +62,10 @@
                  size, (char*)(addr - stm_object_pages),
                  (uintptr_t)(addr - stm_object_pages) / 4096UL));
 
+        /* set stm_flags to 0 in seg0 so that major gc will see them
+           as not visited during sweeping */
+        ((struct object_s*)addr)->stm_flags = 0;
+
         return (stm_char*)(addr - stm_object_pages);
     }
 
@@ -91,6 +93,8 @@
              size, (char*)(addr - stm_object_pages),
              (uintptr_t)(addr - stm_object_pages) / 4096UL));
 
+    ((struct object_s*)addr)->stm_flags = 0;
+
     spinlock_release(lock_growth_large);
     return (stm_char*)(addr - stm_object_pages);
 }
@@ -101,7 +105,9 @@
     stm_char *p = allocate_outside_nursery_large(size_rounded_up);
     object_t *o = (object_t *)p;
 
-    // sharing seg0 needs to be current:
+    /* Sharing seg0 needs to be current, because in core.c handle_segfault_in_page,
+       we depend on simply copying the page from seg0 if it was never accessed by
+       anyone so far (we only run in seg1 <= seg < NB_SEGMENT). */
     assert(STM_SEGMENT->segment_num == 0);
     memset(REAL_ADDRESS(STM_SEGMENT->segment_base, o), 0, size_rounded_up);
     o->stm_flags = GCFLAG_WRITE_BARRIER;
@@ -140,6 +146,7 @@
     }
 
     s_mutex_unlock();
+    exec_local_finalizers();
 }
 
 
@@ -382,10 +389,10 @@
         */
 
         /* only for new, uncommitted objects:
-           If 'tl' is currently running, its 'associated_segment_num'
+           If 'tl' is currently running, its 'last_associated_segment_num'
            field is the segment number that contains the correct
            version of its overflowed objects. */
-        char *segment_base = get_segment_base(tl->associated_segment_num);
+        char *segment_base = get_segment_base(tl->last_associated_segment_num);
 
         struct stm_shadowentry_s *current = tl->shadowstack;
         struct stm_shadowentry_s *base = tl->shadowstack_base;
@@ -402,6 +409,7 @@
 
     /* also visit all objs in the rewind-shadowstack */
     long i;
+    assert(get_priv_segment(0)->transaction_state == TS_NONE);
     for (i = 1; i < NB_SEGMENTS; i++) {
         if (get_priv_segment(i)->transaction_state != TS_NONE) {
             mark_visit_possibly_new_object(
@@ -445,10 +453,13 @@
                 assert(realobj = (struct object_s*)REAL_ADDRESS(pseg->pub.segment_base, item));
                 assert(realobj->stm_flags & GCFLAG_WB_EXECUTED);
 
-                /* clear VISITED and ensure WB_EXECUTED in seg0 */
+                /* clear VISITED (garbage) and ensure WB_EXECUTED in seg0 */
                 mark_visited_test_and_clear(item);
                 realobj = (struct object_s*)REAL_ADDRESS(stm_object_pages, item);
                 realobj->stm_flags |= GCFLAG_WB_EXECUTED;
+
+                /* make sure this flag is cleared as well */
+                realobj->stm_flags &= ~GCFLAG_FINALIZATION_ORDERING;
             }));
     }
 #pragma pop_macro("STM_SEGMENT")
@@ -475,6 +486,7 @@
            This is the case for transactions where
                MINOR_NOTHING_TO_DO() == true
            but they still did write-barriers on objects
+           (the objs are still in modified_old_objects list)
         */
         lst = pseg->objects_pointing_to_nursery;
         if (!list_is_empty(lst)) {
@@ -509,6 +521,7 @@
 #pragma pop_macro("STM_PSEGMENT")
 }
 
+
 static inline bool largemalloc_keep_object_at(char *data)
 {
     /* this is called by _stm_largemalloc_sweep() */
@@ -569,16 +582,19 @@
 #ifndef NDEBUG
     /* check that all segments are at the same revision: */
     cl = get_priv_segment(0)->last_commit_log_entry;
-    for (long i = 1; i < NB_SEGMENTS; i++) {
-        assert(get_priv_segment(i)->last_commit_log_entry == cl);
+    for (long i = 0; i < NB_SEGMENTS; i++) {
+        struct stm_priv_segment_info_s *pseg = get_priv_segment(i);
+        assert(pseg->last_commit_log_entry == cl);
     }
 #endif
 
     /* if there is only one element, we don't have to do anything: */
     cl = &commit_log_root;
 
-    if (cl->next == NULL || cl->next == INEV_RUNNING)
+    if (cl->next == NULL || cl->next == INEV_RUNNING) {
+        assert(get_priv_segment(0)->last_commit_log_entry == cl);
         return;
+    }
 
     bool was_inev = false;
     uint64_t rev_num = -1;
@@ -645,10 +661,18 @@
     LIST_CREATE(marked_objects_to_trace);
     mark_visit_from_modified_objects();
     mark_visit_from_roots();
+    mark_visit_from_finalizer_pending();
+
+    /* finalizer support: will mark as visited all objects with a
+       finalizer and all objects reachable from there, and also moves
+       some objects from 'objects_with_finalizers' to 'run_finalizers'. */
+    deal_with_objects_with_finalizers();
+
     LIST_FREE(marked_objects_to_trace);
 
-    /* weakrefs */
+    /* weakrefs and execute old light finalizers */
     stm_visit_old_weakrefs();
+    deal_with_old_objects_with_finalizers();
 
     /* cleanup */
     clean_up_segment_lists();
diff --git a/rpython/translator/stm/src_stm/stm/gcpage.h b/rpython/translator/stm/src_stm/stm/gcpage.h
--- a/rpython/translator/stm/src_stm/stm/gcpage.h
+++ b/rpython/translator/stm/src_stm/stm/gcpage.h
@@ -1,5 +1,4 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
-
 /* Granularity when grabbing more unused pages: take 20 at a time */
 #define GCPAGE_NUM_PAGES   20
 
@@ -9,7 +8,7 @@
 #define GC_MAJOR_COLLECT       1.82
 
 
-
+static struct list_s *testing_prebuilt_objs;
 static char *uninitialized_page_start;   /* within segment 0 */
 static char *uninitialized_page_stop;
 
diff --git a/rpython/translator/stm/src_stm/stm/largemalloc.c b/rpython/translator/stm/src_stm/stm/largemalloc.c
--- a/rpython/translator/stm/src_stm/stm/largemalloc.c
+++ b/rpython/translator/stm/src_stm/stm/largemalloc.c
@@ -616,6 +616,7 @@
         /* use the callback to know if 'chunk' contains an object that
            survives or dies */
         if (!_largemalloc_sweep_keep(chunk)) {
+            dprintf(("dies: %p\n", (char*)((char*)&chunk->d - stm_object_pages)));
             _large_free(chunk);     /* dies */
         }
         chunk = mnext;
diff --git a/rpython/translator/stm/src_stm/stm/list.c b/rpython/translator/stm/src_stm/stm/list.c
--- a/rpython/translator/stm/src_stm/stm/list.c
+++ b/rpython/translator/stm/src_stm/stm/list.c
@@ -30,6 +30,21 @@
     return lst;
 }
 
+static struct list_s *list_extend(struct list_s *lst, struct list_s *lst2,
+                                  uintptr_t slicestart)
+{
+    if (lst2->count <= slicestart)
+        return lst;
+    uintptr_t baseindex = lst->count;
+    lst->count = baseindex + lst2->count - slicestart;
+    uintptr_t lastindex = lst->count - 1;
+    if (lastindex > lst->last_allocated)
+        lst = _list_grow(lst, lastindex);
+    memcpy(lst->items + baseindex, lst2->items + slicestart,
+           (lst2->count - slicestart) * sizeof(uintptr_t));
+    return lst;
+}
+
 
 /************************************************************/
 
diff --git a/rpython/translator/stm/src_stm/stm/list.h b/rpython/translator/stm/src_stm/stm/list.h
--- a/rpython/translator/stm/src_stm/stm/list.h
+++ b/rpython/translator/stm/src_stm/stm/list.h
@@ -1,7 +1,6 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
 #include 
 #include 
-
 /************************************************************/
 
 struct list_s {
@@ -97,6 +96,10 @@
     return &lst->items[index];
 }
 
+static struct list_s *list_extend(struct list_s *lst, struct list_s *lst2,
+                                  uintptr_t slicestart);
+
+
 #define LIST_FOREACH_R(lst, TYPE, CODE)         \
     do {                                        \
         struct list_s *_lst = (lst);            \
diff --git a/rpython/translator/stm/src_stm/stm/nursery.c b/rpython/translator/stm/src_stm/stm/nursery.c
--- a/rpython/translator/stm/src_stm/stm/nursery.c
+++ b/rpython/translator/stm/src_stm/stm/nursery.c
@@ -37,6 +37,10 @@
         tree_contains(STM_PSEGMENT->young_outside_nursery, (uintptr_t)obj));
 }
 
+static inline bool _is_from_same_transaction(object_t *obj) {
+    return _is_young(obj) || (obj->stm_flags & GCFLAG_WB_EXECUTED);
+}
+
 long stm_can_move(object_t *obj)
 {
     /* 'long' return value to avoid using 'bool' in the public interface */
@@ -226,6 +230,22 @@
     }
 }
 
+static void collect_objs_still_young_but_with_finalizers(void)
+{
+    struct list_s *lst = STM_PSEGMENT->finalizers->objects_with_finalizers;
+    uintptr_t i, total = list_count(lst);
+
+    for (i = STM_PSEGMENT->finalizers->count_non_young; i < total; i++) {
+
+        object_t *o = (object_t *)list_item(lst, i);
+        minor_trace_if_young(&o);
+
+        /* was not actually movable */
+        assert(o == (object_t *)list_item(lst, i));
+    }
+    STM_PSEGMENT->finalizers->count_non_young = total;
+}
+
 
 static size_t throw_away_nursery(struct stm_priv_segment_info_s *pseg)
 {
@@ -296,12 +316,16 @@
 
     collect_roots_in_nursery();
 
+    if (STM_PSEGMENT->finalizers != NULL)
+        collect_objs_still_young_but_with_finalizers();
+
     collect_oldrefs_to_nursery();
 
     /* now all surviving nursery objects have been moved out */
     acquire_privatization_lock(STM_SEGMENT->segment_num);
     stm_move_young_weakrefs();
     release_privatization_lock(STM_SEGMENT->segment_num);
+    deal_with_young_objects_with_finalizers();
 
     assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery));
 
@@ -429,11 +453,16 @@
     int original_num = STM_SEGMENT->segment_num;
     long i;
 
+    assert(_has_mutex());
+
     /* including the sharing seg0 */
     for (i = 0; i < NB_SEGMENTS; i++) {
         set_gs_register(get_segment_base(i));
 
-        if (!_stm_validate()) {
+        bool ok = _stm_validate();
+        assert(get_priv_segment(i)->last_commit_log_entry->next == NULL
+               || get_priv_segment(i)->last_commit_log_entry->next == INEV_RUNNING);
+        if (!ok) {
             assert(i != 0);     /* sharing seg0 should never need an abort */
 
             if (STM_PSEGMENT->transaction_state == TS_NONE) {
diff --git a/rpython/translator/stm/src_stm/stm/nursery.h b/rpython/translator/stm/src_stm/stm/nursery.h
--- a/rpython/translator/stm/src_stm/stm/nursery.h
+++ b/rpython/translator/stm/src_stm/stm/nursery.h
@@ -1,5 +1,4 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
-
 #define NSE_SIGPAUSE   _STM_NSE_SIGNAL_MAX
 #define NSE_SIGABORT   _STM_NSE_SIGNAL_ABORT
 
diff --git a/rpython/translator/stm/src_stm/stm/pages.c b/rpython/translator/stm/src_stm/stm/pages.c
--- a/rpython/translator/stm/src_stm/stm/pages.c
+++ b/rpython/translator/stm/src_stm/stm/pages.c
@@ -2,7 +2,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
 #endif
-
 #include 
 /************************************************************/
 struct {
diff --git a/rpython/translator/stm/src_stm/stm/pages.h b/rpython/translator/stm/src_stm/stm/pages.h
--- a/rpython/translator/stm/src_stm/stm/pages.h
+++ b/rpython/translator/stm/src_stm/stm/pages.h
@@ -4,7 +4,6 @@
       logical page
   We have virtual pages: one virtual address can point in some
       virtual page. We have NB_SEGMENTS virtual pages per logical page.
-
   Each virtual page is either accessible, or PAGE_NO_ACCESS (and then
   has no underlying memory).
 
diff --git a/rpython/translator/stm/src_stm/stm/setup.c b/rpython/translator/stm/src_stm/stm/setup.c
--- a/rpython/translator/stm/src_stm/stm/setup.c
+++ b/rpython/translator/stm/src_stm/stm/setup.c
@@ -2,7 +2,6 @@
 #ifndef _STM_CORE_H_
 # error "must be compiled via stmgc.c"
 #endif
-
 #include 
 #include            /* For O_* constants */
 
@@ -109,6 +108,8 @@
         pr->nursery_objects_shadows = tree_create();
         pr->callbacks_on_commit_and_abort[0] = tree_create();
         pr->callbacks_on_commit_and_abort[1] = tree_create();
+        pr->young_objects_with_light_finalizers = list_create();
+        pr->old_objects_with_light_finalizers = list_create();
 
         pr->last_commit_log_entry = &commit_log_root;
         pr->pub.transaction_read_version = 0xff;
@@ -127,6 +128,8 @@
     setup_nursery();
     setup_gcpage();
     setup_pages();
+    setup_forksupport();
+    setup_finalizer();
 
     set_gs_register(get_segment_base(0));
 }
@@ -153,6 +156,8 @@
         tree_free(pr->nursery_objects_shadows);
         tree_free(pr->callbacks_on_commit_and_abort[0]);
         tree_free(pr->callbacks_on_commit_and_abort[1]);
+        list_free(pr->young_objects_with_light_finalizers);
+        list_free(pr->old_objects_with_light_finalizers);
     }
 
     munmap(stm_object_pages, TOTAL_MEMORY);
@@ -160,6 +165,7 @@
     commit_log_root.next = NULL; /* xxx:free them */
     commit_log_root.segment_num = -1;
 
+    teardown_finalizer();
     teardown_sync();
     teardown_gcpage();
     teardown_smallmalloc();
@@ -223,14 +229,15 @@
         tl->prev = stm_all_thread_locals->prev;
         stm_all_thread_locals->prev->next = tl;
         stm_all_thread_locals->prev = tl;
-        num = (tl->prev->associated_segment_num) % (NB_SEGMENTS-1);
+        num = (tl->prev->last_associated_segment_num) % (NB_SEGMENTS-1);
     }
     tl->thread_local_obj = NULL;
 
     /* assign numbers consecutively, but that's for tests; we could also
        assign the same number to all of them and they would get their own
        numbers automatically. */
-    tl->associated_segment_num = num + 1;
+    tl->associated_segment_num = -1;
+    tl->last_associated_segment_num = num + 1;
     *_get_cpth(tl) = pthread_self();
     _init_shadow_stack(tl);
     set_gs_register(get_segment_base(num + 1));
diff --git a/rpython/translator/stm/src_stm/stm/setup.h b/rpython/translator/stm/src_stm/stm/setup.h
--- a/rpython/translator/stm/src_stm/stm/setup.h
+++ b/rpython/translator/stm/src_stm/stm/setup.h
@@ -2,7 +2,6 @@
 static void setup_mmap(char *reason);
 static void setup_protection_settings(void);
 static pthread_t *_get_cpth(stm_thread_local_t *);
-
 #ifndef NDEBUG
 static __thread long _stm_segfault_expected = 0;
 #define DEBUG_EXPECT_SEGFAULT(v) do {if (v) _stm_segfault_expected++; else _stm_segfault_expected--;} while (0)
diff --git a/rpython/translator/stm/src_stm/stm/smallmalloc.c b/rpython/translator/stm/src_stm/stm/smallmalloc.c
--- a/rpython/translator/stm/src_stm/stm/smallmalloc.c
+++ b/rpython/translator/stm/src_stm/stm/smallmalloc.c
@@ -3,7 +3,6 @@
 # error "must be compiled via stmgc.c"
 #endif
 
-
 #define PAGE_SMSIZE_START   0
 #define PAGE_SMSIZE_END     NB_SHARED_PAGES
 
@@ -65,7 +64,8 @@
         uninitialized_page_stop -= decrease_by;
         first_small_uniform_loc = uninitialized_page_stop - stm_object_pages;
 
-        if (!_stm_largemalloc_resize_arena(uninitialized_page_stop - uninitialized_page_start))
+        char *base = stm_object_pages + END_NURSERY_PAGE * 4096UL;
+        if (!_stm_largemalloc_resize_arena(uninitialized_page_stop - base))
             goto out_of_memory;
 
         /* make writable in sharing seg */
@@ -176,13 +176,17 @@
 
     increment_total_allocated(size);
 
-    if (UNLIKELY(result == NULL))
+    if (UNLIKELY(result == NULL)) {
+        char *addr = _allocate_small_slowpath(size);
+        ((struct object_s*)addr)->stm_flags = 0;
         return (stm_char*)
-            (_allocate_small_slowpath(size) - stm_object_pages);
+            (addr - stm_object_pages);
+    }
 
     *fl = result->next;
     /* dprintf(("allocate_outside_nursery_small(%lu): %p\n", */
     /*          size, (char*)((char *)result - stm_object_pages))); */
+    ((struct object_s*)result)->stm_flags = 0;
     return (stm_char*)
         ((char *)result - stm_object_pages);
 }
@@ -197,6 +201,10 @@
     memset(REAL_ADDRESS(STM_SEGMENT->segment_base, o), 0, size_rounded_up);
     o->stm_flags = GCFLAG_WRITE_BARRIER;
 
+    if (testing_prebuilt_objs == NULL)
+        testing_prebuilt_objs = list_create();
+    LIST_APPEND(testing_prebuilt_objs, o);
+
     dprintf(("_stm_allocate_old_small(%lu): %p, seg=%d, page=%lu\n",
              size_rounded_up, p,
              get_segment_of_linear_address(stm_object_pages + (uintptr_t)p),
@@ -205,6 +213,7 @@
     return o;
 }
 
+
 /************************************************************/
 
 static inline bool _smallmalloc_sweep_keep(char *p)
diff --git a/rpython/translator/stm/src_stm/stm/smallmalloc.h b/rpython/translator/stm/src_stm/stm/smallmalloc.h
--- a/rpython/translator/stm/src_stm/stm/smallmalloc.h
+++ b/rpython/translator/stm/src_stm/stm/smallmalloc.h
@@ -1,5 +1,4 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
-
 /* Outside the nursery, we are taking from the highest addresses
    complete pages, one at a time, which uniformly contain objects of
    size "8 * N" for some N in range(2, GC_N_SMALL_REQUESTS).  We are
diff --git a/rpython/translator/stm/src_stm/stm/sync.c b/rpython/translator/stm/src_stm/stm/sync.c
--- a/rpython/translator/stm/src_stm/stm/sync.c
+++ b/rpython/translator/stm/src_stm/stm/sync.c
@@ -110,7 +110,7 @@
     assert(_has_mutex());
     assert(_is_tl_registered(tl));
 
-    int num = tl->associated_segment_num - 1; // 0..NB_SEG-1
+    int num = tl->last_associated_segment_num - 1; // 0..NB_SEG-1
     OPT_ASSERT(num >= 0);
     if (sync_ctl.in_use1[num+1] == 0) {
         /* fast-path: we can get the same segment number than the one
@@ -130,8 +130,9 @@
         num = (num+1) % (NB_SEGMENTS-1);
         if (sync_ctl.in_use1[num+1] == 0) {
             /* we're getting 'num', a different number. */
-            dprintf(("acquired different segment: %d->%d\n", tl->associated_segment_num, num+1));
-            tl->associated_segment_num = num+1;
+            dprintf(("acquired different segment: %d->%d\n",
+                     tl->last_associated_segment_num, num+1));
+            tl->last_associated_segment_num = num+1;
             set_gs_register(get_segment_base(num+1));
             goto got_num;
         }
@@ -147,6 +148,7 @@
     sync_ctl.in_use1[num+1] = 1;
     assert(STM_SEGMENT->segment_num == num+1);
     assert(STM_SEGMENT->running_thread == NULL);
+    tl->associated_segment_num = tl->last_associated_segment_num;
     STM_SEGMENT->running_thread = tl;
     return true;
 }
@@ -158,10 +160,12 @@
     cond_signal(C_SEGMENT_FREE);
 
     assert(STM_SEGMENT->running_thread == tl);
+    assert(tl->associated_segment_num == tl->last_associated_segment_num);
+    tl->associated_segment_num = -1;
     STM_SEGMENT->running_thread = NULL;
 
-    assert(sync_ctl.in_use1[tl->associated_segment_num] == 1);
-    sync_ctl.in_use1[tl->associated_segment_num] = 0;
+    assert(sync_ctl.in_use1[tl->last_associated_segment_num] == 1);
+    sync_ctl.in_use1[tl->last_associated_segment_num] = 0;
 }
 
 __attribute__((unused))
@@ -172,9 +176,16 @@
 
 bool _stm_in_transaction(stm_thread_local_t *tl)
 {
-    int num = tl->associated_segment_num;
-    assert(1 <= num && num < NB_SEGMENTS);
-    return get_segment(num)->running_thread == tl;
+    if (tl->associated_segment_num == -1) {
+        return false;
+    }
+    else {
+        int num = tl->associated_segment_num;
+        OPT_ASSERT(1 <= num && num < NB_SEGMENTS);
+        OPT_ASSERT(num == tl->last_associated_segment_num);
+        OPT_ASSERT(get_segment(num)->running_thread == tl);
+        return true;
+    }
 }
 
 void _stm_test_switch(stm_thread_local_t *tl)
@@ -182,6 +193,7 @@
     assert(_stm_in_transaction(tl));
     set_gs_register(get_segment_base(tl->associated_segment_num));
     assert(STM_SEGMENT->running_thread == tl);
+    exec_local_finalizers();
 }
 
 void _stm_test_switch_segment(int segnum)
diff --git a/rpython/translator/stm/src_stm/stm/sync.h b/rpython/translator/stm/src_stm/stm/sync.h
--- a/rpython/translator/stm/src_stm/stm/sync.h
+++ b/rpython/translator/stm/src_stm/stm/sync.h
@@ -1,7 +1,6 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
 static void setup_sync(void);
 static void teardown_sync(void);
-
 enum cond_type_e {
     C_AT_SAFE_POINT,
     C_REQUEST_REMOVED,
diff --git a/rpython/translator/stm/src_stm/stmgc.c b/rpython/translator/stm/src_stm/stmgc.c
--- a/rpython/translator/stm/src_stm/stmgc.c
+++ b/rpython/translator/stm/src_stm/stmgc.c
@@ -16,7 +16,7 @@
 #include "stm/extra.h"
 #include "stm/fprintcolor.h"
 #include "stm/rewind_setjmp.h"
-
+#include "stm/finalizer.h"
 #include "stm/misc.c"
 #include "stm/list.c"
 #include "stm/smallmalloc.c"
@@ -28,9 +28,11 @@
 #include "stm/nursery.c"
 #include "stm/weakref.c"
 #include "stm/sync.c"
+#include "stm/forksupport.c"
 #include "stm/setup.c"
 #include "stm/hash_id.c"
 #include "stm/core.c"
 #include "stm/extra.c"
 #include "stm/fprintcolor.c"
 #include "stm/rewind_setjmp.c"
+#include "stm/finalizer.c"
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -2,7 +2,6 @@
 #ifndef _STMGC_H
 #define _STMGC_H
 
-
 /* ==================== INTERNAL ==================== */
 
 /* See "API" below. */
@@ -68,6 +67,7 @@
     long last_abort__bytes_in_nursery;
     /* the next fields are handled internally by the library */
     int associated_segment_num;
+    int last_associated_segment_num;
     struct stm_thread_local_s *prev, *next;
     void *creating_pthread[2];
 } stm_thread_local_t;
@@ -270,6 +270,31 @@
 void stm_validate(void);
 
 
+
+/* Support for light finalizers.  This is a simple version of
+   finalizers that guarantees not to do anything fancy, like not
+   resurrecting objects. */
+extern void (*stmcb_light_finalizer)(object_t *);
+void stm_enable_light_finalizer(object_t *);
+
+/* Support for regular finalizers.  Unreachable objects with
+   finalizers are kept alive, as well as everything they point to, and
+   stmcb_finalizer() is called after the major GC.  If there are
+   several objects with finalizers that reference each other in a
+   well-defined order (i.e. there are no cycles), then they are
+   finalized in order from outermost to innermost (i.e. starting with
+   the ones that are unreachable even from others).
+
+   For objects that have been created by the current transaction, if a
+   major GC runs while that transaction is alive and finds the object
+   unreachable, the finalizer is called immediately in the same
+   transaction.  For older objects, the finalizer is called from a
+   random thread between regular transactions, in a new custom
+   transaction. */
+extern void (*stmcb_finalizer)(object_t *);
+object_t *stm_allocate_with_finalizer(ssize_t size_rounded_up);
+
+
 /* dummies for now: */
 __attribute__((always_inline))
 static inline void stm_write_card(object_t *obj, uintptr_t index)

From noreply at buildbot.pypy.org  Wed Feb 25 11:53:26 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 25 Feb 2015 11:53:26 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c8: remove now-implemented dummies
Message-ID: <20150225105326.2F12F1C13B0@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: stmgc-c8
Changeset: r76120:687a216f86e7
Date: 2015-02-25 10:58 +0100
http://bitbucket.org/pypy/pypy/changeset/687a216f86e7/

Log:	remove now-implemented dummies

diff --git a/rpython/translator/stm/src_stm/stmgcintf.c b/rpython/translator/stm/src_stm/stmgcintf.c
--- a/rpython/translator/stm/src_stm/stmgcintf.c
+++ b/rpython/translator/stm/src_stm/stmgcintf.c
@@ -10,13 +10,6 @@
 __thread uintptr_t pypy_stm_nursery_low_fill_mark_saved;
 
 
-/* C8: not implemented properly yet: */
-void (*stmcb_light_finalizer)(object_t *o);
-void (*stmcb_finalizer)(object_t *o);
-/* C8: not implemented properly yet ^^^^^^^^^^^^^^^^^^ */
-
-
-
 extern Signed pypy_stmcb_size_rounded_up(void*);
 extern void pypy_stmcb_get_card_base_itemsize(void*, uintptr_t[]);
 extern void pypy_stmcb_trace(void*, void(*)(void*));
diff --git a/rpython/translator/stm/src_stm/stmgcintf.h b/rpython/translator/stm/src_stm/stmgcintf.h
--- a/rpython/translator/stm/src_stm/stmgcintf.h
+++ b/rpython/translator/stm/src_stm/stmgcintf.h
@@ -45,13 +45,7 @@
     uintptr_t odd_number;  /* marker odd number, or 0 if marker is missing */
     object_t *object;      /* marker object, or NULL if marker is missing */
 } stm_loc_marker_t;
-extern void (*stmcb_light_finalizer)(object_t *o);
-extern void (*stmcb_finalizer)(object_t *o);
-static inline object_t *stm_allocate_with_finalizer(ssize_t size_rounded_up) {
-    return stm_allocate(size_rounded_up);
-}
-static inline void stm_enable_light_finalizer(object_t *o) {};
-static inline int stm_set_timing_log(const char *profiling_file_name,
+static inline int stm_set_timing_log(const char *profiling_file_name, int fork_mode,
                        int expand_marker(stm_loc_marker_t *, char *, int)) {return 0;}
 /* C8: not implemented properly yet ^^^^^^^^^^^^^^^^^^ */
 

From noreply at buildbot.pypy.org  Wed Feb 25 11:53:27 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 25 Feb 2015 11:53:27 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c8: disable hashtable for now
Message-ID: <20150225105327.5039D1C13B0@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: stmgc-c8
Changeset: r76121:5c981f82e4af
Date: 2015-02-25 11:41 +0100
http://bitbucket.org/pypy/pypy/changeset/5c981f82e4af/

Log:	disable hashtable for now

diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -252,7 +252,8 @@
                                       'freelist': _ll_hashtable_freelist,
                                         'lookup': _ll_hashtable_lookup,
                                       'writeobj': _ll_hashtable_writeobj})
-NULL_HASHTABLE = lltype.nullptr(_HASHTABLE_OBJ)
+# NULL_HASHTABLE = lltype.nullptr(_HASHTABLE_OBJ)
+NULL_HASHTABLE = None
 
 def _ll_hashtable_trace(gc, obj, callback, arg):
     from rpython.memory.gctransform.stmframework import get_visit_function
@@ -272,23 +273,45 @@
 def create_hashtable():
     if not we_are_translated():
         return HashtableForTest()      # for tests
-    rgc.register_custom_light_finalizer(_HASHTABLE_OBJ, lambda_hashtable_finlz)
-    rgc.register_custom_trace_hook(_HASHTABLE_OBJ, lambda_hashtable_trace)
-    # Pass a null pointer to _STM_HASHTABLE_ENTRY to stm_hashtable_create().
-    # Make sure we see a malloc() of it, so that its typeid is correctly
-    # initialized.  It can be done in a NonConstant(False) path so that
-    # the C compiler will actually drop it.
-    if _false:
-        p = lltype.malloc(_STM_HASHTABLE_ENTRY)
-    else:
-        p = lltype.nullptr(_STM_HASHTABLE_ENTRY)
-    h = lltype.malloc(_HASHTABLE_OBJ)
-    h.ll_raw_hashtable = lltype.nullptr(_STM_HASHTABLE_P.TO)
-    h.ll_raw_hashtable = llop.stm_hashtable_create(_STM_HASHTABLE_P, p)
-    return h
+    return HashtableEmulation()
+    # rgc.register_custom_light_finalizer(_HASHTABLE_OBJ, lambda_hashtable_finlz)
+    # rgc.register_custom_trace_hook(_HASHTABLE_OBJ, lambda_hashtable_trace)
+    # # Pass a null pointer to _STM_HASHTABLE_ENTRY to stm_hashtable_create().
+    # # Make sure we see a malloc() of it, so that its typeid is correctly
+    # # initialized.  It can be done in a NonConstant(False) path so that
+    # # the C compiler will actually drop it.
+    # if _false:
+    #     p = lltype.malloc(_STM_HASHTABLE_ENTRY)
+    # else:
+    #     p = lltype.nullptr(_STM_HASHTABLE_ENTRY)
+    # h = lltype.malloc(_HASHTABLE_OBJ)
+    # h.ll_raw_hashtable = lltype.nullptr(_STM_HASHTABLE_P.TO)
+    # h.ll_raw_hashtable = llop.stm_hashtable_create(_STM_HASHTABLE_P, p)
+    # return h
 
 NULL_GCREF = lltype.nullptr(llmemory.GCREF.TO)
 
+class HashtableEmulation(object):
+    def __init__(self):
+        self._content = {}      # dict {integer: GCREF}
+
+    def get(self, key):
+        return self._content.get(key, NULL_GCREF)
+
+    def set(self, key, value):
+        if value:
+            self._content[key] = value
+        else:
+            try:
+                del self._content[key]
+            except KeyError:
+                pass
+
+    def len(self):
+        return len(self._content)
+
+
+
 class HashtableForTest(object):
     def __init__(self):
         self._content = {}      # dict {integer: GCREF}

From noreply at buildbot.pypy.org  Wed Feb 25 11:53:28 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 25 Feb 2015 11:53:28 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c8: fake some things to make jit
	translation work
Message-ID: <20150225105328.85DA51C13B0@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: stmgc-c8
Changeset: r76122:d28565a39d2f
Date: 2015-02-25 11:52 +0100
http://bitbucket.org/pypy/pypy/changeset/d28565a39d2f/

Log:	fake some things to make jit translation work

diff --git a/rpython/translator/stm/src_stm/stmgcintf.c b/rpython/translator/stm/src_stm/stmgcintf.c
--- a/rpython/translator/stm/src_stm/stmgcintf.c
+++ b/rpython/translator/stm/src_stm/stmgcintf.c
@@ -10,6 +10,28 @@
 __thread uintptr_t pypy_stm_nursery_low_fill_mark_saved;
 
 
+/* C8: not implemented properly yet: */
+char _stm_write_slowpath_card_extra(object_t *obj)
+{
+    /* the PyPy JIT calls this function directly if it finds that an
+       array doesn't have the GCFLAG_CARDS_SET */
+    stm_write(obj);
+    return 0;
+}
+
+long _stm_write_slowpath_card_extra_base(void)
+{
+    /* for the PyPy JIT: _stm_write_slowpath_card_extra_base[obj >> 4]
+       is the byte that must be set to CARD_MARKED.  The logic below
+       does the same, but more explicitly. */
+    /* result should only appear in JIT-paths that are never executed
+       because the above func returns false */
+    return 42;
+}
+/* C8: not implemented properly yet ^^^^^^^^^^^^^^^^^^ */
+
+
+
 extern Signed pypy_stmcb_size_rounded_up(void*);
 extern void pypy_stmcb_get_card_base_itemsize(void*, uintptr_t[]);
 extern void pypy_stmcb_trace(void*, void(*)(void*));
diff --git a/rpython/translator/stm/src_stm/stmgcintf.h b/rpython/translator/stm/src_stm/stmgcintf.h
--- a/rpython/translator/stm/src_stm/stmgcintf.h
+++ b/rpython/translator/stm/src_stm/stmgcintf.h
@@ -39,6 +39,11 @@
 long _pypy_stm_count(void);
 
 /* C8: not implemented properly yet: */
+extern void stmcb_commit_soon(void);
+#define _STM_CARD_SIZE                 32     /* must be >= 32 */
+#define _STM_CARD_MARKED 100
+char _stm_write_slowpath_card_extra(object_t *obj);
+long _stm_write_slowpath_card_extra_base(void);
 typedef struct {
     stm_thread_local_t *tl;
     char *segment_base;    /* base to interpret the 'object' below */

From noreply at buildbot.pypy.org  Wed Feb 25 12:01:31 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 25 Feb 2015 12:01:31 +0100 (CET)
Subject: [pypy-commit] pypy default: fix test
Message-ID: <20150225110131.55A631C13B0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76123:648124294bf5
Date: 2015-02-25 12:01 +0100
http://bitbucket.org/pypy/pypy/changeset/648124294bf5/

Log:	fix test

diff --git a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -178,7 +178,11 @@
         assert p[3] == 'y'
         assert p[4] == 'Z'
         assert p[5] == 'z'
-        assert allblocks == [(rawstart, rawstart + 6)]
+        # 'allblocks' should be one block of length 6 + 15
+        # (15 = alignment - 1) containing the range(rawstart, rawstart + 6)
+        [(blockstart, blockend)] = allblocks
+        assert blockend == blockstart + 6 + (mc.ALIGN_MATERIALIZE - 1)
+        assert blockstart <= rawstart < rawstart + 6 <= blockend
         assert puts == [(rawstart + 2, ['a', 'b', 'c', 'd']),
                         (rawstart + 4, ['e', 'f', 'g'])]
 

From noreply at buildbot.pypy.org  Wed Feb 25 12:08:36 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 25 Feb 2015 12:08:36 +0100 (CET)
Subject: [pypy-commit] pypy default: wat
Message-ID: <20150225110836.648CD1C13AE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76124:c195a8bc4ff5
Date: 2015-02-25 12:08 +0100
http://bitbucket.org/pypy/pypy/changeset/c195a8bc4ff5/

Log:	wat

diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py
--- a/pypy/tool/pytest/apptest.py
+++ b/pypy/tool/pytest/apptest.py
@@ -35,8 +35,8 @@
         try:
             target(*args)
         except OperationError, e:
-            if self.config.option.verbose:
-                raise
+            #if self.config.option.verbose:
+            #    raise
             tb = sys.exc_info()[2]
             if e.match(space, space.w_KeyboardInterrupt):
                 raise KeyboardInterrupt, KeyboardInterrupt(), tb

From noreply at buildbot.pypy.org  Wed Feb 25 12:16:39 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed, 25 Feb 2015 12:16:39 +0100 (CET)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <20150225111639.F089E1C13C9@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3k
Changeset: r76125:1ea84a73908b
Date: 2015-02-25 12:16 +0100
http://bitbucket.org/pypy/pypy/changeset/1ea84a73908b/

Log:	hg merge default

diff too long, truncating to 2000 out of 3584 lines

diff --git a/lib_pypy/_gdbm.py b/lib_pypy/_gdbm.py
--- a/lib_pypy/_gdbm.py
+++ b/lib_pypy/_gdbm.py
@@ -20,9 +20,11 @@
 } datum;
 
 datum gdbm_fetch(void*, datum);
+datum pygdbm_fetch(void*, char*, int);
 int gdbm_delete(void*, datum);
 int gdbm_store(void*, datum, datum, int);
 int gdbm_exists(void*, datum);
+int pygdbm_exists(void*, char*, int);
 
 int gdbm_reorganize(void*);
 
@@ -37,19 +39,29 @@
 ''')
 
 try:
+    verify_code = '''
+    #include "gdbm.h"
+
+    static datum pygdbm_fetch(GDBM_FILE gdbm_file, char *dptr, int dsize) {
+        datum key = {dptr, dsize};
+        return gdbm_fetch(gdbm_file, key);
+    }
+
+    static int pygdbm_exists(GDBM_FILE gdbm_file, char *dptr, int dsize) {
+        datum key = {dptr, dsize};
+        return gdbm_exists(gdbm_file, key);
+    }
+    
+    '''
     if sys.platform.startswith('freebsd'):
         import os.path
         _localbase = os.environ.get('LOCALBASE', '/usr/local')
-        lib = ffi.verify('''
-        #include "gdbm.h"
-        ''', libraries=['gdbm'],
+        lib = ffi.verify(verify_code, libraries=['gdbm'],
              include_dirs=[os.path.join(_localbase, 'include')],
              library_dirs=[os.path.join(_localbase, 'lib')]
         )
     else:
-        lib = ffi.verify('''
-        #include "gdbm.h"
-        ''', libraries=['gdbm'])
+        lib = ffi.verify(verify_code, libraries=['gdbm'])
 except cffi.VerificationError as e:
     # distutils does not preserve the actual message,
     # but the verification is simple enough that the
@@ -59,6 +71,13 @@
 class error(IOError):
     pass
 
+def _checkstr(key):
+    if isinstance(key, unicode):
+        key = key.encode("ascii")
+    if not isinstance(key, str):
+        raise TypeError("gdbm mappings have string indices only")
+    return key
+
 def _fromstr(key):
     if isinstance(key, str):
         key = key.encode(sys.getdefaultencoding())
@@ -108,12 +127,14 @@
 
     def __contains__(self, key):
         self._check_closed()
-        return lib.gdbm_exists(self.ll_dbm, _fromstr(key))
+        key = _checkstr(key)
+        return lib.pygdbm_exists(self.ll_dbm, key, len(key))
     has_key = __contains__
 
     def get(self, key, default=None):
         self._check_closed()
-        drec = lib.gdbm_fetch(self.ll_dbm, _fromstr(key))
+        key = _checkstr(key)        
+        drec = lib.pygdbm_fetch(self.ll_dbm, key, len(key))
         if not drec.dptr:
             return default
         res = bytes(ffi.buffer(drec.dptr, drec.dsize))
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -36,7 +36,8 @@
    "PyPy home directory".  The arguments are:
 
    * ``home``: NULL terminated path to an executable inside the pypy directory
-     (can be a .so name, can be made up)
+     (can be a .so name, can be made up).  Used to look up the standard
+     library, and is also set as ``sys.executable``.
 
    * ``verbose``: if non-zero, it will print error messages to stderr
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -6,8 +6,8 @@
 .. startrev: 397b96217b85
 
 
-Fix non-blocking file reads sometimes raising EAGAIN even though they
-have buffered data waiting (b1c4fcb04a42)
+Non-blocking file reads sometimes raised EAGAIN even though they
+had buffered data waiting, fixed in b1c4fcb04a42
 
 
 .. branch: vmprof
@@ -18,3 +18,19 @@
 
 .. branch: stdlib-2.7.9
 Update stdlib to version 2.7.9
+
+.. branch: fix-kqueue-error2
+Fix exception being raised by kqueue.control (CPython compatibility)
+
+.. branch: gitignore
+
+.. branch: framestate2
+Refactor rpython.flowspace.framestate.FrameState.
+
+.. branch: alt_errno
+Add an alternative location to save LastError, errno around ctypes,
+cffi external calls so things like pdb will not overwrite it
+
+.. branch: nonquadratic-heapcache
+Speed up the warmup times of the JIT by removing a quadratic algorithm in the
+heapcache.
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -113,6 +113,9 @@
         space.call_function(w_pathsetter, w_path)
         # import site
         try:
+            space.setattr(space.getbuiltinmodule('sys'),
+                          space.wrap('executable'),
+                          space.wrap(home))
             import_ = space.getattr(space.getbuiltinmodule('builtins'),
                                     space.wrap('__import__'))
             space.call_function(import_, space.wrap('site'))
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -1,5 +1,5 @@
 #! /usr/bin/env python
-# App-level version of py.py.
+# This is pure Python code that handles the main entry point into "pypy".
 # See test/test_app_main.
 
 # Missing vs CPython: -b, -d, -v, -x, -3
@@ -158,11 +158,14 @@
             current = group
     raise SystemExit
 
+def get_sys_executable():
+    return getattr(sys, 'executable', 'pypy')
+
 def print_help(*args):
     import os
     initstdio()
     print('usage: %s [option] ... [-c cmd | -m mod | file | -] [arg] ...' % (
-        sys.executable,))
+        get_sys_executable(),))
     print(USAGE1, end='')
     if 'pypyjit' in sys.builtin_module_names:
         print("--jit options: advanced JIT options: try 'off' or 'help'")
@@ -174,7 +177,7 @@
     try:
         import pypyjit
     except ImportError:
-        print("No jit support in %s" % (sys.executable,), file=sys.stderr)
+        print("No jit support in %s" % (get_sys_executable(),), file=sys.stderr)
         return
     items = sorted(pypyjit.defaults.items())
     print('Advanced JIT options: a comma-separated list of OPTION=VALUE:')
@@ -213,7 +216,7 @@
         raise SystemExit
     if 'pypyjit' not in sys.builtin_module_names:
         initstdio()
-        print("Warning: No jit support in %s" % (sys.executable,),
+        print("Warning: No jit support in %s" % (get_sys_executable(),),
               file=sys.stderr)
     else:
         import pypyjit
@@ -224,8 +227,8 @@
 
 def print_error(msg):
     print(msg, file=sys.stderr)
-    print('usage: %s [options]' % (sys.executable,), file=sys.stderr)
-    print('Try `%s -h` for more information.' % (sys.executable,), file=sys.stderr)
+    print('usage: %s [options]' % (get_sys_executable(),), file=sys.stderr)
+    print('Try `%s -h` for more information.' % (get_sys_executable(),), file=sys.stderr)
 
 def fdopen(fd, mode, bufsize=-1):
     try:
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -373,7 +373,7 @@
         config = make_config(None)
         space = make_objspace(config)
         w_executable = space.wrap('executable')
-        assert space.str_w(space.getattr(space.sys, w_executable)) == 'py.py'
+        assert space.findattr(space.sys, w_executable) is None
         space.setattr(space.sys, w_executable, space.wrap('foobar'))
         assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
         space.startup()
diff --git a/pypy/interpreter/test/test_targetpypy.py b/pypy/interpreter/test/test_targetpypy.py
--- a/pypy/interpreter/test/test_targetpypy.py
+++ b/pypy/interpreter/test/test_targetpypy.py
@@ -8,7 +8,7 @@
         entry_point = get_entry_point(config)[0]
         entry_point(['pypy-c' , '-S', '-c', 'print 3'])
 
-def test_exeucte_source(space):
+def test_execute_source(space):
     _, d = create_entry_point(space, None)
     execute_source = d['pypy_execute_source']
     lls = rffi.str2charp("import sys; sys.modules['xyz'] = 3")
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -210,6 +210,6 @@
         space.threadlocals.leave_thread(space)
 
 def invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata):
-    cerrno._errno_after(rffi.RFFI_ERR_ALL)
+    cerrno._errno_after(rffi.RFFI_ERR_ALL | rffi.RFFI_ALT_ERRNO)
     _invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata)
-    cerrno._errno_before(rffi.RFFI_ERR_ALL)
+    cerrno._errno_before(rffi.RFFI_ERR_ALL | rffi.RFFI_ALT_ERRNO)
diff --git a/pypy/module/_cffi_backend/cerrno.py b/pypy/module/_cffi_backend/cerrno.py
--- a/pypy/module/_cffi_backend/cerrno.py
+++ b/pypy/module/_cffi_backend/cerrno.py
@@ -13,18 +13,18 @@
 _errno_after  = rposix._errno_after
 
 def get_errno(space):
-    return space.wrap(rposix.get_saved_errno())
+    return space.wrap(rposix.get_saved_alterrno())
 
 @unwrap_spec(errno=int)
 def set_errno(space, errno):
-    rposix.set_saved_errno(errno)
+    rposix.set_saved_alterrno(errno)
 
 # ____________________________________________________________
 
 @unwrap_spec(code=int)
 def getwinerror(space, code=-1):
-    from rpython.rlib.rwin32 import GetLastError_saved, FormatError
+    from rpython.rlib.rwin32 import GetLastError_alt_saved, FormatError
     if code == -1:
-        code = GetLastError_saved()
+        code = GetLastError_alt_saved()
     message = FormatError(code)
     return space.newtuple([space.wrap(code), space.wrap(message)])
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2716,6 +2716,14 @@
     assert data == b"Xhello\n"
     posix.close(fdr)
 
+def test_errno_saved():
+    set_errno(42)
+    # a random function that will reset errno to 0 (at least on non-windows)
+    import os; os.stat('.')
+    #
+    res = get_errno()
+    assert res == 42
+
 def test_GetLastError():
     if sys.platform != "win32":
         py.test.skip("GetLastError(): only for Windows")
diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py
--- a/pypy/module/_rawffi/array.py
+++ b/pypy/module/_rawffi/array.py
@@ -15,7 +15,7 @@
 from pypy.module._rawffi.interp_rawffi import unpack_shape_with_length
 from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr
 from rpython.rlib.rarithmetic import r_uint
-from rpython.rlib import rgc
+from rpython.rlib import rgc, clibffi
 
 
 class W_Array(W_DataShape):
@@ -84,14 +84,11 @@
 
 class W_ArrayInstance(W_DataInstance):
     def __init__(self, space, shape, length, address=r_uint(0)):
-        # Workaround for a strange behavior of libffi: make sure that
-        # we always have at least 8 bytes.  For W_ArrayInstances that are
-        # used as the result value of a function call, ffi_call() writes
-        # 8 bytes into it even if the function's result type asks for less.
-        # This strange behavior is documented.
         memsize = shape.size * length
-        if memsize < 8:
-            memsize = 8
+        # For W_ArrayInstances that are used as the result value of a
+        # function call, ffi_call() writes 8 bytes into it even if the
+        # function's result type asks for less.
+        memsize = clibffi.adjust_return_size(memsize)
         W_DataInstance.__init__(self, space, memsize, address)
         self.length = length
         self.shape = shape
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -498,6 +498,7 @@
         try:
             if self.resshape is not None:
                 result = self.resshape.allocate(space, 1, autofree=True)
+                # adjust_return_size() was used here on result.ll_buffer
                 self.ptr.call(args_ll, result.ll_buffer)
                 return space.wrap(result)
             else:
@@ -611,19 +612,19 @@
     return space.wrap(W_CDLL(space, name, cdll))
 
 def get_errno(space):
-    return space.wrap(rposix.get_saved_errno())
+    return space.wrap(rposix.get_saved_alterrno())
 
 def set_errno(space, w_errno):
-    rposix.set_saved_errno(space.int_w(w_errno))
+    rposix.set_saved_alterrno(space.int_w(w_errno))
 
 if sys.platform == 'win32':
     # see also
     # https://bitbucket.org/pypy/pypy/issue/1944/ctypes-on-windows-getlasterror
     def get_last_error(space):
-        return space.wrap(rwin32.GetLastError_saved())
+        return space.wrap(rwin32.GetLastError_alt_saved())
     @unwrap_spec(error=int)
     def set_last_error(space, error):
-        rwin32.SetLastError_saved(error)
+        rwin32.SetLastError_alt_saved(error)
 else:
     # always have at least a dummy version of these functions
     # (https://bugs.pypy.org/issue1242)
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -4,6 +4,8 @@
 from rpython.rlib.rarithmetic import intmask, widen, r_uint
 from rpython.rlib import rpoll, rsocket, rthread, rweakref
 from rpython.rlib.ropenssl import *
+from pypy.module._socket import interp_socket
+from rpython.rlib._rsocket_rffi import MAX_FD_SIZE
 from rpython.rlib.rposix import get_saved_errno
 from rpython.rlib.rweakref import RWeakValueDictionary
 from rpython.rlib.objectmodel import specialize, compute_unique_id
@@ -16,7 +18,6 @@
 from pypy.interpreter.unicodehelper import fsdecode
 from pypy.module._ssl.ssl_data import (
     LIBRARY_CODES_TO_NAMES, ERROR_CODES_TO_NAMES)
-from pypy.module._socket import interp_socket
 
 
 # user defined constants
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -236,6 +236,9 @@
 
     def test_npn_protocol(self):
         import socket, _ssl, gc
+        if not _ssl.HAS_NPN:
+            skip("NPN requires OpenSSL 1.0.1 or greater")
+
         ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
         ctx._set_npn_protocols(b'\x08http/1.1\x06spdy/2')
         ss = ctx._wrap_socket(self.s, True,
@@ -304,12 +307,13 @@
             os.path.dirname(__file__), 'dh512.pem'))
 
     def test_load_cert_chain(self):
-        import _ssl
+        import _ssl, errno
         ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
         ctx.load_cert_chain(self.keycert)
         ctx.load_cert_chain(self.cert, self.key)
-        raises(IOError, ctx.load_cert_chain, "inexistent.pem")
-        raises(_ssl.SSLError, ctx.load_cert_chain, self.badcert)
+        exc = raises(IOError, ctx.load_cert_chain, "inexistent.pem")
+        assert exc.value.errno == errno.ENOENT
+        exc = raises(_ssl.SSLError, ctx.load_cert_chain, self.badcert)
         raises(_ssl.SSLError, ctx.load_cert_chain, self.emptycert)
         # Password protected key and cert
         raises(_ssl.SSLError, ctx.load_cert_chain, self.cert_protected,
@@ -367,12 +371,14 @@
         assert ctx.cert_store_stats() == {'x509_ca': 0, 'crl': 0, 'x509': 1}
 
     def test_load_dh_params(self):
-        import _ssl
+        import _ssl, errno
         ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
         ctx.load_dh_params(self.dh512)
         raises(TypeError, ctx.load_dh_params)
         raises(TypeError, ctx.load_dh_params, None)
         raises(_ssl.SSLError, ctx.load_dh_params, self.keycert)
+        exc = raises(IOError, ctx.load_dh_params, "inexistent.pem")
+        assert exc.value.errno == errno.ENOENT
 
     def test_set_ecdh_curve(self):
         import _ssl
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,7 +29,7 @@
 #define PY_VERSION		"3.2.5"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "2.6.0"
+#define PYPY_VERSION "2.6.0-alpha0"
 
 /* Subversion Revision number of this file (not of the repository).
  * Empty since Mercurial migration. */
diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -202,7 +202,7 @@
         assert loop.match_by_id('cfficall', """
             p96 = force_token()
             setfield_gc(p0, p96, descr=)
-            f97 = call_release_gil(27, i59, 1.0, 3, descr=)
+            f97 = call_release_gil(91, i59, 1.0, 3, descr=)
             guard_not_forced(descr=...)
             guard_no_exception(descr=...)
         """, ignore_ops=['guard_not_invalidated'])
diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py
--- a/pypy/module/pypyjit/test_pypy_c/test_thread.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py
@@ -67,21 +67,10 @@
         i58 = call_release_gil(0, _, i37, 1, descr=)
         guard_not_forced(descr=...)
         guard_no_exception(descr=...)
-        i59 = int_is_true(i58)
-        guard_true(i59, descr=...)
-        i60 = int_sub(i44, 1)
-        p62 = force_token()
-        setfield_gc(p0, p62, descr=)
-        i63 = call_release_gil(0, _, i37, 0, descr=)
-        guard_not_forced(descr=...)
-        guard_no_exception(descr=...)
-        i64 = int_is_true(i63)
-        guard_false(i64, descr=...)
-        p65 = force_token()
-        setfield_gc(p0, p65, descr=)
-        call_release_gil(0, _, i37, descr=)
-        guard_not_forced(descr=...)
-        guard_no_exception(descr=...)
+        i58 = int_sub(i44, 1)
+        i59 = call(ConstClass(RPyThreadReleaseLock), i37, descr=)
+        i60 = int_is_true(i59)
+        guard_false(i60, descr=...)
         guard_not_invalidated(descr=...)
         --TICK--
         jump(..., descr=...)
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -57,7 +57,6 @@
         'getsizeof'             : 'vm.getsizeof',
         'intern'                : 'vm.intern',
 
-        'executable'            : 'space.wrap("py.py")',
         'api_version'           : 'version.get_api_version(space)',
         'version_info'          : 'version.get_version_info(space)',
         'version'               : 'version.get_version(space)',
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -497,7 +497,7 @@
         assert isinstance(sys.builtin_module_names, tuple)
         assert isinstance(sys.copyright, str)
         #assert isinstance(sys.exec_prefix, str) -- not present!
-        assert isinstance(sys.executable, str)
+        #assert isinstance(sys.executable, str)
         assert isinstance(sys.hexversion, int)
         assert isinstance(sys.maxsize, int)
         assert isinstance(sys.maxunicode, int)
@@ -515,6 +515,12 @@
         assert vi[3] in ("alpha", "beta", "candidate", "final")
         assert isinstance(vi[4], int)
 
+    def test_reload_doesnt_override_sys_executable(self):
+        import sys
+        sys.executable = 'from_test_sysmodule'
+        reload(sys)
+        assert sys.executable == 'from_test_sysmodule'
+
     def test_settrace(self):
         import sys
         counts = []
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c b/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
--- a/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
+++ b/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
@@ -272,7 +272,11 @@
 {
 	double x, sum=0.0, dx=(b-a)/(double)nstep;
 	for(x=a+0.5*dx; (b-x)*(x-a)>0.0; x+=dx)
+    {   
+        double y = f(x);
+        printf("f(x)=%.1f\n", y);
 		sum += f(x);
+    }
 	return sum/(double)nstep;
 }
 
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py b/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py
@@ -138,6 +138,7 @@
         integrate.restype = c_double
 
         def func(x):
+            print 'calculating x**2 of',x
             return x**2
 
         result = integrate(0.0, 1.0, CALLBACK(func), 10)
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -168,7 +168,6 @@
 if cConfig.has_gettimeofday:
     c_gettimeofday = external('gettimeofday', [rffi.VOIDP, rffi.VOIDP], rffi.INT)
 TM_P = lltype.Ptr(tm)
-c_clock = external('clock', [rffi.TIME_TP], clock_t)
 c_time = external('time', [rffi.TIME_TP], rffi.TIME_T)
 c_gmtime = external('gmtime', [rffi.TIME_TP], TM_P,
                     save_err=rffi.RFFI_SAVE_ERRNO)
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -199,8 +199,7 @@
         magic = importing._get_long(buf[:4])
         timestamp = importing._get_long(buf[4:8])
         if not self.can_use_pyc(space, filename, magic, timestamp):
-            return self.import_py_file(space, modname, filename[:-1], buf,
-                                       pkgpath)
+            return None
         buf = buf[8:] # XXX ugly copy, should use sequential read instead
         w_mod = w(Module(space, w(modname)))
         real_name = self.filename + os.path.sep + self.corr_zname(filename)
@@ -249,7 +248,6 @@
     def load_module(self, space, fullname):
         w = space.wrap
         filename = self.make_filename(fullname)
-        last_exc = None
         for compiled, is_package, ext in ENUMERATE_EXTS:
             fname = filename + ext
             try:
@@ -268,19 +266,18 @@
                     pkgpath = None
                 try:
                     if compiled:
-                        return self.import_pyc_file(space, fullname, fname,
-                                                    buf, pkgpath)
+                        w_result = self.import_pyc_file(space, fullname, fname,
+                                                        buf, pkgpath)
+                        if w_result is not None:
+                            return w_result
                     else:
                         return self.import_py_file(space, fullname, fname,
                                                    buf, pkgpath)
-                except OperationError, e:
-                    last_exc = e
+                except:
                     w_mods = space.sys.get('modules')
-                space.call_method(w_mods, 'pop', w(fullname), space.w_None)
-        if last_exc:
-            raise OperationError(get_error(space), last_exc.get_w_value(space))
-        # should never happen I think
-        return space.w_None
+                    space.call_method(w_mods, 'pop', w(fullname), space.w_None)
+                    raise
+        raise oefmt(get_error(space), "can't find module '%s'", fullname)
 
     @unwrap_spec(filename='str0')
     def get_data(self, space, filename):
diff --git a/pypy/module/zipimport/test/test_zipimport.py b/pypy/module/zipimport/test/test_zipimport.py
--- a/pypy/module/zipimport/test/test_zipimport.py
+++ b/pypy/module/zipimport/test/test_zipimport.py
@@ -195,7 +195,8 @@
         m0 ^= 0x04
         test_pyc = bytes([m0]) + self.get_pyc()[1:]
         self.writefile("uu.pyc", test_pyc)
-        raises(ImportError, "__import__('uu', globals(), locals(), [])")
+        raises(zipimport.ZipImportError,
+               "__import__('uu', globals(), locals(), [])")
         assert 'uu' not in sys.modules
 
     def test_force_py(self):
@@ -381,6 +382,11 @@
         finally:
             os.remove(filename)
 
+    def test_import_exception(self):
+        self.writefile('x1test.py', '1/0')
+        self.writefile('x1test/__init__.py', 'raise ValueError')
+        raises(ValueError, __import__, 'x1test', None, None, [])
+
 
 if os.sep != '/':
     class AppTestNativePathSep(AppTestZipimport):
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -567,9 +567,11 @@
 
     def descr_bit_length(self, space):
         val = self.intval
+        bits = 0
         if val < 0:
-            val = -val
-        bits = 0
+            # warning, "-val" overflows here
+            val = -((val + 1) >> 1)
+            bits = 1
         while val:
             bits += 1
             val >>= 1
diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py
--- a/pypy/objspace/std/test/test_intobject.py
+++ b/pypy/objspace/std/test/test_intobject.py
@@ -476,11 +476,20 @@
             (10, 4),
             (150, 8),
             (-1, 1),
+            (-2, 2),
+            (-3, 2),
+            (-4, 3),
             (-10, 4),
             (-150, 8),
         ]:
             assert val.bit_length() == bits
 
+    def test_bit_length_max(self):
+        import sys
+        val = -sys.maxint-1
+        bits = 32 if val == -2147483648 else 64
+        assert val.bit_length() == bits
+
     def test_int_real(self):
         class A(int): pass
         b = A(5).real
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -12,8 +12,7 @@
 from rpython.flowspace.argument import CallSpec
 from rpython.flowspace.model import (Constant, Variable, Block, Link,
     c_last_exception, const, FSException)
-from rpython.flowspace.framestate import (FrameState, recursively_unflatten,
-    recursively_flatten)
+from rpython.flowspace.framestate import FrameState
 from rpython.flowspace.specialcase import (rpython_print_item,
     rpython_print_newline)
 from rpython.flowspace.operation import op
@@ -278,6 +277,7 @@
     "cmp_exc_match",
     ]
 
+
 class FlowContext(object):
     def __init__(self, graph, code):
         self.graph = graph
@@ -307,112 +307,91 @@
 
         The locals are ordered according to self.pycode.signature.
         """
-        self.valuestackdepth = code.co_nlocals
-        self.locals_stack_w = [None] * (code.co_stacksize + code.co_nlocals)
+        self.nlocals = code.co_nlocals
+        self.locals_w = [None] * code.co_nlocals
+        self.stack = []
+
+    @property
+    def stackdepth(self):
+        return len(self.stack)
 
     def pushvalue(self, w_object):
-        depth = self.valuestackdepth
-        self.locals_stack_w[depth] = w_object
-        self.valuestackdepth = depth + 1
+        self.stack.append(w_object)
 
     def popvalue(self):
-        depth = self.valuestackdepth - 1
-        assert depth >= self.pycode.co_nlocals, "pop from empty value stack"
-        w_object = self.locals_stack_w[depth]
-        self.locals_stack_w[depth] = None
-        self.valuestackdepth = depth
-        return w_object
+        return self.stack.pop()
 
     def peekvalue(self, index_from_top=0):
         # NOTE: top of the stack is peekvalue(0).
-        index = self.valuestackdepth + ~index_from_top
-        assert index >= self.pycode.co_nlocals, (
-            "peek past the bottom of the stack")
-        return self.locals_stack_w[index]
+        index = ~index_from_top
+        return self.stack[index]
 
     def settopvalue(self, w_object, index_from_top=0):
-        index = self.valuestackdepth + ~index_from_top
-        assert index >= self.pycode.co_nlocals, (
-            "settop past the bottom of the stack")
-        self.locals_stack_w[index] = w_object
+        index = ~index_from_top
+        self.stack[index] = w_object
 
     def popvalues(self, n):
-        values_w = [self.popvalue() for i in range(n)]
-        values_w.reverse()
+        if n == 0:
+            return []
+        values_w = self.stack[-n:]
+        del self.stack[-n:]
         return values_w
 
-    def dropvalues(self, n):
-        finaldepth = self.valuestackdepth - n
-        for n in range(finaldepth, self.valuestackdepth):
-            self.locals_stack_w[n] = None
-        self.valuestackdepth = finaldepth
-
     def dropvaluesuntil(self, finaldepth):
-        for n in range(finaldepth, self.valuestackdepth):
-            self.locals_stack_w[n] = None
-        self.valuestackdepth = finaldepth
-
-    def save_locals_stack(self):
-        return self.locals_stack_w[:self.valuestackdepth]
-
-    def restore_locals_stack(self, items_w):
-        self.locals_stack_w[:len(items_w)] = items_w
-        self.dropvaluesuntil(len(items_w))
+        del self.stack[finaldepth:]
 
     def getstate(self, next_offset):
-        # getfastscope() can return real None, for undefined locals
-        data = self.save_locals_stack()
-        if self.last_exception is None:
-            data.append(Constant(None))
-            data.append(Constant(None))
-        else:
-            data.append(self.last_exception.w_type)
-            data.append(self.last_exception.w_value)
-        recursively_flatten(data)
-        return FrameState(data, self.blockstack[:], next_offset)
+        return FrameState(self.locals_w[:], self.stack[:],
+                self.last_exception, self.blockstack[:], next_offset)
 
     def setstate(self, state):
         """ Reset the context to the given frame state. """
-        data = state.mergeable[:]
-        recursively_unflatten(data)
-        self.restore_locals_stack(data[:-2])  # Nones == undefined locals
-        if data[-2] == Constant(None):
-            assert data[-1] == Constant(None)
-            self.last_exception = None
-        else:
-            self.last_exception = FSException(data[-2], data[-1])
+        self.locals_w = state.locals_w[:]
+        self.stack = state.stack[:]
+        self.last_exception = state.last_exception
         self.blockstack = state.blocklist[:]
+        self._normalize_raise_signals()
+
+    def _normalize_raise_signals(self):
+        st = self.stack
+        for i in range(len(st)):
+            if isinstance(st[i], RaiseImplicit):
+                st[i] = Raise(st[i].w_exc)
 
     def guessbool(self, w_condition):
         if isinstance(w_condition, Constant):
             return w_condition.value
         return self.recorder.guessbool(self, w_condition)
 
-    def record(self, spaceop):
+    def maybe_merge(self):
         recorder = self.recorder
         if getattr(recorder, 'final_state', None) is not None:
             self.mergeblock(recorder.crnt_block, recorder.final_state)
             raise StopFlowing
+
+    def record(self, spaceop):
         spaceop.offset = self.last_offset
-        recorder.append(spaceop)
+        self.recorder.append(spaceop)
 
     def do_op(self, op):
+        self.maybe_merge()
         self.record(op)
         self.guessexception(op.canraise)
         return op.result
 
-    def guessexception(self, exceptions, force=False):
+    def guessexception(self, exceptions):
         """
         Catch possible exceptions implicitly.
         """
         if not exceptions:
             return
-        if not force and not any(isinstance(block, (ExceptBlock, FinallyBlock))
-                                 for block in self.blockstack):
-            # The implicit exception wouldn't be caught and would later get
-            # removed, so don't bother creating it.
-            return
-        self.recorder.guessexception(self, *exceptions)
+        # Implicit exceptions are ignored unless they are caught explicitly
+        if self.has_exc_handler():
+            self.recorder.guessexception(self, *exceptions)
+
+    def has_exc_handler(self):
+        return any(isinstance(block, (ExceptBlock, FinallyBlock))
+                for block in self.blockstack)
 
     def build_flow(self):
         graph = self.graph
@@ -430,35 +409,8 @@
             while True:
                 next_offset = self.handle_bytecode(next_offset)
                 self.recorder.final_state = self.getstate(next_offset)
-
-        except RaiseImplicit as e:
-            w_exc = e.w_exc
-            if isinstance(w_exc.w_type, Constant):
-                exc_cls = w_exc.w_type.value
-            else:
-                exc_cls = Exception
-            msg = "implicit %s shouldn't occur" % exc_cls.__name__
-            w_type = Constant(AssertionError)
-            w_value = Constant(AssertionError(msg))
-            link = Link([w_type, w_value], self.graph.exceptblock)
-            self.recorder.crnt_block.closeblock(link)
-
-        except Raise as e:
-            w_exc = e.w_exc
-            if w_exc.w_type == const(ImportError):
-                msg = 'import statement always raises %s' % e
-                raise ImportError(msg)
-            link = Link([w_exc.w_type, w_exc.w_value], self.graph.exceptblock)
-            self.recorder.crnt_block.closeblock(link)
-
         except StopFlowing:
             pass
-
-        except Return as exc:
-            w_result = exc.w_value
-            link = Link([w_result], self.graph.returnblock)
-            self.recorder.crnt_block.closeblock(link)
-
         except FlowingError as exc:
             if exc.ctx is None:
                 exc.ctx = self
@@ -476,14 +428,8 @@
             if newstate is not None:
                 break
         else:
-            newstate = currentstate.copy()
-            newblock = SpamBlock(newstate)
-            # unconditionally link the current block to the newblock
-            outputargs = currentstate.getoutputargs(newstate)
-            link = Link(outputargs, newblock)
-            currentblock.closeblock(link)
+            newblock = self.make_next_block(currentblock, currentstate)
             candidates.insert(0, newblock)
-            self.pendingblocks.append(newblock)
             return
 
         if newstate.matches(block.framestate):
@@ -493,7 +439,7 @@
 
         newblock = SpamBlock(newstate)
         varnames = self.pycode.co_varnames
-        for name, w_value in zip(varnames, newstate.mergeable):
+        for name, w_value in zip(varnames, newstate.locals_w):
             if isinstance(w_value, Variable):
                 w_value.rename(name)
         # unconditionally link the current block to the newblock
@@ -513,11 +459,21 @@
         candidates.insert(0, newblock)
         self.pendingblocks.append(newblock)
 
+    def make_next_block(self, block, state):
+        newstate = state.copy()
+        newblock = SpamBlock(newstate)
+        # unconditionally link the current block to the newblock
+        outputargs = state.getoutputargs(newstate)
+        link = Link(outputargs, newblock)
+        block.closeblock(link)
+        self.pendingblocks.append(newblock)
+        return newblock
+
     # hack for unrolling iterables, don't use this
     def replace_in_stack(self, oldvalue, newvalue):
         w_new = Constant(newvalue)
-        stack_items_w = self.locals_stack_w
-        for i in range(self.valuestackdepth - 1, self.pycode.co_nlocals - 1, -1):
+        stack_items_w = self.stack
+        for i in range(self.stackdepth - 1, - 1, -1):
             w_v = stack_items_w[i]
             if isinstance(w_v, Constant):
                 if w_v.value is oldvalue:
@@ -541,7 +497,7 @@
             if isinstance(signal, block.handles):
                 return block.handle(self, signal)
             block.cleanupstack(self)
-        return signal.nomoreblocks()
+        return signal.nomoreblocks(self)
 
     def getlocalvarname(self, index):
         return self.pycode.co_varnames[index]
@@ -870,7 +826,7 @@
             op.simple_call(w_exitfunc, w_None, w_None, w_None).eval(self)
 
     def LOAD_FAST(self, varindex):
-        w_value = self.locals_stack_w[varindex]
+        w_value = self.locals_w[varindex]
         if w_value is None:
             raise FlowingError("Local variable referenced before assignment")
         self.pushvalue(w_value)
@@ -915,7 +871,7 @@
     def STORE_FAST(self, varindex):
         w_newvalue = self.popvalue()
         assert w_newvalue is not None
-        self.locals_stack_w[varindex] = w_newvalue
+        self.locals_w[varindex] = w_newvalue
         if isinstance(w_newvalue, Variable):
             w_newvalue.rename(self.getlocalvarname(varindex))
 
@@ -1128,11 +1084,11 @@
         op.simple_call(w_append_meth, w_value).eval(self)
 
     def DELETE_FAST(self, varindex):
-        if self.locals_stack_w[varindex] is None:
+        if self.locals_w[varindex] is None:
             varname = self.getlocalvarname(varindex)
             message = "local variable '%s' referenced before assignment"
             raise UnboundLocalError(message, varname)
-        self.locals_stack_w[varindex] = None
+        self.locals_w[varindex] = None
 
     def STORE_MAP(self, oparg):
         w_key = self.popvalue()
@@ -1220,25 +1176,32 @@
                 WHY_CONTINUE,   Continue
                 WHY_YIELD       not needed
     """
-    def nomoreblocks(self):
+    def nomoreblocks(self, ctx):
         raise BytecodeCorruption("misplaced bytecode - should not return")
 
+    def __eq__(self, other):
+        return type(other) is type(self) and other.args == self.args
+
 
 class Return(FlowSignal):
     """Signals a 'return' statement.
-    Argument is the wrapped object to return."""
-
+    Argument is the wrapped object to return.
+    """
     def __init__(self, w_value):
         self.w_value = w_value
 
-    def nomoreblocks(self):
-        raise Return(self.w_value)
+    def nomoreblocks(self, ctx):
+        w_result = self.w_value
+        link = Link([w_result], ctx.graph.returnblock)
+        ctx.recorder.crnt_block.closeblock(link)
+        raise StopFlowing
 
-    def state_unpack_variables(self):
+    @property
+    def args(self):
         return [self.w_value]
 
     @staticmethod
-    def state_pack_variables(w_value):
+    def rebuild(w_value):
         return Return(w_value)
 
 class Raise(FlowSignal):
@@ -1248,28 +1211,48 @@
     def __init__(self, w_exc):
         self.w_exc = w_exc
 
-    def nomoreblocks(self):
-        raise self
+    def nomoreblocks(self, ctx):
+        w_exc = self.w_exc
+        if w_exc.w_type == const(ImportError):
+            msg = 'import statement always raises %s' % self
+            raise ImportError(msg)
+        link = Link([w_exc.w_type, w_exc.w_value], ctx.graph.exceptblock)
+        ctx.recorder.crnt_block.closeblock(link)
+        raise StopFlowing
 
-    def state_unpack_variables(self):
+    @property
+    def args(self):
         return [self.w_exc.w_type, self.w_exc.w_value]
 
-    @staticmethod
-    def state_pack_variables(w_type, w_value):
-        return Raise(FSException(w_type, w_value))
+    @classmethod
+    def rebuild(cls, w_type, w_value):
+        return cls(FSException(w_type, w_value))
 
 class RaiseImplicit(Raise):
     """Signals an exception raised implicitly"""
+    def nomoreblocks(self, ctx):
+        w_exc = self.w_exc
+        if isinstance(w_exc.w_type, Constant):
+            exc_cls = w_exc.w_type.value
+        else:
+            exc_cls = Exception
+        msg = "implicit %s shouldn't occur" % exc_cls.__name__
+        w_type = Constant(AssertionError)
+        w_value = Constant(AssertionError(msg))
+        link = Link([w_type, w_value], ctx.graph.exceptblock)
+        ctx.recorder.crnt_block.closeblock(link)
+        raise StopFlowing
 
 
 class Break(FlowSignal):
     """Signals a 'break' statement."""
 
-    def state_unpack_variables(self):
+    @property
+    def args(self):
         return []
 
     @staticmethod
-    def state_pack_variables():
+    def rebuild():
         return Break.singleton
 
 Break.singleton = Break()
@@ -1281,11 +1264,12 @@
     def __init__(self, jump_to):
         self.jump_to = jump_to
 
-    def state_unpack_variables(self):
+    @property
+    def args(self):
         return [const(self.jump_to)]
 
     @staticmethod
-    def state_pack_variables(w_jump_to):
+    def rebuild(w_jump_to):
         return Continue(w_jump_to.value)
 
 
@@ -1295,21 +1279,21 @@
 
     def __init__(self, ctx, handlerposition):
         self.handlerposition = handlerposition
-        self.valuestackdepth = ctx.valuestackdepth
+        self.stackdepth = ctx.stackdepth
 
     def __eq__(self, other):
         return (self.__class__ is other.__class__ and
                 self.handlerposition == other.handlerposition and
-                self.valuestackdepth == other.valuestackdepth)
+                self.stackdepth == other.stackdepth)
 
     def __ne__(self, other):
         return not (self == other)
 
     def __hash__(self):
-        return hash((self.handlerposition, self.valuestackdepth))
+        return hash((self.handlerposition, self.stackdepth))
 
     def cleanupstack(self, ctx):
-        ctx.dropvaluesuntil(self.valuestackdepth)
+        ctx.dropvaluesuntil(self.stackdepth)
 
     def handle(self, ctx, unroller):
         raise NotImplementedError
diff --git a/rpython/flowspace/framestate.py b/rpython/flowspace/framestate.py
--- a/rpython/flowspace/framestate.py
+++ b/rpython/flowspace/framestate.py
@@ -1,21 +1,50 @@
-from rpython.flowspace.model import Variable, Constant
+from rpython.flowspace.model import Variable, Constant, FSException
 from rpython.rlib.unroll import SpecTag
 
+def _copy(v):
+    from rpython.flowspace.flowcontext import FlowSignal
+    if isinstance(v, Variable):
+        return Variable(v)
+    elif isinstance(v, FlowSignal):
+        vars = [_copy(var) for var in v.args]
+        return v.rebuild(*vars)
+    else:
+        return v
+
+def _union(seq1, seq2):
+    return [union(v1, v2) for v1, v2 in zip(seq1, seq2)]
+
 
 class FrameState(object):
-    def __init__(self, mergeable, blocklist, next_offset):
-        self.mergeable = mergeable
+    def __init__(self, locals_w, stack, last_exception, blocklist, next_offset):
+        self.locals_w = locals_w
+        self.stack = stack
+        self.last_exception = last_exception
         self.blocklist = blocklist
         self.next_offset = next_offset
+        self._mergeable = None
+
+    @property
+    def mergeable(self):
+        if self._mergeable is not None:
+            return self._mergeable
+        self._mergeable = data = self.locals_w + self.stack
+        if self.last_exception is None:
+            data.append(Constant(None))
+            data.append(Constant(None))
+        else:
+            data.append(self.last_exception.w_type)
+            data.append(self.last_exception.w_value)
+        recursively_flatten(data)
+        return data
 
     def copy(self):
         "Make a copy of this state in which all Variables are fresh."
-        newstate = []
-        for w in self.mergeable:
-            if isinstance(w, Variable):
-                w = Variable(w)
-            newstate.append(w)
-        return FrameState(newstate, self.blocklist, self.next_offset)
+        exc = self.last_exception
+        if exc is not None:
+            exc = FSException(_copy(exc.w_type), _copy(exc.w_value))
+        return FrameState(map(_copy, self.locals_w), map(_copy, self.stack),
+                exc, self.blocklist, self.next_offset)
 
     def getvariables(self):
         return [w for w in self.mergeable if isinstance(w, Variable)]
@@ -33,18 +62,31 @@
                 return False
         return True
 
+    def _exc_args(self):
+        if self.last_exception is None:
+            return [Constant(None), Constant(None)]
+        else:
+            return [self.last_exception.w_type,
+                    self.last_exception.w_value]
+
     def union(self, other):
         """Compute a state that is at least as general as both self and other.
            A state 'a' is more general than a state 'b' if all Variables in 'b'
            are also Variables in 'a', but 'a' may have more Variables.
         """
-        newstate = []
         try:
-            for w1, w2 in zip(self.mergeable, other.mergeable):
-                newstate.append(union(w1, w2))
+            locals = _union(self.locals_w, other.locals_w)
+            stack = _union(self.stack, other.stack)
+            if self.last_exception is None and other.last_exception is None:
+                exc = None
+            else:
+                args1 = self._exc_args()
+                args2 = other._exc_args()
+                exc = FSException(union(args1[0], args2[0]),
+                        union(args1[1], args2[1]))
         except UnionError:
             return None
-        return FrameState(newstate, self.blocklist, self.next_offset)
+        return FrameState(locals, stack, exc, self.blocklist, self.next_offset)
 
     def getoutputargs(self, targetstate):
         "Return the output arguments needed to link self to targetstate."
@@ -61,6 +103,7 @@
 
 def union(w1, w2):
     "Union of two variables or constants."
+    from rpython.flowspace.flowcontext import FlowSignal
     if w1 == w2:
         return w1
     if w1 is None or w2 is None:
@@ -69,38 +112,21 @@
     if isinstance(w1, Variable) or isinstance(w2, Variable):
         return Variable()  # new fresh Variable
     if isinstance(w1, Constant) and isinstance(w2, Constant):
-        # FlowSignal represent stack unrollers in the stack.
-        # They should not be merged because they will be unwrapped.
-        # This is needed for try:except: and try:finally:, though
-        # it makes the control flow a bit larger by duplicating the
-        # handlers.
-        dont_merge_w1 = w1 in UNPICKLE_TAGS or isinstance(w1.value, SpecTag)
-        dont_merge_w2 = w2 in UNPICKLE_TAGS or isinstance(w2.value, SpecTag)
-        if dont_merge_w1 or dont_merge_w2:
+        if isinstance(w1.value, SpecTag) or isinstance(w2.value, SpecTag):
             raise UnionError
         else:
             return Variable()  # generalize different constants
+    if isinstance(w1, FlowSignal) and isinstance(w2, FlowSignal):
+        if type(w1) is not type(w2):
+            raise UnionError
+        vars = [union(v1, v2) for v1, v2 in zip(w1.args, w2.args)]
+        return w1.rebuild(*vars)
+    if isinstance(w1, FlowSignal) or isinstance(w2, FlowSignal):
+        raise UnionError
     raise TypeError('union of %r and %r' % (w1.__class__.__name__,
                                             w2.__class__.__name__))
 
 
-# ____________________________________________________________
-#
-# We have to flatten out the state of the frame into a list of
-# Variables and Constants.  This is done above by collecting the
-# locals and the items on the value stack, but the latter may contain
-# FlowSignal.  We have to handle these specially, because
-# some of them hide references to more Variables and Constants.
-# The trick is to flatten ("pickle") them into the list so that the
-# extra Variables show up directly in the list too.
-
-class PickleTag:
-    pass
-
-PICKLE_TAGS = {}
-UNPICKLE_TAGS = {}
-
-
 def recursively_flatten(lst):
     from rpython.flowspace.flowcontext import FlowSignal
     i = 0
@@ -109,22 +135,4 @@
         if not isinstance(unroller, FlowSignal):
             i += 1
         else:
-            vars = unroller.state_unpack_variables()
-            key = unroller.__class__, len(vars)
-            try:
-                tag = PICKLE_TAGS[key]
-            except KeyError:
-                tag = PICKLE_TAGS[key] = Constant(PickleTag())
-                UNPICKLE_TAGS[tag] = key
-            lst[i:i + 1] = [tag] + vars
-
-
-def recursively_unflatten(lst):
-    for i in xrange(len(lst) - 1, -1, -1):
-        item = lst[i]
-        if item in UNPICKLE_TAGS:
-            unrollerclass, argcount = UNPICKLE_TAGS[item]
-            arguments = lst[i + 1:i + 1 + argcount]
-            del lst[i + 1:i + 1 + argcount]
-            unroller = unrollerclass.state_pack_variables(*arguments)
-            lst[i] = unroller
+            lst[i:i + 1] = unroller.args
diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py
--- a/rpython/flowspace/operation.py
+++ b/rpython/flowspace/operation.py
@@ -517,7 +517,7 @@
                     ctx.replace_in_stack(it, next_unroller)
                     return const(v)
         w_item = ctx.do_op(self)
-        ctx.guessexception([StopIteration, RuntimeError], force=True)
+        ctx.recorder.guessexception(ctx, StopIteration, RuntimeError)
         return w_item
 
 class GetAttr(SingleDispatchMixin, HLOperation):
diff --git a/rpython/flowspace/pygraph.py b/rpython/flowspace/pygraph.py
--- a/rpython/flowspace/pygraph.py
+++ b/rpython/flowspace/pygraph.py
@@ -11,10 +11,10 @@
 
     def __init__(self, func, code):
         from rpython.flowspace.flowcontext import SpamBlock
-        data = [None] * code.co_nlocals
+        locals = [None] * code.co_nlocals
         for i in range(code.formalargcount):
-            data[i] = Variable(code.co_varnames[i])
-        state = FrameState(data + [Constant(None), Constant(None)], [], 0)
+            locals[i] = Variable(code.co_varnames[i])
+        state = FrameState(locals, [], None, [], 0)
         initialblock = SpamBlock(state)
         super(PyGraph, self).__init__(self._sanitize_funcname(func), initialblock)
         self.func = func
diff --git a/rpython/flowspace/test/test_flowcontext.py b/rpython/flowspace/test/test_flowcontext.py
new file mode 100644
--- /dev/null
+++ b/rpython/flowspace/test/test_flowcontext.py
@@ -0,0 +1,15 @@
+""" Unit tests for flowcontext.py """
+import pytest
+from rpython.flowspace.model import Variable, FSException
+from rpython.flowspace.flowcontext import (
+    Return, Raise, RaiseImplicit, Continue, Break)
+
+ at pytest.mark.parametrize('signal', [
+    Return(Variable()),
+    Raise(FSException(Variable(), Variable())),
+    RaiseImplicit(FSException(Variable(), Variable())),
+    Break(),
+    Continue(42),
+])
+def test_signals(signal):
+    assert signal.rebuild(*signal.args) == signal
diff --git a/rpython/flowspace/test/test_framestate.py b/rpython/flowspace/test/test_framestate.py
--- a/rpython/flowspace/test/test_framestate.py
+++ b/rpython/flowspace/test/test_framestate.py
@@ -15,7 +15,7 @@
         ctx = FlowContext(graph, code)
         # hack the frame
         ctx.setstate(graph.startblock.framestate)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Constant(None)
+        ctx.locals_w[-1] = Constant(None)
         return ctx
 
     def func_simple(x):
@@ -31,7 +31,7 @@
     def test_neq_hacked_framestate(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable()
+        ctx.locals_w[-1] = Variable()
         fs2 = ctx.getstate(0)
         assert not fs1.matches(fs2)
 
@@ -44,7 +44,7 @@
     def test_union_on_hacked_framestates(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable()
+        ctx.locals_w[-1] = Variable()
         fs2 = ctx.getstate(0)
         assert fs1.union(fs2).matches(fs2)  # fs2 is more general
         assert fs2.union(fs1).matches(fs2)  # fs2 is more general
@@ -52,7 +52,7 @@
     def test_restore_frame(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable()
+        ctx.locals_w[-1] = Variable()
         ctx.setstate(fs1)
         assert fs1.matches(ctx.getstate(0))
 
@@ -71,26 +71,25 @@
     def test_getoutputargs(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable()
+        ctx.locals_w[-1] = Variable()
         fs2 = ctx.getstate(0)
         outputargs = fs1.getoutputargs(fs2)
         # 'x' -> 'x' is a Variable
         # locals_w[n-1] -> locals_w[n-1] is Constant(None)
-        assert outputargs == [ctx.locals_stack_w[0], Constant(None)]
+        assert outputargs == [ctx.locals_w[0], Constant(None)]
 
     def test_union_different_constants(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Constant(42)
+        ctx.locals_w[-1] = Constant(42)
         fs2 = ctx.getstate(0)
         fs3 = fs1.union(fs2)
         ctx.setstate(fs3)
-        assert isinstance(ctx.locals_stack_w[ctx.pycode.co_nlocals-1],
-                          Variable)   # generalized
+        assert isinstance(ctx.locals_w[-1], Variable)   # generalized
 
     def test_union_spectag(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Constant(SpecTag())
+        ctx.locals_w[-1] = Constant(SpecTag())
         fs2 = ctx.getstate(0)
         assert fs1.union(fs2) is None   # UnionError
diff --git a/rpython/jit/backend/arm/arch.py b/rpython/jit/backend/arm/arch.py
--- a/rpython/jit/backend/arm/arch.py
+++ b/rpython/jit/backend/arm/arch.py
@@ -1,4 +1,3 @@
-FUNC_ALIGN = 8
 WORD = 4
 DOUBLE_WORD = 8
 
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -4,7 +4,7 @@
 
 from rpython.jit.backend.arm import conditions as c, registers as r
 from rpython.jit.backend.arm import shift
-from rpython.jit.backend.arm.arch import (WORD, DOUBLE_WORD, FUNC_ALIGN,
+from rpython.jit.backend.arm.arch import (WORD, DOUBLE_WORD,
     JITFRAME_FIXED_SIZE)
 from rpython.jit.backend.arm.codebuilder import InstrBuilder, OverwritingBuilder
 from rpython.jit.backend.arm.locations import imm, StackLocation, get_fp_offset
@@ -484,10 +484,6 @@
         self.mc.BL(target)
         return startpos
 
-    def align(self):
-        while(self.mc.currpos() % FUNC_ALIGN != 0):
-            self.mc.writechar(chr(0))
-
     def gen_func_epilog(self, mc=None, cond=c.AL):
         gcrootmap = self.cpu.gc_ll_descr.gcrootmap
         if mc is None:
@@ -557,7 +553,7 @@
         debug_stop('jit-backend-ops')
 
     def _call_header(self):
-        self.align()
+        assert self.currpos() == 0
         self.gen_func_prolog()
 
     def _call_header_with_stack_check(self):
diff --git a/rpython/jit/backend/arm/callbuilder.py b/rpython/jit/backend/arm/callbuilder.py
--- a/rpython/jit/backend/arm/callbuilder.py
+++ b/rpython/jit/backend/arm/callbuilder.py
@@ -176,11 +176,14 @@
 
     def write_real_errno(self, save_err):
         if save_err & rffi.RFFI_READSAVED_ERRNO:
-            # Just before a call, read 'rpy_errno' and write it into the
+            # Just before a call, read '*_errno' and write it into the
             # real 'errno'.  The r0-r3 registers contain arguments to the
             # future call; the r5-r7 registers contain various stuff.
             # We still have r8-r12.
-            rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu)
+            if save_err & rffi.RFFI_ALT_ERRNO:
+                rpy_errno = llerrno.get_alt_errno_offset(self.asm.cpu)
+            else:
+                rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu)
             p_errno = llerrno.get_p_errno_offset(self.asm.cpu)
             self.mc.LDR_ri(r.r9.value, r.sp.value,
                            self.asm.saved_threadlocal_addr + self.current_sp)
@@ -199,10 +202,13 @@
     def read_real_errno(self, save_err):
         if save_err & rffi.RFFI_SAVE_ERRNO:
             # Just after a call, read the real 'errno' and save a copy of
-            # it inside our thread-local 'rpy_errno'.  Registers r8-r12
+            # it inside our thread-local '*_errno'.  Registers r8-r12
             # are unused here, and registers r2-r3 never contain anything
             # after the call.
-            rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu)
+            if save_err & rffi.RFFI_ALT_ERRNO:
+                rpy_errno = llerrno.get_alt_errno_offset(self.asm.cpu)
+            else:
+                rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu)
             p_errno = llerrno.get_p_errno_offset(self.asm.cpu)
             self.mc.LDR_ri(r.r3.value, r.sp.value,
                            self.asm.saved_threadlocal_addr)
diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py
--- a/rpython/jit/backend/arm/codebuilder.py
+++ b/rpython/jit/backend/arm/codebuilder.py
@@ -1,7 +1,7 @@
 from rpython.jit.backend.arm import conditions as cond
 from rpython.jit.backend.arm import registers as reg
 from rpython.jit.backend.arm import support
-from rpython.jit.backend.arm.arch import (WORD, FUNC_ALIGN, PC_OFFSET)
+from rpython.jit.backend.arm.arch import WORD, PC_OFFSET
 from rpython.jit.backend.arm.instruction_builder import define_instructions
 from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin
 from rpython.rlib.objectmodel import we_are_translated
@@ -29,14 +29,9 @@
 
 
 class AbstractARMBuilder(object):
-
     def __init__(self, arch_version=7):
         self.arch_version = arch_version
 
-    def align(self):
-        while(self.currpos() % FUNC_ALIGN != 0):
-            self.writechar(chr(0))
-
     def NOP(self):
         self.MOV_rr(0, 0)
 
@@ -467,21 +462,6 @@
                 f.write(data[i])
             f.close()
 
-    # XXX remove and setup aligning in llsupport
-    def materialize(self, asmmemmgr, allblocks, gcrootmap=None):
-        size = self.get_relative_pos() + WORD
-        malloced = asmmemmgr.malloc(size, size + 7)
-        allblocks.append(malloced)
-        rawstart = malloced[0]
-        while(rawstart % FUNC_ALIGN != 0):
-            rawstart += 1
-        self.copy_to_raw_memory(rawstart)
-        if self.gcroot_markers is not None:
-            assert gcrootmap is not None
-            for pos, mark in self.gcroot_markers:
-                gcrootmap.put(rawstart + pos, mark)
-        return rawstart
-
     def clear_cache(self, addr):
         if we_are_translated():
             startaddr = rffi.cast(llmemory.Address, addr)
diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py
--- a/rpython/jit/backend/llsupport/asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/asmmemmgr.py
@@ -208,6 +208,8 @@
                    ('data', lltype.FixedSizeArray(lltype.Char, SUBBLOCK_SIZE)))
     SUBBLOCK_PTR.TO.become(SUBBLOCK)
 
+    ALIGN_MATERIALIZE = 16
+
     gcroot_markers = None
 
     def __init__(self, translated=None):
@@ -303,9 +305,12 @@
 
     def materialize(self, asmmemmgr, allblocks, gcrootmap=None):
         size = self.get_relative_pos()
+        align = self.ALIGN_MATERIALIZE
+        size += align - 1
         malloced = asmmemmgr.malloc(size, size)
         allblocks.append(malloced)
         rawstart = malloced[0]
+        rawstart = (rawstart + align - 1) & (-align)
         self.copy_to_raw_memory(rawstart)
         if self.gcroot_markers is not None:
             assert gcrootmap is not None
diff --git a/rpython/jit/backend/llsupport/llerrno.py b/rpython/jit/backend/llsupport/llerrno.py
--- a/rpython/jit/backend/llsupport/llerrno.py
+++ b/rpython/jit/backend/llsupport/llerrno.py
@@ -18,19 +18,41 @@
         return 3 * WORD
 
 
+def get_debug_saved_alterrno(cpu):
+    return cpu._debug_errno_container[4]
+
+def set_debug_saved_alterrno(cpu, nerrno):
+    assert nerrno >= 0
+    cpu._debug_errno_container[4] = nerrno
+
+def get_alt_errno_offset(cpu):
+    if cpu.translate_support_code:
+        from rpython.rlib import rthread
+        return rthread.tlfield_alt_errno.getoffset()
+    else:
+        return 4 * WORD
+
+
 def get_debug_saved_lasterror(cpu):
-    return cpu._debug_errno_container[4]
+    return cpu._debug_errno_container[5]
 
 def set_debug_saved_lasterror(cpu, nerrno):
     assert nerrno >= 0
-    cpu._debug_errno_container[4] = nerrno
+    cpu._debug_errno_container[5] = nerrno
 
 def get_rpy_lasterror_offset(cpu):
     if cpu.translate_support_code:
         from rpython.rlib import rthread
         return rthread.tlfield_rpy_lasterror.getoffset()
     else:
-        return 4 * WORD
+        return 5 * WORD
+
+def get_alt_lasterror_offset(cpu):
+    if cpu.translate_support_code:
+        from rpython.rlib import rthread
+        return rthread.tlfield_alt_lasterror.getoffset()
+    else:
+        return 6 * WORD
 
 
 def _fetch_addr_errno():
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -63,7 +63,7 @@
                                               ad.lendescr, FLAG_FLOAT)
         self.setup()
         self._debug_errno_container = lltype.malloc(
-            rffi.CArray(lltype.Signed), 5, flavor='raw', zero=True,
+            rffi.CArray(lltype.Signed), 7, flavor='raw', zero=True,
             track_allocation=False)
 
     def getarraydescr_for_frame(self, type):
diff --git a/rpython/jit/backend/llsupport/test/zrpy_releasegil_test.py b/rpython/jit/backend/llsupport/test/zrpy_releasegil_test.py
--- a/rpython/jit/backend/llsupport/test/zrpy_releasegil_test.py
+++ b/rpython/jit/backend/llsupport/test/zrpy_releasegil_test.py
@@ -98,6 +98,7 @@
         self.run('close_stack')
         assert 'call_release_gil' in udir.join('TestCompileFramework.log').read()
 
+    # XXX this should also test get/set_alterrno ?
     def define_get_set_errno(self):
         eci = ExternalCompilationInfo(
             post_include_bits=[r'''
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -47,6 +47,7 @@
 
     add_loop_instructions = ['overload for a specific cpu']
     bridge_loop_instructions = ['overload for a specific cpu']
+    bridge_loop_instructions_alternative = None   # or another possible answer
 
     def execute_operation(self, opname, valueboxes, result_type, descr=None):
         inputargs, operations = self._get_single_operation_list(opname,
@@ -2948,7 +2949,11 @@
         calldescr = self.cpu._calldescr_dynamic_for_tests([types.slong]*7,
                                                           types.slong)
         #
-        for saveerr in [rffi.RFFI_ERR_NONE, rffi.RFFI_SAVE_ERRNO]:
+        for saveerr in [rffi.RFFI_ERR_NONE,
+                        rffi.RFFI_SAVE_ERRNO,
+                        rffi.RFFI_ERR_NONE | rffi.RFFI_ALT_ERRNO,
+                        rffi.RFFI_SAVE_ERRNO | rffi.RFFI_ALT_ERRNO,
+                        ]:
             faildescr = BasicFailDescr(1)
             inputargs = [BoxInt() for i in range(7)]
             i1 = BoxInt()
@@ -2965,15 +2970,23 @@
             self.cpu.compile_loop(inputargs, ops, looptoken)
             #
             llerrno.set_debug_saved_errno(self.cpu, 24)
+            llerrno.set_debug_saved_alterrno(self.cpu, 25)
             deadframe = self.cpu.execute_token(looptoken, 9, 8, 7, 6, 5, 4, 3)
             original_result = self.cpu.get_int_value(deadframe, 0)
             result = llerrno.get_debug_saved_errno(self.cpu)
-            print 'saveerr =', saveerr, ': got result =', result
+            altresult = llerrno.get_debug_saved_alterrno(self.cpu)
+            print 'saveerr =', saveerr, ': got result =', result, \
+                  'altresult =', altresult
             #
-            if saveerr == rffi.RFFI_SAVE_ERRNO:
-                assert result == 42      # from the C code
-            else:
-                assert result == 24      # not touched
+            expected = {
+                rffi.RFFI_ERR_NONE: (24, 25),
+                rffi.RFFI_SAVE_ERRNO: (42, 25),
+                rffi.RFFI_ERR_NONE | rffi.RFFI_ALT_ERRNO: (24, 25),
+                rffi.RFFI_SAVE_ERRNO | rffi.RFFI_ALT_ERRNO: (24, 42),
+            }
+            # expected (24, 25) as originally set, with possibly one
+            # of the two changed to 42 by the assembler code
+            assert (result, altresult) == expected[saveerr]
             assert original_result == 3456789
 
     def test_call_release_gil_readsaved_errno(self):
@@ -3007,7 +3020,11 @@
         calldescr = self.cpu._calldescr_dynamic_for_tests([types.slong]*7,
                                                           types.slong)
         #
-        for saveerr in [rffi.RFFI_READSAVED_ERRNO, rffi.RFFI_ZERO_ERRNO_BEFORE]:
+        for saveerr in [rffi.RFFI_READSAVED_ERRNO,
+                        rffi.RFFI_ZERO_ERRNO_BEFORE,
+                        rffi.RFFI_READSAVED_ERRNO   | rffi.RFFI_ALT_ERRNO,
+                        rffi.RFFI_ZERO_ERRNO_BEFORE | rffi.RFFI_ALT_ERRNO,
+                        ]:
             faildescr = BasicFailDescr(1)
             inputargs = [BoxInt() for i in range(7)]
             i1 = BoxInt()
@@ -3024,12 +3041,17 @@
             self.cpu.compile_loop(inputargs, ops, looptoken)
             #
             llerrno.set_debug_saved_errno(self.cpu, 24)
+            llerrno.set_debug_saved_alterrno(self.cpu, 25)
             deadframe = self.cpu.execute_token(looptoken, 9, 8, 7, 6, 5, 4, 3)
             result = self.cpu.get_int_value(deadframe, 0)
             assert llerrno.get_debug_saved_errno(self.cpu) == 24
+            assert llerrno.get_debug_saved_alterrno(self.cpu) == 25
             #
-            if saveerr == rffi.RFFI_READSAVED_ERRNO:
-                assert result == 24 + 345678900
+            if saveerr & rffi.RFFI_READSAVED_ERRNO:
+                if saveerr & rffi.RFFI_ALT_ERRNO:
+                    assert result == 25 + 345678900
+                else:
+                    assert result == 24 + 345678900
             else:
                 assert result == 0  + 345678900
 
@@ -3064,7 +3086,10 @@
                                                           types.slong)
         #
         for saveerr in [rffi.RFFI_SAVE_ERRNO,  # but not _LASTERROR
-                        rffi.RFFI_SAVE_LASTERROR]:
+                        rffi.RFFI_SAVE_ERRNO | rffi.RFFI_ALT_ERRNO,
+                        rffi.RFFI_SAVE_LASTERROR,
+                        rffi.RFFI_SAVE_LASTERROR | rffi.RFFI_ALT_ERRNO,
+                        ]:
             faildescr = BasicFailDescr(1)
             inputargs = [BoxInt() for i in range(7)]
             i1 = BoxInt()
@@ -3125,7 +3150,9 @@
         calldescr = self.cpu._calldescr_dynamic_for_tests([types.slong]*7,
                                                           types.slong)
         #
-        for saveerr in [rffi.RFFI_READSAVED_LASTERROR]:
+        for saveerr in [rffi.RFFI_READSAVED_LASTERROR,
+                        rffi.RFFI_READSAVED_LASTERROR | rffi.RFFI_ALT_ERRNO,
+                       ]:
             faildescr = BasicFailDescr(1)
             inputargs = [BoxInt() for i in range(7)]
             i1 = BoxInt()
@@ -3198,7 +3225,10 @@
         calldescr = self.cpu._calldescr_dynamic_for_tests([types.slong]*7,
                                                           types.slong)
         #
-        for saveerr in [rffi.RFFI_ERR_ALL]:
+        for saveerr in [rffi.RFFI_ERR_ALL,
+                        rffi.RFFI_ERR_ALL | rffi.RFFI_ALT_ERRNO, 
+                       ]:
+            use_alt_errno = saveerr & rffi.RFFI_ALT_ERRNO
             faildescr = BasicFailDescr(1)
             inputargs = [BoxInt() for i in range(7)]
             i1 = BoxInt()
@@ -3214,7 +3244,10 @@
             looptoken = JitCellToken()
             self.cpu.compile_loop(inputargs, ops, looptoken)
             #
-            llerrno.set_debug_saved_errno(self.cpu, 8)
+            if use_alt_errno:
+                llerrno.set_debug_saved_alterrno(self.cpu, 8)
+            else:
+                llerrno.set_debug_saved_errno(self.cpu, 8)
             llerrno.set_debug_saved_lasterror(self.cpu, 9)
             deadframe = self.cpu.execute_token(looptoken, 1, 2, 3, 4, 5, 6, 7)
             result = self.cpu.get_int_value(deadframe, 0)
@@ -4252,7 +4285,9 @@
         # XXX we have to check the precise assembler, otherwise
         # we don't quite know if borders are correct
 
-        def checkops(mc, ops):
+        def checkops(mc, ops, alt_ops=None):
+            if len(mc) != len(ops) and alt_ops is not None:
+                ops = alt_ops
             assert len(mc) == len(ops)
             for i in range(len(mc)):
                 if ops[i] == '*':
@@ -4267,7 +4302,8 @@
             data = ctypes.string_at(bridge_info.asmaddr, bridge_info.asmlen)
             mc = list(machine_code_dump(data, bridge_info.asmaddr, cpuname))
             lines = [line for line in mc if line.count('\t') >= 2]
-            checkops(lines, self.bridge_loop_instructions)
+            checkops(lines, self.bridge_loop_instructions,
+                            self.bridge_loop_instructions_alternative)
         except ObjdumpNotFound:
             py.test.skip("requires (g)objdump")
 
diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py
--- a/rpython/jit/backend/x86/callbuilder.py
+++ b/rpython/jit/backend/x86/callbuilder.py
@@ -196,21 +196,27 @@
             SetLastError_addr = self.asm.cpu.cast_adr_to_int(adr)
             assert isinstance(self, CallBuilder32)    # Windows 32-bit only
             #
-            rpy_lasterror = llerrno.get_rpy_lasterror_offset(self.asm.cpu)
+            if save_err & rffi.RFFI_ALT_ERRNO:
+                lasterror = llerrno.get_alt_lasterror_offset(self.asm.cpu)
+            else:
+                lasterror = llerrno.get_rpy_lasterror_offset(self.asm.cpu)
             tlofsreg = self.get_tlofs_reg()    # => esi, callee-saved
             self.save_stack_position()         # => edi, callee-saved
-            mc.PUSH_m((tlofsreg.value, rpy_lasterror))
+            mc.PUSH_m((tlofsreg.value, lasterror))
             mc.CALL(imm(SetLastError_addr))
             # restore the stack position without assuming a particular
             # calling convention of _SetLastError()
             self.mc.MOV(esp, self.saved_stack_position_reg)
 
         if save_err & rffi.RFFI_READSAVED_ERRNO:
-            # Just before a call, read 'rpy_errno' and write it into the
+            # Just before a call, read '*_errno' and write it into the
             # real 'errno'.  Most registers are free here, including the
             # callee-saved ones, except 'ebx' and except the ones used to
             # pass the arguments on x86-64.
-            rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu)
+            if save_err & rffi.RFFI_ALT_ERRNO:
+                rpy_errno = llerrno.get_alt_errno_offset(self.asm.cpu)
+            else:
+                rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu)
             p_errno = llerrno.get_p_errno_offset(self.asm.cpu)
             tlofsreg = self.get_tlofs_reg()    # => esi or r12, callee-saved
             if IS_X86_32:
@@ -234,11 +240,14 @@
 
         if save_err & rffi.RFFI_SAVE_ERRNO:
             # Just after a call, read the real 'errno' and save a copy of
-            # it inside our thread-local 'rpy_errno'.  Most registers are
+            # it inside our thread-local '*_errno'.  Most registers are
             # free here, including the callee-saved ones, except 'ebx'.
             # The tlofs register might have been loaded earlier and is
             # callee-saved, so it does not need to be reloaded.
-            rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu)
+            if save_err & rffi.RFFI_ALT_ERRNO:
+                rpy_errno = llerrno.get_alt_errno_offset(self.asm.cpu)
+            else:
+                rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu)
             p_errno = llerrno.get_p_errno_offset(self.asm.cpu)
             tlofsreg = self.get_tlofs_reg()   # => esi or r12 (possibly reused)
             mc.MOV_rm(edi.value, (tlofsreg.value, p_errno))
@@ -256,13 +265,16 @@
             GetLastError_addr = self.asm.cpu.cast_adr_to_int(adr)
             assert isinstance(self, CallBuilder32)    # Windows 32-bit only
             #
-            rpy_lasterror = llerrno.get_rpy_lasterror_offset(self.asm.cpu)
+            if save_err & rffi.RFFI_ALT_ERRNO:
+                lasterror = llerrno.get_alt_lasterror_offset(self.asm.cpu)
+            else:
+                lasterror = llerrno.get_rpy_lasterror_offset(self.asm.cpu)
             self.save_result_value(save_edx=True)   # save eax/edx/xmm0
             self.result_value_saved_early = True
             mc.CALL(imm(GetLastError_addr))
             #
             tlofsreg = self.get_tlofs_reg()    # => esi (possibly reused)
-            mc.MOV32_mr((tlofsreg.value, rpy_lasterror), eax.value)
+            mc.MOV32_mr((tlofsreg.value, lasterror), eax.value)
 
     def move_real_result_and_call_reacqgil_addr(self, fastgil):
         from rpython.jit.backend.x86 import rx86
@@ -314,8 +326,8 @@
             # in 'ebx'), and if not, we fall back to 'reacqgil_addr'.
             mc.J_il8(rx86.Conditions['NE'], 0)
             jne_location = mc.get_relative_pos()
-            # here, ecx is zero (so rpy_fastgil was in 'released' state
-            # before the XCHG, but the XCHG acquired it by writing 1)
+            # here, ecx (=old_value) is zero (so rpy_fastgil was in 'released'
+            # state before the XCHG, but the XCHG acquired it by writing 1)
             rst = gcrootmap.get_root_stack_top_addr()
             mc = self.mc
             mc.CMP(ebx, heap(rst))
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -339,14 +339,21 @@
             self.possibly_free_var(arg)
 
     def flush_loop(self):
+        # Force the code to be aligned to a multiple of 16.  Also,
         # rare case: if the loop is too short, or if we are just after
-        # a GUARD_NOT_INVALIDATED, pad with NOPs.  Important!  This must
-        # be called to ensure that there are enough bytes produced,
-        # because GUARD_NOT_INVALIDATED or redirect_call_assembler()
-        # will maybe overwrite them.
+        # a GUARD_NOT_INVALIDATED, we need to make sure we insert enough
+        # NOPs.  This is important to ensure that there are enough bytes
+        # produced, because GUARD_NOT_INVALIDATED or
+        # redirect_call_assembler() will maybe overwrite them.  (In that
+        # rare case we don't worry too much about alignment.)
         mc = self.assembler.mc
-        while mc.get_relative_pos() < self.min_bytes_before_label:
-            mc.NOP()
+        current_pos = mc.get_relative_pos()
+        target_pos = (current_pos + 15) & ~15
+        target_pos = max(target_pos, self.min_bytes_before_label)
+        insert_nops = target_pos - current_pos
+        assert 0 <= insert_nops <= 15
+        for c in mc.MULTIBYTE_NOPs[insert_nops]:
+            mc.writechar(c)
 
     def loc(self, v):
         if v is None: # xxx kludgy
diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -671,11 +671,39 @@
 def invert_condition(cond_num):
     return cond_num ^ 1
 
+
 class X86_32_CodeBuilder(AbstractX86CodeBuilder):
     WORD = 4
 
     PMOVMSKB_rx = xmminsn('\x66', rex_nw, '\x0F\xD7', register(1, 8), register(2), '\xC0')
 
+    # multibyte nops, from 0 to 15 bytes
+    MULTIBYTE_NOPs = [
+        '',
+        '\x90',                          # nop
+        '\x66\x90',                      # xchg ax, ax
+        '\x8d\x76\x00',                  # lea    0x0(%esi),%esi
+        '\x8d\x74\x26\x00',              # lea    0x0(%esi,%eiz,1),%esi
+        '\x90\x8d\x74\x26\x00',          # nop; lea 0x0(%esi,%eiz,1),%esi
+        '\x8d\xb6\x00\x00\x00\x00',      # lea    0x0(%esi),%esi
+        '\x8d\xb4\x26\x00\x00\x00\x00',  # lea    0x0(%esi,%eiz,1),%esi
+        ('\x90'                          # nop
+         '\x8d\xb4\x26\x00\x00\x00\x00'),#   lea    0x0(%esi,%eiz,1),%esi
+        ('\x89\xf6'                      # mov    %esi,%esi
+         '\x8d\xbc\x27\x00\x00\x00\x00'),#   lea    0x0(%edi,%eiz,1),%edi
+        ('\x8d\x76\x00'                  # lea    0x0(%esi),%esi
+         '\x8d\xbc\x27\x00\x00\x00\x00'),#   lea    0x0(%edi,%eiz,1),%edi
+        ('\x8d\x74\x26\x00'              # lea    0x0(%esi,%eiz,1),%esi
+         '\x8d\xbc\x27\x00\x00\x00\x00'),#   lea    0x0(%edi,%eiz,1),%edi
+        ('\x8d\xb6\x00\x00\x00\x00'      # lea    0x0(%esi),%esi
+         '\x8d\xbf\x00\x00\x00\x00'),    #   lea    0x0(%edi),%edi
+        ('\x8d\xb6\x00\x00\x00\x00'      # lea    0x0(%esi),%esi
+         '\x8d\xbc\x27\x00\x00\x00\x00'),#   lea    0x0(%edi,%eiz,1),%edi
+        ('\x8d\xb4\x26\x00\x00\x00\x00'  # lea    0x0(%esi,%eiz,1),%esi
+         '\x8d\xbc\x27\x00\x00\x00\x00'),#   lea    0x0(%edi,%eiz,1),%edi
+        ('\xeb\x0d' + '\x90' * 13)]      # jmp +x0d; a bunch of nops
+
+
 class X86_64_CodeBuilder(AbstractX86CodeBuilder):
     WORD = 8
 
@@ -706,6 +734,24 @@
         else:
             self.MOV_ri64(reg, immed)
 
+    # multibyte nops, from 0 to 15 bytes
+    MULTIBYTE_NOPs = ([
+        '',
+        '\x90',                          # nop
+        '\x66\x90',                      # xchg ax, ax
+        '\x0f\x1f\x00',                  # nopl   (%rax)
+        '\x0f\x1f\x40\x00',              # nopl   0x0(%rax)
+        '\x0f\x1f\x44\x00\x00',          # nopl   0x0(%rax,%rax,1)
+        '\x66\x0f\x1f\x44\x00\x00',      # nopw   0x0(%rax,%rax,1)
+        '\x0f\x1f\x80\x00\x00\x00\x00',  # nopl   0x0(%rax)
+        ('\x0f\x1f\x84\x00\x00\x00\x00'  # nopl   0x0(%rax,%rax,1)
+         '\x00'),
+        ('\x66\x0f\x1f\x84\x00\x00\x00'  # nopw   0x0(%rax,%rax,1)
+         '\x00\x00')] +
+        ['\x66' * _i + '\x2e\x0f\x1f'    # nopw   %cs:0x0(%rax,%rax,1)
+         '\x84\x00\x00\x00\x00\x00' for _i in range(1, 7)])
+
+
 def define_modrm_modes(insnname_template, before_modrm, after_modrm=[], regtype='GPR'):
     def add_insn(code, *modrm):
         args = before_modrm + list(modrm)
diff --git a/rpython/jit/backend/x86/test/test_runner.py b/rpython/jit/backend/x86/test/test_runner.py
--- a/rpython/jit/backend/x86/test/test_runner.py
+++ b/rpython/jit/backend/x86/test/test_runner.py
@@ -30,12 +30,24 @@
     # for the individual tests see
     # ====> ../../test/runner_test.py
 
-    add_loop_instructions = ['mov', 'add', 'test', 'je', 'jmp']
     if WORD == 4:
-        bridge_loop_instructions = ['cmp', 'jge', 'mov', 'mov', 'call', 'jmp']
+        add_loop_instructions = ['mov',
+                                 'lea',    # a nop, for the label
+                                 'add', 'test', 'je', 'jmp',
+                                 'nop']    # padding
+        bridge_loop_instructions = ['cmp', 'jge', 'mov', 'mov', 'call', 'jmp',
+                                    'lea', 'lea']   # padding
     else:
-        bridge_loop_instructions = ['cmp', 'jge', 'mov', 'mov', 'mov', 'mov',
-                                    'call', 'mov', 'jmp']
+        add_loop_instructions = ['mov',
+                                 'nop',    # for the label
+                                 'add', 'test', 'je', 'jmp',
+                                 'data32']   # padding
+        bridge_loop_instructions = [
+            'cmp', 'jge', 'mov', 'mov', 'mov', 'mov', 'call', 'mov', 'jmp',
+            'nop']      # padding
+        bridge_loop_instructions_alternative = [
+            'cmp', 'jge', 'mov', 'mov', 'mov', 'call', 'mov', 'jmp',
+            'nop']      # padding
 
     def get_cpu(self):
         cpu = CPU(rtyper=None, stats=FakeStats())
diff --git a/rpython/jit/backend/x86/test/test_rx86.py b/rpython/jit/backend/x86/test/test_rx86.py
--- a/rpython/jit/backend/x86/test/test_rx86.py
+++ b/rpython/jit/backend/x86/test/test_rx86.py
@@ -229,3 +229,9 @@
     s = CodeBuilder64()
     s.MOVSD_xj(xmm2, 0x01234567)
     assert s.getvalue() == '\xF2\x0F\x10\x14\x25\x67\x45\x23\x01'
+
+def test_multibyte_nops():
+    for cls in [X86_64_CodeBuilder, X86_32_CodeBuilder]:
+        assert len(cls.MULTIBYTE_NOPs) == 16
+        for i in range(16):
+            assert len(cls.MULTIBYTE_NOPs[i]) == i
diff --git a/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py b/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
--- a/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
+++ b/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
@@ -317,7 +317,9 @@
                 # CALL_j is actually relative, so tricky to test
                 (instrname == 'CALL' and argmodes == 'j') or
                 # SET_ir must be tested manually
-                (instrname == 'SET' and argmodes == 'ir')
+                (instrname == 'SET' and argmodes == 'ir') or
+                # MULTIBYTE_NOPs can't easily be tested the same way

From noreply at buildbot.pypy.org  Wed Feb 25 12:29:36 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed, 25 Feb 2015 12:29:36 +0100 (CET)
Subject: [pypy-commit] pypy default: Redo a6099e49c9ec: Add option
 --raise-operr to pytest that shows the interp-level OperationError when an
 app-level test fails.
Message-ID: <20150225112936.F280B1C13C9@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: 
Changeset: r76126:a910c6548369
Date: 2015-02-25 12:29 +0100
http://bitbucket.org/pypy/pypy/changeset/a910c6548369/

Log:	Redo a6099e49c9ec: Add option --raise-operr to pytest that shows the
	interp-level OperationError when an app-level test fails.

	This commit was backed out in 1ec5a5fa19da and redone differently
	previously in 96036c3af238, which itself was reverted in
	c195a8bc4ff5.

diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -36,7 +36,7 @@
 def pytest_addoption(parser):
     from rpython.conftest import pytest_addoption
     pytest_addoption(parser)
-    
+
     group = parser.getgroup("pypy options")
     group.addoption('-A', '--runappdirect', action="store_true",
            default=False, dest="runappdirect",
@@ -44,6 +44,9 @@
     group.addoption('--direct', action="store_true",
            default=False, dest="rundirect",
            help="run pexpect tests directly")
+    group.addoption('--raise-operr', action="store_true",
+            default=False, dest="raise_operr",
+            help="Show the interp-level OperationError in app-level tests")
 
 def pytest_funcarg__space(request):
     from pypy.tool.pytest.objspace import gettestobjspace
diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py
--- a/pypy/tool/pytest/apptest.py
+++ b/pypy/tool/pytest/apptest.py
@@ -35,8 +35,8 @@
         try:
             target(*args)
         except OperationError, e:
-            #if self.config.option.verbose:
-            #    raise
+            if self.config.option.raise_operr:
+                raise
             tb = sys.exc_info()[2]
             if e.match(space, space.w_KeyboardInterrupt):
                 raise KeyboardInterrupt, KeyboardInterrupt(), tb

From noreply at buildbot.pypy.org  Wed Feb 25 13:15:09 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 25 Feb 2015 13:15:09 +0100 (CET)
Subject: [pypy-commit] stmgc default: c8: add some marker macros and
 stop/resume_all_other_threads
Message-ID: <20150225121509.D55271C02BD@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: 
Changeset: r1664:e32b94df8ecb
Date: 2015-02-25 13:15 +0100
http://bitbucket.org/pypy/stmgc/changeset/e32b94df8ecb/

Log:	c8: add some marker macros and stop/resume_all_other_threads

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1089,6 +1089,27 @@
 }
 
 
+void stm_stop_all_other_threads(void)
+{
+    if (!stm_is_inevitable())         /* may still abort */
+        _stm_become_inevitable("stop_all_other_threads");
+
+    s_mutex_lock();
+    synchronize_all_threads(STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE);
+    s_mutex_unlock();
+}
+
+void stm_resume_all_other_threads(void)
+{
+    /* this calls 'committed_globally_unique_transaction()' even though
+       we're not committing now.  It's a way to piggyback on the existing
+       implementation for stm_become_globally_unique_transaction(). */
+    s_mutex_lock();
+    committed_globally_unique_transaction();
+    s_mutex_unlock();
+}
+
+
 
 static inline void _synchronize_fragment(stm_char *frag, ssize_t frag_size)
 {
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -269,6 +269,37 @@
 void stm_become_globally_unique_transaction(stm_thread_local_t *tl, const char *msg);
 void stm_validate(void);
 
+/* Temporarily stop all the other threads, by waiting until they
+   reach a safe-point.  Don't nest the calls to stop/resume and make sure
+   that resume is called.  The current transaction is turned inevitable. */
+void stm_stop_all_other_threads(void);
+void stm_resume_all_other_threads(void);
+
+
+/* Convenience macros to push the markers into the shadowstack */
+#define STM_PUSH_MARKER(tl, odd_num, p)   do {  \
+    uintptr_t _odd_num = (odd_num);             \
+    assert(_odd_num & 1);                       \
+    STM_PUSH_ROOT(tl, _odd_num);                \
+    STM_PUSH_ROOT(tl, p);                       \
+} while (0)
+
+#define STM_POP_MARKER(tl)   ({                 \
+    object_t *_popped = STM_POP_ROOT_RET(tl);   \
+    STM_POP_ROOT_RET(tl);                       \
+    _popped;                                    \
+})
+
+#define STM_UPDATE_MARKER_NUM(tl, odd_num)  do {                \
+    uintptr_t _odd_num = (odd_num);                             \
+    assert(_odd_num & 1);                                       \
+    struct stm_shadowentry_s *_ss = (tl).shadowstack - 2;       \
+    while (!(((uintptr_t)(_ss->ss)) & 1)) {                     \
+        _ss--;                                                  \
+        assert(_ss >= (tl).shadowstack_base);                   \
+    }                                                           \
+    _ss->ss = (object_t *)_odd_num;                             \
+} while (0)
 
 
 /* Support for light finalizers.  This is a simple version of
diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -78,6 +78,8 @@
 bool _check_abort_transaction(void);
 bool _check_become_inevitable(stm_thread_local_t *tl);
 bool _check_become_globally_unique_transaction(stm_thread_local_t *tl);
+bool _check_stop_all_other_threads(void);
+void stm_resume_all_other_threads(void);
 int stm_is_inevitable(void);
 long current_segment_num(void);
 
@@ -225,6 +227,10 @@
     CHECKED(stm_become_globally_unique_transaction(tl, "TESTGUT"));
 }
 
+bool _check_stop_all_other_threads(void) {
+    CHECKED(stm_stop_all_other_threads());
+}
+
 bool _check_stm_validate(void) {
     CHECKED(stm_validate());
 }
@@ -668,3 +674,11 @@
         tl = self.tls[self.current_thread]
         if lib._check_become_globally_unique_transaction(tl):
             raise Conflict()
+
+    def stop_all_other_threads(self):
+        if lib._check_stop_all_other_threads():
+            raise Conflict()
+
+    def resume_all_other_threads(self):
+        if lib.stm_resume_all_other_threads():
+            raise Conflict()
diff --git a/c8/test/test_extra.py b/c8/test/test_extra.py
--- a/c8/test/test_extra.py
+++ b/c8/test/test_extra.py
@@ -192,3 +192,24 @@
         assert lib.stm_is_inevitable()
         #
         py.test.raises(Conflict, self.switch, 0)
+
+    def test_stm_stop_all_other_threads_1(self):
+        self.start_transaction()
+        #
+        self.switch(1)
+        self.start_transaction()
+        self.stop_all_other_threads()
+        assert lib.stm_is_inevitable()
+        #
+        py.test.raises(Conflict, self.switch, 0)
+
+    def test_stm_stop_all_other_threads_2(self):
+        self.start_transaction()
+        #
+        self.switch(1)
+        self.start_transaction()
+        self.stop_all_other_threads()
+        self.resume_all_other_threads()
+        assert lib.stm_is_inevitable()
+        #
+        self.switch(0)   # no conflict

From noreply at buildbot.pypy.org  Wed Feb 25 13:32:52 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 25 Feb 2015 13:32:52 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c8: add some bits to the fake
 stm-hashtable to make pypy translate
Message-ID: <20150225123252.7F72D1C02BD@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: stmgc-c8
Changeset: r76127:28ebc96dd613
Date: 2015-02-25 13:20 +0100
http://bitbucket.org/pypy/pypy/changeset/28ebc96dd613/

Log:	add some bits to the fake stm-hashtable to make pypy translate

diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -310,6 +310,28 @@
     def len(self):
         return len(self._content)
 
+    def list(self):
+        items = []
+        for key in self._content.keys():
+            items.append(self.lookup(key))
+        count = len(items)
+        return items, count
+
+    def freelist(self, array):
+        pass
+
+    def lookup(self, key):
+        return EntryObjectEmulation(self, key)
+
+    def writeobj(self, entry, nvalue):
+        self.set(entry.key, nvalue)
+
+class EntryObjectEmulation(object):
+    def __init__(self, hashtable, key):
+        self.hashtable = hashtable
+        self.key = key
+        self.index = r_uint(key)
+        self.object = hashtable.get(key)
 
 
 class HashtableForTest(object):

From noreply at buildbot.pypy.org  Wed Feb 25 13:32:53 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 25 Feb 2015 13:32:53 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c8: import stmgc e32b94df8ecb
Message-ID: <20150225123253.EA67D1C02BD@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: stmgc-c8
Changeset: r76128:7e650b854871
Date: 2015-02-25 13:20 +0100
http://bitbucket.org/pypy/pypy/changeset/7e650b854871/

Log:	import stmgc e32b94df8ecb

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-8c547a55af94
+e32b94df8ecb
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -1089,6 +1089,27 @@
 }
 
 
+void stm_stop_all_other_threads(void)
+{
+    if (!stm_is_inevitable())         /* may still abort */
+        _stm_become_inevitable("stop_all_other_threads");
+
+    s_mutex_lock();
+    synchronize_all_threads(STOP_OTHERS_AND_BECOME_GLOBALLY_UNIQUE);
+    s_mutex_unlock();
+}
+
+void stm_resume_all_other_threads(void)
+{
+    /* this calls 'committed_globally_unique_transaction()' even though
+       we're not committing now.  It's a way to piggyback on the existing
+       implementation for stm_become_globally_unique_transaction(). */
+    s_mutex_lock();
+    committed_globally_unique_transaction();
+    s_mutex_unlock();
+}
+
+
 
 static inline void _synchronize_fragment(stm_char *frag, ssize_t frag_size)
 {
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -269,6 +269,37 @@
 void stm_become_globally_unique_transaction(stm_thread_local_t *tl, const char *msg);
 void stm_validate(void);
 
+/* Temporarily stop all the other threads, by waiting until they
+   reach a safe-point.  Don't nest the calls to stop/resume and make sure
+   that resume is called.  The current transaction is turned inevitable. */
+void stm_stop_all_other_threads(void);
+void stm_resume_all_other_threads(void);
+
+
+/* Convenience macros to push the markers into the shadowstack */
+#define STM_PUSH_MARKER(tl, odd_num, p)   do {  \
+    uintptr_t _odd_num = (odd_num);             \
+    assert(_odd_num & 1);                       \
+    STM_PUSH_ROOT(tl, _odd_num);                \
+    STM_PUSH_ROOT(tl, p);                       \
+} while (0)
+
+#define STM_POP_MARKER(tl)   ({                 \
+    object_t *_popped = STM_POP_ROOT_RET(tl);   \
+    STM_POP_ROOT_RET(tl);                       \
+    _popped;                                    \
+})
+
+#define STM_UPDATE_MARKER_NUM(tl, odd_num)  do {                \
+    uintptr_t _odd_num = (odd_num);                             \
+    assert(_odd_num & 1);                                       \
+    struct stm_shadowentry_s *_ss = (tl).shadowstack - 2;       \
+    while (!(((uintptr_t)(_ss->ss)) & 1)) {                     \
+        _ss--;                                                  \
+        assert(_ss >= (tl).shadowstack_base);                   \
+    }                                                           \
+    _ss->ss = (object_t *)_odd_num;                             \
+} while (0)
 
 
 /* Support for light finalizers.  This is a simple version of
diff --git a/rpython/translator/stm/src_stm/stmgcintf.h b/rpython/translator/stm/src_stm/stmgcintf.h
--- a/rpython/translator/stm/src_stm/stmgcintf.h
+++ b/rpython/translator/stm/src_stm/stmgcintf.h
@@ -38,6 +38,8 @@
 
 long _pypy_stm_count(void);
 
+
+
 /* C8: not implemented properly yet: */
 extern void stmcb_commit_soon(void);
 #define _STM_CARD_SIZE                 32     /* must be >= 32 */

From noreply at buildbot.pypy.org  Wed Feb 25 13:37:58 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed, 25 Feb 2015 13:37:58 +0100 (CET)
Subject: [pypy-commit] pypy py3k-memoryview: hg merge py3k
Message-ID: <20150225123759.06DB81C1186@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3k-memoryview
Changeset: r76129:e3bdc0312b1d
Date: 2015-02-25 12:45 +0100
http://bitbucket.org/pypy/pypy/changeset/e3bdc0312b1d/

Log:	hg merge py3k

diff too long, truncating to 2000 out of 73193 lines

diff --git a/.gitignore b/.gitignore
--- a/.gitignore
+++ b/.gitignore
@@ -7,7 +7,10 @@
 
 bin/pypy-c
 include/*.h
+include/numpy/
 lib_pypy/ctypes_config_cache/_[^_]*_*.py
+libpypy-c.*
+pypy-c
 pypy/_cache
 pypy/doc/*.html
 pypy/doc/config/*.html
@@ -18,4 +21,5 @@
 pypy/translator/c/src/dtoa.o
 pypy/translator/goal/pypy-c
 pypy/translator/goal/target*-c
-release/
\ No newline at end of file
+release/
+rpython/_cache/
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -7,10 +7,7 @@
 9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
 ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm
 20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0
-20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0
-0000000000000000000000000000000000000000 release-2.3.0
 394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3
 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1
 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1
-32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1
-0000000000000000000000000000000000000000 release-2.2=3.1
+10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -28,7 +28,7 @@
     DEALINGS IN THE SOFTWARE.
 
 
-PyPy Copyright holders 2003-2014
+PyPy Copyright holders 2003-2015
 ----------------------------------- 
 
 Except when otherwise stated (look for LICENSE files or information at
@@ -42,19 +42,19 @@
   Amaury Forgeot d'Arc
   Samuele Pedroni
   Alex Gaynor
+  Brian Kearns
+  Matti Picus
+  Philip Jenvey
   Michael Hudson
   David Schneider
-  Matti Picus
-  Brian Kearns
-  Philip Jenvey
   Holger Krekel
   Christian Tismer
   Hakan Ardo
   Benjamin Peterson
   Manuel Jacob
+  Ronan Lamy
   Anders Chrigstrom
   Eric van Riet Paap
-  Ronan Lamy
   Wim Lavrijsen
   Richard Emslie
   Alexander Schremmer
@@ -68,9 +68,9 @@
   Camillo Bruni
   Laura Creighton
   Toon Verwaest
+  Romain Guillebert
   Leonardo Santagada
   Seo Sanghyeon
-  Romain Guillebert
   Justin Peel
   Ronny Pfannschmidt
   David Edelsohn
@@ -91,15 +91,16 @@
   Michal Bendowski
   Jan de Mooij
   stian
+  Tyler Wade
   Michael Foord
   Stephan Diehl
-  Tyler Wade
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
   Patrick Maupin
   Bob Ippolito
   Bruno Gola
+  David Malcolm
   Jean-Paul Calderone
   Timo Paulssen
   Squeaky
@@ -108,18 +109,19 @@
   Marius Gedminas
   Martin Matusiak
   Konstantin Lopuhin
+  Wenzhu Man
   John Witulski
-  Wenzhu Man
+  Laurence Tratt
+  Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
   Mark Pearse
   Simon Cross
-  Ivan Sichmann Freitas
   Andreas Stührk
+  Stefano Rivera
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Stefano Rivera
   Paweł Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -129,7 +131,6 @@
   tav
   Taavi Burns
   Georg Brandl
-  Laurence Tratt
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -141,13 +142,12 @@
   Jeremy Thurgood
   Rami Chowdhury
   Tobias Pape
-  David Malcolm
   Eugene Oden
   Henry Mason
   Vasily Kuznetsov
   Preston Timmons
+  David Ripton
   Jeff Terrace
-  David Ripton
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
@@ -166,13 +166,16 @@
   Gintautas Miliauskas
   Michael Twomey
   Lucian Branescu Mihaila
+  Yichao Yu
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
   Karl Bartel
+  Wouter van Heyst
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  anatoly techtonik
   Stuart Williams
   Jasper Schulz
   Christian Hudon
@@ -182,12 +185,11 @@
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
-  anatoly techtonik
   Neil Shepperd
+  Stanislaw Halik
   Mikael Schönenberg
   Elmo M?ntynen
   Jonathan David Riehl
-  Stanislaw Halik
   Anders Qvist
   Corbin Simpson
   Chirag Jadwani
@@ -196,10 +198,13 @@
   Vincent Legoll
   Alan McIntyre
   Alexander Sedov
+  Attila Gobi
   Christopher Pope
   Christian Tismer 
   Marc Abramowitz
   Dan Stromberg
+  Arjun Naik
+  Valentina Mukhamedzhanova
   Stefano Parmesan
   Alexis Daboville
   Jens-Uwe Mager
@@ -213,8 +218,6 @@
   Sylvain Thenault
   Nathan Taylor
   Vladimir Kryachko
-  Arjun Naik
-  Attila Gobi
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -222,22 +225,23 @@
   Ryan Gonzalez
   Ian Foote
   Kristjan Valur Jonsson
+  David Lievens
   Neil Blakey-Milner
   Lutz Paelike
   Lucio Torre
   Lars Wassermann
-  Valentina Mukhamedzhanova
   Henrik Vendelbo
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
   Sergey Kishchenko
-  Yichao Yu
   Ignas Mikalajunas
   Christoph Gerum
   Martin Blais
   Lene Wagner
   Tomo Cocoa
+  Toni Mattis
+  Lucas Stadler
   roberto at goyle
   Yury V. Zaytsev
   Anna Katrina Dominguez
@@ -265,23 +269,30 @@
   Stephan Busemann
   Rafał Gałczyński
   Christian Muirhead
+  Berker Peksag
   James Lan
   shoma hosaka
-  Daniel Neuh?user
-  Matthew Miller
+  Daniel Neuhäuser
+  Ben Mather
+  halgari
+  Boglarka Vezer
+  Chris Pressey
   Buck Golemon
   Konrad Delong
   Dinu Gherman
   Chris Lambacher
   coolbutuseless at gmail.com
+  Jim Baker
   Rodrigo Araújo
-  Jim Baker
+  Nikolaos-Digenis Karagiannis
   James Robert
   Armin Ronacher
   Brett Cannon
+  Donald Stufft
   yrttyr
   aliceinwire
   OlivierBlanvillain
+  Dan Sanders
   Zooko Wilcox-O Hearn
   Tomer Chachamu
   Christopher Groskopf
@@ -295,6 +306,7 @@
   Markus Unterwaditzer
   Even Wiik Thomassen
   jbs
+  squeaky
   soareschen
   Kurt Griffiths
   Mike Bayer
@@ -306,6 +318,7 @@
   Anna Ravencroft
   Dan Crosta
   Julien Phalip
+  Roman Podoliaka
   Dan Loewenherz
 
   Heinrich-Heine University, Germany 
diff --git a/lib-python/2.7/CGIHTTPServer.py b/lib-python/2.7/CGIHTTPServer.py
--- a/lib-python/2.7/CGIHTTPServer.py
+++ b/lib-python/2.7/CGIHTTPServer.py
@@ -106,16 +106,16 @@
     def run_cgi(self):
         """Execute a CGI script."""
         dir, rest = self.cgi_info
-
-        i = rest.find('/')
+        path = dir + '/' + rest
+        i = path.find('/', len(dir)+1)
         while i >= 0:
-            nextdir = rest[:i]
-            nextrest = rest[i+1:]
+            nextdir = path[:i]
+            nextrest = path[i+1:]
 
             scriptdir = self.translate_path(nextdir)
             if os.path.isdir(scriptdir):
                 dir, rest = nextdir, nextrest
-                i = rest.find('/')
+                i = path.find('/', len(dir)+1)
             else:
                 break
 
diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py
--- a/lib-python/2.7/Cookie.py
+++ b/lib-python/2.7/Cookie.py
@@ -56,7 +56,7 @@
    >>> C = Cookie.SmartCookie()
 
 [Note: Long-time users of Cookie.py will remember using
-Cookie.Cookie() to create an Cookie object.  Although deprecated, it
+Cookie.Cookie() to create a Cookie object.  Although deprecated, it
 is still supported by the code.  See the Backward Compatibility notes
 for more information.]
 
@@ -426,6 +426,8 @@
                    "version" : "Version",
                    }
 
+    _flags = {'secure', 'httponly'}
+
     def __init__(self):
         # Set defaults
         self.key = self.value = self.coded_value = None
@@ -529,9 +531,11 @@
 _LegalCharsPatt  = r"[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=]"
 _CookiePattern = re.compile(
     r"(?x)"                       # This is a Verbose pattern
+    r"\s*"                        # Optional whitespace at start of cookie
     r"(?P"                   # Start of group 'key'
     ""+ _LegalCharsPatt +"+?"     # Any word of at least one letter, nongreedy
     r")"                          # End of group 'key'
+    r"("                          # Optional group: there may not be a value.
     r"\s*=\s*"                    # Equal Sign
     r"(?P"                   # Start of group 'val'
     r'"(?:[^\\"]|\\.)*"'            # Any doublequoted string
@@ -540,7 +544,9 @@
     r"|"                            # or
     ""+ _LegalCharsPatt +"*"        # Any word or empty string
     r")"                          # End of group 'val'
-    r"\s*;?"                      # Probably ending in a semi-colon
+    r")?"                         # End of optional value group
+    r"\s*"                        # Any number of spaces.
+    r"(\s+|;|$)"                  # Ending either at space, semicolon, or EOS.
     )
 
 
@@ -585,8 +591,12 @@
 
     def __setitem__(self, key, value):
         """Dictionary style assignment."""
-        rval, cval = self.value_encode(value)
-        self.__set(key, rval, cval)
+        if isinstance(value, Morsel):
+            # allow assignment of constructed Morsels (e.g. for pickling)
+            dict.__setitem__(self, key, value)
+        else:
+            rval, cval = self.value_encode(value)
+            self.__set(key, rval, cval)
     # end __setitem__
 
     def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"):
@@ -641,7 +651,7 @@
 
         while 0 <= i < n:
             # Start looking for a cookie
-            match = patt.search(str, i)
+            match = patt.match(str, i)
             if not match: break          # No more cookies
 
             K,V = match.group("key"), match.group("val")
@@ -656,8 +666,12 @@
                     M[ K[1:] ] = V
             elif K.lower() in Morsel._reserved:
                 if M:
-                    M[ K ] = _unquote(V)
-            else:
+                    if V is None:
+                        if K.lower() in Morsel._flags:
+                            M[K] = True
+                    else:
+                        M[K] = _unquote(V)
+            elif V is not None:
                 rval, cval = self.value_decode(V)
                 self.__set(K, rval, cval)
                 M = self[K]
diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py
--- a/lib-python/2.7/SocketServer.py
+++ b/lib-python/2.7/SocketServer.py
@@ -416,8 +416,12 @@
         self.socket = socket.socket(self.address_family,
                                     self.socket_type)
         if bind_and_activate:
-            self.server_bind()
-            self.server_activate()
+            try:
+                self.server_bind()
+                self.server_activate()
+            except:
+                self.server_close()
+                raise
 
     def server_bind(self):
         """Called by constructor to bind the socket.
diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py
--- a/lib-python/2.7/_abcoll.py
+++ b/lib-python/2.7/_abcoll.py
@@ -143,7 +143,7 @@
     methods except for __contains__, __iter__ and __len__.
 
     To override the comparisons (presumably for speed, as the
-    semantics are fixed), all you have to do is redefine __le__ and
+    semantics are fixed), redefine __le__ and __ge__,
     then the other operations will automatically follow suit.
     """
 
diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py
--- a/lib-python/2.7/argparse.py
+++ b/lib-python/2.7/argparse.py
@@ -1089,7 +1089,14 @@
         # parse all the remaining options into the namespace
         # store any unrecognized options on the object, so that the top
         # level parser can decide what to do with them
-        namespace, arg_strings = parser.parse_known_args(arg_strings, namespace)
+
+        # In case this subparser defines new defaults, we parse them
+        # in a new namespace object and then update the original
+        # namespace for the relevant parts.
+        subnamespace, arg_strings = parser.parse_known_args(arg_strings, None)
+        for key, value in vars(subnamespace).items():
+            setattr(namespace, key, value)
+
         if arg_strings:
             vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, [])
             getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings)
diff --git a/lib-python/2.7/asynchat.py b/lib-python/2.7/asynchat.py
--- a/lib-python/2.7/asynchat.py
+++ b/lib-python/2.7/asynchat.py
@@ -46,12 +46,17 @@
 you - by calling your self.found_terminator() method.
 """
 
+import asyncore
+import errno
 import socket
-import asyncore
 from collections import deque
 from sys import py3kwarning
 from warnings import filterwarnings, catch_warnings
 
+_BLOCKING_IO_ERRORS = (errno.EAGAIN, errno.EALREADY, errno.EINPROGRESS,
+                       errno.EWOULDBLOCK)
+
+
 class async_chat (asyncore.dispatcher):
     """This is an abstract class.  You must derive from this class, and add
     the two methods collect_incoming_data() and found_terminator()"""
@@ -109,6 +114,8 @@
         try:
             data = self.recv (self.ac_in_buffer_size)
         except socket.error, why:
+            if why.args[0] in _BLOCKING_IO_ERRORS:
+                return
             self.handle_error()
             return
 
diff --git a/lib-python/2.7/bsddb/test/test_queue.py b/lib-python/2.7/bsddb/test/test_queue.py
--- a/lib-python/2.7/bsddb/test/test_queue.py
+++ b/lib-python/2.7/bsddb/test/test_queue.py
@@ -10,6 +10,7 @@
 
 #----------------------------------------------------------------------
 
+ at unittest.skip("fails on Windows; see issue 22943")
 class SimpleQueueTestCase(unittest.TestCase):
     def setUp(self):
         self.filename = get_new_database_path()
diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -17,6 +17,10 @@
 except ImportError:
     assert '__pypy__' not in _sys.builtin_module_names
     newdict = lambda _ : {}
+try:
+    from __pypy__ import reversed_dict
+except ImportError:
+    reversed_dict = lambda d: reversed(d.keys())
 
 try:
     from thread import get_ident as _get_ident
@@ -29,142 +33,35 @@
 ################################################################################
 
 class OrderedDict(dict):
-    'Dictionary that remembers insertion order'
-    # An inherited dict maps keys to values.
-    # The inherited dict provides __getitem__, __len__, __contains__, and get.
-    # The remaining methods are order-aware.
-    # Big-O running times for all methods are the same as regular dictionaries.
+    '''Dictionary that remembers insertion order.
 
-    # The internal self.__map dict maps keys to links in a doubly linked list.
-    # The circular doubly linked list starts and ends with a sentinel element.
-    # The sentinel element never gets deleted (this simplifies the algorithm).
-    # Each link is stored as a list of length three:  [PREV, NEXT, KEY].
+    In PyPy all dicts are ordered anyway.  This is mostly useful as a
+    placeholder to mean "this dict must be ordered even on CPython".
 
-    def __init__(self, *args, **kwds):
-        '''Initialize an ordered dictionary.  The signature is the same as
-        regular dictionaries, but keyword arguments are not recommended because
-        their insertion order is arbitrary.
-
-        '''
-        if len(args) > 1:
-            raise TypeError('expected at most 1 arguments, got %d' % len(args))
-        try:
-            self.__root
-        except AttributeError:
-            self.__root = root = []                     # sentinel node
-            root[:] = [root, root, None]
-            self.__map = {}
-        self.__update(*args, **kwds)
-
-    def __setitem__(self, key, value, dict_setitem=dict.__setitem__):
-        'od.__setitem__(i, y) <==> od[i]=y'
-        # Setting a new item creates a new link at the end of the linked list,
-        # and the inherited dictionary is updated with the new key/value pair.
-        if key not in self:
-            root = self.__root
-            last = root[0]
-            last[1] = root[0] = self.__map[key] = [last, root, key]
-        return dict_setitem(self, key, value)
-
-    def __delitem__(self, key, dict_delitem=dict.__delitem__):
-        'od.__delitem__(y) <==> del od[y]'
-        # Deleting an existing item uses self.__map to find the link which gets
-        # removed by updating the links in the predecessor and successor nodes.
-        dict_delitem(self, key)
-        link_prev, link_next, _ = self.__map.pop(key)
-        link_prev[1] = link_next                        # update link_prev[NEXT]
-        link_next[0] = link_prev                        # update link_next[PREV]
-
-    def __iter__(self):
-        'od.__iter__() <==> iter(od)'
-        # Traverse the linked list in order.
-        root = self.__root
-        curr = root[1]                                  # start at the first node
-        while curr is not root:
-            yield curr[2]                               # yield the curr[KEY]
-            curr = curr[1]                              # move to next node
+    Known difference: iterating over an OrderedDict which is being
+    concurrently modified raises RuntimeError in PyPy.  In CPython
+    instead we get some behavior that appears reasonable in some
+    cases but is nonsensical in other cases.  This is officially
+    forbidden by the CPython docs, so we forbid it explicitly for now.
+    '''
 
     def __reversed__(self):
-        'od.__reversed__() <==> reversed(od)'
-        # Traverse the linked list in reverse order.
-        root = self.__root
-        curr = root[0]                                  # start at the last node
-        while curr is not root:
-            yield curr[2]                               # yield the curr[KEY]
-            curr = curr[0]                              # move to previous node
-
-    def clear(self):
-        'od.clear() -> None.  Remove all items from od.'
-        root = self.__root
-        root[:] = [root, root, None]
-        self.__map.clear()
-        dict.clear(self)
-
-    # -- the following methods do not depend on the internal structure --
-
-    def keys(self):
-        'od.keys() -> list of keys in od'
-        return list(self)
-
-    def values(self):
-        'od.values() -> list of values in od'
-        return [self[key] for key in self]
-
-    def items(self):
-        'od.items() -> list of (key, value) pairs in od'
-        return [(key, self[key]) for key in self]
-
-    def iterkeys(self):
-        'od.iterkeys() -> an iterator over the keys in od'
-        return iter(self)
-
-    def itervalues(self):
-        'od.itervalues -> an iterator over the values in od'
-        for k in self:
-            yield self[k]
-
-    def iteritems(self):
-        'od.iteritems -> an iterator over the (key, value) pairs in od'
-        for k in self:
-            yield (k, self[k])
-
-    update = MutableMapping.update
-
-    __update = update # let subclasses override update without breaking __init__
-
-    __marker = object()
-
-    def pop(self, key, default=__marker):
-        '''od.pop(k[,d]) -> v, remove specified key and return the corresponding
-        value.  If key is not found, d is returned if given, otherwise KeyError
-        is raised.
-
-        '''
-        if key in self:
-            result = self[key]
-            del self[key]
-            return result
-        if default is self.__marker:
-            raise KeyError(key)
-        return default
-
-    def setdefault(self, key, default=None):
-        'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
-        if key in self:
-            return self[key]
-        self[key] = default
-        return default
+        return reversed_dict(self)
 
     def popitem(self, last=True):
         '''od.popitem() -> (k, v), return and remove a (key, value) pair.
         Pairs are returned in LIFO order if last is true or FIFO order if false.
 
         '''
-        if not self:
-            raise KeyError('dictionary is empty')
-        key = next(reversed(self) if last else iter(self))
-        value = self.pop(key)
-        return key, value
+        if last:
+            return dict.popitem(self)
+        else:
+            it = dict.__iter__(self)
+            try:
+                k = it.next()
+            except StopIteration:
+                raise KeyError('dictionary is empty')
+            return (k, self.pop(k))
 
     def __repr__(self, _repr_running={}):
         'od.__repr__() <==> repr(od)'
@@ -183,8 +80,6 @@
         'Return state information for pickling'
         items = [[k, self[k]] for k in self]
         inst_dict = vars(self).copy()
-        for k in vars(OrderedDict()):
-            inst_dict.pop(k, None)
         if inst_dict:
             return (self.__class__, (items,), inst_dict)
         return self.__class__, (items,)
@@ -193,17 +88,6 @@
         'od.copy() -> a shallow copy of od'
         return self.__class__(self)
 
-    @classmethod
-    def fromkeys(cls, iterable, value=None):
-        '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S.
-        If not specified, the value defaults to None.
-
-        '''
-        self = cls()
-        for key in iterable:
-            self[key] = value
-        return self
-
     def __eq__(self, other):
         '''od.__eq__(y) <==> od==y.  Comparison to another OD is order-sensitive
         while comparison to a regular mapping is order-insensitive.
diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py
--- a/lib-python/2.7/cookielib.py
+++ b/lib-python/2.7/cookielib.py
@@ -1719,12 +1719,12 @@
     def __repr__(self):
         r = []
         for cookie in self: r.append(repr(cookie))
-        return "<%s[%s]>" % (self.__class__, ", ".join(r))
+        return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r))
 
     def __str__(self):
         r = []
         for cookie in self: r.append(str(cookie))
-        return "<%s[%s]>" % (self.__class__, ", ".join(r))
+        return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r))
 
 
 # derives from IOError for backwards-compatibility with Python 2.4.0
diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py
@@ -2,7 +2,6 @@
 import array
 import gc
 import unittest
-from ctypes.test import xfail
 
 class X(Structure):
     _fields_ = [("c_int", c_int)]
@@ -11,7 +10,6 @@
         self._init_called = True
 
 class Test(unittest.TestCase):
-    @xfail
     def test_fom_buffer(self):
         a = array.array("i", range(16))
         x = (c_int * 16).from_buffer(a)
@@ -34,10 +32,9 @@
         del a; gc.collect(); gc.collect(); gc.collect()
         self.assertEqual(x[:], expected)
 
-        self.assertRaises(TypeError,
+        self.assertRaises((TypeError, ValueError),
                           (c_char * 16).from_buffer, "a" * 16)
 
-    @xfail
     def test_fom_buffer_with_offset(self):
         a = array.array("i", range(16))
         x = (c_int * 15).from_buffer(a, sizeof(c_int))
@@ -46,7 +43,6 @@
         self.assertRaises(ValueError, lambda: (c_int * 16).from_buffer(a, sizeof(c_int)))
         self.assertRaises(ValueError, lambda: (c_int * 1).from_buffer(a, 16 * sizeof(c_int)))
 
-    @xfail
     def test_from_buffer_copy(self):
         a = array.array("i", range(16))
         x = (c_int * 16).from_buffer_copy(a)
@@ -71,7 +67,6 @@
         x = (c_char * 16).from_buffer_copy("a" * 16)
         self.assertEqual(x[:], "a" * 16)
 
-    @xfail
     def test_fom_buffer_copy_with_offset(self):
         a = array.array("i", range(16))
         x = (c_int * 15).from_buffer_copy(a, sizeof(c_int))
diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py
--- a/lib-python/2.7/ctypes/test/test_pointers.py
+++ b/lib-python/2.7/ctypes/test/test_pointers.py
@@ -7,6 +7,8 @@
                  c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float]
 python_types = [int, int, int, int, int, long,
                 int, long, long, long, float, float]
+LargeNamedType = type('T' * 2 ** 25, (Structure,), {})
+large_string = 'T' * 2 ** 25
 
 class PointersTestCase(unittest.TestCase):
 
@@ -188,5 +190,11 @@
             mth = WINFUNCTYPE(None)(42, "name", (), None)
             self.assertEqual(bool(mth), True)
 
+    def test_pointer_type_name(self):
+        self.assertTrue(POINTER(LargeNamedType))
+
+    def test_pointer_type_str_name(self):
+        self.assertTrue(POINTER(large_string))
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_python_api.py b/lib-python/2.7/ctypes/test/test_python_api.py
--- a/lib-python/2.7/ctypes/test/test_python_api.py
+++ b/lib-python/2.7/ctypes/test/test_python_api.py
@@ -46,8 +46,8 @@
     # This test is unreliable, because it is possible that code in
     # unittest changes the refcount of the '42' integer.  So, it
     # is disabled by default.
-    @requires("refcount")
     def test_PyInt_Long(self):
+        requires("refcount")
         ref42 = grc(42)
         pythonapi.PyInt_FromLong.restype = py_object
         self.assertEqual(pythonapi.PyInt_FromLong(42), 42)
diff --git a/lib-python/2.7/ctypes/test/test_win32.py b/lib-python/2.7/ctypes/test/test_win32.py
--- a/lib-python/2.7/ctypes/test/test_win32.py
+++ b/lib-python/2.7/ctypes/test/test_win32.py
@@ -38,8 +38,11 @@
 
 @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test')
 class FunctionCallTestCase(unittest.TestCase):
-    @requires("SEH")
+    @unittest.skipUnless('MSC' in sys.version, "SEH only supported by MSC")
+    @unittest.skipIf(sys.executable.endswith('_d.exe'),
+                     "SEH not enabled in debug builds")
     def test_SEH(self):
+        requires("SEH")
         # Call functions with invalid arguments, and make sure
         # that access violations are trapped and raise an
         # exception.
@@ -87,9 +90,29 @@
 
         dll = CDLL(_ctypes_test.__file__)
 
-        pt = POINT(10, 10)
-        rect = RECT(0, 0, 20, 20)
-        self.assertEqual(1, dll.PointInRect(byref(rect), pt))
+        pt = POINT(15, 25)
+        left = c_long.in_dll(dll, 'left')
+        top = c_long.in_dll(dll, 'top')
+        right = c_long.in_dll(dll, 'right')
+        bottom = c_long.in_dll(dll, 'bottom')
+        rect = RECT(left, top, right, bottom)
+        PointInRect = dll.PointInRect
+        PointInRect.argtypes = [POINTER(RECT), POINT]
+        self.assertEqual(1, PointInRect(byref(rect), pt))
+
+        ReturnRect = dll.ReturnRect
+        ReturnRect.argtypes = [c_int, RECT, POINTER(RECT), POINT, RECT,
+                               POINTER(RECT), POINT, RECT]
+        ReturnRect.restype = RECT
+        for i in range(4):
+            ret = ReturnRect(i, rect, pointer(rect), pt, rect,
+                         byref(rect), pt, rect)
+            # the c function will check and modify ret if something is
+            # passed in improperly
+            self.assertEqual(ret.left, left.value)
+            self.assertEqual(ret.right, right.value)
+            self.assertEqual(ret.top, top.value)
+            self.assertEqual(ret.bottom, bottom.value)
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/decimal.py b/lib-python/2.7/decimal.py
--- a/lib-python/2.7/decimal.py
+++ b/lib-python/2.7/decimal.py
@@ -136,7 +136,6 @@
 
 __version__ = '1.70'    # Highest version of the spec this complies with
 
-import copy as _copy
 import math as _math
 import numbers as _numbers
 
@@ -3665,6 +3664,8 @@
         if self._is_special:
             sign = _format_sign(self._sign, spec)
             body = str(self.copy_abs())
+            if spec['type'] == '%':
+                body += '%'
             return _format_align(sign, body, spec)
 
         # a type of None defaults to 'g' or 'G', depending on context
@@ -6033,7 +6034,10 @@
         format_dict['decimal_point'] = '.'
 
     # record whether return type should be str or unicode
-    format_dict['unicode'] = isinstance(format_spec, unicode)
+    try:
+        format_dict['unicode'] = isinstance(format_spec, unicode)
+    except NameError:
+        format_dict['unicode'] = False
 
     return format_dict
 
diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py
--- a/lib-python/2.7/distutils/__init__.py
+++ b/lib-python/2.7/distutils/__init__.py
@@ -15,5 +15,5 @@
 # Updated automatically by the Python release process.
 #
 #--start constants--
-__version__ = "2.7.8"
+__version__ = "2.7.9"
 #--end constants--
diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py
--- a/lib-python/2.7/distutils/command/build_ext.py
+++ b/lib-python/2.7/distutils/command/build_ext.py
@@ -245,7 +245,7 @@
         # Python's library directory must be appended to library_dirs
         # See Issues: #1600860, #4366
         if (sysconfig.get_config_var('Py_ENABLE_SHARED')):
-            if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")):
+            if not sysconfig.python_build:
                 # building third party extensions
                 self.library_dirs.append(sysconfig.get_config_var('LIBDIR'))
             else:
diff --git a/lib-python/2.7/distutils/command/upload.py b/lib-python/2.7/distutils/command/upload.py
--- a/lib-python/2.7/distutils/command/upload.py
+++ b/lib-python/2.7/distutils/command/upload.py
@@ -136,8 +136,8 @@
 
         # Build up the MIME payload for the POST data
         boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'
-        sep_boundary = '\n--' + boundary
-        end_boundary = sep_boundary + '--'
+        sep_boundary = '\r\n--' + boundary
+        end_boundary = sep_boundary + '--\r\n'
         body = StringIO.StringIO()
         for key, value in data.items():
             # handle multiple entries for the same name
@@ -151,14 +151,13 @@
                     fn = ""
 
                 body.write(sep_boundary)
-                body.write('\nContent-Disposition: form-data; name="%s"'%key)
+                body.write('\r\nContent-Disposition: form-data; name="%s"' % key)
                 body.write(fn)
-                body.write("\n\n")
+                body.write("\r\n\r\n")
                 body.write(value)
                 if value and value[-1] == '\r':
                     body.write('\n')  # write an extra newline (lurve Macs)
         body.write(end_boundary)
-        body.write("\n")
         body = body.getvalue()
 
         self.announce("Submitting %s to %s" % (filename, self.repository), log.INFO)
diff --git a/lib-python/2.7/distutils/file_util.py b/lib-python/2.7/distutils/file_util.py
--- a/lib-python/2.7/distutils/file_util.py
+++ b/lib-python/2.7/distutils/file_util.py
@@ -85,7 +85,8 @@
     (os.symlink) instead of copying: set it to "hard" or "sym"; if it is
     None (the default), files are copied.  Don't set 'link' on systems that
     don't support it: 'copy_file()' doesn't check if hard or symbolic
-    linking is available.
+    linking is available. If hardlink fails, falls back to
+    _copy_file_contents().
 
     Under Mac OS, uses the native file copy function in macostools; on
     other systems, uses '_copy_file_contents()' to copy file contents.
@@ -137,24 +138,31 @@
     # (Unix only, of course, but that's the caller's responsibility)
     if link == 'hard':
         if not (os.path.exists(dst) and os.path.samefile(src, dst)):
-            os.link(src, dst)
+            try:
+                os.link(src, dst)
+                return (dst, 1)
+            except OSError:
+                # If hard linking fails, fall back on copying file
+                # (some special filesystems don't support hard linking
+                #  even under Unix, see issue #8876).
+                pass
     elif link == 'sym':
         if not (os.path.exists(dst) and os.path.samefile(src, dst)):
             os.symlink(src, dst)
+            return (dst, 1)
 
     # Otherwise (non-Mac, not linking), copy the file contents and
     # (optionally) copy the times and mode.
-    else:
-        _copy_file_contents(src, dst)
-        if preserve_mode or preserve_times:
-            st = os.stat(src)
+    _copy_file_contents(src, dst)
+    if preserve_mode or preserve_times:
+        st = os.stat(src)
 
-            # According to David Ascher , utime() should be done
-            # before chmod() (at least under NT).
-            if preserve_times:
-                os.utime(dst, (st[ST_ATIME], st[ST_MTIME]))
-            if preserve_mode:
-                os.chmod(dst, S_IMODE(st[ST_MODE]))
+        # According to David Ascher , utime() should be done
+        # before chmod() (at least under NT).
+        if preserve_times:
+            os.utime(dst, (st[ST_ATIME], st[ST_MTIME]))
+        if preserve_mode:
+            os.chmod(dst, S_IMODE(st[ST_MODE]))
 
     return (dst, 1)
 
diff --git a/lib-python/2.7/distutils/sysconfig_cpython.py b/lib-python/2.7/distutils/sysconfig_cpython.py
--- a/lib-python/2.7/distutils/sysconfig_cpython.py
+++ b/lib-python/2.7/distutils/sysconfig_cpython.py
@@ -165,7 +165,8 @@
             # version and build tools may not support the same set
             # of CPU architectures for universal builds.
             global _config_vars
-            if not _config_vars.get('CUSTOMIZED_OSX_COMPILER', ''):
+            # Use get_config_var() to ensure _config_vars is initialized.
+            if not get_config_var('CUSTOMIZED_OSX_COMPILER'):
                 import _osx_support
                 _osx_support.customize_compiler(_config_vars)
                 _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True'
diff --git a/lib-python/2.7/distutils/tests/test_bdist_rpm.py b/lib-python/2.7/distutils/tests/test_bdist_rpm.py
--- a/lib-python/2.7/distutils/tests/test_bdist_rpm.py
+++ b/lib-python/2.7/distutils/tests/test_bdist_rpm.py
@@ -25,6 +25,7 @@
 """
 
 class BuildRpmTestCase(support.TempdirManager,
+                       support.EnvironGuard,
                        support.LoggingSilencer,
                        unittest.TestCase):
 
@@ -50,6 +51,7 @@
     def test_quiet(self):
         # let's create a package
         tmp_dir = self.mkdtemp()
+        os.environ['HOME'] = tmp_dir   # to confine dir '.rpmdb' creation
         pkg_dir = os.path.join(tmp_dir, 'foo')
         os.mkdir(pkg_dir)
         self.write_file((pkg_dir, 'setup.py'), SETUP_PY)
@@ -92,6 +94,7 @@
     def test_no_optimize_flag(self):
         # let's create a package that brakes bdist_rpm
         tmp_dir = self.mkdtemp()
+        os.environ['HOME'] = tmp_dir   # to confine dir '.rpmdb' creation
         pkg_dir = os.path.join(tmp_dir, 'foo')
         os.mkdir(pkg_dir)
         self.write_file((pkg_dir, 'setup.py'), SETUP_PY)
diff --git a/lib-python/2.7/distutils/tests/test_dist.py b/lib-python/2.7/distutils/tests/test_dist.py
--- a/lib-python/2.7/distutils/tests/test_dist.py
+++ b/lib-python/2.7/distutils/tests/test_dist.py
@@ -11,7 +11,7 @@
 from distutils.dist import Distribution, fix_help_options
 from distutils.cmd import Command
 import distutils.dist
-from test.test_support import TESTFN, captured_stdout, run_unittest
+from test.test_support import TESTFN, captured_stdout, run_unittest, unlink
 from distutils.tests import support
 
 
@@ -64,6 +64,7 @@
         with open(TESTFN, "w") as f:
             f.write("[global]\n")
             f.write("command_packages = foo.bar, splat")
+        self.addCleanup(unlink, TESTFN)
 
         files = [TESTFN]
         sys.argv.append("build")
diff --git a/lib-python/2.7/distutils/tests/test_file_util.py b/lib-python/2.7/distutils/tests/test_file_util.py
--- a/lib-python/2.7/distutils/tests/test_file_util.py
+++ b/lib-python/2.7/distutils/tests/test_file_util.py
@@ -8,6 +8,11 @@
 from distutils.tests import support
 from test.test_support import run_unittest
 
+
+requires_os_link = unittest.skipUnless(hasattr(os, "link"),
+                                       "test requires os.link()")
+
+
 class FileUtilTestCase(support.TempdirManager, unittest.TestCase):
 
     def _log(self, msg, *args):
@@ -74,6 +79,44 @@
         copy_file(foo, dst_dir)
         self.assertTrue(os.path.exists(os.path.join(dst_dir, 'foo')))
 
+    @requires_os_link
+    def test_copy_file_hard_link(self):
+        with open(self.source, 'w') as f:
+            f.write('some content')
+        st = os.stat(self.source)
+        copy_file(self.source, self.target, link='hard')
+        st2 = os.stat(self.source)
+        st3 = os.stat(self.target)
+        self.assertTrue(os.path.samestat(st, st2), (st, st2))
+        self.assertTrue(os.path.samestat(st2, st3), (st2, st3))
+        with open(self.source, 'r') as f:
+            self.assertEqual(f.read(), 'some content')
+
+    @requires_os_link
+    def test_copy_file_hard_link_failure(self):
+        # If hard linking fails, copy_file() falls back on copying file
+        # (some special filesystems don't support hard linking even under
+        #  Unix, see issue #8876).
+        with open(self.source, 'w') as f:
+            f.write('some content')
+        st = os.stat(self.source)
+        def _os_link(*args):
+            raise OSError(0, "linking unsupported")
+        old_link = os.link
+        os.link = _os_link
+        try:
+            copy_file(self.source, self.target, link='hard')
+        finally:
+            os.link = old_link
+        st2 = os.stat(self.source)
+        st3 = os.stat(self.target)
+        self.assertTrue(os.path.samestat(st, st2), (st, st2))
+        self.assertFalse(os.path.samestat(st2, st3), (st2, st3))
+        for fn in (self.source, self.target):
+            with open(fn, 'r') as f:
+                self.assertEqual(f.read(), 'some content')
+
+
 def test_suite():
     return unittest.makeSuite(FileUtilTestCase)
 
diff --git a/lib-python/2.7/distutils/tests/test_sysconfig.py b/lib-python/2.7/distutils/tests/test_sysconfig.py
--- a/lib-python/2.7/distutils/tests/test_sysconfig.py
+++ b/lib-python/2.7/distutils/tests/test_sysconfig.py
@@ -3,6 +3,9 @@
 import test
 import unittest
 import shutil
+import subprocess
+import sys
+import textwrap
 
 from distutils import sysconfig
 from distutils.tests import support
@@ -99,6 +102,24 @@
         self.assertEqual(global_sysconfig.get_config_var('LDSHARED'), sysconfig.get_config_var('LDSHARED'))
         self.assertEqual(global_sysconfig.get_config_var('CC'), sysconfig.get_config_var('CC'))
 
+    def test_customize_compiler_before_get_config_vars(self):
+        # Issue #21923: test that a Distribution compiler
+        # instance can be called without an explicit call to
+        # get_config_vars().
+        with open(TESTFN, 'w') as f:
+            f.writelines(textwrap.dedent('''\
+                from distutils.core import Distribution
+                config = Distribution().get_command_obj('config')
+                # try_compile may pass or it may fail if no compiler
+                # is found but it should not raise an exception.
+                rc = config.try_compile('int x;')
+                '''))
+        p = subprocess.Popen([str(sys.executable), TESTFN],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                universal_newlines=True)
+        outs, errs = p.communicate()
+        self.assertEqual(0, p.returncode, "Subprocess failed: " + outs)
 
 
 def test_suite():
diff --git a/lib-python/2.7/distutils/tests/test_upload.py b/lib-python/2.7/distutils/tests/test_upload.py
--- a/lib-python/2.7/distutils/tests/test_upload.py
+++ b/lib-python/2.7/distutils/tests/test_upload.py
@@ -119,7 +119,7 @@
         # what did we send ?
         self.assertIn('dédé', self.last_open.req.data)
         headers = dict(self.last_open.req.headers)
-        self.assertEqual(headers['Content-length'], '2085')
+        self.assertEqual(headers['Content-length'], '2159')
         self.assertTrue(headers['Content-type'].startswith('multipart/form-data'))
         self.assertEqual(self.last_open.req.get_method(), 'POST')
         self.assertEqual(self.last_open.req.get_full_url(),
diff --git a/lib-python/2.7/distutils/unixccompiler.py b/lib-python/2.7/distutils/unixccompiler.py
--- a/lib-python/2.7/distutils/unixccompiler.py
+++ b/lib-python/2.7/distutils/unixccompiler.py
@@ -58,7 +58,7 @@
     executables = {'preprocessor' : None,
                    'compiler'     : ["cc"],
                    'compiler_so'  : ["cc"],
-                   'compiler_cxx' : ["cc"],
+                   'compiler_cxx' : ["c++"],  # pypy: changed, 'cc' is bogus
                    'linker_so'    : ["cc", "-shared"],
                    'linker_exe'   : ["cc"],
                    'archiver'     : ["ar", "-cr"],
diff --git a/lib-python/2.7/doctest.py b/lib-python/2.7/doctest.py
--- a/lib-python/2.7/doctest.py
+++ b/lib-python/2.7/doctest.py
@@ -216,7 +216,7 @@
                 # get_data() opens files as 'rb', so one must do the equivalent
                 # conversion as universal newlines would do.
                 return file_contents.replace(os.linesep, '\n'), filename
-    with open(filename) as f:
+    with open(filename, 'U') as f:
         return f.read(), filename
 
 # Use sys.stdout encoding for ouput.
diff --git a/lib-python/2.7/email/feedparser.py b/lib-python/2.7/email/feedparser.py
--- a/lib-python/2.7/email/feedparser.py
+++ b/lib-python/2.7/email/feedparser.py
@@ -49,8 +49,8 @@
     simple abstraction -- it parses until EOF closes the current message.
     """
     def __init__(self):
-        # The last partial line pushed into this object.
-        self._partial = ''
+        # Chunks of the last partial line pushed into this object.
+        self._partial = []
         # The list of full, pushed lines, in reverse order
         self._lines = []
         # The stack of false-EOF checking predicates.
@@ -66,8 +66,8 @@
 
     def close(self):
         # Don't forget any trailing partial line.
-        self._lines.append(self._partial)
-        self._partial = ''
+        self.pushlines(''.join(self._partial).splitlines(True))
+        self._partial = []
         self._closed = True
 
     def readline(self):
@@ -95,8 +95,29 @@
 
     def push(self, data):
         """Push some new data into this object."""
-        # Handle any previous leftovers
-        data, self._partial = self._partial + data, ''
+        # Crack into lines, but preserve the linesep characters on the end of each
+        parts = data.splitlines(True)
+
+        if not parts or not parts[0].endswith(('\n', '\r')):
+            # No new complete lines, so just accumulate partials
+            self._partial += parts
+            return
+
+        if self._partial:
+            # If there are previous leftovers, complete them now
+            self._partial.append(parts[0])
+            parts[0:1] = ''.join(self._partial).splitlines(True)
+            del self._partial[:]
+
+        # If the last element of the list does not end in a newline, then treat
+        # it as a partial line.  We only check for '\n' here because a line
+        # ending with '\r' might be a line that was split in the middle of a
+        # '\r\n' sequence (see bugs 1555570 and 1721862).
+        if not parts[-1].endswith('\n'):
+            self._partial = [parts.pop()]
+        self.pushlines(parts)
+
+    def pushlines(self, lines):
         # Crack into lines, but preserve the newlines on the end of each
         parts = NLCRE_crack.split(data)
         # The *ahem* interesting behaviour of re.split when supplied grouping
diff --git a/lib-python/2.7/email/mime/nonmultipart.py b/lib-python/2.7/email/mime/nonmultipart.py
--- a/lib-python/2.7/email/mime/nonmultipart.py
+++ b/lib-python/2.7/email/mime/nonmultipart.py
@@ -12,7 +12,7 @@
 
 

 class MIMENonMultipart(MIMEBase):
-    """Base class for MIME multipart/* type messages."""
+    """Base class for MIME non-multipart type messages."""
 
     def attach(self, payload):
         # The public API prohibits attaching multiple subparts to MIMEBase
diff --git a/lib-python/2.7/email/test/test_email.py b/lib-python/2.7/email/test/test_email.py
--- a/lib-python/2.7/email/test/test_email.py
+++ b/lib-python/2.7/email/test/test_email.py
@@ -11,6 +11,7 @@
 import warnings
 import textwrap
 from cStringIO import StringIO
+from random import choice
 
 import email
 
@@ -2578,16 +2579,64 @@
             bsf.push(il)
             nt += n
             n1 = 0
-            while True:
-                ol = bsf.readline()
-                if ol == NeedMoreData:
-                    break
+            for ol in iter(bsf.readline, NeedMoreData):
                 om.append(ol)
                 n1 += 1
             self.assertEqual(n, n1)
         self.assertEqual(len(om), nt)
         self.assertEqual(''.join([il for il, n in imt]), ''.join(om))
 
+    def test_push_random(self):
+        from email.feedparser import BufferedSubFile, NeedMoreData
+
+        n = 10000
+        chunksize = 5
+        chars = 'abcd \t\r\n'
+
+        s = ''.join(choice(chars) for i in range(n)) + '\n'
+        target = s.splitlines(True)
+
+        bsf = BufferedSubFile()
+        lines = []
+        for i in range(0, len(s), chunksize):
+            chunk = s[i:i+chunksize]
+            bsf.push(chunk)
+            lines.extend(iter(bsf.readline, NeedMoreData))
+        self.assertEqual(lines, target)
+
+
+class TestFeedParsers(TestEmailBase):
+
+    def parse(self, chunks):
+        from email.feedparser import FeedParser
+        feedparser = FeedParser()
+        for chunk in chunks:
+            feedparser.feed(chunk)
+        return feedparser.close()
+
+    def test_newlines(self):
+        m = self.parse(['a:\nb:\rc:\r\nd:\n'])
+        self.assertEqual(m.keys(), ['a', 'b', 'c', 'd'])
+        m = self.parse(['a:\nb:\rc:\r\nd:'])
+        self.assertEqual(m.keys(), ['a', 'b', 'c', 'd'])
+        m = self.parse(['a:\rb', 'c:\n'])
+        self.assertEqual(m.keys(), ['a', 'bc'])
+        m = self.parse(['a:\r', 'b:\n'])
+        self.assertEqual(m.keys(), ['a', 'b'])
+        m = self.parse(['a:\r', '\nb:\n'])
+        self.assertEqual(m.keys(), ['a', 'b'])
+
+    def test_long_lines(self):
+        # Expected peak memory use on 32-bit platform: 4*N*M bytes.
+        M, N = 1000, 20000
+        m = self.parse(['a:b\n\n'] + ['x'*M] * N)
+        self.assertEqual(m.items(), [('a', 'b')])
+        self.assertEqual(m.get_payload(), 'x'*M*N)
+        m = self.parse(['a:b\r\r'] + ['x'*M] * N)
+        self.assertEqual(m.items(), [('a', 'b')])
+        self.assertEqual(m.get_payload(), 'x'*M*N)
+        m = self.parse(['a:\r', 'b: '] + ['x'*M] * N)
+        self.assertEqual(m.items(), [('a', ''), ('b', 'x'*M*N)])
 
 
 class TestParsers(TestEmailBase):
@@ -3180,7 +3229,6 @@
         self.assertEqual(res, '=?iso-8859-2?q?abc?=')
         self.assertIsInstance(res, str)
 
-
 # Test RFC 2231 header parameters (en/de)coding
 class TestRFC2231(TestEmailBase):
     def test_get_param(self):
diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/ensurepip/__init__.py
@@ -0,0 +1,227 @@
+#!/usr/bin/env python2
+from __future__ import print_function
+
+import os
+import os.path
+import pkgutil
+import shutil
+import sys
+import tempfile
+
+
+__all__ = ["version", "bootstrap"]
+
+
+_SETUPTOOLS_VERSION = "7.0"
+
+_PIP_VERSION = "1.5.6"
+
+# pip currently requires ssl support, so we try to provide a nicer
+# error message when that is missing (http://bugs.python.org/issue19744)
+_MISSING_SSL_MESSAGE = ("pip {} requires SSL/TLS".format(_PIP_VERSION))
+try:
+    import ssl
+except ImportError:
+    ssl = None
+
+    def _require_ssl_for_pip():
+        raise RuntimeError(_MISSING_SSL_MESSAGE)
+else:
+    def _require_ssl_for_pip():
+        pass
+
+_PROJECTS = [
+    ("setuptools", _SETUPTOOLS_VERSION),
+    ("pip", _PIP_VERSION),
+]
+
+
+def _run_pip(args, additional_paths=None):
+    # Add our bundled software to the sys.path so we can import it
+    if additional_paths is not None:
+        sys.path = additional_paths + sys.path
+
+    # Install the bundled software
+    import pip
+    pip.main(args)
+
+
+def version():
+    """
+    Returns a string specifying the bundled version of pip.
+    """
+    return _PIP_VERSION
+
+
+def _disable_pip_configuration_settings():
+    # We deliberately ignore all pip environment variables
+    # when invoking pip
+    # See http://bugs.python.org/issue19734 for details
+    keys_to_remove = [k for k in os.environ if k.startswith("PIP_")]
+    for k in keys_to_remove:
+        del os.environ[k]
+    # We also ignore the settings in the default pip configuration file
+    # See http://bugs.python.org/issue20053 for details
+    os.environ['PIP_CONFIG_FILE'] = os.devnull
+
+
+def bootstrap(root=None, upgrade=False, user=False,
+              altinstall=False, default_pip=True,
+              verbosity=0):
+    """
+    Bootstrap pip into the current Python installation (or the given root
+    directory).
+
+    Note that calling this function will alter both sys.path and os.environ.
+    """
+    if altinstall and default_pip:
+        raise ValueError("Cannot use altinstall and default_pip together")
+
+    _require_ssl_for_pip()
+    _disable_pip_configuration_settings()
+
+    # By default, installing pip and setuptools installs all of the
+    # following scripts (X.Y == running Python version):
+    #
+    #   pip, pipX, pipX.Y, easy_install, easy_install-X.Y
+    #
+    # pip 1.5+ allows ensurepip to request that some of those be left out
+    if altinstall:
+        # omit pip, pipX and easy_install
+        os.environ["ENSUREPIP_OPTIONS"] = "altinstall"
+    elif not default_pip:
+        # omit pip and easy_install
+        os.environ["ENSUREPIP_OPTIONS"] = "install"
+
+    tmpdir = tempfile.mkdtemp()
+    try:
+        # Put our bundled wheels into a temporary directory and construct the
+        # additional paths that need added to sys.path
+        additional_paths = []
+        for project, version in _PROJECTS:
+            wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version)
+            whl = pkgutil.get_data(
+                "ensurepip",
+                "_bundled/{}".format(wheel_name),
+            )
+            with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
+                fp.write(whl)
+
+            additional_paths.append(os.path.join(tmpdir, wheel_name))
+
+        # Construct the arguments to be passed to the pip command
+        args = ["install", "--no-index", "--find-links", tmpdir]
+        if root:
+            args += ["--root", root]
+        if upgrade:
+            args += ["--upgrade"]
+        if user:
+            args += ["--user"]
+        if verbosity:
+            args += ["-" + "v" * verbosity]
+
+        _run_pip(args + [p[0] for p in _PROJECTS], additional_paths)
+    finally:
+        shutil.rmtree(tmpdir, ignore_errors=True)
+
+
+def _uninstall_helper(verbosity=0):
+    """Helper to support a clean default uninstall process on Windows
+
+    Note that calling this function may alter os.environ.
+    """
+    # Nothing to do if pip was never installed, or has been removed
+    try:
+        import pip
+    except ImportError:
+        return
+
+    # If the pip version doesn't match the bundled one, leave it alone
+    if pip.__version__ != _PIP_VERSION:
+        msg = ("ensurepip will only uninstall a matching version "
+               "({!r} installed, {!r} bundled)")
+        print(msg.format(pip.__version__, _PIP_VERSION), file=sys.stderr)
+        return
+
+    _require_ssl_for_pip()
+    _disable_pip_configuration_settings()
+
+    # Construct the arguments to be passed to the pip command
+    args = ["uninstall", "-y"]
+    if verbosity:
+        args += ["-" + "v" * verbosity]
+
+    _run_pip(args + [p[0] for p in reversed(_PROJECTS)])
+
+
+def _main(argv=None):
+    if ssl is None:
+        print("Ignoring ensurepip failure: {}".format(_MISSING_SSL_MESSAGE),
+              file=sys.stderr)
+        return
+
+    import argparse
+    parser = argparse.ArgumentParser(prog="python -m ensurepip")
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="pip {}".format(version()),
+        help="Show the version of pip that is bundled with this Python.",
+    )
+    parser.add_argument(
+        "-v", "--verbose",
+        action="count",
+        default=0,
+        dest="verbosity",
+        help=("Give more output. Option is additive, and can be used up to 3 "
+              "times."),
+    )
+    parser.add_argument(
+        "-U", "--upgrade",
+        action="store_true",
+        default=False,
+        help="Upgrade pip and dependencies, even if already installed.",
+    )
+    parser.add_argument(
+        "--user",
+        action="store_true",
+        default=False,
+        help="Install using the user scheme.",
+    )
+    parser.add_argument(
+        "--root",
+        default=None,
+        help="Install everything relative to this alternate root directory.",
+    )
+    parser.add_argument(
+        "--altinstall",
+        action="store_true",
+        default=False,
+        help=("Make an alternate install, installing only the X.Y versioned"
+              "scripts (Default: pipX, pipX.Y, easy_install-X.Y)"),
+    )
+    parser.add_argument(
+        "--default-pip",
+        action="store_true",
+        default=True,
+        dest="default_pip",
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--no-default-pip",
+        action="store_false",
+        dest="default_pip",
+        help=("Make a non default install, installing only the X and X.Y "
+              "versioned scripts."),
+    )
+
+    args = parser.parse_args(argv)
+
+    bootstrap(
+        root=args.root,
+        upgrade=args.upgrade,
+        user=args.user,
+        verbosity=args.verbosity,
+        altinstall=args.altinstall,
+        default_pip=args.default_pip,
+    )
diff --git a/lib-python/2.7/ensurepip/__main__.py b/lib-python/2.7/ensurepip/__main__.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/ensurepip/__main__.py
@@ -0,0 +1,4 @@
+import ensurepip
+
+if __name__ == "__main__":
+    ensurepip._main()
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..097ab43430d4c1302b0be353a8c16407c370693b
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..fa1d1054da1dab98f8906555d31a9fda271b3a85
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_uninstall.py b/lib-python/2.7/ensurepip/_uninstall.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/ensurepip/_uninstall.py
@@ -0,0 +1,30 @@
+"""Basic pip uninstallation support, helper for the Windows uninstaller"""
+
+import argparse
+import ensurepip
+
+
+def _main(argv=None):
+    parser = argparse.ArgumentParser(prog="python -m ensurepip._uninstall")
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="pip {}".format(ensurepip.version()),
+        help="Show the version of pip this will attempt to uninstall.",
+    )
+    parser.add_argument(
+        "-v", "--verbose",
+        action="count",
+        default=0,
+        dest="verbosity",
+        help=("Give more output. Option is additive, and can be used up to 3 "
+              "times."),
+    )
+
+    args = parser.parse_args(argv)
+
+    ensurepip._uninstall_helper(verbosity=args.verbosity)
+
+
+if __name__ == "__main__":
+    _main()
diff --git a/lib-python/2.7/glob.py b/lib-python/2.7/glob.py
--- a/lib-python/2.7/glob.py
+++ b/lib-python/2.7/glob.py
@@ -35,11 +35,16 @@
     patterns.
 
     """
+    dirname, basename = os.path.split(pathname)
     if not has_magic(pathname):
-        if os.path.lexists(pathname):
-            yield pathname
+        if basename:
+            if os.path.lexists(pathname):
+                yield pathname
+        else:
+            # Patterns ending with a slash should match only directories
+            if os.path.isdir(dirname):
+                yield pathname
         return
-    dirname, basename = os.path.split(pathname)
     if not dirname:
         for name in glob1(os.curdir, basename):
             yield name
diff --git a/lib-python/2.7/gzip.py b/lib-python/2.7/gzip.py
--- a/lib-python/2.7/gzip.py
+++ b/lib-python/2.7/gzip.py
@@ -164,9 +164,16 @@
     def _write_gzip_header(self):
         self.fileobj.write('\037\213')             # magic header
         self.fileobj.write('\010')                 # compression method
-        fname = os.path.basename(self.name)
-        if fname.endswith(".gz"):
-            fname = fname[:-3]
+        try:
+            # RFC 1952 requires the FNAME field to be Latin-1. Do not
+            # include filenames that cannot be represented that way.
+            fname = os.path.basename(self.name)
+            if not isinstance(fname, str):
+                fname = fname.encode('latin-1')
+            if fname.endswith('.gz'):
+                fname = fname[:-3]
+        except UnicodeEncodeError:
+            fname = ''
         flags = 0
         if fname:
             flags = FNAME
diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py
--- a/lib-python/2.7/hashlib.py
+++ b/lib-python/2.7/hashlib.py
@@ -15,8 +15,9 @@
 
 md5(), sha1(), sha224(), sha256(), sha384(), and sha512()
 
-More algorithms may be available on your platform but the above are
-guaranteed to exist.
+More algorithms may be available on your platform but the above are guaranteed
+to exist.  See the algorithms_guaranteed and algorithms_available attributes
+to find out what algorithm names can be passed to new().
 
 NOTE: If you want the adler32 or crc32 hash functions they are available in
 the zlib module.
@@ -58,9 +59,14 @@
 # always available algorithm is added.
 __always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')
 
+algorithms_guaranteed = set(__always_supported)
+algorithms_available = set(__always_supported)
+
 algorithms = __always_supported
 
-__all__ = __always_supported + ('new', 'algorithms', 'pbkdf2_hmac')
+__all__ = __always_supported + ('new', 'algorithms_guaranteed',
+                                'algorithms_available', 'algorithms',
+                                'pbkdf2_hmac')
 
 
 def __get_builtin_constructor(name):
@@ -128,6 +134,8 @@
     import _hashlib
     new = __hash_new
     __get_hash = __get_openssl_constructor
+    algorithms_available = algorithms_available.union(
+        _hashlib.openssl_md_meth_names)
 except ImportError:
     new = __py_new
     __get_hash = __get_builtin_constructor
diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py
--- a/lib-python/2.7/httplib.py
+++ b/lib-python/2.7/httplib.py
@@ -215,6 +215,10 @@
 # maximal line length when calling readline().
 _MAXLINE = 65536
 
+# maximum amount of headers accepted
+_MAXHEADERS = 100
+
+
 class HTTPMessage(mimetools.Message):
 
     def addheader(self, key, value):
@@ -271,6 +275,8 @@
         elif self.seekable:
             tell = self.fp.tell
         while True:
+            if len(hlist) > _MAXHEADERS:
+                raise HTTPException("got more than %d headers" % _MAXHEADERS)
             if tell:
                 try:
                     startofline = tell()
@@ -1185,21 +1191,29 @@
 
         def __init__(self, host, port=None, key_file=None, cert_file=None,
                      strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
-                     source_address=None):
+                     source_address=None, context=None):
             HTTPConnection.__init__(self, host, port, strict, timeout,
                                     source_address)
             self.key_file = key_file
             self.cert_file = cert_file
+            if context is None:
+                context = ssl._create_default_https_context()
+            if key_file or cert_file:
+                context.load_cert_chain(cert_file, key_file)
+            self._context = context
 
         def connect(self):
             "Connect to a host on a given (SSL) port."
 
-            sock = self._create_connection((self.host, self.port),
-                                          self.timeout, self.source_address)
+            HTTPConnection.connect(self)
+
             if self._tunnel_host:
-                self.sock = sock
-                self._tunnel()
-            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+                server_hostname = self._tunnel_host
+            else:
+                server_hostname = self.host
+
+            self.sock = self._context.wrap_socket(self.sock,
+                                                  server_hostname=server_hostname)
 
     __all__.append("HTTPSConnection")
 
@@ -1214,14 +1228,15 @@
         _connection_class = HTTPSConnection
 
         def __init__(self, host='', port=None, key_file=None, cert_file=None,
-                     strict=None):
+                     strict=None, context=None):
             # provide a default host, pass the X509 cert info
 
             # urf. compensate for bad input.
             if port == 0:
                 port = None
             self._setup(self._connection_class(host, port, key_file,
-                                               cert_file, strict))
+                                               cert_file, strict,
+                                               context=context))
 
             # we never actually use these for anything, but we keep them
             # here for compatibility with post-1.5.2 CVS.
diff --git a/lib-python/2.7/idlelib/Bindings.py b/lib-python/2.7/idlelib/Bindings.py
--- a/lib-python/2.7/idlelib/Bindings.py
+++ b/lib-python/2.7/idlelib/Bindings.py
@@ -75,7 +75,8 @@
    ('!_Auto-open Stack Viewer', '<>'),
    ]),
  ('options', [
-   ('_Configure IDLE...', '<>'),
+   ('Configure _IDLE', '<>'),
+   ('Configure _Extensions', '<>'),
    None,
    ]),
  ('help', [
diff --git a/lib-python/2.7/idlelib/CallTipWindow.py b/lib-python/2.7/idlelib/CallTipWindow.py
--- a/lib-python/2.7/idlelib/CallTipWindow.py
+++ b/lib-python/2.7/idlelib/CallTipWindow.py
@@ -2,9 +2,8 @@
 
 After ToolTip.py, which uses ideas gleaned from PySol
 Used by the CallTips IDLE extension.
-
 """
-from Tkinter import *
+from Tkinter import Toplevel, Label, LEFT, SOLID, TclError
 
 HIDE_VIRTUAL_EVENT_NAME = "<>"
 HIDE_SEQUENCES = ("", "")
@@ -133,35 +132,28 @@
         return bool(self.tipwindow)
 
 
-def _calltip_window(parent):
-    root = Tk()
-    root.title("Test calltips")
-    width, height, x, y = list(map(int, re.split('[x+]', parent.geometry())))
-    root.geometry("+%d+%d"%(x, y + 150))
+def _calltip_window(parent):  # htest #
+    from Tkinter import Toplevel, Text, LEFT, BOTH
 
-    class MyEditWin: # comparenceptually an editor_window
-        def __init__(self):
-            text = self.text = Text(root)
-            text.pack(side=LEFT, fill=BOTH, expand=1)
-            text.insert("insert", "string.split")
-            root.update()
-            self.calltip = CallTip(text)
+    top = Toplevel(parent)
+    top.title("Test calltips")
+    top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200,
+                  parent.winfo_rooty() + 150))
+    text = Text(top)
+    text.pack(side=LEFT, fill=BOTH, expand=1)
+    text.insert("insert", "string.split")
+    top.update()
+    calltip = CallTip(text)
 
-            text.event_add("<>", "(")
-            text.event_add("<>", ")")
-            text.bind("<>", self.calltip_show)
-            text.bind("<>", self.calltip_hide)
-
-            text.focus_set()
-            root.mainloop()
-
-        def calltip_show(self, event):
-            self.calltip.showtip("Hello world", "insert", "end")
-
-        def calltip_hide(self, event):
-            self.calltip.hidetip()
-
-    editwin = MyEditWin()
+    def calltip_show(event):
+        calltip.showtip("(s=Hello world)", "insert", "end")
+    def calltip_hide(event):
+        calltip.hidetip()
+    text.event_add("<>", "(")
+    text.event_add("<>", ")")
+    text.bind("<>", calltip_show)
+    text.bind("<>", calltip_hide)
+    text.focus_set()
 
 if __name__=='__main__':
     from idlelib.idle_test.htest import run
diff --git a/lib-python/2.7/idlelib/ClassBrowser.py b/lib-python/2.7/idlelib/ClassBrowser.py
--- a/lib-python/2.7/idlelib/ClassBrowser.py
+++ b/lib-python/2.7/idlelib/ClassBrowser.py
@@ -19,6 +19,9 @@
 from idlelib.TreeWidget import TreeNode, TreeItem, ScrolledCanvas
 from idlelib.configHandler import idleConf
 
+file_open = None  # Method...Item and Class...Item use this.
+# Normally PyShell.flist.open, but there is no PyShell.flist for htest.
+
 class ClassBrowser:
 
     def __init__(self, flist, name, path, _htest=False):
@@ -27,6 +30,9 @@
         """
         _htest - bool, change box when location running htest.
         """
+        global file_open
+        if not _htest:
+            file_open = PyShell.flist.open
         self.name = name
         self.file = os.path.join(path[0], self.name + ".py")
         self._htest = _htest
@@ -101,7 +107,7 @@
             return []
         try:
             dict = pyclbr.readmodule_ex(name, [dir] + sys.path)
-        except ImportError, msg:
+        except ImportError:
             return []
         items = []
         self.classes = {}
@@ -170,7 +176,7 @@
     def OnDoubleClick(self):
         if not os.path.exists(self.file):
             return
-        edit = PyShell.flist.open(self.file)
+        edit = file_open(self.file)
         if hasattr(self.cl, 'lineno'):
             lineno = self.cl.lineno
             edit.gotoline(lineno)
@@ -206,7 +212,7 @@
     def OnDoubleClick(self):
         if not os.path.exists(self.file):
             return
-        edit = PyShell.flist.open(self.file)
+        edit = file_open(self.file)
         edit.gotoline(self.cl.methods[self.name])
 
 def _class_browser(parent): #Wrapper for htest
@@ -221,8 +227,9 @@
     dir, file = os.path.split(file)
     name = os.path.splitext(file)[0]
     flist = PyShell.PyShellFileList(parent)
+    global file_open
+    file_open = flist.open
     ClassBrowser(flist, name, [dir], _htest=True)
-    parent.mainloop()
 
 if __name__ == "__main__":
     from idlelib.idle_test.htest import run
diff --git a/lib-python/2.7/idlelib/ColorDelegator.py b/lib-python/2.7/idlelib/ColorDelegator.py
--- a/lib-python/2.7/idlelib/ColorDelegator.py
+++ b/lib-python/2.7/idlelib/ColorDelegator.py
@@ -2,7 +2,6 @@
 import re
 import keyword
 import __builtin__
-from Tkinter import *
 from idlelib.Delegator import Delegator
 from idlelib.configHandler import idleConf
 
@@ -34,7 +33,6 @@
 
 prog = re.compile(make_pat(), re.S)
 idprog = re.compile(r"\s+(\w+)", re.S)
-asprog = re.compile(r".*?\b(as)\b")
 
 class ColorDelegator(Delegator):
 
@@ -42,7 +40,6 @@
         Delegator.__init__(self)
         self.prog = prog
         self.idprog = idprog
-        self.asprog = asprog
         self.LoadTagDefs()
 
     def setdelegate(self, delegate):
@@ -74,7 +71,6 @@
             "DEFINITION": idleConf.GetHighlight(theme, "definition"),
             "SYNC": {'background':None,'foreground':None},
             "TODO": {'background':None,'foreground':None},
-            "BREAK": idleConf.GetHighlight(theme, "break"),
             "ERROR": idleConf.GetHighlight(theme, "error"),
             # The following is used by ReplaceDialog:
             "hit": idleConf.GetHighlight(theme, "hit"),
@@ -216,22 +212,6 @@
                                     self.tag_add("DEFINITION",
                                                  head + "+%dc" % a,
                                                  head + "+%dc" % b)
-                            elif value == "import":
-                                # color all the "as" words on same line, except
-                                # if in a comment; cheap approximation to the
-                                # truth
-                                if '#' in chars:
-                                    endpos = chars.index('#')
-                                else:
-                                    endpos = len(chars)
-                                while True:
-                                    m1 = self.asprog.match(chars, b, endpos)
-                                    if not m1:
-                                        break
-                                    a, b = m1.span(1)
-                                    self.tag_add("KEYWORD",
-                                                 head + "+%dc" % a,
-                                                 head + "+%dc" % b)
                     m = self.prog.search(chars, m.end())
                 if "SYNC" in self.tag_names(next + "-1c"):
                     head = next
@@ -255,20 +235,23 @@
         for tag in self.tagdefs.keys():
             self.tag_remove(tag, "1.0", "end")
 
-def _color_delegator(parent):
+def _color_delegator(parent):  # htest #
+    from Tkinter import Toplevel, Text
     from idlelib.Percolator import Percolator
-    root = Tk()
-    root.title("Test ColorDelegator")
-    width, height, x, y = list(map(int, re.split('[x+]', parent.geometry())))
-    root.geometry("+%d+%d"%(x, y + 150))
-    source = "if somename: x = 'abc' # comment\nprint"
-    text = Text(root, background="white")
+
+    top = Toplevel(parent)
+    top.title("Test ColorDelegator")
+    top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200,
+                  parent.winfo_rooty() + 150))
+    source = "if somename: x = 'abc' # comment\nprint\n"
+    text = Text(top, background="white")
+    text.pack(expand=1, fill="both")
     text.insert("insert", source)

From noreply at buildbot.pypy.org  Wed Feb 25 13:38:00 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed, 25 Feb 2015 13:38:00 +0100 (CET)
Subject: [pypy-commit] pypy py3k-memoryview: Fix translation.
Message-ID: <20150225123800.395BF1C1186@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3k-memoryview
Changeset: r76130:b413d6c02268
Date: 2015-02-25 13:37 +0100
http://bitbucket.org/pypy/pypy/changeset/b413d6c02268/

Log:	Fix translation.

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -25,6 +25,8 @@
                                       reds=['items', 'w_iterator'])
 
 
+# It seems there's no way to do it without top-level-functions.
+
 @specialize.memo()
 def _does_override_buffer_w(type):
     return type.buffer_w != W_Root.buffer_w
@@ -33,6 +35,18 @@
 def _does_override_buffer_w_ex(type):
     return type.buffer_w_ex != W_Root.buffer_w_ex
 
+ at specialize.argtype(0)
+def W_Root_buffer_w(self, space, flags):
+    if _does_override_buffer_w_ex(self.__class__):
+        return self.buffer_w_ex(space, flags)[0]
+    return self._buffer(space, flags).buffer_w(space, flags)
+
+ at specialize.argtype(0)
+def W_Root_buffer_w_ex(self, space, flags):
+    if _does_override_buffer_w(self.__class__):
+        return self.buffer_w(space, flags), 'B', 1
+    return self._buffer(space, flags).buffer_w_ex(space, flags)
+
 
 class W_Root(object):
     """This is the abstract root class of all wrapped objects that live
@@ -203,17 +217,11 @@
     def immutable_unique_id(self, space):
         return None
 
-    @specialize.argtype(0)
     def buffer_w(self, space, flags):
-        if _does_override_buffer_w_ex(self.__class__):
-            return self.buffer_w_ex(space, flags)[0]
-        return self._buffer(space, flags).buffer_w(space, flags)
+        return W_Root_buffer_w(self, space, flags)
 
-    @specialize.argtype(0)
     def buffer_w_ex(self, space, flags):
-        if _does_override_buffer_w(self.__class__):
-            return self.buffer_w(space, flags), 'B', 1
-        return self._buffer(space, flags).buffer_w_ex(space, flags)
+        return W_Root_buffer_w_ex(self, space, flags)
 
     def _buffer(self, space, flags):
         w_impl = space.lookup(self, '__buffer__')

From noreply at buildbot.pypy.org  Wed Feb 25 15:27:01 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed, 25 Feb 2015 15:27:01 +0100 (CET)
Subject: [pypy-commit] pypy py3k-memoryview: Test and fix memory slices with
 non-bytes format.
Message-ID: <20150225142701.8AC901C009F@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3k-memoryview
Changeset: r76131:ed7ac9b0bbd8
Date: 2015-02-25 15:26 +0100
http://bitbucket.org/pypy/pypy/changeset/ed7ac9b0bbd8/

Log:	Test and fix memory slices with non-bytes format.

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -92,7 +92,7 @@
         else:
             buf = SubBuffer(self.buf, start * self.itemsize,
                             size * self.itemsize)
-            return W_MemoryView(buf)
+            return W_MemoryView(buf, self.format, self.itemsize)
 
     def descr_setitem(self, space, w_index, w_obj):
         self._check_released(space)
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -131,12 +131,27 @@
     def test_int_array_buffer(self):
         import array
         m = memoryview(array.array('i', list(range(10))))
+        assert m.format == 'i'
+        assert m.itemsize == 4
         assert len(m) == 10
         assert len(m.tobytes()) == 40
         assert m[0] == b'\x00\x00\x00\x00'
         m[0] = b'\x00\x00\x00\x01'
         assert m[0] == b'\x00\x00\x00\x01'
 
+    def test_int_array_slice(self):
+        import array
+        m = memoryview(array.array('i', list(range(10))))
+        slice = m[2:8]
+        assert slice.format == 'i'
+        assert slice.itemsize == 4
+        assert len(slice) == 6
+        assert len(slice.tobytes()) == 24
+        assert slice[0] in (b'\x00\x00\x00\x02', b'\x02\x00\x00\x00')
+        slice[0] = b'\x00\x00\x00\x01'
+        assert slice[0] == b'\x00\x00\x00\x01'
+        assert m[2] == b'\x00\x00\x00\x01'
+
     def test_pypy_raw_address_base(self):
         raises(ValueError, memoryview(b"foobar")._pypy_raw_address)
         e = raises(ValueError, memoryview(bytearray(b"foobar"))._pypy_raw_address)

From noreply at buildbot.pypy.org  Wed Feb 25 15:35:32 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 25 Feb 2015 15:35:32 +0100 (CET)
Subject: [pypy-commit] pypy default: (fijal, arigo)  Document the env vars
Message-ID: <20150225143532.81D441C135B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76132:ff8c9e514653
Date: 2015-02-25 15:35 +0100
http://bitbucket.org/pypy/pypy/changeset/ff8c9e514653/

Log:	(fijal, arigo) Document the env vars

diff --git a/rpython/doc/logging.rst b/rpython/doc/logging.rst
new file mode 100644
--- /dev/null
+++ b/rpython/doc/logging.rst
@@ -0,0 +1,55 @@
+Logging environment variables
+=============================
+
+PyPy, and all other RPython programs, support some special environment
+variables used to tweak various advanced parameters.
+
+
+Garbage collector
+-----------------
+
+Right now the default GC is (an incremental version of) MiniMark__.
+It has :ref:`a number of environment variables
+` that can be tweaked.  Their default
+value should be ok for most usages.
+
+.. __: garbage_collection.html#minimark-gc
+
+
+PYPYLOG
+-------
+
+The ``PYPYLOG`` environment variable enables debugging output.  For
+example::
+
+   PYPYLOG=jit:log
+
+means enabling all debugging output from the JIT, and writing to a
+file called ``log``.  More precisely, the condition ``jit`` means
+enabling output of all sections whose name start with ``jit``; other
+interesting names to use here are ``gc`` to get output from the GC, or
+``jit-backend`` to get only output from the JIT's machine code
+backend.  It is possible to use several prefixes, like in the
+following example::
+
+   PYPYLOG=jit-log-opt,jit-backend:log
+
+which outputs sections containing to the optimized loops plus anything
+produced from the JIT backend.  The above example is what you need for
+jitviewer_.
+
+.. _jitviewer: https://bitbucket.org/pypy/jitviewer
+
+The filename can be given as ``-`` to dump the log to stderr.
+
+As a special case, the value ``PYPYLOG=+filename`` means that only
+the section markers are written (for any section).  This is mostly
+only useful for ``rpython/tool/logparser.py``.
+
+
+PYPYSTM
+-------
+
+Only available in ``pypy-stm``.  Names a log file into which the
+PyPy-STM will output contention information.  Can be read with
+``pypy/stm/print_stm_log.py``.

From noreply at buildbot.pypy.org  Wed Feb 25 15:37:44 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 25 Feb 2015 15:37:44 +0100 (CET)
Subject: [pypy-commit] pypy default: remove unused file (?)
Message-ID: <20150225143744.94D101C009F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r76133:422ced33da44
Date: 2015-02-25 16:36 +0200
http://bitbucket.org/pypy/pypy/changeset/422ced33da44/

Log:	remove unused file (?)

diff --git a/rpython/translator/goal/order.py b/rpython/translator/goal/order.py
deleted file mode 100644
--- a/rpython/translator/goal/order.py
+++ /dev/null
@@ -1,56 +0,0 @@
-import sys
-import os
-
-RTYPERORDER = os.getenv('RTYPERORDER').split(',')
-if len(RTYPERORDER) == 2:
-    module_list = RTYPERORDER[1]
-else:
-    module_list = 'module-list'
-
-lst = open(module_list, 'r')
-try:
-    print "reading module-list: %s" % module_list
-    prefixes = lst.readlines()
-finally:
-    lst.close()
-
-prefixes = [line.strip() for line in prefixes]
-prefixes = [line for line in prefixes if line and not line.startswith('#')]
-
-NOMATCH = sys.maxint
-
-def order(annotator, pending):
-    cache = {}
-    annotated = annotator.annotated
-    def indx(block):
-        func = annotated[block]
-        module = func.__module__
-        if module is None:
-            module = 'None'
-        tag = "%s:%s" % (module, func.__name__)
-        try:
-            return cache[tag]
-        except KeyError:
-            match = NOMATCH
-            i = 0
-            for pfx in prefixes:
-                if tag.startswith(pfx):
-                    if match == NOMATCH:
-                        match = i
-                    else:
-                        if len(pfx) > len(prefixes[match]):
-                            match = i
-                i += 1
-            cache[tag] = match, module
-            return match
-
-    pending.sort(lambda blk1, blk2: cmp(indx(blk1), indx(blk2)))
-
-    cur_module = ['$']
-
-    def track(block):
-        module = annotated[block].__module__
-        if module != cur_module[0]:
-            print "--- Specializing blocks in module: %s" % module
-            cur_module[0] = module
-    return track

From noreply at buildbot.pypy.org  Wed Feb 25 15:40:08 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 25 Feb 2015 15:40:08 +0100 (CET)
Subject: [pypy-commit] pypy default: link.
Message-ID: <20150225144008.5C8D21C1186@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76134:c80440153082
Date: 2015-02-25 15:39 +0100
http://bitbucket.org/pypy/pypy/changeset/c80440153082/

Log:	link.

diff --git a/rpython/doc/index.rst b/rpython/doc/index.rst
--- a/rpython/doc/index.rst
+++ b/rpython/doc/index.rst
@@ -36,6 +36,7 @@
    :maxdepth: 1
 
    arm
+   logging
 
 
 Writing your own interpreter in RPython

From noreply at buildbot.pypy.org  Wed Feb 25 16:40:36 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed, 25 Feb 2015 16:40:36 +0100 (CET)
Subject: [pypy-commit] pypy py3k: 2to3
Message-ID: <20150225154036.33E3A1C1177@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3k
Changeset: r76135:817e1b862c7c
Date: 2015-02-25 16:32 +0100
http://bitbucket.org/pypy/pypy/changeset/817e1b862c7c/

Log:	2to3

diff --git a/lib_pypy/_gdbm.py b/lib_pypy/_gdbm.py
--- a/lib_pypy/_gdbm.py
+++ b/lib_pypy/_gdbm.py
@@ -72,9 +72,9 @@
     pass
 
 def _checkstr(key):
-    if isinstance(key, unicode):
+    if isinstance(key, str):
         key = key.encode("ascii")
-    if not isinstance(key, str):
+    if not isinstance(key, bytes):
         raise TypeError("gdbm mappings have string indices only")
     return key
 

From noreply at buildbot.pypy.org  Wed Feb 25 16:40:37 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed, 25 Feb 2015 16:40:37 +0100 (CET)
Subject: [pypy-commit] pypy py3k: 2to3
Message-ID: <20150225154037.74BF41C1177@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3k
Changeset: r76136:66a2427025a2
Date: 2015-02-25 16:36 +0100
http://bitbucket.org/pypy/pypy/changeset/66a2427025a2/

Log:	2to3

diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -517,6 +517,7 @@
 
     def test_reload_doesnt_override_sys_executable(self):
         import sys
+        from imp import reload
         sys.executable = 'from_test_sysmodule'
         reload(sys)
         assert sys.executable == 'from_test_sysmodule'

From noreply at buildbot.pypy.org  Wed Feb 25 16:40:38 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed, 25 Feb 2015 16:40:38 +0100 (CET)
Subject: [pypy-commit] pypy py3k: 2to3
Message-ID: <20150225154038.AC8E41C1177@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3k
Changeset: r76137:8d66bee0307a
Date: 2015-02-25 16:38 +0100
http://bitbucket.org/pypy/pypy/changeset/8d66bee0307a/

Log:	2to3

diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py
--- a/pypy/objspace/std/test/test_intobject.py
+++ b/pypy/objspace/std/test/test_intobject.py
@@ -486,7 +486,7 @@
 
     def test_bit_length_max(self):
         import sys
-        val = -sys.maxint-1
+        val = -sys.maxsize-1
         bits = 32 if val == -2147483648 else 64
         assert val.bit_length() == bits
 

From noreply at buildbot.pypy.org  Wed Feb 25 17:06:39 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed, 25 Feb 2015 17:06:39 +0100 (CET)
Subject: [pypy-commit] pypy py3k-memoryview: Close to-be-merged branch.
Message-ID: <20150225160639.060CE1C1177@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3k-memoryview
Changeset: r76138:86039a9a7ca3
Date: 2015-02-25 16:47 +0100
http://bitbucket.org/pypy/pypy/changeset/86039a9a7ca3/

Log:	Close to-be-merged branch.


From noreply at buildbot.pypy.org  Wed Feb 25 17:06:41 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed, 25 Feb 2015 17:06:41 +0100 (CET)
Subject: [pypy-commit] pypy py3k: hg merge py3k-memoryview
Message-ID: <20150225160641.5149F1C1177@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3k
Changeset: r76139:1091e0c8103b
Date: 2015-02-25 16:52 +0100
http://bitbucket.org/pypy/pypy/changeset/1091e0c8103b/

Log:	hg merge py3k-memoryview

	This fixes issue #1542.

diff --git a/lib-python/3/test/test_memoryview.py b/lib-python/3/test/test_memoryview.py
--- a/lib-python/3/test/test_memoryview.py
+++ b/lib-python/3/test/test_memoryview.py
@@ -50,7 +50,6 @@
         m = None
         self.assertEqual(getrefcount(b), oldrefcount)
 
-    @unittest.skip('XXX: https://bugs.pypy.org/issue1542')
     def test_getitem(self):
         for tp in self._types:
             self.check_getitem_with_type(tp)
@@ -75,7 +74,6 @@
         m = None
         self.assertEqual(getrefcount(b), oldrefcount)
 
-    @unittest.skip('XXX: https://bugs.pypy.org/issue1542')
     def test_setitem_writable(self):
         if not self.rw_type:
             return
@@ -128,7 +126,6 @@
             with self.assertRaises(TypeError):
                 del m[1:4]
 
-    @unittest.skip('XXX: https://bugs.pypy.org/issue1542')
     def test_tobytes(self):
         for tp in self._types:
             m = self._view(tp(self._source))
@@ -145,7 +142,6 @@
             l = m.tolist()
             self.assertEqual(l, list(b"abcdef"))
 
-    @unittest.skip('XXX: https://bugs.pypy.org/issue1542')
     def test_compare(self):
         # memoryviews can compare for equality with other objects
         # having the buffer interface.
@@ -193,7 +189,6 @@
         m = self.check_attributes_with_type(self.ro_type)
         self.assertEqual(m.readonly, True)
 
-    @unittest.skip('XXX: https://bugs.pypy.org/issue1542')
     def test_attributes_writable(self):
         if not self.rw_type:
             return
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -25,6 +25,29 @@
                                       reds=['items', 'w_iterator'])
 
 
+# It seems there's no way to do it without top-level-functions.
+
+ at specialize.memo()
+def _does_override_buffer_w(type):
+    return type.buffer_w != W_Root.buffer_w
+
+ at specialize.memo()
+def _does_override_buffer_w_ex(type):
+    return type.buffer_w_ex != W_Root.buffer_w_ex
+
+ at specialize.argtype(0)
+def W_Root_buffer_w(self, space, flags):
+    if _does_override_buffer_w_ex(self.__class__):
+        return self.buffer_w_ex(space, flags)[0]
+    return self._buffer(space, flags).buffer_w(space, flags)
+
+ at specialize.argtype(0)
+def W_Root_buffer_w_ex(self, space, flags):
+    if _does_override_buffer_w(self.__class__):
+        return self.buffer_w(space, flags), 'B', 1
+    return self._buffer(space, flags).buffer_w_ex(space, flags)
+
+
 class W_Root(object):
     """This is the abstract root class of all wrapped objects that live
     in a 'normal' object space like StdObjSpace."""
@@ -195,11 +218,17 @@
         return None
 
     def buffer_w(self, space, flags):
+        return W_Root_buffer_w(self, space, flags)
+
+    def buffer_w_ex(self, space, flags):
+        return W_Root_buffer_w_ex(self, space, flags)
+
+    def _buffer(self, space, flags):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
             w_result = space.get_and_call_function(w_impl, self)
             if space.isinstance_w(w_result, space.w_memoryview):
-                return w_result.buffer_w(space, flags)
+                return w_result
         raise TypeError
 
     def bytes_w(self, space):
@@ -1368,6 +1397,15 @@
             raise oefmt(self.w_TypeError,
                         "'%T' does not support the buffer interface", w_obj)
 
+    def buffer_w_ex(self, w_obj, flags):
+        # New buffer interface, returns a buffer based on flags (PyObject_GetBuffer)
+        # Returns extra information: (buffer, typecode, itemsize)
+        try:
+            return w_obj.buffer_w_ex(self, flags)
+        except TypeError:
+            raise oefmt(self.w_TypeError,
+                        "'%T' does not support the buffer interface", w_obj)
+
     def readbuf_w(self, w_obj):
         # Old buffer interface, returns a readonly buffer (PyObject_AsReadBuffer)
         try:
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -3,7 +3,6 @@
 from pypy.interpreter.gateway import unwrap_spec, interp2app
 from pypy.interpreter.typedef import TypeDef, make_weakref_descr
 from pypy.module._cffi_backend import cdataobj, ctypeptr, ctypearray
-from pypy.objspace.std.memoryobject import _buffer_setitem
 
 from rpython.rlib.buffer import Buffer
 from rpython.rtyper.annlowlevel import llstr
@@ -41,8 +40,6 @@
         copy_string_to_raw(llstr(string), raw_cdata, 0, len(string))
 
 
-# Override the typedef to narrow down the interface that's exposed to app-level
-
 class MiniBuffer(W_Root):
     def __init__(self, buffer, keepalive=None):
         self.buffer = buffer
@@ -63,7 +60,18 @@
         return space.wrapbytes(res)
 
     def descr_setitem(self, space, w_index, w_newstring):
-        _buffer_setitem(space, self.buffer, w_index, w_newstring)
+        start, stop, step, size = space.decode_index4(w_index,
+                                                      self.buffer.getlength())
+        if step not in (0, 1):
+            raise oefmt(space.w_NotImplementedError, "")
+        value = space.buffer_w(w_newstring, space.BUF_CONTIG_RO)
+        if value.getlength() != size:
+            raise oefmt(space.w_ValueError,
+                        "cannot modify size of memoryview object")
+        if step == 0:  # index only
+            self.buffer.setitem(start, value.getitem(0))
+        elif step == 1:
+            self.buffer.setslice(start, value.as_str())
 
 
 MiniBuffer.typedef = TypeDef(
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -59,10 +59,12 @@
 
 
 def descr_itemsize(space, self):
+    assert isinstance(self, W_ArrayBase)
     return space.wrap(self.itemsize)
 
 
 def descr_typecode(space, self):
+    assert isinstance(self, W_ArrayBase)
     return space.wrap(self.typecode)
 
 arr_eq_driver = jit.JitDriver(name='array_eq_driver', greens=['comp_func'],
@@ -135,8 +137,8 @@
         self.len = 0
         self.allocated = 0
 
-    def buffer_w(self, space, flags):
-        return ArrayBuffer(self, False)
+    def buffer_w_ex(self, space, flags):
+        return ArrayBuffer(self, False), self.typecode, self.itemsize
 
     def descr_append(self, space, w_x):
         """ append(x)
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -411,7 +411,16 @@
     def test_buffer(self):
         a = self.array('h', b'Hi')
         buf = memoryview(a)
-        assert buf[1] == b'i'
+        assert buf[0] == b'Hi'
+        raises(IndexError, 'buf[1]')
+        assert buf.tobytes() == b'Hi'
+        #assert buf.tolist() == [26952]
+        assert buf.format == 'h'
+        assert buf.itemsize == 2
+        assert buf.shape == (1,)
+        assert buf.ndim == 1
+        assert buf.strides == (2,)
+        assert not buf.readonly
 
     def test_buffer_write(self):
         a = self.array('b', b'hello')
@@ -431,6 +440,11 @@
         a.fromstring(b'some extra text')
         assert buf[:] == b'foobarbazsome extra text'
 
+    def test_memview_multi_tobytes(self):
+        a = self.array('i', list(b"abcdef"))
+        m = memoryview(a)
+        assert m.tobytes() == a.tobytes()
+
     def test_list_methods(self):
         assert repr(self.array('i')) == "array('i')"
         assert repr(self.array('i', [1, 2, 3])) == "array('i', [1, 2, 3])"
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -255,5 +255,3 @@
         return w_obj
     buffer = space.buffer_w(w_obj, space.BUF_FULL_RO)
     return space.wrapbytes(buffer.as_str())
-    
-
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -633,6 +633,7 @@
             "ctypes not implemented yet"))
 
     def buffer_w(self, space, flags):
+        # XXX format isn't always 'B' probably
         return self.implementation.get_buffer(space, True)
 
     def descr_get_data(self, space):
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -10,39 +10,25 @@
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 
 
-def _buffer_setitem(space, buf, w_index, w_obj):
-    if buf.readonly:
-        raise oefmt(space.w_TypeError, "cannot modify read-only memory")
-    start, stop, step, size = space.decode_index4(w_index, buf.getlength())
-    if step not in (0, 1):
-        raise oefmt(space.w_NotImplementedError, "")
-    value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
-    if value.getlength() != size:
-        raise oefmt(space.w_ValueError,
-                    "cannot modify size of memoryview object")
-    if step == 0:  # index only
-        buf.setitem(start, value.getitem(0))
-    elif step == 1:
-        buf.setslice(start, value.as_str())
-
-
 class W_MemoryView(W_Root):
     """Implement the built-in 'memoryview' type as a wrapper around
     an interp-level buffer.
     """
 
-    def __init__(self, buf):
+    def __init__(self, buf, format='B', itemsize=1):
         assert isinstance(buf, Buffer)
         self.buf = buf
+        self.format = format
+        self.itemsize = itemsize
 
-    def buffer_w(self, space, flags):
+    def buffer_w_ex(self, space, flags):
         self._check_released(space)
         space.check_buf_flags(flags, self.buf.readonly)
-        return self.buf
+        return self.buf, self.format, self.itemsize
 
     @staticmethod
     def descr_new_memoryview(space, w_subtype, w_object):
-        return W_MemoryView(space.buffer_w(w_object, space.BUF_FULL_RO))
+        return W_MemoryView(*space.buffer_w_ex(w_object, space.BUF_FULL_RO))
 
     def _make_descr__cmp(name):
         def descr__cmp(self, space, w_other):
@@ -71,10 +57,12 @@
     descr_ne = _make_descr__cmp('ne')
 
     def as_str(self):
-        return self.buf.as_str()
+        buf = self.buf
+        n_bytes = buf.getlength()
+        return buf.getslice(0, n_bytes, 1, n_bytes)
 
     def getlength(self):
-        return self.buf.getlength()
+        return self.buf.getlength() // self.itemsize
 
     def descr_tobytes(self, space):
         self._check_released(space)
@@ -83,9 +71,12 @@
     def descr_tolist(self, space):
         self._check_released(space)
         buf = self.buf
+        if self.format != 'B':
+            raise oefmt(space.w_NotImplementedError,
+                        "tolist() only supports byte views")
         result = []
         for i in range(buf.getlength()):
-            result.append(space.wrap(ord(buf.getitem(i))))
+            result.append(space.wrap(ord(buf.getitem(i)[0])))
         return space.newlist(result)
 
     def descr_getitem(self, space, w_index):
@@ -94,26 +85,39 @@
         if step not in (0, 1):
             raise oefmt(space.w_NotImplementedError, "")
         if step == 0:  # index only
-            return space.wrapbytes(self.buf.getitem(start))
+            a = start * self.itemsize
+            b = a + self.itemsize
+            return space.wrapbytes(
+                ''.join([self.buf.getitem(i) for i in range(a, b)]))
         else:
-            buf = SubBuffer(self.buf, start, size)
-            return W_MemoryView(buf)
+            buf = SubBuffer(self.buf, start * self.itemsize,
+                            size * self.itemsize)
+            return W_MemoryView(buf, self.format, self.itemsize)
 
     def descr_setitem(self, space, w_index, w_obj):
         self._check_released(space)
-        _buffer_setitem(space, self.buf, w_index, w_obj)
+        if self.buf.readonly:
+            raise oefmt(space.w_TypeError, "cannot modify read-only memory")
+        start, stop, step, size = space.decode_index4(w_index, self.getlength())
+        if step not in (0, 1):
+            raise oefmt(space.w_NotImplementedError, "")
+        value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
+        if value.getlength() != size * self.itemsize:
+            raise oefmt(space.w_ValueError,
+                        "cannot modify size of memoryview object")
+        self.buf.setslice(start * self.itemsize, value.as_str())
 
     def descr_len(self, space):
         self._check_released(space)
-        return space.wrap(self.buf.getlength())
+        return space.wrap(self.getlength())
 
     def w_get_format(self, space):
         self._check_released(space)
-        return space.wrap("B")
+        return space.wrap(self.format)
 
     def w_get_itemsize(self, space):
         self._check_released(space)
-        return space.wrap(1)
+        return space.wrap(self.itemsize)
 
     def w_get_ndim(self, space):
         self._check_released(space)
@@ -129,7 +133,7 @@
 
     def w_get_strides(self, space):
         self._check_released(space)
-        return space.newtuple([space.wrap(1)])
+        return space.newtuple([space.wrap(self.itemsize)])
 
     def w_get_suboffsets(self, space):
         self._check_released(space)
@@ -147,8 +151,8 @@
 
     def _check_released(self, space):
         if self.buf is None:
-            raise OperationError(space.w_ValueError, space.wrap(
-                    "operation forbidden on released memoryview object"))
+            raise oefmt(space.w_ValueError,
+                        "operation forbidden on released memoryview object")
 
     def descr_enter(self, space):
         self._check_released(space)
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -128,8 +128,31 @@
         raises(ValueError, bytes, v)
         assert "released memory" in repr(v)
 
+    def test_int_array_buffer(self):
+        import array
+        m = memoryview(array.array('i', list(range(10))))
+        assert m.format == 'i'
+        assert m.itemsize == 4
+        assert len(m) == 10
+        assert len(m.tobytes()) == 40
+        assert m[0] == b'\x00\x00\x00\x00'
+        m[0] = b'\x00\x00\x00\x01'
+        assert m[0] == b'\x00\x00\x00\x01'
+
+    def test_int_array_slice(self):
+        import array
+        m = memoryview(array.array('i', list(range(10))))
+        slice = m[2:8]
+        assert slice.format == 'i'
+        assert slice.itemsize == 4
+        assert len(slice) == 6
+        assert len(slice.tobytes()) == 24
+        assert slice[0] in (b'\x00\x00\x00\x02', b'\x02\x00\x00\x00')
+        slice[0] = b'\x00\x00\x00\x01'
+        assert slice[0] == b'\x00\x00\x00\x01'
+        assert m[2] == b'\x00\x00\x00\x01'
+
     def test_pypy_raw_address_base(self):
         raises(ValueError, memoryview(b"foobar")._pypy_raw_address)
         e = raises(ValueError, memoryview(bytearray(b"foobar"))._pypy_raw_address)
         assert 'BytearrayBuffer' in str(e.value)
-

From noreply at buildbot.pypy.org  Wed Feb 25 17:06:42 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed, 25 Feb 2015 17:06:42 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Introduce whatsnew-pypy3-head.rst and
 change test_whatsnew to use it.
Message-ID: <20150225160642.83FAA1C1177@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3k
Changeset: r76140:fdf7176248d9
Date: 2015-02-25 17:03 +0100
http://bitbucket.org/pypy/pypy/changeset/fdf7176248d9/

Log:	Introduce whatsnew-pypy3-head.rst and change test_whatsnew to use
	it.

diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py
--- a/pypy/doc/test/test_whatsnew.py
+++ b/pypy/doc/test/test_whatsnew.py
@@ -85,7 +85,7 @@
     #whatsnew_list = doc.listdir('whatsnew-*.rst')
     #whatsnew_list.sort()
     #last_whatsnew = whatsnew_list[-1].read()
-    last_whatsnew = doc.join('whatsnew-head.rst').read()
+    last_whatsnew = doc.join('whatsnew-pypy3-head.rst').read()
     startrev, documented = parse_doc(last_whatsnew)
     merged, branch = get_merged_branches(ROOT, startrev, '')
     merged.discard('default')
@@ -100,5 +100,5 @@
     print '\n'.join(not_merged)
     print
     assert not not_documented
-    if branch == 'default':
+    if branch == 'py3k':
         assert not not_merged
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -0,0 +1,6 @@
+========================
+What's new in PyPy3 2.4+
+========================
+
+.. this is the revision after pypy3-release-2.4.x was branched
+.. startrev: 3f967c2be00e

From noreply at buildbot.pypy.org  Wed Feb 25 17:06:43 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed, 25 Feb 2015 17:06:43 +0100 (CET)
Subject: [pypy-commit] pypy py3k: Document branch py3k-memoryview.
Message-ID: <20150225160643.AC2251C1177@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3k
Changeset: r76141:9074a59edfb1
Date: 2015-02-25 17:05 +0100
http://bitbucket.org/pypy/pypy/changeset/9074a59edfb1/

Log:	Document branch py3k-memoryview.

diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -4,3 +4,7 @@
 
 .. this is the revision after pypy3-release-2.4.x was branched
 .. startrev: 3f967c2be00e
+
+.. branch: py3k-memoryview
+
+Implement new memoryview features.

From noreply at buildbot.pypy.org  Wed Feb 25 18:44:53 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 25 Feb 2015 18:44:53 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: Try quite hard to move all the logic to C
Message-ID: <20150225174453.EF1311C02BD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r76142:254acb1e7174
Date: 2015-02-25 19:44 +0200
http://bitbucket.org/pypy/pypy/changeset/254acb1e7174/

Log:	Try quite hard to move all the logic to C

diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py
--- a/rpython/jit/backend/llsupport/codemap.py
+++ b/rpython/jit/backend/llsupport/codemap.py
@@ -17,17 +17,26 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 
-INITIAL_SIZE = 1000
-GROWTH_FACTOR = 4
-
-INT_LIST = lltype.Array(lltype.Signed) # raw, but with length
+INT_LIST = rffi.CArray(lltype.Signed)
 
 CODEMAP = lltype.Struct(
-    'codemap',
+    'pypy_codemap_item',
     ('addr', lltype.Signed),
     ('machine_code_size', lltype.Signed),
-    ('bytecode_info', lltype.Ptr(INT_LIST)))
-CODEMAP_LIST = lltype.Array(CODEMAP)
+    ('bytecode_info_size', lltype.Signed),
+    ('bytecode_info', lltype.Ptr(INT_LIST)),
+    hints=dict(external=True, c_name='pypy_codemap_item'))
+CODEMAP_LIST = rffi.CArray(CODEMAP)
+
+CODEMAP_STORAGE = lltype.Struct(
+    'pypy_codemap_storage',
+    ('jit_addr_map_used', lltype.Signed),
+    ('jit_frame_depth_map_used', lltype.Signed),
+    ('jit_codemap_used', lltype.Signed),
+    ('jit_addr_map', lltype.Ptr(INT_LIST)),
+    ('jit_frame_depth_map', lltype.Ptr(INT_LIST)),
+    ('jit_codemap', lltype.Ptr(CODEMAP_LIST)),
+    hints=dict(external=True, c_name='pypy_codemap_storage'))
 
 CODEMAP_GCARRAY = lltype.GcArray(CODEMAP)
 
@@ -36,23 +45,141 @@
 eci = ExternalCompilationInfo(post_include_bits=["""
 RPY_EXTERN volatile int pypy_codemap_currently_invalid;
 RPY_EXTERN void pypy_codemap_invalid_set(int);
+
+typedef struct pypy_codemap_item {
+   long addr, machine_code_size, bytecode_info_size;
+   long* bytecode_info;
+} pypy_codemap_item;
+
+typedef struct pypy_codemap_storage {
+   long jit_addr_map_used;
+   long jit_frame_depth_map_used;
+   long jit_codemap_used;
+   long* jit_addr_map;
+   long* jit_frame_depth_map;
+   pypy_codemap_item* jit_codemap;
+} pypy_codemap_storage;
+
+RPY_EXTERN pypy_codemap_storage *pypy_get_codemap_storage();
+RPY_EXTERN long pypy_jit_stack_depth_at_loc(long loc);
+RPY_EXTERN long pypy_find_codemap_at_addr(long addr);
+RPY_EXTERN long pypy_jit_start_addr(void);
+RPY_EXTERN long pypy_jit_end_addr(void);
+RPY_EXTERN long pypy_yield_codemap_at_addr(long, long, long*);
+
 """], separate_module_sources=["""
 volatile int pypy_codemap_currently_invalid = 0;
 
+static pypy_codemap_storage pypy_cs_g;
+
+long bisect_right(long *a, long x, long hi)
+{
+    long lo, mid;
+    lo = 0;
+    while (lo < hi) {
+        mid = (lo+hi) / 2;
+        if (x < a[mid]) { hi = mid; }
+        else { lo = mid+1; }
+    }
+    return lo;
+}
+
+long bisect_right_addr(pypy_codemap_item *a, long x, long hi)
+{
+    long lo, mid;
+    lo = 0;
+    while (lo < hi) {
+        mid = (lo+hi) / 2;
+        if (x < a[mid].addr) { hi = mid; }
+        else { lo = mid+1; }
+    }
+    return lo;
+}
+
+long pypy_jit_stack_depth_at_loc(long loc)
+{
+    long pos;
+    pos = bisect_right(pypy_cs_g.jit_addr_map, loc,
+                       pypy_cs_g.jit_addr_map_used);
+    if (pos == 0 || pos == pypy_cs_g.jit_addr_map_used)
+        return -1;
+    return pypy_cs_g.jit_frame_depth_map[pos - 1];
+}
+
+long pypy_find_codemap_at_addr(long addr)
+{
+    return bisect_right_addr(pypy_cs_g.jit_codemap, addr,
+                             pypy_cs_g.jit_codemap_used) - 1;
+}
+
+long pypy_jit_start_addr(void)
+{
+    return pypy_cs_g.jit_addr_map[0];
+}
+
+long pypy_jit_end_addr(void)
+{
+    return pypy_cs_g.jit_addr_map[pypy_cs_g.jit_addr_map_used - 1];
+}
+
+long pypy_yield_codemap_at_addr(long codemap_no, long addr,
+                                long* current_pos_addr)
+{
+    // will return consecutive unique_ids from codemap, starting from position
+    // `pos` until addr
+    pypy_codemap_item *codemap = &(pypy_cs_g.jit_codemap[codemap_no]);
+    long current_pos = *current_pos_addr;
+    long start_addr = codemap->addr;
+    long rel_addr = addr - start_addr;
+    long next_start, next_stop;
+
+    while (1) {
+        if (current_pos >= codemap->bytecode_info_size)
+            return 0;
+        next_start = codemap->bytecode_info[current_pos + 1];
+        if (next_start > rel_addr)
+            return 0;
+        next_stop = codemap->bytecode_info[current_pos + 2];
+        if (next_stop > rel_addr) {
+            *current_pos_addr = current_pos + 4;
+            return codemap->bytecode_info[current_pos];
+        }
+        // we need to skip potentially more than one
+        current_pos = codemap->bytecode_info[current_pos + 3];
+    }
+}
+
+pypy_codemap_storage *pypy_get_codemap_storage(void)
+{
+    return &pypy_cs_g;
+}
+
 void pypy_codemap_invalid_set(int value)
 {
     pypy_codemap_currently_invalid = value;
 }
 """])
 
-ll_pypy_codemap_invalid_set = rffi.llexternal('pypy_codemap_invalid_set',
-                                              [rffi.INT], lltype.Void,
-                                              compilation_info=eci,
-                                              releasegil=False)
+def llexternal(name, args, res):
+    return rffi.llexternal(name, args, res, compilation_info=eci,
+                           releasegil=False)
+
+ll_pypy_codemap_invalid_set = llexternal('pypy_codemap_invalid_set',
+                                         [rffi.INT], lltype.Void)
+pypy_get_codemap_storage = llexternal('pypy_get_codemap_storage',
+                                      [], lltype.Ptr(CODEMAP_STORAGE))
+stack_depth_at_loc = llexternal('pypy_jit_stack_depth_at_loc',
+                                [lltype.Signed], lltype.Signed)
+find_codemap_at_addr = llexternal('pypy_find_codemap_at_addr',
+                                  [lltype.Signed], lltype.Signed)
+yield_bytecode_at_addr = llexternal('pypy_yield_codemap_at_addr',
+                                    [lltype.Signed, lltype.Signed,
+                                     rffi.CArrayPtr(lltype.Signed)],
+                                     lltype.Signed)
 
 def pypy_codemap_invalid_set(val):
-    if we_are_translated():
-        ll_pypy_codemap_invalid_set(val)
+    #if we_are_translated():
+    ll_pypy_codemap_invalid_set(val)
 
 @specialize.ll()
 def copy_item(source, dest, si, di, baseline=0):
@@ -66,23 +193,28 @@
     # XXX this code has wrong complexity, we should come up with a better
     #     data structure ideally
     _mixin_ = True
+    jit_addr_map_allocated = 0
+    jit_codemap_allocated = 0
+    jit_frame_depth_map_allocated = 0
 
     @specialize.arg(1)
     def extend_with(self, name, to_insert, pos, baseline=0):
         # first check if we need to reallocate
-        used = getattr(self, name + '_used')
-        allocated = len(getattr(self, name))
-        lst = getattr(self, name)
+        g = pypy_get_codemap_storage()
+        used = getattr(g, name + '_used')
+        allocated = getattr(self, name + '_allocated')
+        lst = getattr(g, name)
         if used + len(to_insert) > allocated or pos != used:
             old_lst = lst
             if used + len(to_insert) > allocated:
-                new_size = max(4 * len(old_lst),
-                               (len(old_lst) + len(to_insert)) * 2)
+                new_size = max(4 * allocated,
+                               (allocated + len(to_insert)) * 2)
             else:
-                new_size = len(old_lst)
+                new_size = allocated
             lst = lltype.malloc(lltype.typeOf(lst).TO, new_size,
                                 flavor='raw',
-                                track_allocation=self.track_allocation)
+                                track_allocation=False)
+            setattr(self, name + '_allocated', new_size)
             for i in range(0, pos):
                 copy_item(old_lst, lst, i, i)
             j = 0
@@ -97,16 +229,14 @@
         else:
             for i in range(len(to_insert)):
                 copy_item(to_insert, lst, i, i + pos, baseline)
-        pypy_codemap_invalid_set(1)
-        setattr(self, name, lst)
-        setattr(self, name + '_used', len(to_insert) + used)
-        pypy_codemap_invalid_set(0)
+        setattr(g, name, lst)
+        setattr(g, name + '_used', len(to_insert) + used)
 
     @specialize.arg(1)
     def remove(self, name, start, end):
-        pypy_codemap_invalid_set(1)
-        lst = getattr(self, name)
-        used = getattr(self, name + '_used')
+        g = pypy_get_codemap_storage()
+        lst = getattr(g, name)
+        used = getattr(g, name + '_used')
         j = end
         for i in range(start, used - (end - start)):
             info = lltype.nullptr(INT_LIST)
@@ -118,164 +248,105 @@
                 if info:
                     lltype.free(info, flavor='raw', track_allocation=False)
             j += 1
-        setattr(self, name + '_used', used - (end - start))
-        pypy_codemap_invalid_set(0)
+        setattr(g, name + '_used', used - (end - start))
+
+    def free(self):
+        g = pypy_get_codemap_storage()
+        # if setup has not been called
+        if g.jit_addr_map_used:
+            lltype.free(g.jit_addr_map, flavor='raw', track_allocation=False)
+            g.jit_addr_map_used = 0
+            g.jit_addr_map = lltype.nullptr(INT_LIST)
+        i = 0
+        while i < g.jit_codemap_used:
+            lltype.free(g.jit_codemap[i].bytecode_info, flavor='raw',
+                        track_allocation=False)
+            i += 1
+        if g.jit_codemap_used:
+            lltype.free(g.jit_codemap, flavor='raw',
+                        track_allocation=False)
+            g.jit_codemap_used = 0
+            g.jit_codemap = lltype.nullptr(CODEMAP_LIST)
+        if g.jit_frame_depth_map_used:
+            lltype.free(g.jit_frame_depth_map, flavor='raw',
+                        track_allocation=False)
+            g.jit_frame_depth_map_used = 0
+            g.jit_frame_depth_map = lltype.nullptr(INT_LIST)
+
+    @specialize.arg(1)
+    def free_lst(self, name, lst):
+        if lst:
+            lltype.free(lst, flavor='raw', track_allocation=False)
 
 class CodemapStorage(ListStorageMixin):
     """ An immortal wrapper around underlaying jit codemap data
     """
-    track_allocation = False
-    jit_addr_map = lltype.nullptr(INT_LIST)
-    jit_addr_map_used = 0
-    jit_codemap = lltype.nullptr(CODEMAP_LIST)
-    jit_codemap_used = 0
-    jit_frame_depth_map = lltype.nullptr(INT_LIST)
-    jit_frame_depth_map_used = 0
-    
-    def __init__(self):
-        global _codemap
-
-        _codemap = self # a global singleton, for @entrypoint, self it's
-        # a prebuilt constant anyway
-
     def setup(self):
-        self.jit_addr_map = lltype.malloc(INT_LIST, INITIAL_SIZE, flavor='raw',
-                                          track_allocation=False)
-        self.jit_addr_map_used = 0
-        self.jit_codemap = lltype.malloc(CODEMAP_LIST, INITIAL_SIZE,
-                                         flavor='raw',
-                                         track_allocation=False)
-        self.jit_codemap_used = 0
-        self.jit_frame_depth_map = lltype.malloc(INT_LIST, INITIAL_SIZE,
-                                                 flavor='raw',
-                                                 track_allocation=False)
-        self.jit_frame_depth_map_used = 0
-
-    @specialize.arg(1)
-    def free_lst(self, name, lst):
-        lltype.free(lst, flavor='raw', track_allocation=False)
-
-    def __del__(self):
-        self.free()
-
-    def free(self):
-        # if setup has not been called
-        if not self.jit_addr_map:
-            return
-        lltype.free(self.jit_addr_map, flavor='raw')
-        i = 0
-        while i < self.jit_codemap_used:
-            lltype.free(self.jit_codemap[i].bytecode_info, flavor='raw')
-            i += 1
-        lltype.free(self.jit_codemap, flavor='raw')
-        lltype.free(self.jit_frame_depth_map, flavor='raw')
-        self.jit_adr_map = lltype.nullptr(INT_LIST)
+        g = pypy_get_codemap_storage()
+        if g.jit_addr_map_used != 0:
+             # someone failed to call free(), in tests only anyway
+             self.free()
 
     def free_asm_block(self, start, stop):
         # fix up jit_addr_map
-        jit_adr_start = bisect_left(self.jit_addr_map, start,
-                                    self.jit_addr_map_used)
-        jit_adr_stop = bisect_left(self.jit_addr_map, stop,
-                                   self.jit_addr_map_used)
+        g = pypy_get_codemap_storage()
+        jit_adr_start = bisect_left(g.jit_addr_map, start,
+                                    g.jit_addr_map_used)
+        jit_adr_stop = bisect_left(g.jit_addr_map, stop,
+                                   g.jit_addr_map_used)
+        pypy_codemap_invalid_set(1)
         self.remove('jit_addr_map', jit_adr_start, jit_adr_stop)
         self.remove('jit_frame_depth_map', jit_adr_start, jit_adr_stop)
         # fix up codemap
         # (there should only be zero or one codemap entry in that range,
         # but still we use a range to distinguish between zero and one)
-        codemap_adr_start = bisect_left_addr(self.jit_codemap, start,
-                                             self.jit_codemap_used)
-        codemap_adr_stop = bisect_left_addr(self.jit_codemap, stop,
-                                            self.jit_codemap_used)
+        codemap_adr_start = bisect_left_addr(g.jit_codemap, start,
+                                             g.jit_codemap_used)
+        codemap_adr_stop = bisect_left_addr(g.jit_codemap, stop,
+                                            g.jit_codemap_used)
         self.remove('jit_codemap', codemap_adr_start, codemap_adr_stop)
+        pypy_codemap_invalid_set(0)
 
     def register_frame_depth_map(self, rawstart, frame_positions,
                                  frame_assignments):
         if not frame_positions:
             return
-        if (not self.jit_addr_map_used or
-            rawstart > self.jit_addr_map[self.jit_addr_map_used - 1]):
-            start = self.jit_addr_map_used
+        pypy_codemap_invalid_set(1)
+        g = pypy_get_codemap_storage()
+        if (not g.jit_addr_map_used or
+            rawstart > g.jit_addr_map[g.jit_addr_map_used - 1]):
+            start = g.jit_addr_map_used
             self.extend_with('jit_addr_map', frame_positions,
-                             self.jit_addr_map_used, rawstart)
+                             g.jit_addr_map_used, rawstart)
             self.extend_with('jit_frame_depth_map', frame_assignments,
-                             self.jit_frame_depth_map_used)
+                             g.jit_frame_depth_map_used)
         else:
-            start = bisect_left(self.jit_addr_map, rawstart,
-                                self.jit_addr_map_used)
+            start = bisect_left(g.jit_addr_map, rawstart,
+                                g.jit_addr_map_used)
             self.extend_with('jit_addr_map', frame_positions, start, rawstart)
             self.extend_with('jit_frame_depth_map', frame_assignments,
                              start)
+        pypy_codemap_invalid_set(0)
 
     def register_codemap(self, codemap):
         start = codemap[0]
-        pos = bisect_left_addr(self.jit_codemap, start, self.jit_codemap_used)
+        g = pypy_get_codemap_storage()
+        pos = bisect_left_addr(g.jit_codemap, start, g.jit_codemap_used)
         items = lltype.malloc(INT_LIST, len(codemap[2]), flavor='raw',
-                             track_allocation=False)
+                              track_allocation=False)
         for i in range(len(codemap[2])):
             items[i] = codemap[2][i]
         s = lltype.malloc(CODEMAP_GCARRAY, 1)
         s[0].addr = codemap[0]
         s[0].machine_code_size = codemap[1]
         s[0].bytecode_info = items
+        s[0].bytecode_info_size = len(codemap[2])
+        pypy_codemap_invalid_set(1)
         self.extend_with('jit_codemap', s, pos)
+        pypy_codemap_invalid_set(0)
 
- at jit_entrypoint([lltype.Signed], lltype.Signed,
-                c_name='pypy_jit_stack_depth_at_loc')
- at rgc.no_collect
-def stack_depth_at_loc(loc):
-    global _codemap
-
-    pos = bisect_right(_codemap.jit_addr_map, loc, _codemap.jit_addr_map_used)
-    if pos == 0 or pos == _codemap.jit_addr_map_used:
-        return -1
-    return _codemap.jit_frame_depth_map[pos - 1]
-
- at jit_entrypoint([], lltype.Signed, c_name='pypy_jit_start_addr')
-def jit_start_addr():
-    global _codemap
-
-    return _codemap.jit_addr_map[0]
-
- at jit_entrypoint([], lltype.Signed, c_name='pypy_jit_end_addr')
-def jit_end_addr():
-    global _codemap
-
-    return _codemap.jit_addr_map[_codemap.jit_addr_map_used - 1]
-
- at jit_entrypoint([lltype.Signed], lltype.Signed,
-                c_name='pypy_find_codemap_at_addr')
-def find_codemap_at_addr(addr):
-    global _codemap
-
-    res = bisect_right_addr(_codemap.jit_codemap, addr,
-                            _codemap.jit_codemap_used) - 1
-    return res
-
- at jit_entrypoint([lltype.Signed, lltype.Signed,
-                 rffi.CArrayPtr(lltype.Signed)], lltype.Signed,
-                 c_name='pypy_yield_codemap_at_addr')
-def yield_bytecode_at_addr(codemap_no, addr, current_pos_addr):
-    """ will return consecutive unique_ids from codemap, starting from position
-    `pos` until addr
-    """
-    global _codemap
-
-    codemap = _codemap.jit_codemap[codemap_no]
-    current_pos = current_pos_addr[0]
-    start_addr = codemap.addr
-    rel_addr = addr - start_addr
-    while True:
-        if current_pos >= len(codemap.bytecode_info):
-            return 0
-        next_start = codemap.bytecode_info[current_pos + 1]
-        if next_start > rel_addr:
-            return 0
-        next_stop = codemap.bytecode_info[current_pos + 2]
-        if next_stop > rel_addr:
-            current_pos_addr[0] = current_pos + 4
-            return codemap.bytecode_info[current_pos]
-        # we need to skip potentially more than one
-        current_pos = codemap.bytecode_info[current_pos + 3]
+    def finish_once(self):
+        self.free()
 
 def unpack_traceback(addr):
     codemap_pos = find_codemap_at_addr(addr)
@@ -341,4 +412,3 @@
             item = self.l[i * 4 + 3] # end in l
             assert item > 0
         return (addr, size, self.l) # XXX compact self.l
-
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -79,6 +79,9 @@
     def setup(self):
         pass
 
+    def finish_once(self):
+        self.codemap.finish_once()
+
     def _setup_frame_realloc(self, translate_support_code):
         FUNC_TP = lltype.Ptr(lltype.FuncType([llmemory.GCREF, lltype.Signed],
                                              llmemory.GCREF))
diff --git a/rpython/jit/backend/llsupport/test/test_codemap.py b/rpython/jit/backend/llsupport/test/test_codemap.py
--- a/rpython/jit/backend/llsupport/test/test_codemap.py
+++ b/rpython/jit/backend/llsupport/test/test_codemap.py
@@ -1,39 +1,28 @@
 
 from rpython.jit.backend.llsupport.codemap import stack_depth_at_loc
 from rpython.jit.backend.llsupport.codemap import CodemapStorage,\
-     ListStorageMixin, INT_LIST, CodemapBuilder, unpack_traceback
-from rpython.rtyper.lltypesystem import lltype
+     ListStorageMixin, CodemapBuilder, unpack_traceback,\
+     pypy_get_codemap_storage
 
+g = pypy_get_codemap_storage()
 
 def test_list_storage_mixin():
     class X(ListStorageMixin):
-        track_allocation = True
-        
-        def __init__(self):
-            self.x = lltype.malloc(INT_LIST, 4, flavor='raw')
-            self.x_used = 0
-
         def unpack(self):
-            return [self.x[i] for i in range(self.x_used)]
-
-        def free_lst(self, name, lst):
-            lltype.free(lst, flavor='raw')
-        
-        def free(self):
-            lltype.free(self.x, flavor='raw')
+            return [g.jit_addr_map[i] for i in range(g.jit_addr_map_used)]
 
     x = X()
-    x.extend_with('x', [1, 2, 3], 0)
+    x.extend_with('jit_addr_map', [1, 2, 3], 0)
     assert x.unpack() == [1, 2, 3]
-    x.extend_with('x', [4, 5, 6], 3)
+    x.extend_with('jit_addr_map', [4, 5, 6], 3)
     assert x.unpack() == [1, 2, 3, 4, 5, 6]
-    x.extend_with('x', [7, 8, 9], 2, baseline=10)
+    x.extend_with('jit_addr_map', [7, 8, 9], 2, baseline=10)
     assert x.unpack() == [1, 2, 17, 18, 19, 3, 4, 5, 6]
-    x.remove('x', 3, 6)
+    x.remove('jit_addr_map', 3, 6)
     assert x.unpack() == [1, 2, 17, 4, 5, 6]
-    x.extend_with('x', [1] * 6, 6)
+    x.extend_with('jit_addr_map', [1] * 6, 6)
     assert x.unpack() == [1, 2, 17, 4, 5, 6, 1, 1, 1, 1, 1, 1]
-    x.extend_with('x', [10] * 4, 5)
+    x.extend_with('jit_addr_map', [10] * 4, 5)
     assert x.unpack() == [1, 2, 17, 4, 5, 10, 10, 10, 10, 6,
                           1, 1, 1, 1, 1, 1]
     x.free()
@@ -57,6 +46,7 @@
     assert stack_depth_at_loc(5) == 8
     assert stack_depth_at_loc(17) == 9
     assert stack_depth_at_loc(38) == 5
+    codemap.free()
 
 def test_codemaps():
     builder = CodemapBuilder()
@@ -93,3 +83,4 @@
     assert unpack_traceback(275) == [202]
     codemap.free_asm_block(200, 300)
     assert unpack_traceback(225) == []
+    codemap.free()

From noreply at buildbot.pypy.org  Wed Feb 25 18:46:29 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 25 Feb 2015 18:46:29 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: remove the need for get_virtual_ip in
 pypy (that does nothing anyway)
Message-ID: <20150225174629.216D11C02BD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r76143:873a6476e1de
Date: 2015-02-25 19:45 +0200
http://bitbucket.org/pypy/pypy/changeset/873a6476e1de/

Log:	remove the need for get_virtual_ip in pypy (that does nothing
	anyway)

diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -28,14 +28,13 @@
     libraries = ['unwind'],
     
     post_include_bits=["""
-        void* pypy_vmprof_get_virtual_ip(long);
         void pypy_vmprof_init(void);
     """],
     
     separate_module_sources=["""
         void pypy_vmprof_init(void) {
             vmprof_set_mainloop(pypy_execute_frame_trampoline, 0,
-                                pypy_vmprof_get_virtual_ip);
+                                NULL);
         }
     """],
     )
@@ -104,10 +103,6 @@
             return original_execute_frame(frame, w_inputvalue, operr)
 
 
- at entrypoint.entrypoint_lowlevel('main', [lltype.Signed],
-                                'pypy_vmprof_get_virtual_ip', True)
-def get_virtual_ip(unique_id):
-    return rffi.cast(rffi.VOIDP, unique_id)
 
 def write_long_to_string_builder(l, b):
     if sys.maxint == 2147483647:
diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
--- a/pypy/module/_vmprof/src/vmprof.c
+++ b/pypy/module/_vmprof/src/vmprof.c
@@ -176,7 +176,11 @@
           void **arg_ptr = (void**)arg_addr;
           // fprintf(stderr, "stacktrace mainloop: rsp %p   &f2 %p   offset %ld\n", 
           //         sp, arg_addr, mainloop_sp_offset);
-          ip = mainloop_get_virtual_ip(*arg_ptr);
+		  if (mainloop_get_virtual_ip) {
+			  ip = mainloop_get_virtual_ip(*arg_ptr);
+		  } else {
+			  ip = *arg_ptr;
+		  }
         }
 
         result[n++] = ip;

From noreply at buildbot.pypy.org  Wed Feb 25 18:50:20 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 25 Feb 2015 18:50:20 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: rpython fixes
Message-ID: <20150225175020.C13841C0183@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r76144:052d7e5d8a77
Date: 2015-02-25 19:49 +0200
http://bitbucket.org/pypy/pypy/changeset/052d7e5d8a77/

Log:	rpython fixes

diff --git a/rpython/rlib/rbisect.py b/rpython/rlib/rbisect.py
--- a/rpython/rlib/rbisect.py
+++ b/rpython/rlib/rbisect.py
@@ -1,20 +1,16 @@
 
-def bisect_left(a, x, hi=-1):
+def bisect_left(a, x, hi):
     """Return the index in the sorted list 'a' of 'x'.  If 'x' is not in 'a',
     return the index where it can be inserted."""
     lo = 0
-    if hi == -1:
-        hi = len(a)
     while lo < hi:
         mid = (lo+hi)//2
         if a[mid] < x: lo = mid+1
         else: hi = mid
     return lo
 
-def bisect_right(a, x, hi=-1):
+def bisect_right(a, x, hi):
     lo = 0
-    if hi == -1:
-        hi = len(a)
     while lo < hi:
         mid = (lo+hi)//2
         if x < a[mid]: hi = mid
@@ -22,20 +18,16 @@
     return lo
 
 # a copy of the above, but compares the item called 'addr' only
-def bisect_left_addr(a, x, hi=-1):
+def bisect_left_addr(a, x, hi):
     lo = 0
-    if hi == -1:
-        hi = len(a)
     while lo < hi:
         mid = (lo+hi)//2
         if a[mid].addr < x: lo = mid+1
         else: hi = mid
     return lo
 
-def bisect_right_addr(a, x, hi=-1):
+def bisect_right_addr(a, x, hi):
     lo = 0
-    if hi == -1:
-        hi = len(a)
     while lo < hi:
         mid = (lo+hi)//2
         if x < a[mid].addr: hi = mid

From noreply at buildbot.pypy.org  Wed Feb 25 18:56:57 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 25 Feb 2015 18:56:57 +0100 (CET)
Subject: [pypy-commit] pypy vmprof: fix?
Message-ID: <20150225175657.083DB1C0183@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: vmprof
Changeset: r76145:2f9e91643e10
Date: 2015-02-25 19:56 +0200
http://bitbucket.org/pypy/pypy/changeset/2f9e91643e10/

Log:	fix?

diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py
--- a/rpython/translator/c/node.py
+++ b/rpython/translator/c/node.py
@@ -126,8 +126,9 @@
         return self.prefix + name
 
     def verbatim_field_name(self, name):
-        assert name.startswith('c_')   # produced in this way by rffi
-        return name[2:]
+        if name.startswith('c_'):   # produced in this way by rffi
+            return name[2:]
+        return name
 
     def c_struct_field_type(self, name):
         return self.STRUCT._flds[name]

From noreply at buildbot.pypy.org  Wed Feb 25 19:07:49 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed, 25 Feb 2015 19:07:49 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: hg merge py3k
Message-ID: <20150225180749.E9DF41C12E6@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76146:61760b499e7c
Date: 2015-02-25 18:00 +0100
http://bitbucket.org/pypy/pypy/changeset/61760b499e7c/

Log:	hg merge py3k

diff too long, truncating to 2000 out of 3992 lines

diff --git a/lib_pypy/_gdbm.py b/lib_pypy/_gdbm.py
--- a/lib_pypy/_gdbm.py
+++ b/lib_pypy/_gdbm.py
@@ -20,9 +20,11 @@
 } datum;
 
 datum gdbm_fetch(void*, datum);
+datum pygdbm_fetch(void*, char*, int);
 int gdbm_delete(void*, datum);
 int gdbm_store(void*, datum, datum, int);
 int gdbm_exists(void*, datum);
+int pygdbm_exists(void*, char*, int);
 
 int gdbm_reorganize(void*);
 
@@ -37,19 +39,29 @@
 ''')
 
 try:
+    verify_code = '''
+    #include "gdbm.h"
+
+    static datum pygdbm_fetch(GDBM_FILE gdbm_file, char *dptr, int dsize) {
+        datum key = {dptr, dsize};
+        return gdbm_fetch(gdbm_file, key);
+    }
+
+    static int pygdbm_exists(GDBM_FILE gdbm_file, char *dptr, int dsize) {
+        datum key = {dptr, dsize};
+        return gdbm_exists(gdbm_file, key);
+    }
+    
+    '''
     if sys.platform.startswith('freebsd'):
         import os.path
         _localbase = os.environ.get('LOCALBASE', '/usr/local')
-        lib = ffi.verify('''
-        #include "gdbm.h"
-        ''', libraries=['gdbm'],
+        lib = ffi.verify(verify_code, libraries=['gdbm'],
              include_dirs=[os.path.join(_localbase, 'include')],
              library_dirs=[os.path.join(_localbase, 'lib')]
         )
     else:
-        lib = ffi.verify('''
-        #include "gdbm.h"
-        ''', libraries=['gdbm'])
+        lib = ffi.verify(verify_code, libraries=['gdbm'])
 except cffi.VerificationError as e:
     # distutils does not preserve the actual message,
     # but the verification is simple enough that the
@@ -59,6 +71,13 @@
 class error(IOError):
     pass
 
+def _checkstr(key):
+    if isinstance(key, str):
+        key = key.encode("ascii")
+    if not isinstance(key, bytes):
+        raise TypeError("gdbm mappings have string indices only")
+    return key
+
 def _fromstr(key):
     if isinstance(key, str):
         key = key.encode(sys.getdefaultencoding())
@@ -108,12 +127,14 @@
 
     def __contains__(self, key):
         self._check_closed()
-        return lib.gdbm_exists(self.ll_dbm, _fromstr(key))
+        key = _checkstr(key)
+        return lib.pygdbm_exists(self.ll_dbm, key, len(key))
     has_key = __contains__
 
     def get(self, key, default=None):
         self._check_closed()
-        drec = lib.gdbm_fetch(self.ll_dbm, _fromstr(key))
+        key = _checkstr(key)        
+        drec = lib.pygdbm_fetch(self.ll_dbm, key, len(key))
         if not drec.dptr:
             return default
         res = bytes(ffi.buffer(drec.dptr, drec.dsize))
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -36,7 +36,8 @@
    "PyPy home directory".  The arguments are:
 
    * ``home``: NULL terminated path to an executable inside the pypy directory
-     (can be a .so name, can be made up)
+     (can be a .so name, can be made up).  Used to look up the standard
+     library, and is also set as ``sys.executable``.
 
    * ``verbose``: if non-zero, it will print error messages to stderr
 
diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py
--- a/pypy/doc/test/test_whatsnew.py
+++ b/pypy/doc/test/test_whatsnew.py
@@ -85,7 +85,7 @@
     #whatsnew_list = doc.listdir('whatsnew-*.rst')
     #whatsnew_list.sort()
     #last_whatsnew = whatsnew_list[-1].read()
-    last_whatsnew = doc.join('whatsnew-head.rst').read()
+    last_whatsnew = doc.join('whatsnew-pypy3-head.rst').read()
     startrev, documented = parse_doc(last_whatsnew)
     merged, branch = get_merged_branches(ROOT, startrev, '')
     merged.discard('default')
@@ -100,5 +100,5 @@
     print '\n'.join(not_merged)
     print
     assert not not_documented
-    if branch == 'default':
+    if branch == 'py3k':
         assert not not_merged
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -6,8 +6,8 @@
 .. startrev: 397b96217b85
 
 
-Fix non-blocking file reads sometimes raising EAGAIN even though they
-have buffered data waiting (b1c4fcb04a42)
+Non-blocking file reads sometimes raised EAGAIN even though they
+had buffered data waiting, fixed in b1c4fcb04a42
 
 
 .. branch: vmprof
@@ -18,3 +18,19 @@
 
 .. branch: stdlib-2.7.9
 Update stdlib to version 2.7.9
+
+.. branch: fix-kqueue-error2
+Fix exception being raised by kqueue.control (CPython compatibility)
+
+.. branch: gitignore
+
+.. branch: framestate2
+Refactor rpython.flowspace.framestate.FrameState.
+
+.. branch: alt_errno
+Add an alternative location to save LastError, errno around ctypes,
+cffi external calls so things like pdb will not overwrite it
+
+.. branch: nonquadratic-heapcache
+Speed up the warmup times of the JIT by removing a quadratic algorithm in the
+heapcache.
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -0,0 +1,10 @@
+========================
+What's new in PyPy3 2.4+
+========================
+
+.. this is the revision after pypy3-release-2.4.x was branched
+.. startrev: 3f967c2be00e
+
+.. branch: py3k-memoryview
+
+Implement new memoryview features.
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -113,6 +113,9 @@
         space.call_function(w_pathsetter, w_path)
         # import site
         try:
+            space.setattr(space.getbuiltinmodule('sys'),
+                          space.wrap('executable'),
+                          space.wrap(home))
             import_ = space.getattr(space.getbuiltinmodule('builtins'),
                                     space.wrap('__import__'))
             space.call_function(import_, space.wrap('site'))
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -1,5 +1,5 @@
 #! /usr/bin/env python
-# App-level version of py.py.
+# This is pure Python code that handles the main entry point into "pypy".
 # See test/test_app_main.
 
 # Missing vs CPython: -b, -d, -v, -x, -3
@@ -158,11 +158,14 @@
             current = group
     raise SystemExit
 
+def get_sys_executable():
+    return getattr(sys, 'executable', 'pypy')
+
 def print_help(*args):
     import os
     initstdio()
     print('usage: %s [option] ... [-c cmd | -m mod | file | -] [arg] ...' % (
-        sys.executable,))
+        get_sys_executable(),))
     print(USAGE1, end='')
     if 'pypyjit' in sys.builtin_module_names:
         print("--jit options: advanced JIT options: try 'off' or 'help'")
@@ -174,7 +177,7 @@
     try:
         import pypyjit
     except ImportError:
-        print("No jit support in %s" % (sys.executable,), file=sys.stderr)
+        print("No jit support in %s" % (get_sys_executable(),), file=sys.stderr)
         return
     items = sorted(pypyjit.defaults.items())
     print('Advanced JIT options: a comma-separated list of OPTION=VALUE:')
@@ -213,7 +216,7 @@
         raise SystemExit
     if 'pypyjit' not in sys.builtin_module_names:
         initstdio()
-        print("Warning: No jit support in %s" % (sys.executable,),
+        print("Warning: No jit support in %s" % (get_sys_executable(),),
               file=sys.stderr)
     else:
         import pypyjit
@@ -224,8 +227,8 @@
 
 def print_error(msg):
     print(msg, file=sys.stderr)
-    print('usage: %s [options]' % (sys.executable,), file=sys.stderr)
-    print('Try `%s -h` for more information.' % (sys.executable,), file=sys.stderr)
+    print('usage: %s [options]' % (get_sys_executable(),), file=sys.stderr)
+    print('Try `%s -h` for more information.' % (get_sys_executable(),), file=sys.stderr)
 
 def fdopen(fd, mode, bufsize=-1):
     try:
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -25,6 +25,29 @@
                                       reds=['items', 'w_iterator'])
 
 
+# It seems there's no way to do it without top-level-functions.
+
+ at specialize.memo()
+def _does_override_buffer_w(type):
+    return type.buffer_w != W_Root.buffer_w
+
+ at specialize.memo()
+def _does_override_buffer_w_ex(type):
+    return type.buffer_w_ex != W_Root.buffer_w_ex
+
+ at specialize.argtype(0)
+def W_Root_buffer_w(self, space, flags):
+    if _does_override_buffer_w_ex(self.__class__):
+        return self.buffer_w_ex(space, flags)[0]
+    return self._buffer(space, flags).buffer_w(space, flags)
+
+ at specialize.argtype(0)
+def W_Root_buffer_w_ex(self, space, flags):
+    if _does_override_buffer_w(self.__class__):
+        return self.buffer_w(space, flags), 'B', 1
+    return self._buffer(space, flags).buffer_w_ex(space, flags)
+
+
 class W_Root(object):
     """This is the abstract root class of all wrapped objects that live
     in a 'normal' object space like StdObjSpace."""
@@ -195,11 +218,17 @@
         return None
 
     def buffer_w(self, space, flags):
+        return W_Root_buffer_w(self, space, flags)
+
+    def buffer_w_ex(self, space, flags):
+        return W_Root_buffer_w_ex(self, space, flags)
+
+    def _buffer(self, space, flags):
         w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
             w_result = space.get_and_call_function(w_impl, self)
             if space.isinstance_w(w_result, space.w_memoryview):
-                return w_result.buffer_w(space, flags)
+                return w_result
         raise TypeError
 
     def bytes_w(self, space):
@@ -1379,6 +1408,15 @@
             raise oefmt(self.w_TypeError,
                         "'%T' does not support the buffer interface", w_obj)
 
+    def buffer_w_ex(self, w_obj, flags):
+        # New buffer interface, returns a buffer based on flags (PyObject_GetBuffer)
+        # Returns extra information: (buffer, typecode, itemsize)
+        try:
+            return w_obj.buffer_w_ex(self, flags)
+        except TypeError:
+            raise oefmt(self.w_TypeError,
+                        "'%T' does not support the buffer interface", w_obj)
+
     def readbuf_w(self, w_obj):
         # Old buffer interface, returns a readonly buffer (PyObject_AsReadBuffer)
         try:
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -404,7 +404,7 @@
         config = make_config(None)
         space = make_objspace(config)
         w_executable = space.wrap('executable')
-        assert space.str_w(space.getattr(space.sys, w_executable)) == 'py.py'
+        assert space.findattr(space.sys, w_executable) is None
         space.setattr(space.sys, w_executable, space.wrap('foobar'))
         assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
         space.startup()
diff --git a/pypy/interpreter/test/test_targetpypy.py b/pypy/interpreter/test/test_targetpypy.py
--- a/pypy/interpreter/test/test_targetpypy.py
+++ b/pypy/interpreter/test/test_targetpypy.py
@@ -8,7 +8,7 @@
         entry_point = get_entry_point(config)[0]
         entry_point(['pypy-c' , '-S', '-c', 'print 3'])
 
-def test_exeucte_source(space):
+def test_execute_source(space):
     _, d = create_entry_point(space, None)
     execute_source = d['pypy_execute_source']
     lls = rffi.str2charp("import sys; sys.modules['xyz'] = 3")
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -3,7 +3,6 @@
 from pypy.interpreter.gateway import unwrap_spec, interp2app
 from pypy.interpreter.typedef import TypeDef, make_weakref_descr
 from pypy.module._cffi_backend import cdataobj, ctypeptr, ctypearray
-from pypy.objspace.std.memoryobject import _buffer_setitem
 
 from rpython.rlib.buffer import Buffer
 from rpython.rtyper.annlowlevel import llstr
@@ -41,8 +40,6 @@
         copy_string_to_raw(llstr(string), raw_cdata, 0, len(string))
 
 
-# Override the typedef to narrow down the interface that's exposed to app-level
-
 class MiniBuffer(W_Root):
     def __init__(self, buffer, keepalive=None):
         self.buffer = buffer
@@ -63,7 +60,18 @@
         return space.wrapbytes(res)
 
     def descr_setitem(self, space, w_index, w_newstring):
-        _buffer_setitem(space, self.buffer, w_index, w_newstring)
+        start, stop, step, size = space.decode_index4(w_index,
+                                                      self.buffer.getlength())
+        if step not in (0, 1):
+            raise oefmt(space.w_NotImplementedError, "")
+        value = space.buffer_w(w_newstring, space.BUF_CONTIG_RO)
+        if value.getlength() != size:
+            raise oefmt(space.w_ValueError,
+                        "cannot modify size of memoryview object")
+        if step == 0:  # index only
+            self.buffer.setitem(start, value.getitem(0))
+        elif step == 1:
+            self.buffer.setslice(start, value.as_str())
 
 
 MiniBuffer.typedef = TypeDef(
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -210,6 +210,6 @@
         space.threadlocals.leave_thread(space)
 
 def invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata):
-    cerrno._errno_after(rffi.RFFI_ERR_ALL)
+    cerrno._errno_after(rffi.RFFI_ERR_ALL | rffi.RFFI_ALT_ERRNO)
     _invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata)
-    cerrno._errno_before(rffi.RFFI_ERR_ALL)
+    cerrno._errno_before(rffi.RFFI_ERR_ALL | rffi.RFFI_ALT_ERRNO)
diff --git a/pypy/module/_cffi_backend/cerrno.py b/pypy/module/_cffi_backend/cerrno.py
--- a/pypy/module/_cffi_backend/cerrno.py
+++ b/pypy/module/_cffi_backend/cerrno.py
@@ -13,18 +13,18 @@
 _errno_after  = rposix._errno_after
 
 def get_errno(space):
-    return space.wrap(rposix.get_saved_errno())
+    return space.wrap(rposix.get_saved_alterrno())
 
 @unwrap_spec(errno=int)
 def set_errno(space, errno):
-    rposix.set_saved_errno(errno)
+    rposix.set_saved_alterrno(errno)
 
 # ____________________________________________________________
 
 @unwrap_spec(code=int)
 def getwinerror(space, code=-1):
-    from rpython.rlib.rwin32 import GetLastError_saved, FormatError
+    from rpython.rlib.rwin32 import GetLastError_alt_saved, FormatError
     if code == -1:
-        code = GetLastError_saved()
+        code = GetLastError_alt_saved()
     message = FormatError(code)
     return space.newtuple([space.wrap(code), space.wrap(message)])
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2716,6 +2716,14 @@
     assert data == b"Xhello\n"
     posix.close(fdr)
 
+def test_errno_saved():
+    set_errno(42)
+    # a random function that will reset errno to 0 (at least on non-windows)
+    import os; os.stat('.')
+    #
+    res = get_errno()
+    assert res == 42
+
 def test_GetLastError():
     if sys.platform != "win32":
         py.test.skip("GetLastError(): only for Windows")
diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py
--- a/pypy/module/_rawffi/array.py
+++ b/pypy/module/_rawffi/array.py
@@ -15,7 +15,7 @@
 from pypy.module._rawffi.interp_rawffi import unpack_shape_with_length
 from pypy.module._rawffi.interp_rawffi import read_ptr, write_ptr
 from rpython.rlib.rarithmetic import r_uint
-from rpython.rlib import rgc
+from rpython.rlib import rgc, clibffi
 
 
 class W_Array(W_DataShape):
@@ -84,14 +84,11 @@
 
 class W_ArrayInstance(W_DataInstance):
     def __init__(self, space, shape, length, address=r_uint(0)):
-        # Workaround for a strange behavior of libffi: make sure that
-        # we always have at least 8 bytes.  For W_ArrayInstances that are
-        # used as the result value of a function call, ffi_call() writes
-        # 8 bytes into it even if the function's result type asks for less.
-        # This strange behavior is documented.
         memsize = shape.size * length
-        if memsize < 8:
-            memsize = 8
+        # For W_ArrayInstances that are used as the result value of a
+        # function call, ffi_call() writes 8 bytes into it even if the
+        # function's result type asks for less.
+        memsize = clibffi.adjust_return_size(memsize)
         W_DataInstance.__init__(self, space, memsize, address)
         self.length = length
         self.shape = shape
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -498,6 +498,7 @@
         try:
             if self.resshape is not None:
                 result = self.resshape.allocate(space, 1, autofree=True)
+                # adjust_return_size() was used here on result.ll_buffer
                 self.ptr.call(args_ll, result.ll_buffer)
                 return space.wrap(result)
             else:
@@ -611,19 +612,19 @@
     return space.wrap(W_CDLL(space, name, cdll))
 
 def get_errno(space):
-    return space.wrap(rposix.get_saved_errno())
+    return space.wrap(rposix.get_saved_alterrno())
 
 def set_errno(space, w_errno):
-    rposix.set_saved_errno(space.int_w(w_errno))
+    rposix.set_saved_alterrno(space.int_w(w_errno))
 
 if sys.platform == 'win32':
     # see also
     # https://bitbucket.org/pypy/pypy/issue/1944/ctypes-on-windows-getlasterror
     def get_last_error(space):
-        return space.wrap(rwin32.GetLastError_saved())
+        return space.wrap(rwin32.GetLastError_alt_saved())
     @unwrap_spec(error=int)
     def set_last_error(space, error):
-        rwin32.SetLastError_saved(error)
+        rwin32.SetLastError_alt_saved(error)
 else:
     # always have at least a dummy version of these functions
     # (https://bugs.pypy.org/issue1242)
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -4,6 +4,7 @@
 from rpython.rlib.rarithmetic import intmask, widen, r_uint
 from rpython.rlib import rpoll, rsocket, rthread, rweakref
 from rpython.rlib.ropenssl import *
+from rpython.rlib._rsocket_rffi import MAX_FD_SIZE
 from rpython.rlib.rposix import get_saved_errno
 from rpython.rlib.rweakref import RWeakValueDictionary
 from rpython.rlib.objectmodel import specialize, compute_unique_id
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -236,6 +236,9 @@
 
     def test_npn_protocol(self):
         import socket, _ssl, gc
+        if not _ssl.HAS_NPN:
+            skip("NPN requires OpenSSL 1.0.1 or greater")
+
         ctx = _ssl._SSLContext()
         ctx._set_npn_protocols(b'\x08http/1.1\x06spdy/2')
         ss = ctx._wrap_socket(self.s, True,
@@ -306,12 +309,13 @@
             os.path.dirname(__file__), 'dh512.pem'))
 
     def test_load_cert_chain(self):
-        import _ssl
+        import _ssl, errno
         ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
         ctx.load_cert_chain(self.keycert)
         ctx.load_cert_chain(self.cert, self.key)
-        raises(IOError, ctx.load_cert_chain, "inexistent.pem")
-        raises(_ssl.SSLError, ctx.load_cert_chain, self.badcert)
+        exc = raises(IOError, ctx.load_cert_chain, "inexistent.pem")
+        assert exc.value.errno == errno.ENOENT
+        exc = raises(_ssl.SSLError, ctx.load_cert_chain, self.badcert)
         raises(_ssl.SSLError, ctx.load_cert_chain, self.emptycert)
         # Password protected key and cert
         raises(_ssl.SSLError, ctx.load_cert_chain, self.cert_protected,
@@ -370,12 +374,14 @@
         assert ctx.cert_store_stats() == {'x509_ca': 0, 'crl': 0, 'x509': 1}
 
     def test_load_dh_params(self):
-        import _ssl
+        import _ssl, errno
         ctx = _ssl._SSLContext(_ssl.PROTOCOL_TLSv1)
         ctx.load_dh_params(self.dh512)
         raises(TypeError, ctx.load_dh_params)
         raises(TypeError, ctx.load_dh_params, None)
         raises(_ssl.SSLError, ctx.load_dh_params, self.keycert)
+        exc = raises(IOError, ctx.load_dh_params, "inexistent.pem")
+        assert exc.value.errno == errno.ENOENT
 
     def test_set_ecdh_curve(self):
         import _ssl
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -59,10 +59,12 @@
 
 
 def descr_itemsize(space, self):
+    assert isinstance(self, W_ArrayBase)
     return space.wrap(self.itemsize)
 
 
 def descr_typecode(space, self):
+    assert isinstance(self, W_ArrayBase)
     return space.wrap(self.typecode)
 
 arr_eq_driver = jit.JitDriver(name='array_eq_driver', greens=['comp_func'],
@@ -135,8 +137,8 @@
         self.len = 0
         self.allocated = 0
 
-    def buffer_w(self, space, flags):
-        return ArrayBuffer(self, False)
+    def buffer_w_ex(self, space, flags):
+        return ArrayBuffer(self, False), self.typecode, self.itemsize
 
     def descr_append(self, space, w_x):
         """ append(x)
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -413,7 +413,16 @@
     def test_buffer(self):
         a = self.array('h', b'Hi')
         buf = memoryview(a)
-        assert buf[1] == ord('i')
+        assert buf[0] == 26952
+        raises(IndexError, 'buf[1]')
+        assert buf.tobytes() == b'Hi'
+        assert buf.tolist() == [26952]
+        assert buf.format == 'h'
+        assert buf.itemsize == 2
+        assert buf.shape == (1,)
+        assert buf.ndim == 1
+        assert buf.strides == (2,)
+        assert not buf.readonly
 
     def test_buffer_write(self):
         a = self.array('b', b'hello')
@@ -433,6 +442,11 @@
         a.fromstring(b'some extra text')
         assert buf[:] == b'foobarbazsome extra text'
 
+    def test_memview_multi_tobytes(self):
+        a = self.array('i', list(b"abcdef"))
+        m = memoryview(a)
+        assert m.tobytes() == a.tobytes()
+
     def test_list_methods(self):
         assert repr(self.array('i')) == "array('i')"
         assert repr(self.array('i', [1, 2, 3])) == "array('i', [1, 2, 3])"
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -255,5 +255,3 @@
         return w_obj
     buffer = space.buffer_w(w_obj, space.BUF_FULL_RO)
     return space.wrapbytes(buffer.as_str())
-    
-
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,7 +29,7 @@
 #define PY_VERSION		"3.3.5"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "2.6.0"
+#define PYPY_VERSION "2.6.0-alpha0"
 
 /* Subversion Revision number of this file (not of the repository).
  * Empty since Mercurial migration. */
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -633,6 +633,7 @@
             "ctypes not implemented yet"))
 
     def buffer_w(self, space, flags):
+        # XXX format isn't always 'B' probably
         return self.implementation.get_buffer(space, True)
 
     def descr_get_data(self, space):
diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -202,7 +202,7 @@
         assert loop.match_by_id('cfficall', """
             p96 = force_token()
             setfield_gc(p0, p96, descr=)
-            f97 = call_release_gil(27, i59, 1.0, 3, descr=)
+            f97 = call_release_gil(91, i59, 1.0, 3, descr=)
             guard_not_forced(descr=...)
             guard_no_exception(descr=...)
         """, ignore_ops=['guard_not_invalidated'])
diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py
--- a/pypy/module/pypyjit/test_pypy_c/test_thread.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py
@@ -67,21 +67,10 @@
         i58 = call_release_gil(0, _, i37, 1, descr=)
         guard_not_forced(descr=...)
         guard_no_exception(descr=...)
-        i59 = int_is_true(i58)
-        guard_true(i59, descr=...)
-        i60 = int_sub(i44, 1)
-        p62 = force_token()
-        setfield_gc(p0, p62, descr=)
-        i63 = call_release_gil(0, _, i37, 0, descr=)
-        guard_not_forced(descr=...)
-        guard_no_exception(descr=...)
-        i64 = int_is_true(i63)
-        guard_false(i64, descr=...)
-        p65 = force_token()
-        setfield_gc(p0, p65, descr=)
-        call_release_gil(0, _, i37, descr=)
-        guard_not_forced(descr=...)
-        guard_no_exception(descr=...)
+        i58 = int_sub(i44, 1)
+        i59 = call(ConstClass(RPyThreadReleaseLock), i37, descr=)
+        i60 = int_is_true(i59)
+        guard_false(i60, descr=...)
         guard_not_invalidated(descr=...)
         --TICK--
         jump(..., descr=...)
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -56,7 +56,6 @@
         'getsizeof'             : 'vm.getsizeof',
         'intern'                : 'vm.intern',
 
-        'executable'            : 'space.wrap("py.py")',
         'api_version'           : 'version.get_api_version(space)',
         'version_info'          : 'version.get_version_info(space)',
         'version'               : 'version.get_version(space)',
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -511,7 +511,7 @@
         assert isinstance(sys.builtin_module_names, tuple)
         assert isinstance(sys.copyright, str)
         #assert isinstance(sys.exec_prefix, str) -- not present!
-        assert isinstance(sys.executable, str)
+        #assert isinstance(sys.executable, str)
         assert isinstance(sys.hexversion, int)
         assert isinstance(sys.maxsize, int)
         assert isinstance(sys.maxunicode, int)
@@ -565,6 +565,13 @@
         raises(AttributeError, "del ns.spam")
         del ns.y
 
+    def test_reload_doesnt_override_sys_executable(self):
+        import sys
+        from imp import reload
+        sys.executable = 'from_test_sysmodule'
+        reload(sys)
+        assert sys.executable == 'from_test_sysmodule'
+
     def test_settrace(self):
         import sys
         counts = []
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c b/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
--- a/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
+++ b/pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test.c
@@ -272,7 +272,11 @@
 {
 	double x, sum=0.0, dx=(b-a)/(double)nstep;
 	for(x=a+0.5*dx; (b-x)*(x-a)>0.0; x+=dx)
+    {   
+        double y = f(x);
+        printf("f(x)=%.1f\n", y);
 		sum += f(x);
+    }
 	return sum/(double)nstep;
 }
 
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py b/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_callbacks.py
@@ -138,6 +138,7 @@
         integrate.restype = c_double
 
         def func(x):
+            print 'calculating x**2 of',x
             return x**2
 
         result = integrate(0.0, 1.0, CALLBACK(func), 10)
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -171,7 +171,6 @@
 if cConfig.has_gettimeofday:
     c_gettimeofday = external('gettimeofday', [rffi.VOIDP, rffi.VOIDP], rffi.INT)
 TM_P = lltype.Ptr(tm)
-c_clock = external('clock', [rffi.TIME_TP], clock_t)
 c_time = external('time', [rffi.TIME_TP], rffi.TIME_T)
 c_gmtime = external('gmtime', [rffi.TIME_TP], TM_P,
                     save_err=rffi.RFFI_SAVE_ERRNO)
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -199,8 +199,7 @@
         magic = importing._get_long(buf[:4])
         timestamp = importing._get_long(buf[4:8])
         if not self.can_use_pyc(space, filename, magic, timestamp):
-            return self.import_py_file(space, modname, filename[:-1], buf,
-                                       pkgpath)
+            return None
         buf = buf[8:] # XXX ugly copy, should use sequential read instead
         w_mod = w(Module(space, w(modname)))
         real_name = self.filename + os.path.sep + self.corr_zname(filename)
@@ -249,7 +248,6 @@
     def load_module(self, space, fullname):
         w = space.wrap
         filename = self.make_filename(fullname)
-        last_exc = None
         for compiled, is_package, ext in ENUMERATE_EXTS:
             fname = filename + ext
             try:
@@ -268,19 +266,18 @@
                     pkgpath = None
                 try:
                     if compiled:
-                        return self.import_pyc_file(space, fullname, fname,
-                                                    buf, pkgpath)
+                        w_result = self.import_pyc_file(space, fullname, fname,
+                                                        buf, pkgpath)
+                        if w_result is not None:
+                            return w_result
                     else:
                         return self.import_py_file(space, fullname, fname,
                                                    buf, pkgpath)
-                except OperationError, e:
-                    last_exc = e
+                except:
                     w_mods = space.sys.get('modules')
-                space.call_method(w_mods, 'pop', w(fullname), space.w_None)
-        if last_exc:
-            raise OperationError(get_error(space), last_exc.get_w_value(space))
-        # should never happen I think
-        return space.w_None
+                    space.call_method(w_mods, 'pop', w(fullname), space.w_None)
+                    raise
+        raise oefmt(get_error(space), "can't find module '%s'", fullname)
 
     @unwrap_spec(filename='str0')
     def get_data(self, space, filename):
diff --git a/pypy/module/zipimport/test/test_zipimport.py b/pypy/module/zipimport/test/test_zipimport.py
--- a/pypy/module/zipimport/test/test_zipimport.py
+++ b/pypy/module/zipimport/test/test_zipimport.py
@@ -195,7 +195,8 @@
         m0 ^= 0x04
         test_pyc = bytes([m0]) + self.get_pyc()[1:]
         self.writefile("uu.pyc", test_pyc)
-        raises(ImportError, "__import__('uu', globals(), locals(), [])")
+        raises(zipimport.ZipImportError,
+               "__import__('uu', globals(), locals(), [])")
         assert 'uu' not in sys.modules
 
     def test_force_py(self):
@@ -386,6 +387,11 @@
         finally:
             os.remove(filename)
 
+    def test_import_exception(self):
+        self.writefile('x1test.py', '1/0')
+        self.writefile('x1test/__init__.py', 'raise ValueError')
+        raises(ValueError, __import__, 'x1test', None, None, [])
+
 
 if os.sep != '/':
     class AppTestNativePathSep(AppTestZipimport):
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -567,9 +567,11 @@
 
     def descr_bit_length(self, space):
         val = self.intval
+        bits = 0
         if val < 0:
-            val = -val
-        bits = 0
+            # warning, "-val" overflows here
+            val = -((val + 1) >> 1)
+            bits = 1
         while val:
             bits += 1
             val >>= 1
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -11,51 +11,26 @@
 from pypy.interpreter.typedef import TypeDef, GetSetProperty,  make_weakref_descr
 
 
-def _buffer_setitem(space, buf, w_index, w_obj, as_int=False):
-    # This function is also used by _cffi_backend, but cffi.buffer()
-    # works with single byte characters, whereas memory object uses
-    # numbers.
-    if buf.readonly:
-        raise oefmt(space.w_TypeError, "cannot modify read-only memory")
-    start, stop, step, size = space.decode_index4(w_index, buf.getlength())
-    if step == 0:  # index only
-        if as_int:
-            value = chr(space.int_w(w_obj))
-        else:
-            val = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
-            if val.getlength() != 1:
-                raise oefmt(space.w_ValueError,
-                            "cannot modify size of memoryview object")
-            value = val.getitem(0)
-        buf.setitem(start, value)
-    elif step == 1:
-        value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
-        if value.getlength() != size:
-            raise oefmt(space.w_ValueError,
-                        "cannot modify size of memoryview object")
-        buf.setslice(start, value.as_str())
-    else:
-        raise oefmt(space.w_NotImplementedError, "")
-
-
 class W_MemoryView(W_Root):
     """Implement the built-in 'memoryview' type as a wrapper around
     an interp-level buffer.
     """
 
-    def __init__(self, buf):
+    def __init__(self, buf, format='B', itemsize=1):
         assert isinstance(buf, Buffer)
         self.buf = buf
         self._hash = -1
+        self.format = format
+        self.itemsize = itemsize
 
-    def buffer_w(self, space, flags):
+    def buffer_w_ex(self, space, flags):
         self._check_released(space)
         space.check_buf_flags(flags, self.buf.readonly)
-        return self.buf
+        return self.buf, self.format, self.itemsize
 
     @staticmethod
     def descr_new_memoryview(space, w_subtype, w_object):
-        return W_MemoryView(space.buffer_w(w_object, space.BUF_FULL_RO))
+        return W_MemoryView(*space.buffer_w_ex(w_object, space.BUF_FULL_RO))
 
     def _make_descr__cmp(name):
         def descr__cmp(self, space, w_other):
@@ -84,10 +59,12 @@
     descr_ne = _make_descr__cmp('ne')
 
     def as_str(self):
-        return self.buf.as_str()
+        buf = self.buf
+        n_bytes = buf.getlength()
+        return buf.getslice(0, n_bytes, 1, n_bytes)
 
     def getlength(self):
-        return self.buf.getlength()
+        return self.buf.getlength() // self.itemsize
 
     def descr_tobytes(self, space):
         self._check_released(space)
@@ -96,37 +73,53 @@
     def descr_tolist(self, space):
         self._check_released(space)
         buf = self.buf
+        if self.format != 'B':
+            raise oefmt(space.w_NotImplementedError,
+                        "tolist() only supports byte views")
         result = []
         for i in range(buf.getlength()):
-            result.append(space.wrap(ord(buf.getitem(i))))
+            result.append(space.wrap(ord(buf.getitem(i)[0])))
         return space.newlist(result)
 
     def descr_getitem(self, space, w_index):
         self._check_released(space)
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
         if step == 0:  # index only
-            return space.wrap(ord(self.buf.getitem(start)))
+            a = start * self.itemsize
+            b = a + self.itemsize
+            return space.wrapbytes(
+                ''.join([self.buf.getitem(i) for i in range(a, b)]))
         elif step == 1:
-            buf = SubBuffer(self.buf, start, size)
-            return W_MemoryView(buf)
+            buf = SubBuffer(self.buf, start * self.itemsize,
+                            size * self.itemsize)
+            return W_MemoryView(buf, self.format, self.itemsize)
         else:
             raise oefmt(space.w_NotImplementedError, "")
 
     def descr_setitem(self, space, w_index, w_obj):
         self._check_released(space)
-        _buffer_setitem(space, self.buf, w_index, w_obj, as_int=True)
+        if self.buf.readonly:
+            raise oefmt(space.w_TypeError, "cannot modify read-only memory")
+        start, stop, step, size = space.decode_index4(w_index, self.getlength())
+        if step not in (0, 1):
+            raise oefmt(space.w_NotImplementedError, "")
+        value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
+        if value.getlength() != size * self.itemsize:
+            raise oefmt(space.w_ValueError,
+                        "cannot modify size of memoryview object")
+        self.buf.setslice(start * self.itemsize, value.as_str())
 
     def descr_len(self, space):
         self._check_released(space)
-        return space.wrap(self.buf.getlength())
+        return space.wrap(self.getlength())
 
     def w_get_format(self, space):
         self._check_released(space)
-        return space.wrap("B")
+        return space.wrap(self.format)
 
     def w_get_itemsize(self, space):
         self._check_released(space)
-        return space.wrap(1)
+        return space.wrap(self.itemsize)
 
     def w_get_ndim(self, space):
         self._check_released(space)
@@ -142,7 +135,7 @@
 
     def w_get_strides(self, space):
         self._check_released(space)
-        return space.newtuple([space.wrap(1)])
+        return space.newtuple([space.wrap(self.itemsize)])
 
     def w_get_suboffsets(self, space):
         self._check_released(space)
@@ -169,8 +162,8 @@
 
     def _check_released(self, space):
         if self.buf is None:
-            raise OperationError(space.w_ValueError, space.wrap(
-                    "operation forbidden on released memoryview object"))
+            raise oefmt(space.w_ValueError,
+                        "operation forbidden on released memoryview object")
 
     def descr_enter(self, space):
         self._check_released(space)
diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py
--- a/pypy/objspace/std/test/test_intobject.py
+++ b/pypy/objspace/std/test/test_intobject.py
@@ -476,11 +476,20 @@
             (10, 4),
             (150, 8),
             (-1, 1),
+            (-2, 2),
+            (-3, 2),
+            (-4, 3),
             (-10, 4),
             (-150, 8),
         ]:
             assert val.bit_length() == bits
 
+    def test_bit_length_max(self):
+        import sys
+        val = -sys.maxsize-1
+        bits = 32 if val == -2147483648 else 64
+        assert val.bit_length() == bits
+
     def test_int_real(self):
         class A(int): pass
         b = A(5).real
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -133,8 +133,31 @@
         raises(ValueError, bytes, v)
         assert "released memory" in repr(v)
 
+    def test_int_array_buffer(self):
+        import array
+        m = memoryview(array.array('i', list(range(10))))
+        assert m.format == 'i'
+        assert m.itemsize == 4
+        assert len(m) == 10
+        assert len(m.tobytes()) == 40
+        assert m[0] == b'\x00\x00\x00\x00'
+        m[0] = b'\x00\x00\x00\x01'
+        assert m[0] == b'\x00\x00\x00\x01'
+
+    def test_int_array_slice(self):
+        import array
+        m = memoryview(array.array('i', list(range(10))))
+        slice = m[2:8]
+        assert slice.format == 'i'
+        assert slice.itemsize == 4
+        assert len(slice) == 6
+        assert len(slice.tobytes()) == 24
+        assert slice[0] in (b'\x00\x00\x00\x02', b'\x02\x00\x00\x00')
+        slice[0] = b'\x00\x00\x00\x01'
+        assert slice[0] == b'\x00\x00\x00\x01'
+        assert m[2] == b'\x00\x00\x00\x01'
+
     def test_pypy_raw_address_base(self):
         raises(ValueError, memoryview(b"foobar")._pypy_raw_address)
         e = raises(ValueError, memoryview(bytearray(b"foobar"))._pypy_raw_address)
         assert 'BytearrayBuffer' in str(e.value)
-
diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -12,8 +12,7 @@
 from rpython.flowspace.argument import CallSpec
 from rpython.flowspace.model import (Constant, Variable, Block, Link,
     c_last_exception, const, FSException)
-from rpython.flowspace.framestate import (FrameState, recursively_unflatten,
-    recursively_flatten)
+from rpython.flowspace.framestate import FrameState
 from rpython.flowspace.specialcase import (rpython_print_item,
     rpython_print_newline)
 from rpython.flowspace.operation import op
@@ -278,6 +277,7 @@
     "cmp_exc_match",
     ]
 
+
 class FlowContext(object):
     def __init__(self, graph, code):
         self.graph = graph
@@ -307,112 +307,91 @@
 
         The locals are ordered according to self.pycode.signature.
         """
-        self.valuestackdepth = code.co_nlocals
-        self.locals_stack_w = [None] * (code.co_stacksize + code.co_nlocals)
+        self.nlocals = code.co_nlocals
+        self.locals_w = [None] * code.co_nlocals
+        self.stack = []
+
+    @property
+    def stackdepth(self):
+        return len(self.stack)
 
     def pushvalue(self, w_object):
-        depth = self.valuestackdepth
-        self.locals_stack_w[depth] = w_object
-        self.valuestackdepth = depth + 1
+        self.stack.append(w_object)
 
     def popvalue(self):
-        depth = self.valuestackdepth - 1
-        assert depth >= self.pycode.co_nlocals, "pop from empty value stack"
-        w_object = self.locals_stack_w[depth]
-        self.locals_stack_w[depth] = None
-        self.valuestackdepth = depth
-        return w_object
+        return self.stack.pop()
 
     def peekvalue(self, index_from_top=0):
         # NOTE: top of the stack is peekvalue(0).
-        index = self.valuestackdepth + ~index_from_top
-        assert index >= self.pycode.co_nlocals, (
-            "peek past the bottom of the stack")
-        return self.locals_stack_w[index]
+        index = ~index_from_top
+        return self.stack[index]
 
     def settopvalue(self, w_object, index_from_top=0):
-        index = self.valuestackdepth + ~index_from_top
-        assert index >= self.pycode.co_nlocals, (
-            "settop past the bottom of the stack")
-        self.locals_stack_w[index] = w_object
+        index = ~index_from_top
+        self.stack[index] = w_object
 
     def popvalues(self, n):
-        values_w = [self.popvalue() for i in range(n)]
-        values_w.reverse()
+        if n == 0:
+            return []
+        values_w = self.stack[-n:]
+        del self.stack[-n:]
         return values_w
 
-    def dropvalues(self, n):
-        finaldepth = self.valuestackdepth - n
-        for n in range(finaldepth, self.valuestackdepth):
-            self.locals_stack_w[n] = None
-        self.valuestackdepth = finaldepth
-
     def dropvaluesuntil(self, finaldepth):
-        for n in range(finaldepth, self.valuestackdepth):
-            self.locals_stack_w[n] = None
-        self.valuestackdepth = finaldepth
-
-    def save_locals_stack(self):
-        return self.locals_stack_w[:self.valuestackdepth]
-
-    def restore_locals_stack(self, items_w):
-        self.locals_stack_w[:len(items_w)] = items_w
-        self.dropvaluesuntil(len(items_w))
+        del self.stack[finaldepth:]
 
     def getstate(self, next_offset):
-        # getfastscope() can return real None, for undefined locals
-        data = self.save_locals_stack()
-        if self.last_exception is None:
-            data.append(Constant(None))
-            data.append(Constant(None))
-        else:
-            data.append(self.last_exception.w_type)
-            data.append(self.last_exception.w_value)
-        recursively_flatten(data)
-        return FrameState(data, self.blockstack[:], next_offset)
+        return FrameState(self.locals_w[:], self.stack[:],
+                self.last_exception, self.blockstack[:], next_offset)
 
     def setstate(self, state):
         """ Reset the context to the given frame state. """
-        data = state.mergeable[:]
-        recursively_unflatten(data)
-        self.restore_locals_stack(data[:-2])  # Nones == undefined locals
-        if data[-2] == Constant(None):
-            assert data[-1] == Constant(None)
-            self.last_exception = None
-        else:
-            self.last_exception = FSException(data[-2], data[-1])
+        self.locals_w = state.locals_w[:]
+        self.stack = state.stack[:]
+        self.last_exception = state.last_exception
         self.blockstack = state.blocklist[:]
+        self._normalize_raise_signals()
+
+    def _normalize_raise_signals(self):
+        st = self.stack
+        for i in range(len(st)):
+            if isinstance(st[i], RaiseImplicit):
+                st[i] = Raise(st[i].w_exc)
 
     def guessbool(self, w_condition):
         if isinstance(w_condition, Constant):
             return w_condition.value
         return self.recorder.guessbool(self, w_condition)
 
-    def record(self, spaceop):
+    def maybe_merge(self):
         recorder = self.recorder
         if getattr(recorder, 'final_state', None) is not None:
             self.mergeblock(recorder.crnt_block, recorder.final_state)
             raise StopFlowing
+
+    def record(self, spaceop):
         spaceop.offset = self.last_offset
-        recorder.append(spaceop)
+        self.recorder.append(spaceop)
 
     def do_op(self, op):
+        self.maybe_merge()
         self.record(op)
         self.guessexception(op.canraise)
         return op.result
 
-    def guessexception(self, exceptions, force=False):
+    def guessexception(self, exceptions):
         """
         Catch possible exceptions implicitly.
         """
         if not exceptions:
             return
-        if not force and not any(isinstance(block, (ExceptBlock, FinallyBlock))
-                                 for block in self.blockstack):
-            # The implicit exception wouldn't be caught and would later get
-            # removed, so don't bother creating it.
-            return
-        self.recorder.guessexception(self, *exceptions)
+        # Implicit exceptions are ignored unless they are caught explicitly
+        if self.has_exc_handler():
+            self.recorder.guessexception(self, *exceptions)
+
+    def has_exc_handler(self):
+        return any(isinstance(block, (ExceptBlock, FinallyBlock))
+                for block in self.blockstack)
 
     def build_flow(self):
         graph = self.graph
@@ -430,35 +409,8 @@
             while True:
                 next_offset = self.handle_bytecode(next_offset)
                 self.recorder.final_state = self.getstate(next_offset)
-
-        except RaiseImplicit as e:
-            w_exc = e.w_exc
-            if isinstance(w_exc.w_type, Constant):
-                exc_cls = w_exc.w_type.value
-            else:
-                exc_cls = Exception
-            msg = "implicit %s shouldn't occur" % exc_cls.__name__
-            w_type = Constant(AssertionError)
-            w_value = Constant(AssertionError(msg))
-            link = Link([w_type, w_value], self.graph.exceptblock)
-            self.recorder.crnt_block.closeblock(link)
-
-        except Raise as e:
-            w_exc = e.w_exc
-            if w_exc.w_type == const(ImportError):
-                msg = 'import statement always raises %s' % e
-                raise ImportError(msg)
-            link = Link([w_exc.w_type, w_exc.w_value], self.graph.exceptblock)
-            self.recorder.crnt_block.closeblock(link)
-
         except StopFlowing:
             pass
-
-        except Return as exc:
-            w_result = exc.w_value
-            link = Link([w_result], self.graph.returnblock)
-            self.recorder.crnt_block.closeblock(link)
-
         except FlowingError as exc:
             if exc.ctx is None:
                 exc.ctx = self
@@ -476,14 +428,8 @@
             if newstate is not None:
                 break
         else:
-            newstate = currentstate.copy()
-            newblock = SpamBlock(newstate)
-            # unconditionally link the current block to the newblock
-            outputargs = currentstate.getoutputargs(newstate)
-            link = Link(outputargs, newblock)
-            currentblock.closeblock(link)
+            newblock = self.make_next_block(currentblock, currentstate)
             candidates.insert(0, newblock)
-            self.pendingblocks.append(newblock)
             return
 
         if newstate.matches(block.framestate):
@@ -493,7 +439,7 @@
 
         newblock = SpamBlock(newstate)
         varnames = self.pycode.co_varnames
-        for name, w_value in zip(varnames, newstate.mergeable):
+        for name, w_value in zip(varnames, newstate.locals_w):
             if isinstance(w_value, Variable):
                 w_value.rename(name)
         # unconditionally link the current block to the newblock
@@ -513,11 +459,21 @@
         candidates.insert(0, newblock)
         self.pendingblocks.append(newblock)
 
+    def make_next_block(self, block, state):
+        newstate = state.copy()
+        newblock = SpamBlock(newstate)
+        # unconditionally link the current block to the newblock
+        outputargs = state.getoutputargs(newstate)
+        link = Link(outputargs, newblock)
+        block.closeblock(link)
+        self.pendingblocks.append(newblock)
+        return newblock
+
     # hack for unrolling iterables, don't use this
     def replace_in_stack(self, oldvalue, newvalue):
         w_new = Constant(newvalue)
-        stack_items_w = self.locals_stack_w
-        for i in range(self.valuestackdepth - 1, self.pycode.co_nlocals - 1, -1):
+        stack_items_w = self.stack
+        for i in range(self.stackdepth - 1, - 1, -1):
             w_v = stack_items_w[i]
             if isinstance(w_v, Constant):
                 if w_v.value is oldvalue:
@@ -541,7 +497,7 @@
             if isinstance(signal, block.handles):
                 return block.handle(self, signal)
             block.cleanupstack(self)
-        return signal.nomoreblocks()
+        return signal.nomoreblocks(self)
 
     def getlocalvarname(self, index):
         return self.pycode.co_varnames[index]
@@ -870,7 +826,7 @@
             op.simple_call(w_exitfunc, w_None, w_None, w_None).eval(self)
 
     def LOAD_FAST(self, varindex):
-        w_value = self.locals_stack_w[varindex]
+        w_value = self.locals_w[varindex]
         if w_value is None:
             raise FlowingError("Local variable referenced before assignment")
         self.pushvalue(w_value)
@@ -915,7 +871,7 @@
     def STORE_FAST(self, varindex):
         w_newvalue = self.popvalue()
         assert w_newvalue is not None
-        self.locals_stack_w[varindex] = w_newvalue
+        self.locals_w[varindex] = w_newvalue
         if isinstance(w_newvalue, Variable):
             w_newvalue.rename(self.getlocalvarname(varindex))
 
@@ -1128,11 +1084,11 @@
         op.simple_call(w_append_meth, w_value).eval(self)
 
     def DELETE_FAST(self, varindex):
-        if self.locals_stack_w[varindex] is None:
+        if self.locals_w[varindex] is None:
             varname = self.getlocalvarname(varindex)
             message = "local variable '%s' referenced before assignment"
             raise UnboundLocalError(message, varname)
-        self.locals_stack_w[varindex] = None
+        self.locals_w[varindex] = None
 
     def STORE_MAP(self, oparg):
         w_key = self.popvalue()
@@ -1220,25 +1176,32 @@
                 WHY_CONTINUE,   Continue
                 WHY_YIELD       not needed
     """
-    def nomoreblocks(self):
+    def nomoreblocks(self, ctx):
         raise BytecodeCorruption("misplaced bytecode - should not return")
 
+    def __eq__(self, other):
+        return type(other) is type(self) and other.args == self.args
+
 
 class Return(FlowSignal):
     """Signals a 'return' statement.
-    Argument is the wrapped object to return."""
-
+    Argument is the wrapped object to return.
+    """
     def __init__(self, w_value):
         self.w_value = w_value
 
-    def nomoreblocks(self):
-        raise Return(self.w_value)
+    def nomoreblocks(self, ctx):
+        w_result = self.w_value
+        link = Link([w_result], ctx.graph.returnblock)
+        ctx.recorder.crnt_block.closeblock(link)
+        raise StopFlowing
 
-    def state_unpack_variables(self):
+    @property
+    def args(self):
         return [self.w_value]
 
     @staticmethod
-    def state_pack_variables(w_value):
+    def rebuild(w_value):
         return Return(w_value)
 
 class Raise(FlowSignal):
@@ -1248,28 +1211,48 @@
     def __init__(self, w_exc):
         self.w_exc = w_exc
 
-    def nomoreblocks(self):
-        raise self
+    def nomoreblocks(self, ctx):
+        w_exc = self.w_exc
+        if w_exc.w_type == const(ImportError):
+            msg = 'import statement always raises %s' % self
+            raise ImportError(msg)
+        link = Link([w_exc.w_type, w_exc.w_value], ctx.graph.exceptblock)
+        ctx.recorder.crnt_block.closeblock(link)
+        raise StopFlowing
 
-    def state_unpack_variables(self):
+    @property
+    def args(self):
         return [self.w_exc.w_type, self.w_exc.w_value]
 
-    @staticmethod
-    def state_pack_variables(w_type, w_value):
-        return Raise(FSException(w_type, w_value))
+    @classmethod
+    def rebuild(cls, w_type, w_value):
+        return cls(FSException(w_type, w_value))
 
 class RaiseImplicit(Raise):
     """Signals an exception raised implicitly"""
+    def nomoreblocks(self, ctx):
+        w_exc = self.w_exc
+        if isinstance(w_exc.w_type, Constant):
+            exc_cls = w_exc.w_type.value
+        else:
+            exc_cls = Exception
+        msg = "implicit %s shouldn't occur" % exc_cls.__name__
+        w_type = Constant(AssertionError)
+        w_value = Constant(AssertionError(msg))
+        link = Link([w_type, w_value], ctx.graph.exceptblock)
+        ctx.recorder.crnt_block.closeblock(link)
+        raise StopFlowing
 
 
 class Break(FlowSignal):
     """Signals a 'break' statement."""
 
-    def state_unpack_variables(self):
+    @property
+    def args(self):
         return []
 
     @staticmethod
-    def state_pack_variables():
+    def rebuild():
         return Break.singleton
 
 Break.singleton = Break()
@@ -1281,11 +1264,12 @@
     def __init__(self, jump_to):
         self.jump_to = jump_to
 
-    def state_unpack_variables(self):
+    @property
+    def args(self):
         return [const(self.jump_to)]
 
     @staticmethod
-    def state_pack_variables(w_jump_to):
+    def rebuild(w_jump_to):
         return Continue(w_jump_to.value)
 
 
@@ -1295,21 +1279,21 @@
 
     def __init__(self, ctx, handlerposition):
         self.handlerposition = handlerposition
-        self.valuestackdepth = ctx.valuestackdepth
+        self.stackdepth = ctx.stackdepth
 
     def __eq__(self, other):
         return (self.__class__ is other.__class__ and
                 self.handlerposition == other.handlerposition and
-                self.valuestackdepth == other.valuestackdepth)
+                self.stackdepth == other.stackdepth)
 
     def __ne__(self, other):
         return not (self == other)
 
     def __hash__(self):
-        return hash((self.handlerposition, self.valuestackdepth))
+        return hash((self.handlerposition, self.stackdepth))
 
     def cleanupstack(self, ctx):
-        ctx.dropvaluesuntil(self.valuestackdepth)
+        ctx.dropvaluesuntil(self.stackdepth)
 
     def handle(self, ctx, unroller):
         raise NotImplementedError
diff --git a/rpython/flowspace/framestate.py b/rpython/flowspace/framestate.py
--- a/rpython/flowspace/framestate.py
+++ b/rpython/flowspace/framestate.py
@@ -1,21 +1,50 @@
-from rpython.flowspace.model import Variable, Constant
+from rpython.flowspace.model import Variable, Constant, FSException
 from rpython.rlib.unroll import SpecTag
 
+def _copy(v):
+    from rpython.flowspace.flowcontext import FlowSignal
+    if isinstance(v, Variable):
+        return Variable(v)
+    elif isinstance(v, FlowSignal):
+        vars = [_copy(var) for var in v.args]
+        return v.rebuild(*vars)
+    else:
+        return v
+
+def _union(seq1, seq2):
+    return [union(v1, v2) for v1, v2 in zip(seq1, seq2)]
+
 
 class FrameState(object):
-    def __init__(self, mergeable, blocklist, next_offset):
-        self.mergeable = mergeable
+    def __init__(self, locals_w, stack, last_exception, blocklist, next_offset):
+        self.locals_w = locals_w
+        self.stack = stack
+        self.last_exception = last_exception
         self.blocklist = blocklist
         self.next_offset = next_offset
+        self._mergeable = None
+
+    @property
+    def mergeable(self):
+        if self._mergeable is not None:
+            return self._mergeable
+        self._mergeable = data = self.locals_w + self.stack
+        if self.last_exception is None:
+            data.append(Constant(None))
+            data.append(Constant(None))
+        else:
+            data.append(self.last_exception.w_type)
+            data.append(self.last_exception.w_value)
+        recursively_flatten(data)
+        return data
 
     def copy(self):
         "Make a copy of this state in which all Variables are fresh."
-        newstate = []
-        for w in self.mergeable:
-            if isinstance(w, Variable):
-                w = Variable(w)
-            newstate.append(w)
-        return FrameState(newstate, self.blocklist, self.next_offset)
+        exc = self.last_exception
+        if exc is not None:
+            exc = FSException(_copy(exc.w_type), _copy(exc.w_value))
+        return FrameState(map(_copy, self.locals_w), map(_copy, self.stack),
+                exc, self.blocklist, self.next_offset)
 
     def getvariables(self):
         return [w for w in self.mergeable if isinstance(w, Variable)]
@@ -33,18 +62,31 @@
                 return False
         return True
 
+    def _exc_args(self):
+        if self.last_exception is None:
+            return [Constant(None), Constant(None)]
+        else:
+            return [self.last_exception.w_type,
+                    self.last_exception.w_value]
+
     def union(self, other):
         """Compute a state that is at least as general as both self and other.
            A state 'a' is more general than a state 'b' if all Variables in 'b'
            are also Variables in 'a', but 'a' may have more Variables.
         """
-        newstate = []
         try:
-            for w1, w2 in zip(self.mergeable, other.mergeable):
-                newstate.append(union(w1, w2))
+            locals = _union(self.locals_w, other.locals_w)
+            stack = _union(self.stack, other.stack)
+            if self.last_exception is None and other.last_exception is None:
+                exc = None
+            else:
+                args1 = self._exc_args()
+                args2 = other._exc_args()
+                exc = FSException(union(args1[0], args2[0]),
+                        union(args1[1], args2[1]))
         except UnionError:
             return None
-        return FrameState(newstate, self.blocklist, self.next_offset)
+        return FrameState(locals, stack, exc, self.blocklist, self.next_offset)
 
     def getoutputargs(self, targetstate):
         "Return the output arguments needed to link self to targetstate."
@@ -61,6 +103,7 @@
 
 def union(w1, w2):
     "Union of two variables or constants."
+    from rpython.flowspace.flowcontext import FlowSignal
     if w1 == w2:
         return w1
     if w1 is None or w2 is None:
@@ -69,38 +112,21 @@
     if isinstance(w1, Variable) or isinstance(w2, Variable):
         return Variable()  # new fresh Variable
     if isinstance(w1, Constant) and isinstance(w2, Constant):
-        # FlowSignal represent stack unrollers in the stack.
-        # They should not be merged because they will be unwrapped.
-        # This is needed for try:except: and try:finally:, though
-        # it makes the control flow a bit larger by duplicating the
-        # handlers.
-        dont_merge_w1 = w1 in UNPICKLE_TAGS or isinstance(w1.value, SpecTag)
-        dont_merge_w2 = w2 in UNPICKLE_TAGS or isinstance(w2.value, SpecTag)
-        if dont_merge_w1 or dont_merge_w2:
+        if isinstance(w1.value, SpecTag) or isinstance(w2.value, SpecTag):
             raise UnionError
         else:
             return Variable()  # generalize different constants
+    if isinstance(w1, FlowSignal) and isinstance(w2, FlowSignal):
+        if type(w1) is not type(w2):
+            raise UnionError
+        vars = [union(v1, v2) for v1, v2 in zip(w1.args, w2.args)]
+        return w1.rebuild(*vars)
+    if isinstance(w1, FlowSignal) or isinstance(w2, FlowSignal):
+        raise UnionError
     raise TypeError('union of %r and %r' % (w1.__class__.__name__,
                                             w2.__class__.__name__))
 
 
-# ____________________________________________________________
-#
-# We have to flatten out the state of the frame into a list of
-# Variables and Constants.  This is done above by collecting the
-# locals and the items on the value stack, but the latter may contain
-# FlowSignal.  We have to handle these specially, because
-# some of them hide references to more Variables and Constants.
-# The trick is to flatten ("pickle") them into the list so that the
-# extra Variables show up directly in the list too.
-
-class PickleTag:
-    pass
-
-PICKLE_TAGS = {}
-UNPICKLE_TAGS = {}
-
-
 def recursively_flatten(lst):
     from rpython.flowspace.flowcontext import FlowSignal
     i = 0
@@ -109,22 +135,4 @@
         if not isinstance(unroller, FlowSignal):
             i += 1
         else:
-            vars = unroller.state_unpack_variables()
-            key = unroller.__class__, len(vars)
-            try:
-                tag = PICKLE_TAGS[key]
-            except KeyError:
-                tag = PICKLE_TAGS[key] = Constant(PickleTag())
-                UNPICKLE_TAGS[tag] = key
-            lst[i:i + 1] = [tag] + vars
-
-
-def recursively_unflatten(lst):
-    for i in xrange(len(lst) - 1, -1, -1):
-        item = lst[i]
-        if item in UNPICKLE_TAGS:
-            unrollerclass, argcount = UNPICKLE_TAGS[item]
-            arguments = lst[i + 1:i + 1 + argcount]
-            del lst[i + 1:i + 1 + argcount]
-            unroller = unrollerclass.state_pack_variables(*arguments)
-            lst[i] = unroller
+            lst[i:i + 1] = unroller.args
diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py
--- a/rpython/flowspace/operation.py
+++ b/rpython/flowspace/operation.py
@@ -517,7 +517,7 @@
                     ctx.replace_in_stack(it, next_unroller)
                     return const(v)
         w_item = ctx.do_op(self)
-        ctx.guessexception([StopIteration, RuntimeError], force=True)
+        ctx.recorder.guessexception(ctx, StopIteration, RuntimeError)
         return w_item
 
 class GetAttr(SingleDispatchMixin, HLOperation):
diff --git a/rpython/flowspace/pygraph.py b/rpython/flowspace/pygraph.py
--- a/rpython/flowspace/pygraph.py
+++ b/rpython/flowspace/pygraph.py
@@ -11,10 +11,10 @@
 
     def __init__(self, func, code):
         from rpython.flowspace.flowcontext import SpamBlock
-        data = [None] * code.co_nlocals
+        locals = [None] * code.co_nlocals
         for i in range(code.formalargcount):
-            data[i] = Variable(code.co_varnames[i])
-        state = FrameState(data + [Constant(None), Constant(None)], [], 0)
+            locals[i] = Variable(code.co_varnames[i])
+        state = FrameState(locals, [], None, [], 0)
         initialblock = SpamBlock(state)
         super(PyGraph, self).__init__(self._sanitize_funcname(func), initialblock)
         self.func = func
diff --git a/rpython/flowspace/test/test_flowcontext.py b/rpython/flowspace/test/test_flowcontext.py
new file mode 100644
--- /dev/null
+++ b/rpython/flowspace/test/test_flowcontext.py
@@ -0,0 +1,15 @@
+""" Unit tests for flowcontext.py """
+import pytest
+from rpython.flowspace.model import Variable, FSException
+from rpython.flowspace.flowcontext import (
+    Return, Raise, RaiseImplicit, Continue, Break)
+
+ at pytest.mark.parametrize('signal', [
+    Return(Variable()),
+    Raise(FSException(Variable(), Variable())),
+    RaiseImplicit(FSException(Variable(), Variable())),
+    Break(),
+    Continue(42),
+])
+def test_signals(signal):
+    assert signal.rebuild(*signal.args) == signal
diff --git a/rpython/flowspace/test/test_framestate.py b/rpython/flowspace/test/test_framestate.py
--- a/rpython/flowspace/test/test_framestate.py
+++ b/rpython/flowspace/test/test_framestate.py
@@ -15,7 +15,7 @@
         ctx = FlowContext(graph, code)
         # hack the frame
         ctx.setstate(graph.startblock.framestate)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Constant(None)
+        ctx.locals_w[-1] = Constant(None)
         return ctx
 
     def func_simple(x):
@@ -31,7 +31,7 @@
     def test_neq_hacked_framestate(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable()
+        ctx.locals_w[-1] = Variable()
         fs2 = ctx.getstate(0)
         assert not fs1.matches(fs2)
 
@@ -44,7 +44,7 @@
     def test_union_on_hacked_framestates(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable()
+        ctx.locals_w[-1] = Variable()
         fs2 = ctx.getstate(0)
         assert fs1.union(fs2).matches(fs2)  # fs2 is more general
         assert fs2.union(fs1).matches(fs2)  # fs2 is more general
@@ -52,7 +52,7 @@
     def test_restore_frame(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable()
+        ctx.locals_w[-1] = Variable()
         ctx.setstate(fs1)
         assert fs1.matches(ctx.getstate(0))
 
@@ -71,26 +71,25 @@
     def test_getoutputargs(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Variable()
+        ctx.locals_w[-1] = Variable()
         fs2 = ctx.getstate(0)
         outputargs = fs1.getoutputargs(fs2)
         # 'x' -> 'x' is a Variable
         # locals_w[n-1] -> locals_w[n-1] is Constant(None)
-        assert outputargs == [ctx.locals_stack_w[0], Constant(None)]
+        assert outputargs == [ctx.locals_w[0], Constant(None)]
 
     def test_union_different_constants(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Constant(42)
+        ctx.locals_w[-1] = Constant(42)
         fs2 = ctx.getstate(0)
         fs3 = fs1.union(fs2)
         ctx.setstate(fs3)
-        assert isinstance(ctx.locals_stack_w[ctx.pycode.co_nlocals-1],
-                          Variable)   # generalized
+        assert isinstance(ctx.locals_w[-1], Variable)   # generalized
 
     def test_union_spectag(self):
         ctx = self.get_context(self.func_simple)
         fs1 = ctx.getstate(0)
-        ctx.locals_stack_w[ctx.pycode.co_nlocals-1] = Constant(SpecTag())
+        ctx.locals_w[-1] = Constant(SpecTag())
         fs2 = ctx.getstate(0)
         assert fs1.union(fs2) is None   # UnionError
diff --git a/rpython/jit/backend/arm/arch.py b/rpython/jit/backend/arm/arch.py
--- a/rpython/jit/backend/arm/arch.py
+++ b/rpython/jit/backend/arm/arch.py
@@ -1,4 +1,3 @@
-FUNC_ALIGN = 8
 WORD = 4
 DOUBLE_WORD = 8
 
diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py
--- a/rpython/jit/backend/arm/assembler.py
+++ b/rpython/jit/backend/arm/assembler.py
@@ -4,7 +4,7 @@
 
 from rpython.jit.backend.arm import conditions as c, registers as r
 from rpython.jit.backend.arm import shift
-from rpython.jit.backend.arm.arch import (WORD, DOUBLE_WORD, FUNC_ALIGN,
+from rpython.jit.backend.arm.arch import (WORD, DOUBLE_WORD,
     JITFRAME_FIXED_SIZE)
 from rpython.jit.backend.arm.codebuilder import InstrBuilder, OverwritingBuilder
 from rpython.jit.backend.arm.locations import imm, StackLocation, get_fp_offset
@@ -484,10 +484,6 @@
         self.mc.BL(target)
         return startpos
 
-    def align(self):
-        while(self.mc.currpos() % FUNC_ALIGN != 0):
-            self.mc.writechar(chr(0))
-
     def gen_func_epilog(self, mc=None, cond=c.AL):
         gcrootmap = self.cpu.gc_ll_descr.gcrootmap
         if mc is None:
@@ -557,7 +553,7 @@
         debug_stop('jit-backend-ops')
 
     def _call_header(self):
-        self.align()
+        assert self.currpos() == 0
         self.gen_func_prolog()
 
     def _call_header_with_stack_check(self):
diff --git a/rpython/jit/backend/arm/callbuilder.py b/rpython/jit/backend/arm/callbuilder.py
--- a/rpython/jit/backend/arm/callbuilder.py
+++ b/rpython/jit/backend/arm/callbuilder.py
@@ -176,11 +176,14 @@
 
     def write_real_errno(self, save_err):
         if save_err & rffi.RFFI_READSAVED_ERRNO:
-            # Just before a call, read 'rpy_errno' and write it into the
+            # Just before a call, read '*_errno' and write it into the
             # real 'errno'.  The r0-r3 registers contain arguments to the
             # future call; the r5-r7 registers contain various stuff.
             # We still have r8-r12.
-            rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu)
+            if save_err & rffi.RFFI_ALT_ERRNO:
+                rpy_errno = llerrno.get_alt_errno_offset(self.asm.cpu)
+            else:
+                rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu)
             p_errno = llerrno.get_p_errno_offset(self.asm.cpu)
             self.mc.LDR_ri(r.r9.value, r.sp.value,
                            self.asm.saved_threadlocal_addr + self.current_sp)
@@ -199,10 +202,13 @@
     def read_real_errno(self, save_err):
         if save_err & rffi.RFFI_SAVE_ERRNO:
             # Just after a call, read the real 'errno' and save a copy of
-            # it inside our thread-local 'rpy_errno'.  Registers r8-r12
+            # it inside our thread-local '*_errno'.  Registers r8-r12
             # are unused here, and registers r2-r3 never contain anything
             # after the call.
-            rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu)
+            if save_err & rffi.RFFI_ALT_ERRNO:
+                rpy_errno = llerrno.get_alt_errno_offset(self.asm.cpu)
+            else:
+                rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu)
             p_errno = llerrno.get_p_errno_offset(self.asm.cpu)
             self.mc.LDR_ri(r.r3.value, r.sp.value,
                            self.asm.saved_threadlocal_addr)
diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py
--- a/rpython/jit/backend/arm/codebuilder.py
+++ b/rpython/jit/backend/arm/codebuilder.py
@@ -1,7 +1,7 @@
 from rpython.jit.backend.arm import conditions as cond
 from rpython.jit.backend.arm import registers as reg
 from rpython.jit.backend.arm import support
-from rpython.jit.backend.arm.arch import (WORD, FUNC_ALIGN, PC_OFFSET)
+from rpython.jit.backend.arm.arch import WORD, PC_OFFSET
 from rpython.jit.backend.arm.instruction_builder import define_instructions
 from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin
 from rpython.rlib.objectmodel import we_are_translated
@@ -29,14 +29,9 @@
 
 
 class AbstractARMBuilder(object):
-
     def __init__(self, arch_version=7):
         self.arch_version = arch_version
 
-    def align(self):
-        while(self.currpos() % FUNC_ALIGN != 0):
-            self.writechar(chr(0))
-
     def NOP(self):
         self.MOV_rr(0, 0)
 
@@ -467,21 +462,6 @@
                 f.write(data[i])
             f.close()
 
-    # XXX remove and setup aligning in llsupport
-    def materialize(self, asmmemmgr, allblocks, gcrootmap=None):
-        size = self.get_relative_pos() + WORD
-        malloced = asmmemmgr.malloc(size, size + 7)
-        allblocks.append(malloced)
-        rawstart = malloced[0]
-        while(rawstart % FUNC_ALIGN != 0):
-            rawstart += 1
-        self.copy_to_raw_memory(rawstart)
-        if self.gcroot_markers is not None:
-            assert gcrootmap is not None
-            for pos, mark in self.gcroot_markers:
-                gcrootmap.put(rawstart + pos, mark)
-        return rawstart
-
     def clear_cache(self, addr):
         if we_are_translated():
             startaddr = rffi.cast(llmemory.Address, addr)
diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py
--- a/rpython/jit/backend/llsupport/asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/asmmemmgr.py
@@ -208,6 +208,8 @@
                    ('data', lltype.FixedSizeArray(lltype.Char, SUBBLOCK_SIZE)))
     SUBBLOCK_PTR.TO.become(SUBBLOCK)
 
+    ALIGN_MATERIALIZE = 16
+
     gcroot_markers = None
 
     def __init__(self, translated=None):
@@ -303,9 +305,12 @@
 
     def materialize(self, asmmemmgr, allblocks, gcrootmap=None):
         size = self.get_relative_pos()
+        align = self.ALIGN_MATERIALIZE
+        size += align - 1
         malloced = asmmemmgr.malloc(size, size)
         allblocks.append(malloced)
         rawstart = malloced[0]
+        rawstart = (rawstart + align - 1) & (-align)
         self.copy_to_raw_memory(rawstart)
         if self.gcroot_markers is not None:
             assert gcrootmap is not None
diff --git a/rpython/jit/backend/llsupport/llerrno.py b/rpython/jit/backend/llsupport/llerrno.py
--- a/rpython/jit/backend/llsupport/llerrno.py
+++ b/rpython/jit/backend/llsupport/llerrno.py
@@ -18,19 +18,41 @@
         return 3 * WORD
 
 
+def get_debug_saved_alterrno(cpu):
+    return cpu._debug_errno_container[4]
+
+def set_debug_saved_alterrno(cpu, nerrno):
+    assert nerrno >= 0
+    cpu._debug_errno_container[4] = nerrno
+
+def get_alt_errno_offset(cpu):
+    if cpu.translate_support_code:
+        from rpython.rlib import rthread
+        return rthread.tlfield_alt_errno.getoffset()
+    else:
+        return 4 * WORD
+
+
 def get_debug_saved_lasterror(cpu):
-    return cpu._debug_errno_container[4]
+    return cpu._debug_errno_container[5]
 
 def set_debug_saved_lasterror(cpu, nerrno):
     assert nerrno >= 0
-    cpu._debug_errno_container[4] = nerrno
+    cpu._debug_errno_container[5] = nerrno
 
 def get_rpy_lasterror_offset(cpu):
     if cpu.translate_support_code:
         from rpython.rlib import rthread
         return rthread.tlfield_rpy_lasterror.getoffset()
     else:
-        return 4 * WORD
+        return 5 * WORD
+
+def get_alt_lasterror_offset(cpu):
+    if cpu.translate_support_code:
+        from rpython.rlib import rthread
+        return rthread.tlfield_alt_lasterror.getoffset()
+    else:
+        return 6 * WORD
 
 
 def _fetch_addr_errno():
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -63,7 +63,7 @@
                                               ad.lendescr, FLAG_FLOAT)

From noreply at buildbot.pypy.org  Wed Feb 25 19:07:51 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed, 25 Feb 2015 19:07:51 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Fix (partially) memoryview after merging
 py3k (which in turn merged py3k-memoryview).
Message-ID: <20150225180751.290FE1C12E6@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76147:27e66b7853b7
Date: 2015-02-25 19:07 +0100
http://bitbucket.org/pypy/pypy/changeset/27e66b7853b7/

Log:	Fix (partially) memoryview after merging py3k (which in turn merged
	py3k-memoryview).

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -9,6 +9,7 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.typedef import TypeDef, GetSetProperty,  make_weakref_descr
+from pypy.module.struct.formatiterator import UnpackFormatIterator, PackFormatIterator
 
 
 class W_MemoryView(W_Root):
@@ -72,23 +73,20 @@
 
     def descr_tolist(self, space):
         self._check_released(space)
-        buf = self.buf
-        if self.format != 'B':
-            raise oefmt(space.w_NotImplementedError,
-                        "tolist() only supports byte views")
-        result = []
-        for i in range(buf.getlength()):
-            result.append(space.wrap(ord(buf.getitem(i)[0])))
-        return space.newlist(result)
+        # TODO: this probably isn't very fast
+        fmtiter = UnpackFormatIterator(space, self.buf)
+        fmtiter.interpret(self.format * self.getlength())
+        return space.newlist(fmtiter.result_w)
 
     def descr_getitem(self, space, w_index):
         self._check_released(space)
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
         if step == 0:  # index only
-            a = start * self.itemsize
-            b = a + self.itemsize
-            return space.wrapbytes(
-                ''.join([self.buf.getitem(i) for i in range(a, b)]))
+            # TODO: this probably isn't very fast
+            buf = SubBuffer(self.buf, start * self.itemsize, self.itemsize)
+            fmtiter = UnpackFormatIterator(space, buf)
+            fmtiter.interpret(self.format)
+            return fmtiter.result_w[0]
         elif step == 1:
             buf = SubBuffer(self.buf, start * self.itemsize,
                             size * self.itemsize)
@@ -101,13 +99,19 @@
         if self.buf.readonly:
             raise oefmt(space.w_TypeError, "cannot modify read-only memory")
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
-        if step not in (0, 1):
+        if step == 0:  # index only
+            # TODO: this probably isn't very fast
+            fmtiter = PackFormatIterator(space, [w_obj], self.itemsize)
+            fmtiter.interpret(self.format)
+            self.buf.setslice(start * self.itemsize, fmtiter.result.build())
+        elif step == 1:
+            value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
+            if value.getlength() != size * self.itemsize:
+                raise oefmt(space.w_ValueError,
+                            "cannot modify size of memoryview object")
+            self.buf.setslice(start * self.itemsize, value.as_str())
+        else:
             raise oefmt(space.w_NotImplementedError, "")
-        value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
-        if value.getlength() != size * self.itemsize:
-            raise oefmt(space.w_ValueError,
-                        "cannot modify size of memoryview object")
-        self.buf.setslice(start * self.itemsize, value.as_str())
 
     def descr_len(self, space):
         self._check_released(space)
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -140,9 +140,9 @@
         assert m.itemsize == 4
         assert len(m) == 10
         assert len(m.tobytes()) == 40
-        assert m[0] == b'\x00\x00\x00\x00'
-        m[0] = b'\x00\x00\x00\x01'
-        assert m[0] == b'\x00\x00\x00\x01'
+        assert m[0] == 0
+        m[0] = 1
+        assert m[0] == 1
 
     def test_int_array_slice(self):
         import array
@@ -152,10 +152,10 @@
         assert slice.itemsize == 4
         assert len(slice) == 6
         assert len(slice.tobytes()) == 24
-        assert slice[0] in (b'\x00\x00\x00\x02', b'\x02\x00\x00\x00')
-        slice[0] = b'\x00\x00\x00\x01'
-        assert slice[0] == b'\x00\x00\x00\x01'
-        assert m[2] == b'\x00\x00\x00\x01'
+        assert slice[0] == 2
+        slice[0] = 1
+        assert slice[0] == 1
+        assert m[2] == 1
 
     def test_pypy_raw_address_base(self):
         raises(ValueError, memoryview(b"foobar")._pypy_raw_address)

From noreply at buildbot.pypy.org  Wed Feb 25 19:57:39 2015
From: noreply at buildbot.pypy.org (gutworth)
Date: Wed, 25 Feb 2015 19:57:39 +0100 (CET)
Subject: [pypy-commit] pypy default: fix merge_collapse to actually maintain
 the invariant it purports to
Message-ID: <20150225185739.9609A1C1523@cobra.cs.uni-duesseldorf.de>

Author: Benjamin Peterson 
Branch: 
Changeset: r76148:8ea07be32d98
Date: 2015-02-25 13:57 -0500
http://bitbucket.org/pypy/pypy/changeset/8ea07be32d98/

Log:	fix merge_collapse to actually maintain the invariant it purports to

	See de Gouw, Stijn and Rot, Jurriaan and de Boer, Frank S and Bubel,
	Richard and H?hnle, Reiner "OpenJDK?s java.utils.Collection.sort()
	is broken: The good, the bad and the worst case"

	Also https://bugs.python.org/issue23515

diff --git a/rpython/rlib/listsort.py b/rpython/rlib/listsort.py
--- a/rpython/rlib/listsort.py
+++ b/rpython/rlib/listsort.py
@@ -501,7 +501,8 @@
         def merge_collapse(self):
             p = self.pending
             while len(p) > 1:
-                if len(p) >= 3 and p[-3].len <= p[-2].len + p[-1].len:
+                if ((len(p) >= 3 and p[-3].len <= p[-2].len + p[-1].len) or
+                    (len(p) >= 4 and p[-4].len <= p[-3].len + p[-2].len)):
                     if p[-3].len < p[-1].len:
                         self.merge_at(-3)
                     else:

From noreply at buildbot.pypy.org  Wed Feb 25 22:22:05 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 25 Feb 2015 22:22:05 +0100 (CET)
Subject: [pypy-commit] pypy object-dtype: merge default into branch
Message-ID: <20150225212205.045051C02BD@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: object-dtype
Changeset: r76149:503ff7b93510
Date: 2015-02-25 00:18 +0200
http://bitbucket.org/pypy/pypy/changeset/503ff7b93510/

Log:	merge default into branch

diff too long, truncating to 2000 out of 61760 lines

diff --git a/.gitignore b/.gitignore
--- a/.gitignore
+++ b/.gitignore
@@ -7,7 +7,10 @@
 
 bin/pypy-c
 include/*.h
+include/numpy/
 lib_pypy/ctypes_config_cache/_[^_]*_*.py
+libpypy-c.*
+pypy-c
 pypy/_cache
 pypy/doc/*.html
 pypy/doc/config/*.html
@@ -18,4 +21,5 @@
 pypy/translator/c/src/dtoa.o
 pypy/translator/goal/pypy-c
 pypy/translator/goal/target*-c
-release/
\ No newline at end of file
+release/
+rpython/_cache/
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -7,10 +7,7 @@
 9b623bc48b5950cf07184462a0e48f2c4df0d720 pypy-2.1-beta1-arm
 ab0dd631c22015ed88e583d9fdd4c43eebf0be21 pypy-2.1-beta1-arm
 20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0
-20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0
-0000000000000000000000000000000000000000 release-2.3.0
 394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3
 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1
 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1
-32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1
-0000000000000000000000000000000000000000 release-2.2=3.1
+10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -28,7 +28,7 @@
     DEALINGS IN THE SOFTWARE.
 
 
-PyPy Copyright holders 2003-2014
+PyPy Copyright holders 2003-2015
 ----------------------------------- 
 
 Except when otherwise stated (look for LICENSE files or information at
@@ -42,19 +42,19 @@
   Amaury Forgeot d'Arc
   Samuele Pedroni
   Alex Gaynor
+  Brian Kearns
+  Matti Picus
+  Philip Jenvey
   Michael Hudson
   David Schneider
-  Matti Picus
-  Brian Kearns
-  Philip Jenvey
   Holger Krekel
   Christian Tismer
   Hakan Ardo
   Benjamin Peterson
   Manuel Jacob
+  Ronan Lamy
   Anders Chrigstrom
   Eric van Riet Paap
-  Ronan Lamy
   Wim Lavrijsen
   Richard Emslie
   Alexander Schremmer
@@ -68,9 +68,9 @@
   Camillo Bruni
   Laura Creighton
   Toon Verwaest
+  Romain Guillebert
   Leonardo Santagada
   Seo Sanghyeon
-  Romain Guillebert
   Justin Peel
   Ronny Pfannschmidt
   David Edelsohn
@@ -91,15 +91,16 @@
   Michal Bendowski
   Jan de Mooij
   stian
+  Tyler Wade
   Michael Foord
   Stephan Diehl
-  Tyler Wade
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
   Patrick Maupin
   Bob Ippolito
   Bruno Gola
+  David Malcolm
   Jean-Paul Calderone
   Timo Paulssen
   Squeaky
@@ -108,18 +109,19 @@
   Marius Gedminas
   Martin Matusiak
   Konstantin Lopuhin
+  Wenzhu Man
   John Witulski
-  Wenzhu Man
+  Laurence Tratt
+  Ivan Sichmann Freitas
   Greg Price
   Dario Bertini
   Mark Pearse
   Simon Cross
-  Ivan Sichmann Freitas
   Andreas Stührk
+  Stefano Rivera
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
-  Stefano Rivera
   Paweł Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
@@ -129,7 +131,6 @@
   tav
   Taavi Burns
   Georg Brandl
-  Laurence Tratt
   Bert Freudenberg
   Stian Andreassen
   Wanja Saatkamp
@@ -141,13 +142,12 @@
   Jeremy Thurgood
   Rami Chowdhury
   Tobias Pape
-  David Malcolm
   Eugene Oden
   Henry Mason
   Vasily Kuznetsov
   Preston Timmons
+  David Ripton
   Jeff Terrace
-  David Ripton
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
@@ -166,13 +166,16 @@
   Gintautas Miliauskas
   Michael Twomey
   Lucian Branescu Mihaila
+  Yichao Yu
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
   Karl Bartel
+  Wouter van Heyst
   Brian Dorsey
   Victor Stinner
   Andrews Medina
+  anatoly techtonik
   Stuart Williams
   Jasper Schulz
   Christian Hudon
@@ -182,12 +185,11 @@
   Michael Cheng
   Justas Sadzevicius
   Gasper Zejn
-  anatoly techtonik
   Neil Shepperd
+  Stanislaw Halik
   Mikael Schönenberg
   Elmo M?ntynen
   Jonathan David Riehl
-  Stanislaw Halik
   Anders Qvist
   Corbin Simpson
   Chirag Jadwani
@@ -196,10 +198,13 @@
   Vincent Legoll
   Alan McIntyre
   Alexander Sedov
+  Attila Gobi
   Christopher Pope
   Christian Tismer 
   Marc Abramowitz
   Dan Stromberg
+  Arjun Naik
+  Valentina Mukhamedzhanova
   Stefano Parmesan
   Alexis Daboville
   Jens-Uwe Mager
@@ -213,8 +218,6 @@
   Sylvain Thenault
   Nathan Taylor
   Vladimir Kryachko
-  Arjun Naik
-  Attila Gobi
   Jacek Generowicz
   Alejandro J. Cura
   Jacob Oscarson
@@ -222,22 +225,23 @@
   Ryan Gonzalez
   Ian Foote
   Kristjan Valur Jonsson
+  David Lievens
   Neil Blakey-Milner
   Lutz Paelike
   Lucio Torre
   Lars Wassermann
-  Valentina Mukhamedzhanova
   Henrik Vendelbo
   Dan Buch
   Miguel de Val Borro
   Artur Lisiecki
   Sergey Kishchenko
-  Yichao Yu
   Ignas Mikalajunas
   Christoph Gerum
   Martin Blais
   Lene Wagner
   Tomo Cocoa
+  Toni Mattis
+  Lucas Stadler
   roberto at goyle
   Yury V. Zaytsev
   Anna Katrina Dominguez
@@ -265,23 +269,30 @@
   Stephan Busemann
   Rafał Gałczyński
   Christian Muirhead
+  Berker Peksag
   James Lan
   shoma hosaka
-  Daniel Neuh?user
-  Matthew Miller
+  Daniel Neuhäuser
+  Ben Mather
+  halgari
+  Boglarka Vezer
+  Chris Pressey
   Buck Golemon
   Konrad Delong
   Dinu Gherman
   Chris Lambacher
   coolbutuseless at gmail.com
+  Jim Baker
   Rodrigo Araújo
-  Jim Baker
+  Nikolaos-Digenis Karagiannis
   James Robert
   Armin Ronacher
   Brett Cannon
+  Donald Stufft
   yrttyr
   aliceinwire
   OlivierBlanvillain
+  Dan Sanders
   Zooko Wilcox-O Hearn
   Tomer Chachamu
   Christopher Groskopf
@@ -295,6 +306,7 @@
   Markus Unterwaditzer
   Even Wiik Thomassen
   jbs
+  squeaky
   soareschen
   Kurt Griffiths
   Mike Bayer
@@ -306,6 +318,7 @@
   Anna Ravencroft
   Dan Crosta
   Julien Phalip
+  Roman Podoliaka
   Dan Loewenherz
 
   Heinrich-Heine University, Germany 
diff --git a/lib-python/2.7/CGIHTTPServer.py b/lib-python/2.7/CGIHTTPServer.py
--- a/lib-python/2.7/CGIHTTPServer.py
+++ b/lib-python/2.7/CGIHTTPServer.py
@@ -106,16 +106,16 @@
     def run_cgi(self):
         """Execute a CGI script."""
         dir, rest = self.cgi_info
-
-        i = rest.find('/')
+        path = dir + '/' + rest
+        i = path.find('/', len(dir)+1)
         while i >= 0:
-            nextdir = rest[:i]
-            nextrest = rest[i+1:]
+            nextdir = path[:i]
+            nextrest = path[i+1:]
 
             scriptdir = self.translate_path(nextdir)
             if os.path.isdir(scriptdir):
                 dir, rest = nextdir, nextrest
-                i = rest.find('/')
+                i = path.find('/', len(dir)+1)
             else:
                 break
 
diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py
--- a/lib-python/2.7/Cookie.py
+++ b/lib-python/2.7/Cookie.py
@@ -56,7 +56,7 @@
    >>> C = Cookie.SmartCookie()
 
 [Note: Long-time users of Cookie.py will remember using
-Cookie.Cookie() to create an Cookie object.  Although deprecated, it
+Cookie.Cookie() to create a Cookie object.  Although deprecated, it
 is still supported by the code.  See the Backward Compatibility notes
 for more information.]
 
@@ -426,6 +426,8 @@
                    "version" : "Version",
                    }
 
+    _flags = {'secure', 'httponly'}
+
     def __init__(self):
         # Set defaults
         self.key = self.value = self.coded_value = None
@@ -529,9 +531,11 @@
 _LegalCharsPatt  = r"[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=]"
 _CookiePattern = re.compile(
     r"(?x)"                       # This is a Verbose pattern
+    r"\s*"                        # Optional whitespace at start of cookie
     r"(?P"                   # Start of group 'key'
     ""+ _LegalCharsPatt +"+?"     # Any word of at least one letter, nongreedy
     r")"                          # End of group 'key'
+    r"("                          # Optional group: there may not be a value.
     r"\s*=\s*"                    # Equal Sign
     r"(?P"                   # Start of group 'val'
     r'"(?:[^\\"]|\\.)*"'            # Any doublequoted string
@@ -540,7 +544,9 @@
     r"|"                            # or
     ""+ _LegalCharsPatt +"*"        # Any word or empty string
     r")"                          # End of group 'val'
-    r"\s*;?"                      # Probably ending in a semi-colon
+    r")?"                         # End of optional value group
+    r"\s*"                        # Any number of spaces.
+    r"(\s+|;|$)"                  # Ending either at space, semicolon, or EOS.
     )
 
 
@@ -585,8 +591,12 @@
 
     def __setitem__(self, key, value):
         """Dictionary style assignment."""
-        rval, cval = self.value_encode(value)
-        self.__set(key, rval, cval)
+        if isinstance(value, Morsel):
+            # allow assignment of constructed Morsels (e.g. for pickling)
+            dict.__setitem__(self, key, value)
+        else:
+            rval, cval = self.value_encode(value)
+            self.__set(key, rval, cval)
     # end __setitem__
 
     def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"):
@@ -641,7 +651,7 @@
 
         while 0 <= i < n:
             # Start looking for a cookie
-            match = patt.search(str, i)
+            match = patt.match(str, i)
             if not match: break          # No more cookies
 
             K,V = match.group("key"), match.group("val")
@@ -656,8 +666,12 @@
                     M[ K[1:] ] = V
             elif K.lower() in Morsel._reserved:
                 if M:
-                    M[ K ] = _unquote(V)
-            else:
+                    if V is None:
+                        if K.lower() in Morsel._flags:
+                            M[K] = True
+                    else:
+                        M[K] = _unquote(V)
+            elif V is not None:
                 rval, cval = self.value_decode(V)
                 self.__set(K, rval, cval)
                 M = self[K]
diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py
--- a/lib-python/2.7/SocketServer.py
+++ b/lib-python/2.7/SocketServer.py
@@ -416,8 +416,12 @@
         self.socket = socket.socket(self.address_family,
                                     self.socket_type)
         if bind_and_activate:
-            self.server_bind()
-            self.server_activate()
+            try:
+                self.server_bind()
+                self.server_activate()
+            except:
+                self.server_close()
+                raise
 
     def server_bind(self):
         """Called by constructor to bind the socket.
diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py
--- a/lib-python/2.7/_abcoll.py
+++ b/lib-python/2.7/_abcoll.py
@@ -143,7 +143,7 @@
     methods except for __contains__, __iter__ and __len__.
 
     To override the comparisons (presumably for speed, as the
-    semantics are fixed), all you have to do is redefine __le__ and
+    semantics are fixed), redefine __le__ and __ge__,
     then the other operations will automatically follow suit.
     """
 
diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py
--- a/lib-python/2.7/argparse.py
+++ b/lib-python/2.7/argparse.py
@@ -1089,7 +1089,14 @@
         # parse all the remaining options into the namespace
         # store any unrecognized options on the object, so that the top
         # level parser can decide what to do with them
-        namespace, arg_strings = parser.parse_known_args(arg_strings, namespace)
+
+        # In case this subparser defines new defaults, we parse them
+        # in a new namespace object and then update the original
+        # namespace for the relevant parts.
+        subnamespace, arg_strings = parser.parse_known_args(arg_strings, None)
+        for key, value in vars(subnamespace).items():
+            setattr(namespace, key, value)
+
         if arg_strings:
             vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, [])
             getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings)
diff --git a/lib-python/2.7/asynchat.py b/lib-python/2.7/asynchat.py
--- a/lib-python/2.7/asynchat.py
+++ b/lib-python/2.7/asynchat.py
@@ -46,12 +46,17 @@
 you - by calling your self.found_terminator() method.
 """
 
+import asyncore
+import errno
 import socket
-import asyncore
 from collections import deque
 from sys import py3kwarning
 from warnings import filterwarnings, catch_warnings
 
+_BLOCKING_IO_ERRORS = (errno.EAGAIN, errno.EALREADY, errno.EINPROGRESS,
+                       errno.EWOULDBLOCK)
+
+
 class async_chat (asyncore.dispatcher):
     """This is an abstract class.  You must derive from this class, and add
     the two methods collect_incoming_data() and found_terminator()"""
@@ -109,6 +114,8 @@
         try:
             data = self.recv (self.ac_in_buffer_size)
         except socket.error, why:
+            if why.args[0] in _BLOCKING_IO_ERRORS:
+                return
             self.handle_error()
             return
 
diff --git a/lib-python/2.7/bsddb/test/test_queue.py b/lib-python/2.7/bsddb/test/test_queue.py
--- a/lib-python/2.7/bsddb/test/test_queue.py
+++ b/lib-python/2.7/bsddb/test/test_queue.py
@@ -10,6 +10,7 @@
 
 #----------------------------------------------------------------------
 
+ at unittest.skip("fails on Windows; see issue 22943")
 class SimpleQueueTestCase(unittest.TestCase):
     def setUp(self):
         self.filename = get_new_database_path()
diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -17,6 +17,10 @@
 except ImportError:
     assert '__pypy__' not in _sys.builtin_module_names
     newdict = lambda _ : {}
+try:
+    from __pypy__ import reversed_dict
+except ImportError:
+    reversed_dict = lambda d: reversed(d.keys())
 
 try:
     from thread import get_ident as _get_ident
@@ -29,142 +33,35 @@
 ################################################################################
 
 class OrderedDict(dict):
-    'Dictionary that remembers insertion order'
-    # An inherited dict maps keys to values.
-    # The inherited dict provides __getitem__, __len__, __contains__, and get.
-    # The remaining methods are order-aware.
-    # Big-O running times for all methods are the same as regular dictionaries.
+    '''Dictionary that remembers insertion order.
 
-    # The internal self.__map dict maps keys to links in a doubly linked list.
-    # The circular doubly linked list starts and ends with a sentinel element.
-    # The sentinel element never gets deleted (this simplifies the algorithm).
-    # Each link is stored as a list of length three:  [PREV, NEXT, KEY].
+    In PyPy all dicts are ordered anyway.  This is mostly useful as a
+    placeholder to mean "this dict must be ordered even on CPython".
 
-    def __init__(self, *args, **kwds):
-        '''Initialize an ordered dictionary.  The signature is the same as
-        regular dictionaries, but keyword arguments are not recommended because
-        their insertion order is arbitrary.
-
-        '''
-        if len(args) > 1:
-            raise TypeError('expected at most 1 arguments, got %d' % len(args))
-        try:
-            self.__root
-        except AttributeError:
-            self.__root = root = []                     # sentinel node
-            root[:] = [root, root, None]
-            self.__map = {}
-        self.__update(*args, **kwds)
-
-    def __setitem__(self, key, value, dict_setitem=dict.__setitem__):
-        'od.__setitem__(i, y) <==> od[i]=y'
-        # Setting a new item creates a new link at the end of the linked list,
-        # and the inherited dictionary is updated with the new key/value pair.
-        if key not in self:
-            root = self.__root
-            last = root[0]
-            last[1] = root[0] = self.__map[key] = [last, root, key]
-        return dict_setitem(self, key, value)
-
-    def __delitem__(self, key, dict_delitem=dict.__delitem__):
-        'od.__delitem__(y) <==> del od[y]'
-        # Deleting an existing item uses self.__map to find the link which gets
-        # removed by updating the links in the predecessor and successor nodes.
-        dict_delitem(self, key)
-        link_prev, link_next, _ = self.__map.pop(key)
-        link_prev[1] = link_next                        # update link_prev[NEXT]
-        link_next[0] = link_prev                        # update link_next[PREV]
-
-    def __iter__(self):
-        'od.__iter__() <==> iter(od)'
-        # Traverse the linked list in order.
-        root = self.__root
-        curr = root[1]                                  # start at the first node
-        while curr is not root:
-            yield curr[2]                               # yield the curr[KEY]
-            curr = curr[1]                              # move to next node
+    Known difference: iterating over an OrderedDict which is being
+    concurrently modified raises RuntimeError in PyPy.  In CPython
+    instead we get some behavior that appears reasonable in some
+    cases but is nonsensical in other cases.  This is officially
+    forbidden by the CPython docs, so we forbid it explicitly for now.
+    '''
 
     def __reversed__(self):
-        'od.__reversed__() <==> reversed(od)'
-        # Traverse the linked list in reverse order.
-        root = self.__root
-        curr = root[0]                                  # start at the last node
-        while curr is not root:
-            yield curr[2]                               # yield the curr[KEY]
-            curr = curr[0]                              # move to previous node
-
-    def clear(self):
-        'od.clear() -> None.  Remove all items from od.'
-        root = self.__root
-        root[:] = [root, root, None]
-        self.__map.clear()
-        dict.clear(self)
-
-    # -- the following methods do not depend on the internal structure --
-
-    def keys(self):
-        'od.keys() -> list of keys in od'
-        return list(self)
-
-    def values(self):
-        'od.values() -> list of values in od'
-        return [self[key] for key in self]
-
-    def items(self):
-        'od.items() -> list of (key, value) pairs in od'
-        return [(key, self[key]) for key in self]
-
-    def iterkeys(self):
-        'od.iterkeys() -> an iterator over the keys in od'
-        return iter(self)
-
-    def itervalues(self):
-        'od.itervalues -> an iterator over the values in od'
-        for k in self:
-            yield self[k]
-
-    def iteritems(self):
-        'od.iteritems -> an iterator over the (key, value) pairs in od'
-        for k in self:
-            yield (k, self[k])
-
-    update = MutableMapping.update
-
-    __update = update # let subclasses override update without breaking __init__
-
-    __marker = object()
-
-    def pop(self, key, default=__marker):
-        '''od.pop(k[,d]) -> v, remove specified key and return the corresponding
-        value.  If key is not found, d is returned if given, otherwise KeyError
-        is raised.
-
-        '''
-        if key in self:
-            result = self[key]
-            del self[key]
-            return result
-        if default is self.__marker:
-            raise KeyError(key)
-        return default
-
-    def setdefault(self, key, default=None):
-        'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
-        if key in self:
-            return self[key]
-        self[key] = default
-        return default
+        return reversed_dict(self)
 
     def popitem(self, last=True):
         '''od.popitem() -> (k, v), return and remove a (key, value) pair.
         Pairs are returned in LIFO order if last is true or FIFO order if false.
 
         '''
-        if not self:
-            raise KeyError('dictionary is empty')
-        key = next(reversed(self) if last else iter(self))
-        value = self.pop(key)
-        return key, value
+        if last:
+            return dict.popitem(self)
+        else:
+            it = dict.__iter__(self)
+            try:
+                k = it.next()
+            except StopIteration:
+                raise KeyError('dictionary is empty')
+            return (k, self.pop(k))
 
     def __repr__(self, _repr_running={}):
         'od.__repr__() <==> repr(od)'
@@ -183,8 +80,6 @@
         'Return state information for pickling'
         items = [[k, self[k]] for k in self]
         inst_dict = vars(self).copy()
-        for k in vars(OrderedDict()):
-            inst_dict.pop(k, None)
         if inst_dict:
             return (self.__class__, (items,), inst_dict)
         return self.__class__, (items,)
@@ -193,17 +88,6 @@
         'od.copy() -> a shallow copy of od'
         return self.__class__(self)
 
-    @classmethod
-    def fromkeys(cls, iterable, value=None):
-        '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S.
-        If not specified, the value defaults to None.
-
-        '''
-        self = cls()
-        for key in iterable:
-            self[key] = value
-        return self
-
     def __eq__(self, other):
         '''od.__eq__(y) <==> od==y.  Comparison to another OD is order-sensitive
         while comparison to a regular mapping is order-insensitive.
diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py
--- a/lib-python/2.7/cookielib.py
+++ b/lib-python/2.7/cookielib.py
@@ -1719,12 +1719,12 @@
     def __repr__(self):
         r = []
         for cookie in self: r.append(repr(cookie))
-        return "<%s[%s]>" % (self.__class__, ", ".join(r))
+        return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r))
 
     def __str__(self):
         r = []
         for cookie in self: r.append(str(cookie))
-        return "<%s[%s]>" % (self.__class__, ", ".join(r))
+        return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r))
 
 
 # derives from IOError for backwards-compatibility with Python 2.4.0
diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py
@@ -2,7 +2,6 @@
 import array
 import gc
 import unittest
-from ctypes.test import xfail
 
 class X(Structure):
     _fields_ = [("c_int", c_int)]
@@ -11,7 +10,6 @@
         self._init_called = True
 
 class Test(unittest.TestCase):
-    @xfail
     def test_fom_buffer(self):
         a = array.array("i", range(16))
         x = (c_int * 16).from_buffer(a)
@@ -34,10 +32,9 @@
         del a; gc.collect(); gc.collect(); gc.collect()
         self.assertEqual(x[:], expected)
 
-        self.assertRaises(TypeError,
+        self.assertRaises((TypeError, ValueError),
                           (c_char * 16).from_buffer, "a" * 16)
 
-    @xfail
     def test_fom_buffer_with_offset(self):
         a = array.array("i", range(16))
         x = (c_int * 15).from_buffer(a, sizeof(c_int))
@@ -46,7 +43,6 @@
         self.assertRaises(ValueError, lambda: (c_int * 16).from_buffer(a, sizeof(c_int)))
         self.assertRaises(ValueError, lambda: (c_int * 1).from_buffer(a, 16 * sizeof(c_int)))
 
-    @xfail
     def test_from_buffer_copy(self):
         a = array.array("i", range(16))
         x = (c_int * 16).from_buffer_copy(a)
@@ -71,7 +67,6 @@
         x = (c_char * 16).from_buffer_copy("a" * 16)
         self.assertEqual(x[:], "a" * 16)
 
-    @xfail
     def test_fom_buffer_copy_with_offset(self):
         a = array.array("i", range(16))
         x = (c_int * 15).from_buffer_copy(a, sizeof(c_int))
diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py
--- a/lib-python/2.7/ctypes/test/test_pointers.py
+++ b/lib-python/2.7/ctypes/test/test_pointers.py
@@ -7,6 +7,8 @@
                  c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float]
 python_types = [int, int, int, int, int, long,
                 int, long, long, long, float, float]
+LargeNamedType = type('T' * 2 ** 25, (Structure,), {})
+large_string = 'T' * 2 ** 25
 
 class PointersTestCase(unittest.TestCase):
 
@@ -188,5 +190,11 @@
             mth = WINFUNCTYPE(None)(42, "name", (), None)
             self.assertEqual(bool(mth), True)
 
+    def test_pointer_type_name(self):
+        self.assertTrue(POINTER(LargeNamedType))
+
+    def test_pointer_type_str_name(self):
+        self.assertTrue(POINTER(large_string))
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_python_api.py b/lib-python/2.7/ctypes/test/test_python_api.py
--- a/lib-python/2.7/ctypes/test/test_python_api.py
+++ b/lib-python/2.7/ctypes/test/test_python_api.py
@@ -46,8 +46,8 @@
     # This test is unreliable, because it is possible that code in
     # unittest changes the refcount of the '42' integer.  So, it
     # is disabled by default.
-    @requires("refcount")
     def test_PyInt_Long(self):
+        requires("refcount")
         ref42 = grc(42)
         pythonapi.PyInt_FromLong.restype = py_object
         self.assertEqual(pythonapi.PyInt_FromLong(42), 42)
diff --git a/lib-python/2.7/ctypes/test/test_win32.py b/lib-python/2.7/ctypes/test/test_win32.py
--- a/lib-python/2.7/ctypes/test/test_win32.py
+++ b/lib-python/2.7/ctypes/test/test_win32.py
@@ -38,8 +38,11 @@
 
 @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test')
 class FunctionCallTestCase(unittest.TestCase):
-    @requires("SEH")
+    @unittest.skipUnless('MSC' in sys.version, "SEH only supported by MSC")
+    @unittest.skipIf(sys.executable.endswith('_d.exe'),
+                     "SEH not enabled in debug builds")
     def test_SEH(self):
+        requires("SEH")
         # Call functions with invalid arguments, and make sure
         # that access violations are trapped and raise an
         # exception.
@@ -87,9 +90,29 @@
 
         dll = CDLL(_ctypes_test.__file__)
 
-        pt = POINT(10, 10)
-        rect = RECT(0, 0, 20, 20)
-        self.assertEqual(1, dll.PointInRect(byref(rect), pt))
+        pt = POINT(15, 25)
+        left = c_long.in_dll(dll, 'left')
+        top = c_long.in_dll(dll, 'top')
+        right = c_long.in_dll(dll, 'right')
+        bottom = c_long.in_dll(dll, 'bottom')
+        rect = RECT(left, top, right, bottom)
+        PointInRect = dll.PointInRect
+        PointInRect.argtypes = [POINTER(RECT), POINT]
+        self.assertEqual(1, PointInRect(byref(rect), pt))
+
+        ReturnRect = dll.ReturnRect
+        ReturnRect.argtypes = [c_int, RECT, POINTER(RECT), POINT, RECT,
+                               POINTER(RECT), POINT, RECT]
+        ReturnRect.restype = RECT
+        for i in range(4):
+            ret = ReturnRect(i, rect, pointer(rect), pt, rect,
+                         byref(rect), pt, rect)
+            # the c function will check and modify ret if something is
+            # passed in improperly
+            self.assertEqual(ret.left, left.value)
+            self.assertEqual(ret.right, right.value)
+            self.assertEqual(ret.top, top.value)
+            self.assertEqual(ret.bottom, bottom.value)
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/2.7/decimal.py b/lib-python/2.7/decimal.py
--- a/lib-python/2.7/decimal.py
+++ b/lib-python/2.7/decimal.py
@@ -136,7 +136,6 @@
 
 __version__ = '1.70'    # Highest version of the spec this complies with
 
-import copy as _copy
 import math as _math
 import numbers as _numbers
 
@@ -3665,6 +3664,8 @@
         if self._is_special:
             sign = _format_sign(self._sign, spec)
             body = str(self.copy_abs())
+            if spec['type'] == '%':
+                body += '%'
             return _format_align(sign, body, spec)
 
         # a type of None defaults to 'g' or 'G', depending on context
@@ -6033,7 +6034,10 @@
         format_dict['decimal_point'] = '.'
 
     # record whether return type should be str or unicode
-    format_dict['unicode'] = isinstance(format_spec, unicode)
+    try:
+        format_dict['unicode'] = isinstance(format_spec, unicode)
+    except NameError:
+        format_dict['unicode'] = False
 
     return format_dict
 
diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py
--- a/lib-python/2.7/distutils/__init__.py
+++ b/lib-python/2.7/distutils/__init__.py
@@ -15,5 +15,5 @@
 # Updated automatically by the Python release process.
 #
 #--start constants--
-__version__ = "2.7.8"
+__version__ = "2.7.9"
 #--end constants--
diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py
--- a/lib-python/2.7/distutils/command/build_ext.py
+++ b/lib-python/2.7/distutils/command/build_ext.py
@@ -245,7 +245,7 @@
         # Python's library directory must be appended to library_dirs
         # See Issues: #1600860, #4366
         if (sysconfig.get_config_var('Py_ENABLE_SHARED')):
-            if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")):
+            if not sysconfig.python_build:
                 # building third party extensions
                 self.library_dirs.append(sysconfig.get_config_var('LIBDIR'))
             else:
diff --git a/lib-python/2.7/distutils/command/upload.py b/lib-python/2.7/distutils/command/upload.py
--- a/lib-python/2.7/distutils/command/upload.py
+++ b/lib-python/2.7/distutils/command/upload.py
@@ -136,8 +136,8 @@
 
         # Build up the MIME payload for the POST data
         boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'
-        sep_boundary = '\n--' + boundary
-        end_boundary = sep_boundary + '--'
+        sep_boundary = '\r\n--' + boundary
+        end_boundary = sep_boundary + '--\r\n'
         body = StringIO.StringIO()
         for key, value in data.items():
             # handle multiple entries for the same name
@@ -151,14 +151,13 @@
                     fn = ""
 
                 body.write(sep_boundary)
-                body.write('\nContent-Disposition: form-data; name="%s"'%key)
+                body.write('\r\nContent-Disposition: form-data; name="%s"' % key)
                 body.write(fn)
-                body.write("\n\n")
+                body.write("\r\n\r\n")
                 body.write(value)
                 if value and value[-1] == '\r':
                     body.write('\n')  # write an extra newline (lurve Macs)
         body.write(end_boundary)
-        body.write("\n")
         body = body.getvalue()
 
         self.announce("Submitting %s to %s" % (filename, self.repository), log.INFO)
diff --git a/lib-python/2.7/distutils/file_util.py b/lib-python/2.7/distutils/file_util.py
--- a/lib-python/2.7/distutils/file_util.py
+++ b/lib-python/2.7/distutils/file_util.py
@@ -85,7 +85,8 @@
     (os.symlink) instead of copying: set it to "hard" or "sym"; if it is
     None (the default), files are copied.  Don't set 'link' on systems that
     don't support it: 'copy_file()' doesn't check if hard or symbolic
-    linking is available.
+    linking is available. If hardlink fails, falls back to
+    _copy_file_contents().
 
     Under Mac OS, uses the native file copy function in macostools; on
     other systems, uses '_copy_file_contents()' to copy file contents.
@@ -137,24 +138,31 @@
     # (Unix only, of course, but that's the caller's responsibility)
     if link == 'hard':
         if not (os.path.exists(dst) and os.path.samefile(src, dst)):
-            os.link(src, dst)
+            try:
+                os.link(src, dst)
+                return (dst, 1)
+            except OSError:
+                # If hard linking fails, fall back on copying file
+                # (some special filesystems don't support hard linking
+                #  even under Unix, see issue #8876).
+                pass
     elif link == 'sym':
         if not (os.path.exists(dst) and os.path.samefile(src, dst)):
             os.symlink(src, dst)
+            return (dst, 1)
 
     # Otherwise (non-Mac, not linking), copy the file contents and
     # (optionally) copy the times and mode.
-    else:
-        _copy_file_contents(src, dst)
-        if preserve_mode or preserve_times:
-            st = os.stat(src)
+    _copy_file_contents(src, dst)
+    if preserve_mode or preserve_times:
+        st = os.stat(src)
 
-            # According to David Ascher , utime() should be done
-            # before chmod() (at least under NT).
-            if preserve_times:
-                os.utime(dst, (st[ST_ATIME], st[ST_MTIME]))
-            if preserve_mode:
-                os.chmod(dst, S_IMODE(st[ST_MODE]))
+        # According to David Ascher , utime() should be done
+        # before chmod() (at least under NT).
+        if preserve_times:
+            os.utime(dst, (st[ST_ATIME], st[ST_MTIME]))
+        if preserve_mode:
+            os.chmod(dst, S_IMODE(st[ST_MODE]))
 
     return (dst, 1)
 
diff --git a/lib-python/2.7/distutils/sysconfig_cpython.py b/lib-python/2.7/distutils/sysconfig_cpython.py
--- a/lib-python/2.7/distutils/sysconfig_cpython.py
+++ b/lib-python/2.7/distutils/sysconfig_cpython.py
@@ -165,7 +165,8 @@
             # version and build tools may not support the same set
             # of CPU architectures for universal builds.
             global _config_vars
-            if not _config_vars.get('CUSTOMIZED_OSX_COMPILER', ''):
+            # Use get_config_var() to ensure _config_vars is initialized.
+            if not get_config_var('CUSTOMIZED_OSX_COMPILER'):
                 import _osx_support
                 _osx_support.customize_compiler(_config_vars)
                 _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True'
diff --git a/lib-python/2.7/distutils/tests/test_bdist_rpm.py b/lib-python/2.7/distutils/tests/test_bdist_rpm.py
--- a/lib-python/2.7/distutils/tests/test_bdist_rpm.py
+++ b/lib-python/2.7/distutils/tests/test_bdist_rpm.py
@@ -25,6 +25,7 @@
 """
 
 class BuildRpmTestCase(support.TempdirManager,
+                       support.EnvironGuard,
                        support.LoggingSilencer,
                        unittest.TestCase):
 
@@ -50,6 +51,7 @@
     def test_quiet(self):
         # let's create a package
         tmp_dir = self.mkdtemp()
+        os.environ['HOME'] = tmp_dir   # to confine dir '.rpmdb' creation
         pkg_dir = os.path.join(tmp_dir, 'foo')
         os.mkdir(pkg_dir)
         self.write_file((pkg_dir, 'setup.py'), SETUP_PY)
@@ -92,6 +94,7 @@
     def test_no_optimize_flag(self):
         # let's create a package that brakes bdist_rpm
         tmp_dir = self.mkdtemp()
+        os.environ['HOME'] = tmp_dir   # to confine dir '.rpmdb' creation
         pkg_dir = os.path.join(tmp_dir, 'foo')
         os.mkdir(pkg_dir)
         self.write_file((pkg_dir, 'setup.py'), SETUP_PY)
diff --git a/lib-python/2.7/distutils/tests/test_dist.py b/lib-python/2.7/distutils/tests/test_dist.py
--- a/lib-python/2.7/distutils/tests/test_dist.py
+++ b/lib-python/2.7/distutils/tests/test_dist.py
@@ -11,7 +11,7 @@
 from distutils.dist import Distribution, fix_help_options
 from distutils.cmd import Command
 import distutils.dist
-from test.test_support import TESTFN, captured_stdout, run_unittest
+from test.test_support import TESTFN, captured_stdout, run_unittest, unlink
 from distutils.tests import support
 
 
@@ -64,6 +64,7 @@
         with open(TESTFN, "w") as f:
             f.write("[global]\n")
             f.write("command_packages = foo.bar, splat")
+        self.addCleanup(unlink, TESTFN)
 
         files = [TESTFN]
         sys.argv.append("build")
diff --git a/lib-python/2.7/distutils/tests/test_file_util.py b/lib-python/2.7/distutils/tests/test_file_util.py
--- a/lib-python/2.7/distutils/tests/test_file_util.py
+++ b/lib-python/2.7/distutils/tests/test_file_util.py
@@ -8,6 +8,11 @@
 from distutils.tests import support
 from test.test_support import run_unittest
 
+
+requires_os_link = unittest.skipUnless(hasattr(os, "link"),
+                                       "test requires os.link()")
+
+
 class FileUtilTestCase(support.TempdirManager, unittest.TestCase):
 
     def _log(self, msg, *args):
@@ -74,6 +79,44 @@
         copy_file(foo, dst_dir)
         self.assertTrue(os.path.exists(os.path.join(dst_dir, 'foo')))
 
+    @requires_os_link
+    def test_copy_file_hard_link(self):
+        with open(self.source, 'w') as f:
+            f.write('some content')
+        st = os.stat(self.source)
+        copy_file(self.source, self.target, link='hard')
+        st2 = os.stat(self.source)
+        st3 = os.stat(self.target)
+        self.assertTrue(os.path.samestat(st, st2), (st, st2))
+        self.assertTrue(os.path.samestat(st2, st3), (st2, st3))
+        with open(self.source, 'r') as f:
+            self.assertEqual(f.read(), 'some content')
+
+    @requires_os_link
+    def test_copy_file_hard_link_failure(self):
+        # If hard linking fails, copy_file() falls back on copying file
+        # (some special filesystems don't support hard linking even under
+        #  Unix, see issue #8876).
+        with open(self.source, 'w') as f:
+            f.write('some content')
+        st = os.stat(self.source)
+        def _os_link(*args):
+            raise OSError(0, "linking unsupported")
+        old_link = os.link
+        os.link = _os_link
+        try:
+            copy_file(self.source, self.target, link='hard')
+        finally:
+            os.link = old_link
+        st2 = os.stat(self.source)
+        st3 = os.stat(self.target)
+        self.assertTrue(os.path.samestat(st, st2), (st, st2))
+        self.assertFalse(os.path.samestat(st2, st3), (st2, st3))
+        for fn in (self.source, self.target):
+            with open(fn, 'r') as f:
+                self.assertEqual(f.read(), 'some content')
+
+
 def test_suite():
     return unittest.makeSuite(FileUtilTestCase)
 
diff --git a/lib-python/2.7/distutils/tests/test_sysconfig.py b/lib-python/2.7/distutils/tests/test_sysconfig.py
--- a/lib-python/2.7/distutils/tests/test_sysconfig.py
+++ b/lib-python/2.7/distutils/tests/test_sysconfig.py
@@ -3,6 +3,9 @@
 import test
 import unittest
 import shutil
+import subprocess
+import sys
+import textwrap
 
 from distutils import sysconfig
 from distutils.tests import support
@@ -99,6 +102,24 @@
         self.assertEqual(global_sysconfig.get_config_var('LDSHARED'), sysconfig.get_config_var('LDSHARED'))
         self.assertEqual(global_sysconfig.get_config_var('CC'), sysconfig.get_config_var('CC'))
 
+    def test_customize_compiler_before_get_config_vars(self):
+        # Issue #21923: test that a Distribution compiler
+        # instance can be called without an explicit call to
+        # get_config_vars().
+        with open(TESTFN, 'w') as f:
+            f.writelines(textwrap.dedent('''\
+                from distutils.core import Distribution
+                config = Distribution().get_command_obj('config')
+                # try_compile may pass or it may fail if no compiler
+                # is found but it should not raise an exception.
+                rc = config.try_compile('int x;')
+                '''))
+        p = subprocess.Popen([str(sys.executable), TESTFN],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                universal_newlines=True)
+        outs, errs = p.communicate()
+        self.assertEqual(0, p.returncode, "Subprocess failed: " + outs)
 
 
 def test_suite():
diff --git a/lib-python/2.7/distutils/tests/test_upload.py b/lib-python/2.7/distutils/tests/test_upload.py
--- a/lib-python/2.7/distutils/tests/test_upload.py
+++ b/lib-python/2.7/distutils/tests/test_upload.py
@@ -119,7 +119,7 @@
         # what did we send ?
         self.assertIn('dédé', self.last_open.req.data)
         headers = dict(self.last_open.req.headers)
-        self.assertEqual(headers['Content-length'], '2085')
+        self.assertEqual(headers['Content-length'], '2159')
         self.assertTrue(headers['Content-type'].startswith('multipart/form-data'))
         self.assertEqual(self.last_open.req.get_method(), 'POST')
         self.assertEqual(self.last_open.req.get_full_url(),
diff --git a/lib-python/2.7/distutils/unixccompiler.py b/lib-python/2.7/distutils/unixccompiler.py
--- a/lib-python/2.7/distutils/unixccompiler.py
+++ b/lib-python/2.7/distutils/unixccompiler.py
@@ -58,7 +58,7 @@
     executables = {'preprocessor' : None,
                    'compiler'     : ["cc"],
                    'compiler_so'  : ["cc"],
-                   'compiler_cxx' : ["cc"],
+                   'compiler_cxx' : ["c++"],  # pypy: changed, 'cc' is bogus
                    'linker_so'    : ["cc", "-shared"],
                    'linker_exe'   : ["cc"],
                    'archiver'     : ["ar", "-cr"],
diff --git a/lib-python/2.7/doctest.py b/lib-python/2.7/doctest.py
--- a/lib-python/2.7/doctest.py
+++ b/lib-python/2.7/doctest.py
@@ -216,7 +216,7 @@
                 # get_data() opens files as 'rb', so one must do the equivalent
                 # conversion as universal newlines would do.
                 return file_contents.replace(os.linesep, '\n'), filename
-    with open(filename) as f:
+    with open(filename, 'U') as f:
         return f.read(), filename
 
 # Use sys.stdout encoding for ouput.
diff --git a/lib-python/2.7/email/feedparser.py b/lib-python/2.7/email/feedparser.py
--- a/lib-python/2.7/email/feedparser.py
+++ b/lib-python/2.7/email/feedparser.py
@@ -49,8 +49,8 @@
     simple abstraction -- it parses until EOF closes the current message.
     """
     def __init__(self):
-        # The last partial line pushed into this object.
-        self._partial = ''
+        # Chunks of the last partial line pushed into this object.
+        self._partial = []
         # The list of full, pushed lines, in reverse order
         self._lines = []
         # The stack of false-EOF checking predicates.
@@ -66,8 +66,8 @@
 
     def close(self):
         # Don't forget any trailing partial line.
-        self._lines.append(self._partial)
-        self._partial = ''
+        self.pushlines(''.join(self._partial).splitlines(True))
+        self._partial = []
         self._closed = True
 
     def readline(self):
@@ -95,8 +95,29 @@
 
     def push(self, data):
         """Push some new data into this object."""
-        # Handle any previous leftovers
-        data, self._partial = self._partial + data, ''
+        # Crack into lines, but preserve the linesep characters on the end of each
+        parts = data.splitlines(True)
+
+        if not parts or not parts[0].endswith(('\n', '\r')):
+            # No new complete lines, so just accumulate partials
+            self._partial += parts
+            return
+
+        if self._partial:
+            # If there are previous leftovers, complete them now
+            self._partial.append(parts[0])
+            parts[0:1] = ''.join(self._partial).splitlines(True)
+            del self._partial[:]
+
+        # If the last element of the list does not end in a newline, then treat
+        # it as a partial line.  We only check for '\n' here because a line
+        # ending with '\r' might be a line that was split in the middle of a
+        # '\r\n' sequence (see bugs 1555570 and 1721862).
+        if not parts[-1].endswith('\n'):
+            self._partial = [parts.pop()]
+        self.pushlines(parts)
+
+    def pushlines(self, lines):
         # Crack into lines, but preserve the newlines on the end of each
         parts = NLCRE_crack.split(data)
         # The *ahem* interesting behaviour of re.split when supplied grouping
diff --git a/lib-python/2.7/email/mime/nonmultipart.py b/lib-python/2.7/email/mime/nonmultipart.py
--- a/lib-python/2.7/email/mime/nonmultipart.py
+++ b/lib-python/2.7/email/mime/nonmultipart.py
@@ -12,7 +12,7 @@
 
 

 class MIMENonMultipart(MIMEBase):
-    """Base class for MIME multipart/* type messages."""
+    """Base class for MIME non-multipart type messages."""
 
     def attach(self, payload):
         # The public API prohibits attaching multiple subparts to MIMEBase
diff --git a/lib-python/2.7/email/test/test_email.py b/lib-python/2.7/email/test/test_email.py
--- a/lib-python/2.7/email/test/test_email.py
+++ b/lib-python/2.7/email/test/test_email.py
@@ -11,6 +11,7 @@
 import warnings
 import textwrap
 from cStringIO import StringIO
+from random import choice
 
 import email
 
@@ -2578,16 +2579,64 @@
             bsf.push(il)
             nt += n
             n1 = 0
-            while True:
-                ol = bsf.readline()
-                if ol == NeedMoreData:
-                    break
+            for ol in iter(bsf.readline, NeedMoreData):
                 om.append(ol)
                 n1 += 1
             self.assertEqual(n, n1)
         self.assertEqual(len(om), nt)
         self.assertEqual(''.join([il for il, n in imt]), ''.join(om))
 
+    def test_push_random(self):
+        from email.feedparser import BufferedSubFile, NeedMoreData
+
+        n = 10000
+        chunksize = 5
+        chars = 'abcd \t\r\n'
+
+        s = ''.join(choice(chars) for i in range(n)) + '\n'
+        target = s.splitlines(True)
+
+        bsf = BufferedSubFile()
+        lines = []
+        for i in range(0, len(s), chunksize):
+            chunk = s[i:i+chunksize]
+            bsf.push(chunk)
+            lines.extend(iter(bsf.readline, NeedMoreData))
+        self.assertEqual(lines, target)
+
+
+class TestFeedParsers(TestEmailBase):
+
+    def parse(self, chunks):
+        from email.feedparser import FeedParser
+        feedparser = FeedParser()
+        for chunk in chunks:
+            feedparser.feed(chunk)
+        return feedparser.close()
+
+    def test_newlines(self):
+        m = self.parse(['a:\nb:\rc:\r\nd:\n'])
+        self.assertEqual(m.keys(), ['a', 'b', 'c', 'd'])
+        m = self.parse(['a:\nb:\rc:\r\nd:'])
+        self.assertEqual(m.keys(), ['a', 'b', 'c', 'd'])
+        m = self.parse(['a:\rb', 'c:\n'])
+        self.assertEqual(m.keys(), ['a', 'bc'])
+        m = self.parse(['a:\r', 'b:\n'])
+        self.assertEqual(m.keys(), ['a', 'b'])
+        m = self.parse(['a:\r', '\nb:\n'])
+        self.assertEqual(m.keys(), ['a', 'b'])
+
+    def test_long_lines(self):
+        # Expected peak memory use on 32-bit platform: 4*N*M bytes.
+        M, N = 1000, 20000
+        m = self.parse(['a:b\n\n'] + ['x'*M] * N)
+        self.assertEqual(m.items(), [('a', 'b')])
+        self.assertEqual(m.get_payload(), 'x'*M*N)
+        m = self.parse(['a:b\r\r'] + ['x'*M] * N)
+        self.assertEqual(m.items(), [('a', 'b')])
+        self.assertEqual(m.get_payload(), 'x'*M*N)
+        m = self.parse(['a:\r', 'b: '] + ['x'*M] * N)
+        self.assertEqual(m.items(), [('a', ''), ('b', 'x'*M*N)])
 
 
 class TestParsers(TestEmailBase):
@@ -3180,7 +3229,6 @@
         self.assertEqual(res, '=?iso-8859-2?q?abc?=')
         self.assertIsInstance(res, str)
 
-
 # Test RFC 2231 header parameters (en/de)coding
 class TestRFC2231(TestEmailBase):
     def test_get_param(self):
diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/ensurepip/__init__.py
@@ -0,0 +1,227 @@
+#!/usr/bin/env python2
+from __future__ import print_function
+
+import os
+import os.path
+import pkgutil
+import shutil
+import sys
+import tempfile
+
+
+__all__ = ["version", "bootstrap"]
+
+
+_SETUPTOOLS_VERSION = "7.0"
+
+_PIP_VERSION = "1.5.6"
+
+# pip currently requires ssl support, so we try to provide a nicer
+# error message when that is missing (http://bugs.python.org/issue19744)
+_MISSING_SSL_MESSAGE = ("pip {} requires SSL/TLS".format(_PIP_VERSION))
+try:
+    import ssl
+except ImportError:
+    ssl = None
+
+    def _require_ssl_for_pip():
+        raise RuntimeError(_MISSING_SSL_MESSAGE)
+else:
+    def _require_ssl_for_pip():
+        pass
+
+_PROJECTS = [
+    ("setuptools", _SETUPTOOLS_VERSION),
+    ("pip", _PIP_VERSION),
+]
+
+
+def _run_pip(args, additional_paths=None):
+    # Add our bundled software to the sys.path so we can import it
+    if additional_paths is not None:
+        sys.path = additional_paths + sys.path
+
+    # Install the bundled software
+    import pip
+    pip.main(args)
+
+
+def version():
+    """
+    Returns a string specifying the bundled version of pip.
+    """
+    return _PIP_VERSION
+
+
+def _disable_pip_configuration_settings():
+    # We deliberately ignore all pip environment variables
+    # when invoking pip
+    # See http://bugs.python.org/issue19734 for details
+    keys_to_remove = [k for k in os.environ if k.startswith("PIP_")]
+    for k in keys_to_remove:
+        del os.environ[k]
+    # We also ignore the settings in the default pip configuration file
+    # See http://bugs.python.org/issue20053 for details
+    os.environ['PIP_CONFIG_FILE'] = os.devnull
+
+
+def bootstrap(root=None, upgrade=False, user=False,
+              altinstall=False, default_pip=True,
+              verbosity=0):
+    """
+    Bootstrap pip into the current Python installation (or the given root
+    directory).
+
+    Note that calling this function will alter both sys.path and os.environ.
+    """
+    if altinstall and default_pip:
+        raise ValueError("Cannot use altinstall and default_pip together")
+
+    _require_ssl_for_pip()
+    _disable_pip_configuration_settings()
+
+    # By default, installing pip and setuptools installs all of the
+    # following scripts (X.Y == running Python version):
+    #
+    #   pip, pipX, pipX.Y, easy_install, easy_install-X.Y
+    #
+    # pip 1.5+ allows ensurepip to request that some of those be left out
+    if altinstall:
+        # omit pip, pipX and easy_install
+        os.environ["ENSUREPIP_OPTIONS"] = "altinstall"
+    elif not default_pip:
+        # omit pip and easy_install
+        os.environ["ENSUREPIP_OPTIONS"] = "install"
+
+    tmpdir = tempfile.mkdtemp()
+    try:
+        # Put our bundled wheels into a temporary directory and construct the
+        # additional paths that need added to sys.path
+        additional_paths = []
+        for project, version in _PROJECTS:
+            wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version)
+            whl = pkgutil.get_data(
+                "ensurepip",
+                "_bundled/{}".format(wheel_name),
+            )
+            with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
+                fp.write(whl)
+
+            additional_paths.append(os.path.join(tmpdir, wheel_name))
+
+        # Construct the arguments to be passed to the pip command
+        args = ["install", "--no-index", "--find-links", tmpdir]
+        if root:
+            args += ["--root", root]
+        if upgrade:
+            args += ["--upgrade"]
+        if user:
+            args += ["--user"]
+        if verbosity:
+            args += ["-" + "v" * verbosity]
+
+        _run_pip(args + [p[0] for p in _PROJECTS], additional_paths)
+    finally:
+        shutil.rmtree(tmpdir, ignore_errors=True)
+
+
+def _uninstall_helper(verbosity=0):
+    """Helper to support a clean default uninstall process on Windows
+
+    Note that calling this function may alter os.environ.
+    """
+    # Nothing to do if pip was never installed, or has been removed
+    try:
+        import pip
+    except ImportError:
+        return
+
+    # If the pip version doesn't match the bundled one, leave it alone
+    if pip.__version__ != _PIP_VERSION:
+        msg = ("ensurepip will only uninstall a matching version "
+               "({!r} installed, {!r} bundled)")
+        print(msg.format(pip.__version__, _PIP_VERSION), file=sys.stderr)
+        return
+
+    _require_ssl_for_pip()
+    _disable_pip_configuration_settings()
+
+    # Construct the arguments to be passed to the pip command
+    args = ["uninstall", "-y"]
+    if verbosity:
+        args += ["-" + "v" * verbosity]
+
+    _run_pip(args + [p[0] for p in reversed(_PROJECTS)])
+
+
+def _main(argv=None):
+    if ssl is None:
+        print("Ignoring ensurepip failure: {}".format(_MISSING_SSL_MESSAGE),
+              file=sys.stderr)
+        return
+
+    import argparse
+    parser = argparse.ArgumentParser(prog="python -m ensurepip")
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="pip {}".format(version()),
+        help="Show the version of pip that is bundled with this Python.",
+    )
+    parser.add_argument(
+        "-v", "--verbose",
+        action="count",
+        default=0,
+        dest="verbosity",
+        help=("Give more output. Option is additive, and can be used up to 3 "
+              "times."),
+    )
+    parser.add_argument(
+        "-U", "--upgrade",
+        action="store_true",
+        default=False,
+        help="Upgrade pip and dependencies, even if already installed.",
+    )
+    parser.add_argument(
+        "--user",
+        action="store_true",
+        default=False,
+        help="Install using the user scheme.",
+    )
+    parser.add_argument(
+        "--root",
+        default=None,
+        help="Install everything relative to this alternate root directory.",
+    )
+    parser.add_argument(
+        "--altinstall",
+        action="store_true",
+        default=False,
+        help=("Make an alternate install, installing only the X.Y versioned"
+              "scripts (Default: pipX, pipX.Y, easy_install-X.Y)"),
+    )
+    parser.add_argument(
+        "--default-pip",
+        action="store_true",
+        default=True,
+        dest="default_pip",
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--no-default-pip",
+        action="store_false",
+        dest="default_pip",
+        help=("Make a non default install, installing only the X and X.Y "
+              "versioned scripts."),
+    )
+
+    args = parser.parse_args(argv)
+
+    bootstrap(
+        root=args.root,
+        upgrade=args.upgrade,
+        user=args.user,
+        verbosity=args.verbosity,
+        altinstall=args.altinstall,
+        default_pip=args.default_pip,
+    )
diff --git a/lib-python/2.7/ensurepip/__main__.py b/lib-python/2.7/ensurepip/__main__.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/ensurepip/__main__.py
@@ -0,0 +1,4 @@
+import ensurepip
+
+if __name__ == "__main__":
+    ensurepip._main()
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..097ab43430d4c1302b0be353a8c16407c370693b
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..fa1d1054da1dab98f8906555d31a9fda271b3a85
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_uninstall.py b/lib-python/2.7/ensurepip/_uninstall.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/ensurepip/_uninstall.py
@@ -0,0 +1,30 @@
+"""Basic pip uninstallation support, helper for the Windows uninstaller"""
+
+import argparse
+import ensurepip
+
+
+def _main(argv=None):
+    parser = argparse.ArgumentParser(prog="python -m ensurepip._uninstall")
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="pip {}".format(ensurepip.version()),
+        help="Show the version of pip this will attempt to uninstall.",
+    )
+    parser.add_argument(
+        "-v", "--verbose",
+        action="count",
+        default=0,
+        dest="verbosity",
+        help=("Give more output. Option is additive, and can be used up to 3 "
+              "times."),
+    )
+
+    args = parser.parse_args(argv)
+
+    ensurepip._uninstall_helper(verbosity=args.verbosity)
+
+
+if __name__ == "__main__":
+    _main()
diff --git a/lib-python/2.7/glob.py b/lib-python/2.7/glob.py
--- a/lib-python/2.7/glob.py
+++ b/lib-python/2.7/glob.py
@@ -35,11 +35,16 @@
     patterns.
 
     """
+    dirname, basename = os.path.split(pathname)
     if not has_magic(pathname):
-        if os.path.lexists(pathname):
-            yield pathname
+        if basename:
+            if os.path.lexists(pathname):
+                yield pathname
+        else:
+            # Patterns ending with a slash should match only directories
+            if os.path.isdir(dirname):
+                yield pathname
         return
-    dirname, basename = os.path.split(pathname)
     if not dirname:
         for name in glob1(os.curdir, basename):
             yield name
diff --git a/lib-python/2.7/gzip.py b/lib-python/2.7/gzip.py
--- a/lib-python/2.7/gzip.py
+++ b/lib-python/2.7/gzip.py
@@ -164,9 +164,16 @@
     def _write_gzip_header(self):
         self.fileobj.write('\037\213')             # magic header
         self.fileobj.write('\010')                 # compression method
-        fname = os.path.basename(self.name)
-        if fname.endswith(".gz"):
-            fname = fname[:-3]
+        try:
+            # RFC 1952 requires the FNAME field to be Latin-1. Do not
+            # include filenames that cannot be represented that way.
+            fname = os.path.basename(self.name)
+            if not isinstance(fname, str):
+                fname = fname.encode('latin-1')
+            if fname.endswith('.gz'):
+                fname = fname[:-3]
+        except UnicodeEncodeError:
+            fname = ''
         flags = 0
         if fname:
             flags = FNAME
diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py
--- a/lib-python/2.7/hashlib.py
+++ b/lib-python/2.7/hashlib.py
@@ -15,8 +15,9 @@
 
 md5(), sha1(), sha224(), sha256(), sha384(), and sha512()
 
-More algorithms may be available on your platform but the above are
-guaranteed to exist.
+More algorithms may be available on your platform but the above are guaranteed
+to exist.  See the algorithms_guaranteed and algorithms_available attributes
+to find out what algorithm names can be passed to new().
 
 NOTE: If you want the adler32 or crc32 hash functions they are available in
 the zlib module.
@@ -58,9 +59,14 @@
 # always available algorithm is added.
 __always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')
 
+algorithms_guaranteed = set(__always_supported)
+algorithms_available = set(__always_supported)
+
 algorithms = __always_supported
 
-__all__ = __always_supported + ('new', 'algorithms', 'pbkdf2_hmac')
+__all__ = __always_supported + ('new', 'algorithms_guaranteed',
+                                'algorithms_available', 'algorithms',
+                                'pbkdf2_hmac')
 
 
 def __get_builtin_constructor(name):
@@ -128,6 +134,8 @@
     import _hashlib
     new = __hash_new
     __get_hash = __get_openssl_constructor
+    algorithms_available = algorithms_available.union(
+        _hashlib.openssl_md_meth_names)
 except ImportError:
     new = __py_new
     __get_hash = __get_builtin_constructor
diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py
--- a/lib-python/2.7/httplib.py
+++ b/lib-python/2.7/httplib.py
@@ -215,6 +215,10 @@
 # maximal line length when calling readline().
 _MAXLINE = 65536
 
+# maximum amount of headers accepted
+_MAXHEADERS = 100
+
+
 class HTTPMessage(mimetools.Message):
 
     def addheader(self, key, value):
@@ -271,6 +275,8 @@
         elif self.seekable:
             tell = self.fp.tell
         while True:
+            if len(hlist) > _MAXHEADERS:
+                raise HTTPException("got more than %d headers" % _MAXHEADERS)
             if tell:
                 try:
                     startofline = tell()
@@ -1185,21 +1191,29 @@
 
         def __init__(self, host, port=None, key_file=None, cert_file=None,
                      strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
-                     source_address=None):
+                     source_address=None, context=None):
             HTTPConnection.__init__(self, host, port, strict, timeout,
                                     source_address)
             self.key_file = key_file
             self.cert_file = cert_file
+            if context is None:
+                context = ssl._create_default_https_context()
+            if key_file or cert_file:
+                context.load_cert_chain(cert_file, key_file)
+            self._context = context
 
         def connect(self):
             "Connect to a host on a given (SSL) port."
 
-            sock = self._create_connection((self.host, self.port),
-                                          self.timeout, self.source_address)
+            HTTPConnection.connect(self)
+
             if self._tunnel_host:
-                self.sock = sock
-                self._tunnel()
-            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+                server_hostname = self._tunnel_host
+            else:
+                server_hostname = self.host
+
+            self.sock = self._context.wrap_socket(self.sock,
+                                                  server_hostname=server_hostname)
 
     __all__.append("HTTPSConnection")
 
@@ -1214,14 +1228,15 @@
         _connection_class = HTTPSConnection
 
         def __init__(self, host='', port=None, key_file=None, cert_file=None,
-                     strict=None):
+                     strict=None, context=None):
             # provide a default host, pass the X509 cert info
 
             # urf. compensate for bad input.
             if port == 0:
                 port = None
             self._setup(self._connection_class(host, port, key_file,
-                                               cert_file, strict))
+                                               cert_file, strict,
+                                               context=context))
 
             # we never actually use these for anything, but we keep them
             # here for compatibility with post-1.5.2 CVS.
diff --git a/lib-python/2.7/idlelib/Bindings.py b/lib-python/2.7/idlelib/Bindings.py
--- a/lib-python/2.7/idlelib/Bindings.py
+++ b/lib-python/2.7/idlelib/Bindings.py
@@ -75,7 +75,8 @@
    ('!_Auto-open Stack Viewer', '<>'),
    ]),
  ('options', [
-   ('_Configure IDLE...', '<>'),
+   ('Configure _IDLE', '<>'),
+   ('Configure _Extensions', '<>'),
    None,
    ]),
  ('help', [
diff --git a/lib-python/2.7/idlelib/CallTipWindow.py b/lib-python/2.7/idlelib/CallTipWindow.py
--- a/lib-python/2.7/idlelib/CallTipWindow.py
+++ b/lib-python/2.7/idlelib/CallTipWindow.py
@@ -2,9 +2,8 @@
 
 After ToolTip.py, which uses ideas gleaned from PySol
 Used by the CallTips IDLE extension.
-
 """
-from Tkinter import *
+from Tkinter import Toplevel, Label, LEFT, SOLID, TclError
 
 HIDE_VIRTUAL_EVENT_NAME = "<>"
 HIDE_SEQUENCES = ("", "")
@@ -133,35 +132,28 @@
         return bool(self.tipwindow)
 
 
-def _calltip_window(parent):
-    root = Tk()
-    root.title("Test calltips")
-    width, height, x, y = list(map(int, re.split('[x+]', parent.geometry())))
-    root.geometry("+%d+%d"%(x, y + 150))
+def _calltip_window(parent):  # htest #
+    from Tkinter import Toplevel, Text, LEFT, BOTH
 
-    class MyEditWin: # comparenceptually an editor_window
-        def __init__(self):
-            text = self.text = Text(root)
-            text.pack(side=LEFT, fill=BOTH, expand=1)
-            text.insert("insert", "string.split")
-            root.update()
-            self.calltip = CallTip(text)
+    top = Toplevel(parent)
+    top.title("Test calltips")
+    top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200,
+                  parent.winfo_rooty() + 150))
+    text = Text(top)
+    text.pack(side=LEFT, fill=BOTH, expand=1)
+    text.insert("insert", "string.split")
+    top.update()
+    calltip = CallTip(text)
 
-            text.event_add("<>", "(")
-            text.event_add("<>", ")")
-            text.bind("<>", self.calltip_show)
-            text.bind("<>", self.calltip_hide)
-
-            text.focus_set()
-            root.mainloop()
-
-        def calltip_show(self, event):
-            self.calltip.showtip("Hello world", "insert", "end")
-
-        def calltip_hide(self, event):
-            self.calltip.hidetip()
-
-    editwin = MyEditWin()
+    def calltip_show(event):
+        calltip.showtip("(s=Hello world)", "insert", "end")
+    def calltip_hide(event):
+        calltip.hidetip()
+    text.event_add("<>", "(")
+    text.event_add("<>", ")")
+    text.bind("<>", calltip_show)
+    text.bind("<>", calltip_hide)
+    text.focus_set()
 
 if __name__=='__main__':
     from idlelib.idle_test.htest import run
diff --git a/lib-python/2.7/idlelib/ClassBrowser.py b/lib-python/2.7/idlelib/ClassBrowser.py
--- a/lib-python/2.7/idlelib/ClassBrowser.py
+++ b/lib-python/2.7/idlelib/ClassBrowser.py
@@ -19,6 +19,9 @@
 from idlelib.TreeWidget import TreeNode, TreeItem, ScrolledCanvas
 from idlelib.configHandler import idleConf
 
+file_open = None  # Method...Item and Class...Item use this.
+# Normally PyShell.flist.open, but there is no PyShell.flist for htest.
+
 class ClassBrowser:
 
     def __init__(self, flist, name, path, _htest=False):
@@ -27,6 +30,9 @@
         """
         _htest - bool, change box when location running htest.
         """
+        global file_open
+        if not _htest:
+            file_open = PyShell.flist.open
         self.name = name
         self.file = os.path.join(path[0], self.name + ".py")
         self._htest = _htest
@@ -101,7 +107,7 @@
             return []
         try:
             dict = pyclbr.readmodule_ex(name, [dir] + sys.path)
-        except ImportError, msg:
+        except ImportError:
             return []
         items = []
         self.classes = {}
@@ -170,7 +176,7 @@
     def OnDoubleClick(self):
         if not os.path.exists(self.file):
             return
-        edit = PyShell.flist.open(self.file)
+        edit = file_open(self.file)
         if hasattr(self.cl, 'lineno'):
             lineno = self.cl.lineno
             edit.gotoline(lineno)
@@ -206,7 +212,7 @@
     def OnDoubleClick(self):
         if not os.path.exists(self.file):
             return
-        edit = PyShell.flist.open(self.file)
+        edit = file_open(self.file)
         edit.gotoline(self.cl.methods[self.name])
 
 def _class_browser(parent): #Wrapper for htest
@@ -221,8 +227,9 @@
     dir, file = os.path.split(file)
     name = os.path.splitext(file)[0]
     flist = PyShell.PyShellFileList(parent)
+    global file_open
+    file_open = flist.open
     ClassBrowser(flist, name, [dir], _htest=True)
-    parent.mainloop()
 
 if __name__ == "__main__":
     from idlelib.idle_test.htest import run
diff --git a/lib-python/2.7/idlelib/ColorDelegator.py b/lib-python/2.7/idlelib/ColorDelegator.py
--- a/lib-python/2.7/idlelib/ColorDelegator.py
+++ b/lib-python/2.7/idlelib/ColorDelegator.py
@@ -2,7 +2,6 @@
 import re
 import keyword
 import __builtin__
-from Tkinter import *
 from idlelib.Delegator import Delegator
 from idlelib.configHandler import idleConf
 
@@ -34,7 +33,6 @@
 
 prog = re.compile(make_pat(), re.S)
 idprog = re.compile(r"\s+(\w+)", re.S)
-asprog = re.compile(r".*?\b(as)\b")
 
 class ColorDelegator(Delegator):
 
@@ -42,7 +40,6 @@
         Delegator.__init__(self)
         self.prog = prog
         self.idprog = idprog
-        self.asprog = asprog
         self.LoadTagDefs()
 
     def setdelegate(self, delegate):
@@ -74,7 +71,6 @@
             "DEFINITION": idleConf.GetHighlight(theme, "definition"),
             "SYNC": {'background':None,'foreground':None},
             "TODO": {'background':None,'foreground':None},
-            "BREAK": idleConf.GetHighlight(theme, "break"),
             "ERROR": idleConf.GetHighlight(theme, "error"),
             # The following is used by ReplaceDialog:
             "hit": idleConf.GetHighlight(theme, "hit"),
@@ -216,22 +212,6 @@
                                     self.tag_add("DEFINITION",
                                                  head + "+%dc" % a,
                                                  head + "+%dc" % b)
-                            elif value == "import":
-                                # color all the "as" words on same line, except
-                                # if in a comment; cheap approximation to the
-                                # truth
-                                if '#' in chars:
-                                    endpos = chars.index('#')
-                                else:
-                                    endpos = len(chars)
-                                while True:
-                                    m1 = self.asprog.match(chars, b, endpos)
-                                    if not m1:
-                                        break
-                                    a, b = m1.span(1)
-                                    self.tag_add("KEYWORD",
-                                                 head + "+%dc" % a,
-                                                 head + "+%dc" % b)
                     m = self.prog.search(chars, m.end())
                 if "SYNC" in self.tag_names(next + "-1c"):
                     head = next
@@ -255,20 +235,23 @@
         for tag in self.tagdefs.keys():
             self.tag_remove(tag, "1.0", "end")
 
-def _color_delegator(parent):
+def _color_delegator(parent):  # htest #
+    from Tkinter import Toplevel, Text
     from idlelib.Percolator import Percolator
-    root = Tk()
-    root.title("Test ColorDelegator")
-    width, height, x, y = list(map(int, re.split('[x+]', parent.geometry())))
-    root.geometry("+%d+%d"%(x, y + 150))
-    source = "if somename: x = 'abc' # comment\nprint"
-    text = Text(root, background="white")
+
+    top = Toplevel(parent)
+    top.title("Test ColorDelegator")
+    top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200,
+                  parent.winfo_rooty() + 150))
+    source = "if somename: x = 'abc' # comment\nprint\n"
+    text = Text(top, background="white")
+    text.pack(expand=1, fill="both")
     text.insert("insert", source)

From noreply at buildbot.pypy.org  Wed Feb 25 22:22:06 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 25 Feb 2015 22:22:06 +0100 (CET)
Subject: [pypy-commit] pypy object-dtype: fix nonnative and object sort,
 argsort by converting _read, _write static methods
Message-ID: <20150225212206.4346E1C02BD@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: object-dtype
Changeset: r76150:6239ed06f4ed
Date: 2015-02-25 23:22 +0200
http://bitbucket.org/pypy/pypy/changeset/6239ed06f4ed/

Log:	fix nonnative and object sort, argsort by converting _read,_write
	static methods

diff --git a/pypy/module/micronumpy/selection.py b/pypy/module/micronumpy/selection.py
--- a/pypy/module/micronumpy/selection.py
+++ b/pypy/module/micronumpy/selection.py
@@ -13,18 +13,19 @@
 INT_SIZE = rffi.sizeof(lltype.Signed)
 
 all_types = (types.all_float_types + types.all_complex_types +
-             types.all_int_types)
-all_types = [i for i in all_types if not issubclass(i[0], types.Float16)]
+             types.all_int_types + [(types.ObjectType, 'object')])
 all_types = unrolling_iterable(all_types)
 
 
 def make_argsort_function(space, itemtype, comp_type, count=1):
     TP = itemtype.T
     step = rffi.sizeof(TP)
+    reader = itemtype.read_from_storage
+    writer = itemtype.write_to_storage
 
     class Repr(object):
         def __init__(self, index_stride_size, stride_size, size, values,
-                     indexes, index_start, start):
+                     indexes, index_start, start, native):
             self.index_stride_size = index_stride_size
             self.stride_size = stride_size
             self.index_start = index_start
@@ -32,39 +33,34 @@
             self.size = size
             self.values = values
             self.indexes = indexes
+            self.native = native
 
-        def getitem(self, item):
-            if count < 2:
-                v = raw_storage_getitem(TP, self.values, item * self.stride_size
-                                    + self.start)
-            else:
-                v = []
-                for i in range(count):
-                    _v = raw_storage_getitem(TP, self.values, item * self.stride_size
-                                    + self.start + step * i)
-                    v.append(_v)
+        def getitem(self, idx):
+            v = reader(TP, self.values, self.native, idx * self.stride_size + self.start)
             if comp_type == 'int':
                 v = widen(v)
             elif comp_type == 'float':
                 v = float(v)
             elif comp_type == 'complex':
                 v = [float(v[0]),float(v[1])]
+            elif comp_type == 'object':
+                pass
             else:
                 raise NotImplementedError('cannot reach')
             return (v, raw_storage_getitem(lltype.Signed, self.indexes,
-                                           item * self.index_stride_size +
+                                           idx * self.index_stride_size +
                                            self.index_start))
 
         def setitem(self, idx, item):
-            if count < 2:
-                raw_storage_setitem(self.values, idx * self.stride_size +
+            if comp_type == 'object':
+                writer(TP, self.values, self.native, idx * self.stride_size +
+                                self.start, item[0])
+            elif count == 1:
+                writer(TP, self.values, self.native, idx * self.stride_size +
                                 self.start, rffi.cast(TP, item[0]))
             else:
-                i = 0
-                for val in item[0]:
-                    raw_storage_setitem(self.values, idx * self.stride_size +
-                                self.start + i*step, rffi.cast(TP, val))
-                    i += 1
+                writer(TP, self.values, self.native, idx * self.stride_size +
+                                self.start, [rffi.cast(TP, i) for i in item[0]])
             raw_storage_setitem(self.indexes, idx * self.index_stride_size +
                                 self.index_start, item[1])
 
@@ -76,7 +72,7 @@
             values = alloc_raw_storage(size * stride_size,
                                             track_allocation=False)
             Repr.__init__(self, dtype.elsize, stride_size,
-                          size, values, indexes, start, start)
+                          size, values, indexes, start, start, True)
 
         def __del__(self):
             free_raw_storage(self.indexes, track_allocation=False)
@@ -135,11 +131,12 @@
         dtype = descriptor.get_dtype_cache(space).w_longdtype
         index_arr = W_NDimArray.from_shape(space, arr.get_shape(), dtype)
         storage = index_arr.implementation.get_storage()
+        native = arr.dtype.is_native()
         if len(arr.get_shape()) == 1:
             for i in range(arr.get_size()):
                 raw_storage_setitem(storage, i * INT_SIZE, i)
             r = Repr(INT_SIZE, itemsize, arr.get_size(), arr.get_storage(),
-                     storage, 0, arr.start)
+                     storage, 0, arr.start, native)
             ArgSort(r).sort()
         else:
             shape = arr.get_shape()
@@ -160,7 +157,8 @@
                     raw_storage_setitem(storage, i * index_stride_size +
                                         index_state.offset, i)
                 r = Repr(index_stride_size, stride_size, axis_size,
-                         arr.get_storage(), storage, index_state.offset, arr_state.offset)
+                         arr.get_storage(), storage, index_state.offset,
+                         arr_state.offset, native)
                 ArgSort(r).sort()
                 arr_state = arr_iter.next(arr_state)
                 index_state = index_iter.next(index_state)
@@ -185,52 +183,49 @@
 def make_sort_function(space, itemtype, comp_type, count=1):
     TP = itemtype.T
     step = rffi.sizeof(TP)
+    reader = itemtype.read_from_storage
+    writer = itemtype.write_to_storage
 
     class Repr(object):
-        def __init__(self, stride_size, size, values, start):
+        def __init__(self, stride_size, size, values, start, native):
             self.stride_size = stride_size
             self.start = start
             self.size = size
             self.values = values
+            self.native = native
 
-        def getitem(self, item):
-            if count < 2:
-                v = raw_storage_getitem(TP, self.values, item * self.stride_size
-                                    + self.start)
-            else:
-                v = []
-                for i in range(count):
-                    _v = raw_storage_getitem(TP, self.values, item * self.stride_size
-                                    + self.start + step * i)
-                    v.append(_v)
+        def getitem(self, idx):
+            v = reader(TP, self.values, self.native, idx * self.stride_size + self.start)
             if comp_type == 'int':
                 v = widen(v)
             elif comp_type == 'float':
                 v = float(v)
             elif comp_type == 'complex':
                 v = [float(v[0]),float(v[1])]
+            elif comp_type == 'object':
+                pass
             else:
                 raise NotImplementedError('cannot reach')
             return (v)
 
         def setitem(self, idx, item):
-            if count < 2:
-                raw_storage_setitem(self.values, idx * self.stride_size +
+            if comp_type == 'object':
+                writer(TP, self.values, self.native, idx * self.stride_size +
+                                self.start, item)
+            elif count == 1:
+                writer(TP, self.values, self.native, idx * self.stride_size +
                                 self.start, rffi.cast(TP, item))
             else:
-                i = 0
-                for val in item:
-                    raw_storage_setitem(self.values, idx * self.stride_size +
-                                self.start + i*step, rffi.cast(TP, val))
-                    i += 1
+                writer(TP, self.values, self.native, idx * self.stride_size +
+                                self.start, [rffi.cast(TP, i) for i in item])
 
     class ArgArrayRepWithStorage(Repr):
-        def __init__(self, stride_size, size):
+        def __init__(self, stride_size, size, native):
             start = 0
             values = alloc_raw_storage(size * stride_size,
                                             track_allocation=False)
             Repr.__init__(self, stride_size,
-                          size, values, start)
+                          size, values, start, native)
 
         def __del__(self):
             free_raw_storage(self.values, track_allocation=False)
@@ -283,9 +278,10 @@
         else:
             axis = space.int_w(w_axis)
         # create array of indexes
+        native = arr.dtype.is_native()
         if len(arr.get_shape()) == 1:
             r = Repr(itemsize, arr.get_size(), arr.get_storage(),
-                     arr.start)
+                     arr.start, native)
             ArgSort(r).sort()
         else:
             shape = arr.get_shape()
@@ -298,7 +294,8 @@
             stride_size = arr.strides[axis]
             axis_size = arr.shape[axis]
             while not arr_iter.done(arr_state):
-                r = Repr(stride_size, axis_size, arr.get_storage(), arr_state.offset)
+                r = Repr(stride_size, axis_size, arr.get_storage(),
+                                                 arr_state.offset, native)
                 ArgSort(r).sort()
                 arr_state = arr_iter.next(arr_state)
 
@@ -308,9 +305,6 @@
 def sort_array(arr, space, w_axis, w_order):
     cache = space.fromcache(SortCache)  # that populates SortClasses
     itemtype = arr.dtype.itemtype
-    if arr.dtype.byteorder == NPY.OPPBYTE:
-        raise oefmt(space.w_NotImplementedError,
-                    "sorting of non-native byteorder not supported yet")
     for tp in all_types:
         if isinstance(itemtype, tp[0]):
             return cache._lookup(tp)(arr, space, w_axis,
diff --git a/pypy/module/micronumpy/test/test_selection.py b/pypy/module/micronumpy/test/test_selection.py
--- a/pypy/module/micronumpy/test/test_selection.py
+++ b/pypy/module/micronumpy/test/test_selection.py
@@ -6,20 +6,17 @@
         assert array(2.0).argsort() == 0
         nnp = self.non_native_prefix
         for dtype in ['int', 'float', 'int16', 'float32', 'uint64',
-                      nnp + 'i2', complex]:
+                      nnp + 'i2', complex, 'float16']:
             a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
             exp = list(a)
             exp = sorted(range(len(exp)), key=exp.__getitem__)
             c = a.copy()
             res = a.argsort()
-            assert (res == exp).all(), '%r\n%r\n%r' % (a,res,exp)
+            assert (res == exp).all(), 'Failed sortng %r\na=%r\nres=%r\nexp=%r' % (dtype,a,res,exp)
             assert (a == c).all() # not modified
 
-            a = arange(100, dtype=dtype)
-            assert (a.argsort() == a).all()
-        import sys
-        if '__pypy__' in sys.builtin_module_names:
-            raises(NotImplementedError, 'arange(10,dtype="float16").argsort()')
+            #a = arange(100, dtype=dtype)
+            #assert (a.argsort() == a).all()
 
     def test_argsort_ndim(self):
         from numpy import array
@@ -63,14 +60,13 @@
                       'i2', complex]:
             a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
             exp = sorted(list(a))
-            res = a.copy()
-            res.sort()
-            assert (res == exp).all(), '%r\n%r\n%r' % (a,res,exp)
+            a.sort()
+            assert (a == exp).all(), 'Failed sorting %r\n%r\n%r' % (dtype, a, exp)
 
             a = arange(100, dtype=dtype)
             c = a.copy()
             a.sort()
-            assert (a == c).all()
+            assert (a == c).all(), 'Failed sortng %r\na=%r\nc=%r' % (dtype,a,c)
 
     def test_sort_nonnative(self):
         from numpy import array
@@ -79,12 +75,9 @@
             a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
             b = array([-1, 3, 3, 4, 6, 8, 100, 101, 256+20], dtype=dtype)
             c = a.copy()
-            import sys
-            if '__pypy__' in sys.builtin_module_names:
-                exc = raises(NotImplementedError, a.sort)
-                assert exc.value[0].find('supported') >= 0
-            #assert (a == b).all(), \
-            #    'a,orig,dtype %r,%r,%r' % (a,c,dtype)
+            a.sort()
+            assert (a == b).all(), \
+                'a,orig,dtype %r,%r,%r' % (a,c,dtype)
 
 # tests from numpy/tests/test_multiarray.py
     def test_sort_corner_cases(self):
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -30,7 +30,7 @@
 
 
 def simple_unary_op(func):
-    specialize.argtype(1)(func)
+    specialize.argtype(2)(func)
     @functools.wraps(func)
     def dispatcher(self, space, v):
         return self.box(
@@ -66,7 +66,7 @@
     return dispatcher
 
 def raw_unary_op(func):
-    specialize.argtype(1)(func)
+    specialize.argtype(2)(func)
     @functools.wraps(func)
     def dispatcher(self, space, v):
         return func(
@@ -172,30 +172,34 @@
     def default_fromstring(self, space):
         raise NotImplementedError
 
-    def _read(self, storage, i, offset):
-        res = raw_storage_getitem_unaligned(self.T, storage, i + offset)
-        if not self.native:
+    @staticmethod
+    def read_from_storage(T, storage, native, offset):
+        res = raw_storage_getitem_unaligned(T, storage, offset)
+        if not native:
             res = byteswap(res)
         return res
 
-    def _write(self, storage, i, offset, value):
-        if not self.native:
+    @staticmethod
+    def write_to_storage(T, storage, native, offset, value):
+        if not native:
             value = byteswap(value)
-        raw_storage_setitem_unaligned(storage, i + offset, value)
+        raw_storage_setitem_unaligned(storage, offset, value)
 
     def read(self, arr, i, offset, dtype=None):
-        return self.box(self._read(arr.storage, i, offset))
+        return self.box(self.read_from_storage(
+                                self.T, arr.storage, self.native, i + offset))
 
     def read_bool(self, arr, i, offset):
-        return bool(self.for_computation(self._read(arr.storage, i, offset)))
+        return bool(self.for_computation(self.read_from_storage(
+                self.T, arr.storage, self.native, i + offset)))
 
     def store(self, arr, i, offset, box):
-        self._write(arr.storage, i, offset, self.unbox(box))
+        self.write_to_storage(self.T, arr.storage, self.native, i + offset, self.unbox(box))
 
     def fill(self, storage, width, box, start, stop, offset):
         value = self.unbox(box)
         for i in xrange(start, stop, width):
-            self._write(storage, i, offset, value)
+            self.write_to_storage(self.T, storage, self.native, i + offset, value)
 
     def runpack_str(self, space, s):
         v = rffi.cast(self.T, runpack(self.format_code, s))
@@ -967,8 +971,8 @@
         else:
             return x
 
+FLOAT16_STORAGE_T = rffi.USHORT
 class Float16(BaseType, Float):
-    _STORAGE_T = rffi.USHORT
     T = rffi.SHORT
     BoxType = boxes.W_Float16Box
 
@@ -989,24 +993,26 @@
     def byteswap(self, w_v):
         value = self.unbox(w_v)
         hbits = float_pack(value, 2)
-        swapped = byteswap(rffi.cast(self._STORAGE_T, hbits))
+        swapped = byteswap(rffi.cast(FLOAT16_STORAGE_T, hbits))
         return self.box(float_unpack(r_ulonglong(swapped), 2))
 
-    def _read(self, storage, i, offset):
-        hbits = raw_storage_getitem_unaligned(self._STORAGE_T, storage, i + offset)
-        if not self.native:
+    @staticmethod
+    def read_from_storage(T, storage, native, offset):
+        hbits = raw_storage_getitem_unaligned(FLOAT16_STORAGE_T, storage, offset)
+        if not native:
             hbits = byteswap(hbits)
         return float_unpack(r_ulonglong(hbits), 2)
 
-    def _write(self, storage, i, offset, value):
+    @staticmethod
+    def write_to_storage(T, storage, native, offset, value):
         try:
             hbits = float_pack(value, 2)
         except OverflowError:
             hbits = float_pack(rfloat.INFINITY, 2)
-        hbits = rffi.cast(self._STORAGE_T, hbits)
-        if not self.native:
+        hbits = rffi.cast(FLOAT16_STORAGE_T, hbits)
+        if not native:
             hbits = byteswap(hbits)
-        raw_storage_setitem_unaligned(storage, i + offset, hbits)
+        raw_storage_setitem_unaligned(storage, offset, hbits)
 
 class Float32(BaseType, Float):
     T = rffi.FLOAT
@@ -1084,7 +1090,8 @@
         return bool(real) or bool(imag)
 
     def read_bool(self, arr, i, offset):
-        v = self.for_computation(self._read(arr.storage, i, offset))
+        v = self.for_computation(self.read_from_storage(
+                self.T, arr.storage, self.native, i + offset))
         return bool(v[0]) or bool(v[1])
 
     def get_element_size(self):
@@ -1127,33 +1134,36 @@
         assert isinstance(box, self.BoxType)
         return box.real, box.imag
 
-    def _read(self, storage, i, offset):
-        real = raw_storage_getitem_unaligned(self.T, storage, i + offset)
-        imag = raw_storage_getitem_unaligned(self.T, storage, i + offset + rffi.sizeof(self.T))
-        if not self.native:
+    @staticmethod
+    def read_from_storage(T, storage, native, offset):
+        real = raw_storage_getitem_unaligned(T, storage, offset)
+        imag = raw_storage_getitem_unaligned(T, storage, offset + rffi.sizeof(T))
+        if not native:
             real = byteswap(real)
             imag = byteswap(imag)
         return real, imag
 
     def read(self, arr, i, offset, dtype=None):
-        real, imag = self._read(arr.storage, i, offset)
+        real, imag = self.read_from_storage(
+                                self.T, arr.storage, self.native, i + offset)
         return self.box_complex(real, imag)
 
-    def _write(self, storage, i, offset, value):
+    @staticmethod
+    def write_to_storage(T, storage, native, offset, value):
         real, imag = value
-        if not self.native:
+        if not native:
             real = byteswap(real)
             imag = byteswap(imag)
-        raw_storage_setitem_unaligned(storage, i + offset, real)
-        raw_storage_setitem_unaligned(storage, i + offset + rffi.sizeof(self.T), imag)
+        raw_storage_setitem_unaligned(storage, offset, real)
+        raw_storage_setitem_unaligned(storage, offset + rffi.sizeof(T), imag)
 
     def store(self, arr, i, offset, box):
-        self._write(arr.storage, i, offset, self.unbox(box))
+        self.write_to_storage(self.T, arr.storage, self.native, i + offset, self.unbox(box))
 
     def fill(self, storage, width, box, start, stop, offset):
         value = self.unbox(box)
         for i in xrange(start, stop, width):
-            self._write(storage, i, offset, value)
+            self.write_to_storage(self.T, storage, self.native, i + offset, value)
 
     @complex_binary_op
     def add(self, space, v1, v2):
@@ -1642,21 +1652,24 @@
         return boxes.W_ObjectBox(w_item)
 
     def store(self, arr, i, offset, box):
-        self._write(arr.storage, i, offset, self.unbox(box))
+        self.write_to_storage(self.T, arr.storage, self.native, i + offset, self.unbox(box))
 
     def read(self, arr, i, offset, dtype=None):
-        return self.box(self._read(arr.storage, i, offset))
+        return self.box(self.read_from_storage(
+                            self.T, arr.storage, self.native, i + offset))
 
-    def _write(self, storage, i, offset, w_obj):
+    @staticmethod
+    def write_to_storage(T, storage, native, offset, w_obj):
         if we_are_translated():
             value = rffi.cast(lltype.Signed, cast_instance_to_gcref(w_obj))
         else:
             value = len(_all_objs_for_tests)
             _all_objs_for_tests.append(w_obj)
-        raw_storage_setitem_unaligned(storage, i + offset, value)
+        raw_storage_setitem_unaligned(storage, offset, value)
 
-    def _read(self, storage, i, offset):
-        res = raw_storage_getitem_unaligned(self.T, storage, i + offset)
+    @staticmethod
+    def read_from_storage(T, storage, native, offset):
+        res = raw_storage_getitem_unaligned(T, storage, offset)
         if we_are_translated():
             gcref = rffi.cast(llmemory.GCREF, res)
             w_obj = cast_gcref_to_instance(W_Root, gcref)
@@ -1667,13 +1680,16 @@
     def fill(self, storage, width, box, start, stop, offset):
         value = self.unbox(box)
         for i in xrange(start, stop, width):
-            self._write(storage, i, offset, value)
+            self.write_to_storage(self.T, storage, self.native, i + offset, value)
 
     def unbox(self, box):
         assert isinstance(box, self.BoxType)
         return box.w_obj
 
+    @specialize.argtype(1)
     def box(self, w_obj):
+        if we_are_translated():
+            assert isinstance(w_obj, W_Root)
         return self.BoxType(w_obj)
 
     def str_format(self, space, box):
@@ -1685,11 +1701,11 @@
 
     @simple_binary_op
     def add(self, space, v1, v2):
-        return space.add(v1, v2)
+        return space.add(space.wrap(v1), space.wrap(v2))
 
     @raw_binary_op
     def eq(self, space, v1, v2):
-        return space.eq_w(v1, v2)
+        return space.eq_w(space.wrap(v1), space.wrap(v2))
 
 class FlexibleType(BaseType):
     def get_element_size(self):
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -949,7 +949,7 @@
         return dt2
 
     if dt1.num == NPY.OBJECT or dt2.num == NPY.OBJECT:
-        return descriptor.get_dtype_cache(space).w_objectdtype
+        return get_dtype_cache(space).w_objectdtype
 
     # dt1.num should be <= dt2.num
     if dt1.num > dt2.num:
@@ -1065,7 +1065,7 @@
     uint64_dtype = get_dtype_cache(space).w_uint64dtype
     complex_dtype = get_dtype_cache(space).w_complex128dtype
     float_dtype = get_dtype_cache(space).w_float64dtype
-    object_dtype = descriptor.get_dtype_cache(space).w_objectdtype
+    object_dtype = get_dtype_cache(space).w_objectdtype
     if isinstance(w_obj, boxes.W_GenericBox):
         dtype = w_obj.get_dtype(space)
         return find_binop_result_dtype(space, dtype, current_guess)

From noreply at buildbot.pypy.org  Wed Feb 25 22:34:42 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 25 Feb 2015 22:34:42 +0100 (CET)
Subject: [pypy-commit] stmgc default: c8: fix for not executing major gcs
	when requested
Message-ID: <20150225213442.B1A8A1C02BB@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: 
Changeset: r1665:8fc7cb5d20c3
Date: 2015-02-25 16:46 +0100
http://bitbucket.org/pypy/stmgc/changeset/8fc7cb5d20c3/

Log:	c8: fix for not executing major gcs when requested

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -873,7 +873,8 @@
 
 void stm_commit_transaction(void)
 {
-    exec_local_finalizers();
+    major_collection_if_requested();
+    //exec_local_finalizers(); done by ^^^
 
     assert(!_has_mutex());
     assert(STM_PSEGMENT->safe_point == SP_RUNNING);

From noreply at buildbot.pypy.org  Wed Feb 25 22:34:43 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 25 Feb 2015 22:34:43 +0100 (CET)
Subject: [pypy-commit] stmgc default: c8: do previous fix in the right way
Message-ID: <20150225213443.D75EE1C02BB@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: 
Changeset: r1666:1f9775627691
Date: 2015-02-25 18:12 +0100
http://bitbucket.org/pypy/stmgc/changeset/1f9775627691/

Log:	c8: do previous fix in the right way

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -873,8 +873,7 @@
 
 void stm_commit_transaction(void)
 {
-    major_collection_if_requested();
-    //exec_local_finalizers(); done by ^^^
+    exec_local_finalizers();
 
     assert(!_has_mutex());
     assert(STM_PSEGMENT->safe_point == SP_RUNNING);
@@ -897,6 +896,15 @@
 
     stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
 
+    /* if a major collection is required, do it here */
+    if (is_major_collection_requested()) {
+        synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK);
+
+        if (is_major_collection_requested()) {   /* if *still* true */
+            major_collection_now_at_safe_point();
+        }
+    }
+
     commit_finalizers();
 
     invoke_and_clear_user_callbacks(0);   /* for commit */

From noreply at buildbot.pypy.org  Wed Feb 25 22:34:44 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 25 Feb 2015 22:34:44 +0100 (CET)
Subject: [pypy-commit] stmgc default: c8: reduce problem of huge commit logs
 triggering tons of major gcs (see multithread/raytrace benchmark)
Message-ID: <20150225213444.E71541C02BB@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: 
Changeset: r1667:01494a338210
Date: 2015-02-25 18:17 +0100
http://bitbucket.org/pypy/stmgc/changeset/01494a338210/

Log:	c8: reduce problem of huge commit logs triggering tons of major gcs
	(see multithread/raytrace benchmark)

diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -524,6 +524,7 @@
 
 static inline bool largemalloc_keep_object_at(char *data)
 {
+    /* XXX: identical to smallmalloc_keep_object_at()? */
     /* this is called by _stm_largemalloc_sweep() */
     object_t *obj = (object_t *)(data - stm_object_pages);
     //dprintf(("keep obj %p ? -> %d\n", obj, mark_visited_test(obj)));
@@ -646,9 +647,36 @@
     dprintf((" | commit log entries before: %ld\n",
              _stm_count_cl_entries()));
 
+
     /* free all commit log entries. all segments are on the most recent
        revision now. */
+    uint64_t allocd_before = pages_ctl.total_allocated;
     clean_up_commit_log_entries();
+    /* check if freeing the log entries actually freed a considerable
+       amount itself. Then we don't want to also trace the whole heap
+       and just leave major gc right here.
+       The problem is apparent from raytrace.py, but may disappear if
+       we have card marking that also reduces the size of commit log
+       entries */
+    if ((pages_ctl.total_allocated < pages_ctl.total_allocated_bound)
+        && (allocd_before - pages_ctl.total_allocated > 0.3 * allocd_before)) {
+        /* 0.3 should mean that we are at about 50% of the way to the
+           allocated_bound again */
+#ifndef STM_TESTS
+        /* we freed a considerable amount just by freeing commit log entries */
+        pages_ctl.major_collection_requested = false; // reset_m_gc_requested
+
+        dprintf(("STOP AFTER FREEING CL ENTRIES: -%ld\n",
+                 (long)(allocd_before - pages_ctl.total_allocated)));
+        dprintf((" | used after collection:  %ld\n",
+                (long)pages_ctl.total_allocated));
+        dprintf((" `----------------------------------------------\n"));
+        if (must_abort())
+            abort_with_mutex();
+
+        return;
+#endif
+    }
 
     /* only necessary because of assert that fails otherwise (XXX) */
     acquire_all_privatization_locks();

From noreply at buildbot.pypy.org  Wed Feb 25 22:43:49 2015
From: noreply at buildbot.pypy.org (gutworth)
Date: Wed, 25 Feb 2015 22:43:49 +0100 (CET)
Subject: [pypy-commit] pypy default: revert 8ea07be32d98;
 note merge_collapse actually does not necessarily restore all
 invariants
Message-ID: <20150225214349.213AB1C02BB@cobra.cs.uni-duesseldorf.de>

Author: Benjamin Peterson 
Branch: 
Changeset: r76151:11928c1ed7d0
Date: 2015-02-25 16:43 -0500
http://bitbucket.org/pypy/pypy/changeset/11928c1ed7d0/

Log:	revert 8ea07be32d98; note merge_collapse actually does not
	necessarily restore all invariants

diff --git a/rpython/rlib/listsort.py b/rpython/rlib/listsort.py
--- a/rpython/rlib/listsort.py
+++ b/rpython/rlib/listsort.py
@@ -496,13 +496,18 @@
         # 1. len[-3] > len[-2] + len[-1]
         # 2. len[-2] > len[-1]
         #
+        # Note these invariants will not hold for the entire pending array even
+        # after this function completes. [1] This does not affect the
+        # correctness of the overall algorithm.
+        #
+        # [1] http://envisage-project.eu/proving-android-java-and-python-sorting-algorithm-is-broken-and-how-to-fix-it/
+        #
         # See listsort.txt for more info.
 
         def merge_collapse(self):
             p = self.pending
             while len(p) > 1:
-                if ((len(p) >= 3 and p[-3].len <= p[-2].len + p[-1].len) or
-                    (len(p) >= 4 and p[-4].len <= p[-3].len + p[-2].len)):
+                if len(p) >= 3 and p[-3].len <= p[-2].len + p[-1].len:
                     if p[-3].len < p[-1].len:
                         self.merge_at(-3)
                     else:

From noreply at buildbot.pypy.org  Wed Feb 25 22:55:45 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 25 Feb 2015 22:55:45 +0100 (CET)
Subject: [pypy-commit] pypy object-dtype: translation fixes
Message-ID: <20150225215545.7440D1C02BB@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: object-dtype
Changeset: r76152:465c939133e5
Date: 2015-02-25 23:56 +0200
http://bitbucket.org/pypy/pypy/changeset/465c939133e5/

Log:	translation fixes

diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -42,7 +42,7 @@
     return dispatcher
 
 def complex_unary_op(func):
-    specialize.argtype(1)(func)
+    specialize.argtype(2)(func)
     @functools.wraps(func)
     def dispatcher(self, space, v):
         return self.box_complex(
@@ -54,7 +54,7 @@
     return dispatcher
 
 def complex_to_real_unary_op(func):
-    specialize.argtype(1)(func)
+    specialize.argtype(2)(func)
     @functools.wraps(func)
     def dispatcher(self, space, v):
         return self.box_component(
@@ -76,7 +76,7 @@
     return dispatcher
 
 def simple_binary_op(func):
-    specialize.argtype(1, 2)(func)
+    specialize.argtype(2, 3)(func)
     @functools.wraps(func)
     def dispatcher(self, space, v1, v2):
         return self.box(
@@ -90,7 +90,7 @@
     return dispatcher
 
 def complex_binary_op(func):
-    specialize.argtype(1, 2)(func)
+    specialize.argtype(2, 3)(func)
     @functools.wraps(func)
     def dispatcher(self, space, v1, v2):
         return self.box_complex(
@@ -103,7 +103,7 @@
     return dispatcher
 
 def raw_binary_op(func):
-    specialize.argtype(1, 2)(func)
+    specialize.argtype(2, 3)(func)
     @functools.wraps(func)
     def dispatcher(self, space, v1, v2):
         return func(self, space,
@@ -174,6 +174,7 @@
 
     @staticmethod
     def read_from_storage(T, storage, native, offset):
+        assert isinstance(T, lltype.Primitive)
         res = raw_storage_getitem_unaligned(T, storage, offset)
         if not native:
             res = byteswap(res)
@@ -400,7 +401,7 @@
             return 1
         return 0
 
-    @specialize.argtype(1)
+    @specialize.argtype(2)
     def round(self, space, v, decimals=0):
         if decimals != 0:
             return v
@@ -427,7 +428,7 @@
     def default_fromstring(self, space):
         return self.box(0)
 
-    @specialize.argtype(1, 2)
+    @specialize.argtype(2, 3)
     def div(self, space, b1, b2):
         v1 = self.for_computation(self.unbox(b1))
         v2 = self.for_computation(self.unbox(b2))
@@ -439,7 +440,7 @@
                 return self.box(0)
         return self.box(v1 / v2)
 
-    @specialize.argtype(1, 2)
+    @specialize.argtype(2, 3)
     def floordiv(self, space, b1, b2):
         v1 = self.for_computation(self.unbox(b1))
         v2 = self.for_computation(self.unbox(b2))
@@ -516,7 +517,7 @@
     def invert(self, space, v):
         return ~v
 
-    @specialize.argtype(1)
+    @specialize.argtype(2)
     def reciprocal(self, space, v):
         raw = self.for_computation(self.unbox(v))
         ans = 0
@@ -528,7 +529,7 @@
             ans = raw
         return self.box(ans)
 
-    @specialize.argtype(1)
+    @specialize.argtype(2)
     def round(self, space, v, decimals=0):
         raw = self.for_computation(self.unbox(v))
         if decimals < 0:
@@ -755,7 +756,7 @@
     def ceil(self, space, v):
         return math.ceil(v)
 
-    @specialize.argtype(1)
+    @specialize.argtype(2)
     def round(self, space, v, decimals=0):
         raw = self.for_computation(self.unbox(v))
         if rfloat.isinf(raw):
@@ -998,6 +999,7 @@
 
     @staticmethod
     def read_from_storage(T, storage, native, offset):
+        assert isinstance(T, lltype.Primitive)
         hbits = raw_storage_getitem_unaligned(FLOAT16_STORAGE_T, storage, offset)
         if not native:
             hbits = byteswap(hbits)
@@ -1136,6 +1138,7 @@
 
     @staticmethod
     def read_from_storage(T, storage, native, offset):
+        assert isinstance(T, lltype.Primitive)
         real = raw_storage_getitem_unaligned(T, storage, offset)
         imag = raw_storage_getitem_unaligned(T, storage, offset + rffi.sizeof(T))
         if not native:
@@ -1669,6 +1672,7 @@
 
     @staticmethod
     def read_from_storage(T, storage, native, offset):
+        assert isinstance(T, lltype.Primitive)
         res = raw_storage_getitem_unaligned(T, storage, offset)
         if we_are_translated():
             gcref = rffi.cast(llmemory.GCREF, res)
@@ -1726,14 +1730,14 @@
         return builder.build()
 
 def str_unary_op(func):
-    specialize.argtype(1)(func)
+    specialize.argtype(2)(func)
     @functools.wraps(func)
-    def dispatcher(self, v1):
-        return func(self, self.to_str(v1))
+    def dispatcher(self, space, v1):
+        return func(self, space, self.to_str(v1))
     return dispatcher
 
 def str_binary_op(func):
-    specialize.argtype(1, 2)(func)
+    specialize.argtype(2, 3)(func)
     @functools.wraps(func)
     def dispatcher(self, space, v1, v2):
         return func(self, space,

From noreply at buildbot.pypy.org  Thu Feb 26 10:22:58 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 26 Feb 2015 10:22:58 +0100 (CET)
Subject: [pypy-commit] cffi default: issue #177: workaround for some Linux
	kernels
Message-ID: <20150226092258.03F401C1212@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r1651:c7edb1e84eb3
Date: 2015-02-26 10:22 +0100
http://bitbucket.org/cffi/cffi/changeset/c7edb1e84eb3/

Log:	issue #177: workaround for some Linux kernels

diff --git a/c/malloc_closure.h b/c/malloc_closure.h
--- a/c/malloc_closure.h
+++ b/c/malloc_closure.h
@@ -14,6 +14,54 @@
 # endif
 #endif
 
+/* On PaX enable kernels that have MPROTECT enable we can't use PROT_EXEC.
+
+   This is, apparently, an undocumented change to ffi_prep_closure():
+   depending on the Linux kernel we're running on, we must give it a
+   mmap that is either PROT_READ|PROT_WRITE|PROT_EXEC or only
+   PROT_READ|PROT_WRITE.  In the latter case, just trying to obtain a
+   mmap with PROT_READ|PROT_WRITE|PROT_EXEC would kill our process(!),
+   but in that situation libffi is fine with only PROT_READ|PROT_WRITE.
+   There is nothing in the libffi API to know that, though, so we have
+   to guess by parsing /proc/self/status.  "Meh."
+ */
+#ifdef __linux__
+#include 
+
+static int emutramp_enabled = -1;
+
+static int
+emutramp_enabled_check (void)
+{
+    char *buf = NULL;
+    size_t len = 0;
+    FILE *f;
+    int ret;
+    f = fopen ("/proc/self/status", "r");
+    if (f == NULL)
+        return 0;
+    ret = 0;
+
+    while (getline (&buf, &len, f) != -1)
+        if (!strncmp (buf, "PaX:", 4))
+            {
+                char emutramp;
+                if (sscanf (buf, "%*s %*c%c", &emutramp) == 1)
+                    ret = (emutramp == 'E');
+                break;
+            }
+    free (buf);
+    fclose (f);
+    return ret;
+}
+
+#define is_emutramp_enabled() (emutramp_enabled >= 0 ? emutramp_enabled \
+        : (emutramp_enabled = emutramp_enabled_check ()))
+#else
+#define is_emutramp_enabled() 0
+#endif
+
+
 /* 'allocate_num_pages' is dynamically adjusted starting from one
    page.  It grows by a factor of PAGE_ALLOCATION_GROWTH_RATE.  This is
    meant to handle both the common case of not needing a lot of pages,
@@ -77,9 +125,12 @@
     if (item == NULL)
         return;
 #else
+    int prot = PROT_READ | PROT_WRITE | PROT_EXEC;
+    if (is_emutramp_enabled ())
+        prot &= ~PROT_EXEC;
     item = (union mmaped_block *)mmap(NULL,
                         allocate_num_pages * _pagesize,
-                        PROT_READ | PROT_WRITE | PROT_EXEC,
+                        prot,
                         MAP_PRIVATE | MAP_ANONYMOUS,
                         -1,
                         0);

From noreply at buildbot.pypy.org  Thu Feb 26 11:11:32 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Thu, 26 Feb 2015 11:11:32 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Change memoryview.suboffsets to be an
 empty tuple instead of None.
Message-ID: <20150226101132.BC8181C0579@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76153:b9de1d50ab49
Date: 2015-02-25 19:32 +0100
http://bitbucket.org/pypy/pypy/changeset/b9de1d50ab49/

Log:	Change memoryview.suboffsets to be an empty tuple instead of None.

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -144,7 +144,7 @@
     def w_get_suboffsets(self, space):
         self._check_released(space)
         # I've never seen anyone filling this field
-        return space.w_None
+        return space.newtuple([])
 
     def descr_repr(self, space):
         if self.buf is None:
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -44,7 +44,7 @@
 
     def test_suboffsets(self):
         v = memoryview(b"a"*100)
-        assert v.suboffsets == None
+        assert v.suboffsets == ()
 
     def test_compare(self):
         assert memoryview(b"abc") == b"abc"

From noreply at buildbot.pypy.org  Thu Feb 26 11:11:33 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Thu, 26 Feb 2015 11:11:33 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: More fixes to memoryview.
Message-ID: <20150226101133.E45001C0579@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76154:4b6673a65c79
Date: 2015-02-26 11:10 +0100
http://bitbucket.org/pypy/pypy/changeset/4b6673a65c79/

Log:	More fixes to memoryview.

diff --git a/lib-python/3/test/test_memoryview.py b/lib-python/3/test/test_memoryview.py
--- a/lib-python/3/test/test_memoryview.py
+++ b/lib-python/3/test/test_memoryview.py
@@ -11,6 +11,7 @@
 import weakref
 import array
 import io
+from test.support import check_impl_detail
 
 
 try:
@@ -110,10 +111,11 @@
         self.assertRaises(IndexError, setitem, -sys.maxsize, b"a")
         # Wrong index/slice types
         self.assertRaises(TypeError, setitem, 0.0, b"a")
-        self.assertRaises(TypeError, setitem, (0,), b"a")
-        self.assertRaises(TypeError, setitem, (slice(0,1,1), 0), b"a")
-        self.assertRaises(TypeError, setitem, (0, slice(0,1,1)), b"a")
-        self.assertRaises(TypeError, setitem, (0,), b"a")
+        if check_impl_detail():
+            self.assertRaises(TypeError, setitem, (0,), b"a")
+            self.assertRaises(TypeError, setitem, (slice(0,1,1), 0), b"a")
+            self.assertRaises(TypeError, setitem, (0, slice(0,1,1)), b"a")
+            self.assertRaises(TypeError, setitem, (0,), b"a")
         self.assertRaises(TypeError, setitem, "a", b"a")
         # Not implemented: multidimensional slices
         slices = (slice(0,1,1), slice(0,1,2))
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -5,6 +5,7 @@
 
 from rpython.rlib.buffer import Buffer, SubBuffer
 from rpython.rlib.objectmodel import compute_hash
+from rpython.rlib.rstruct.error import StructError
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app
@@ -98,11 +99,18 @@
         self._check_released(space)
         if self.buf.readonly:
             raise oefmt(space.w_TypeError, "cannot modify read-only memory")
+        if space.isinstance_w(w_index, space.w_tuple):
+            raise oefmt(space.w_NotImplementedError, "")
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
         if step == 0:  # index only
             # TODO: this probably isn't very fast
             fmtiter = PackFormatIterator(space, [w_obj], self.itemsize)
-            fmtiter.interpret(self.format)
+            try:
+                fmtiter.interpret(self.format)
+            except StructError as e:
+                raise oefmt(space.w_TypeError,
+                            "memoryview: invalid type for format '%s'",
+                            self.format)
             self.buf.setslice(start * self.itemsize, fmtiter.result.build())
         elif step == 1:
             value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)

From noreply at buildbot.pypy.org  Thu Feb 26 11:36:39 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 26 Feb 2015 11:36:39 +0100 (CET)
Subject: [pypy-commit] pypy default: document those functions
Message-ID: <20150226103639.6FCE01C0557@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r76155:d896f30de3b2
Date: 2015-02-26 12:22 +0200
http://bitbucket.org/pypy/pypy/changeset/d896f30de3b2/

Log:	document those functions

diff --git a/pypy/doc/jit-hooks.rst b/pypy/doc/jit-hooks.rst
--- a/pypy/doc/jit-hooks.rst
+++ b/pypy/doc/jit-hooks.rst
@@ -39,3 +39,18 @@
     Reason is a string, the meaning of other arguments is the same
     as attributes on JitLoopInfo object
 
+.. function:: enable_debug()
+
+    Start recording debugging counters for ``get_stats_snapshot``
+
+.. function:: disable_debug()
+
+    Stop recording debugging counters for ``get_stats_snapshot``
+
+.. function:: get_stats_snapshot()
+
+    Get the jit status in the specific moment in time. Note that this
+    is eager - the attribute access is not lazy, if you need new stats
+    you need to call this function again. You might want to call
+    ``enable_debug`` to get more information
+

From noreply at buildbot.pypy.org  Thu Feb 26 11:36:40 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 26 Feb 2015 11:36:40 +0100 (CET)
Subject: [pypy-commit] pypy default: more docs
Message-ID: <20150226103640.B141C1C0557@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r76156:f1ffbb37232f
Date: 2015-02-26 12:36 +0200
http://bitbucket.org/pypy/pypy/changeset/f1ffbb37232f/

Log:	more docs

diff --git a/pypy/doc/jit-hooks.rst b/pypy/doc/jit-hooks.rst
--- a/pypy/doc/jit-hooks.rst
+++ b/pypy/doc/jit-hooks.rst
@@ -52,5 +52,17 @@
     Get the jit status in the specific moment in time. Note that this
     is eager - the attribute access is not lazy, if you need new stats
     you need to call this function again. You might want to call
-    ``enable_debug`` to get more information
+    ``enable_debug`` to get more information. It returns an instance
+    of ``JitInfoSnapshot``
 
+.. class:: JitInfoSnapshot
+
+    A class describing current snapshot. Usable attributes:
+
+    * ``counters`` - internal JIT integer counters
+
+    * ``counter_times`` - internal JIT float counters, notably time spent
+      TRACING and in the JIT BACKEND
+
+    * ``loop_run_times`` - counters for number of times loops are run, only
+      works when ``enable_debug`` is called.

From noreply at buildbot.pypy.org  Thu Feb 26 11:36:41 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 26 Feb 2015 11:36:41 +0100 (CET)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20150226103641.ECB631C0557@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r76157:954bf2fe475a
Date: 2015-02-26 12:36 +0200
http://bitbucket.org/pypy/pypy/changeset/954bf2fe475a/

Log:	merge

diff --git a/rpython/doc/index.rst b/rpython/doc/index.rst
--- a/rpython/doc/index.rst
+++ b/rpython/doc/index.rst
@@ -36,6 +36,7 @@
    :maxdepth: 1
 
    arm
+   logging
 
 
 Writing your own interpreter in RPython
diff --git a/rpython/rlib/listsort.py b/rpython/rlib/listsort.py
--- a/rpython/rlib/listsort.py
+++ b/rpython/rlib/listsort.py
@@ -496,6 +496,12 @@
         # 1. len[-3] > len[-2] + len[-1]
         # 2. len[-2] > len[-1]
         #
+        # Note these invariants will not hold for the entire pending array even
+        # after this function completes. [1] This does not affect the
+        # correctness of the overall algorithm.
+        #
+        # [1] http://envisage-project.eu/proving-android-java-and-python-sorting-algorithm-is-broken-and-how-to-fix-it/
+        #
         # See listsort.txt for more info.
 
         def merge_collapse(self):

From noreply at buildbot.pypy.org  Thu Feb 26 21:46:46 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu, 26 Feb 2015 21:46:46 +0100 (CET)
Subject: [pypy-commit] stmgc c8-card-marking: make it compile
Message-ID: <20150226204646.606EE1C01CD@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-card-marking
Changeset: r1669:9ccc9bf2dc2d
Date: 2015-02-26 14:51 +0100
http://bitbucket.org/pypy/stmgc/changeset/9ccc9bf2dc2d/

Log:	make it compile

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -576,6 +576,11 @@
 }
 
 
+void _stm_write_slowpath_card(object_t *obj, uintptr_t index)
+{
+    stm_write(obj);
+}
+
 void _stm_write_slowpath(object_t *obj)
 {
     assert(_seems_to_be_running_transaction());
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -38,8 +38,10 @@
     GCFLAG_WRITE_BARRIER = _STM_GCFLAG_WRITE_BARRIER,
     GCFLAG_HAS_SHADOW = 0x02,
     GCFLAG_WB_EXECUTED = 0x04,
-    GCFLAG_VISITED = 0x08,
-    GCFLAG_FINALIZATION_ORDERING = 0x10,
+    GCFLAG_HAS_CARDS = 0x08,
+    GCFLAG_CARDS_SET = _STM_GCFLAG_CARDS_SET,
+    GCFLAG_VISITED = 0x20,
+    GCFLAG_FINALIZATION_ORDERING = 0x40,
 };
 
 
@@ -72,6 +74,7 @@
     struct list_s *modified_old_objects;
 
     struct list_s *objects_pointing_to_nursery;
+    struct list_s *old_objects_with_cards_set;
     struct tree_s *young_outside_nursery;
     struct tree_s *nursery_objects_shadows;
 
diff --git a/c8/stm/misc.c b/c8/stm/misc.c
--- a/c8/stm/misc.c
+++ b/c8/stm/misc.c
@@ -42,6 +42,11 @@
     return (obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) == 0;
 }
 
+bool _stm_was_written_card(object_t *obj)
+{
+    return obj->stm_flags & _STM_GCFLAG_CARDS_SET;
+}
+
 long _stm_count_cl_entries()
 {
     struct stm_commit_log_entry_s *cl = &commit_log_root;
@@ -80,6 +85,13 @@
     return list_count(STM_PSEGMENT->objects_pointing_to_nursery);
 }
 
+long _stm_count_old_objects_with_cards_set(void)
+{
+    if (STM_PSEGMENT->old_objects_with_cards_set == NULL)
+        return -1;
+    return list_count(STM_PSEGMENT->old_objects_with_cards_set);
+}
+
 object_t *_stm_enum_modified_old_objects(long index)
 {
     return (object_t *)list_item(
@@ -92,6 +104,13 @@
         STM_PSEGMENT->objects_pointing_to_nursery, index);
 }
 
+object_t *_stm_enum_old_objects_with_cards_set(long index)
+{
+    return (object_t *)list_item(
+        STM_PSEGMENT->old_objects_with_cards_set, index);
+}
+
+
 static struct stm_commit_log_entry_s *_last_cl_entry;
 static long _last_cl_entry_index;
 void _stm_start_enum_last_cl_entry()
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -104,6 +104,7 @@
         pr->young_weakrefs = list_create();
         pr->old_weakrefs = list_create();
         pr->objects_pointing_to_nursery = list_create();
+        pr->old_objects_with_cards_set = list_create();
         pr->young_outside_nursery = tree_create();
         pr->nursery_objects_shadows = tree_create();
         pr->callbacks_on_commit_and_abort[0] = tree_create();
@@ -147,6 +148,7 @@
         struct stm_priv_segment_info_s *pr = get_priv_segment(i);
         assert(list_is_empty(pr->objects_pointing_to_nursery));
         list_free(pr->objects_pointing_to_nursery);
+        list_free(pr->old_objects_with_cards_set);
         list_free(pr->modified_old_objects);
         assert(list_is_empty(pr->new_objects));
         list_free(pr->new_objects);
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -73,7 +73,7 @@
 } stm_thread_local_t;
 
 #define _STM_GCFLAG_WRITE_BARRIER      0x01
-#define _STM_GCFLAG_CARDS_SET          0x08
+#define _STM_GCFLAG_CARDS_SET          0x10
 #define _STM_FAST_ALLOC           (66*1024)
 #define _STM_NSE_SIGNAL_ABORT             1
 #define _STM_NSE_SIGNAL_MAX               2

From noreply at buildbot.pypy.org  Thu Feb 26 21:46:45 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu, 26 Feb 2015 21:46:45 +0100 (CET)
Subject: [pypy-commit] stmgc c8-card-marking: WIP
Message-ID: <20150226204645.4E1441C01CD@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-card-marking
Changeset: r1668:8a75e2be64ac
Date: 2015-02-26 14:41 +0100
http://bitbucket.org/pypy/stmgc/changeset/8a75e2be64ac/

Log:	WIP

diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -73,11 +73,13 @@
 } stm_thread_local_t;
 
 #define _STM_GCFLAG_WRITE_BARRIER      0x01
+#define _STM_GCFLAG_CARDS_SET          0x08
 #define _STM_FAST_ALLOC           (66*1024)
 #define _STM_NSE_SIGNAL_ABORT             1
 #define _STM_NSE_SIGNAL_MAX               2
 
 void _stm_write_slowpath(object_t *);
+void _stm_write_slowpath_card(object_t *, uintptr_t);
 object_t *_stm_allocate_slowpath(ssize_t);
 object_t *_stm_allocate_external(ssize_t);
 void _stm_become_inevitable(const char*);
@@ -89,7 +91,7 @@
 #include 
 bool _stm_was_read(object_t *obj);
 bool _stm_was_written(object_t *obj);
-
+bool _stm_was_written_card(object_t *obj);
 bool _stm_is_accessible_page(uintptr_t pagenum);
 
 void _stm_test_switch(stm_thread_local_t *tl);
@@ -125,7 +127,8 @@
 object_t *_stm_next_last_cl_entry();
 void _stm_start_enum_last_cl_entry();
 long _stm_count_cl_entries();
-
+long _stm_count_old_objects_with_cards_set(void);
+object_t *_stm_enum_old_objects_with_cards_set(long index);
 uint64_t _stm_total_allocated(void);
 #endif
 
@@ -156,6 +159,12 @@
 
 extern ssize_t stmcb_size_rounded_up(struct object_s *);
 void stmcb_trace(struct object_s *obj, void visit(object_t **));
+/* a special trace-callback that is only called for the marked
+   ranges of indices (using stm_write_card(o, index)) */
+extern void stmcb_trace_cards(struct object_s *, void (object_t **),
+                              uintptr_t start, uintptr_t stop);
+
+
 
 __attribute__((always_inline))
 static inline void stm_read(object_t *obj)
@@ -173,6 +182,14 @@
 
 
 __attribute__((always_inline))
+static inline void stm_write_card(object_t *obj, uintptr_t index)
+{
+    if (UNLIKELY((obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0))
+        _stm_write_slowpath_card(obj, index);
+}
+
+
+__attribute__((always_inline))
 static inline object_t *stm_allocate(ssize_t size_rounded_up)
 {
     OPT_ASSERT(size_rounded_up >= 16);
@@ -327,14 +344,8 @@
 
 
 /* dummies for now: */
-__attribute__((always_inline))
-static inline void stm_write_card(object_t *obj, uintptr_t index)
-{
-    stm_write(obj);
-}
+static inline void stm_flush_timing(stm_thread_local_t *tl, int verbose) {}
 
-
-static inline void stm_flush_timing(stm_thread_local_t *tl, int verbose) {}
 /* ==================== END ==================== */
 
 static void (*stmcb_expand_marker)(char *segment_base, uintptr_t odd_number,
diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -43,6 +43,8 @@
 object_t *stm_allocate_weakref(ssize_t size_rounded_up);
 object_t *stm_allocate_with_finalizer(ssize_t size_rounded_up);
 
+/*void stm_write_card(); use _checked_stm_write_card() instead */
+
 void stm_setup(void);
 void stm_teardown(void);
 void stm_register_thread_local(stm_thread_local_t *tl);
@@ -59,8 +61,10 @@
 ssize_t stmcb_size_rounded_up(struct object_s *obj);
 
 bool _checked_stm_write(object_t *obj);
+bool _checked_stm_write_card(object_t *obj, uintptr_t index);
 bool _stm_was_read(object_t *obj);
 bool _stm_was_written(object_t *obj);
+bool _stm_was_written_card(object_t *obj);
 char *_stm_get_segment_base(long index);
 bool _stm_in_transaction(stm_thread_local_t *tl);
 int _stm_get_flags(object_t *obj);
@@ -118,8 +122,10 @@
 
 long _stm_count_modified_old_objects(void);
 long _stm_count_objects_pointing_to_nursery(void);
+long _stm_count_old_objects_with_cards_set(void);
 object_t *_stm_enum_modified_old_objects(long index);
 object_t *_stm_enum_objects_pointing_to_nursery(long index);
+object_t *_stm_enum_old_objects_with_cards_set(long index);
 object_t *_stm_next_last_cl_entry();
 void _stm_start_enum_last_cl_entry();
 long _stm_count_cl_entries();
@@ -191,6 +197,10 @@
     CHECKED(stm_write(object));
 }
 
+bool _checked_stm_write_card(object_t *object, uintptr_t index) {
+    CHECKED(stm_write_card(object, index));
+}
+
 bool _check_commit_transaction(void) {
     CHECKED(stm_commit_transaction());
 }
@@ -322,6 +332,25 @@
     }
 }
 
+void stmcb_trace_cards(struct object_s *obj, void visit(object_t **),
+                       uintptr_t start, uintptr_t stop)
+{
+    int i;
+    struct myobj_s *myobj = (struct myobj_s*)obj;
+    assert(myobj->type_id != 421419);
+    assert(myobj->type_id != 421418);
+    if (myobj->type_id < 421420) {
+        /* basic case: no references */
+        return;
+    }
+
+    for (i=start; (i < myobj->type_id - 421420) && (i < stop); i++) {
+        object_t **ref = ((object_t **)(myobj + 1)) + i;
+        visit(ref);
+    }
+}
+
+
 long current_segment_num(void)
 {
     return STM_SEGMENT->segment_num;
@@ -506,11 +535,11 @@
         return None
     return map(lib._stm_enum_objects_pointing_to_nursery, range(count))
 
-def old_objects_with_cards():
-    count = lib._stm_count_old_objects_with_cards()
+def old_objects_with_cards_set():
+    count = lib._stm_count_old_objects_with_cards_set()
     if count < 0:
         return None
-    return map(lib._stm_enum_old_objects_with_cards, range(count))
+    return map(lib._stm_enum_old_objects_with_cards_set, range(count))
 
 def last_commit_log_entry_objs():
     lib._stm_start_enum_last_cl_entry()
diff --git a/c8/test/test_card_marking.py b/c8/test/test_card_marking.py
new file mode 100644
--- /dev/null
+++ b/c8/test/test_card_marking.py
@@ -0,0 +1,223 @@
+from support import *
+import py
+
+
+class TestBasic(BaseTest):
+
+    def _collect(self, kind):
+        if kind == 0:
+            stm_minor_collect()
+        elif kind == 1:
+            stm_major_collect()
+        elif kind == 2:
+            self.switch(1)
+            self.start_transaction()
+            stm_major_collect()
+            self.abort_transaction()
+            self.switch(0)
+
+    def test_simple(self):
+        o = stm_allocate_old_refs(1024)
+        self.start_transaction()
+        stm_read(o)
+        stm_write(o)
+        self.commit_transaction()
+
+    def test_simple2(self):
+        o = stm_allocate_old_refs(1024)
+        self.start_transaction()
+        stm_write_card(o, 5)
+        assert not stm_was_written(o) # don't remove GCFLAG_WRITE_BARRIER
+        assert stm_was_written_card(o)
+        self.commit_transaction()
+
+    @py.test.mark.parametrize("k", range(3))
+    def test_overflow(self, k):
+        self.start_transaction()
+        o = stm_allocate_refs(1024)
+
+        self.push_root(o)
+        self._collect(k)
+        o = self.pop_root()
+
+        stm_write_card(o, 5)
+
+        assert o in old_objects_with_cards()
+        assert o not in modified_old_objects() # overflow object
+        assert o not in objects_pointing_to_nursery()
+        # don't remove GCFLAG_WB
+        assert not stm_was_written(o)
+        stm_write(o)
+        assert stm_was_written(o)
+        self.commit_transaction()
+
+    def test_nursery(self):
+        o = stm_allocate_old_refs(200)
+        self.start_transaction()
+        p = stm_allocate(64)
+        stm_set_ref(o, 199, p, True)
+
+        # without a write-barrier:
+        lib._set_ptr(o, 0, ffi.cast("object_t*", -1))
+
+        self.push_root(o)
+        stm_minor_collect()
+        o = self.pop_root()
+
+        lib._set_ptr(o, 0, ffi.NULL)
+
+        pn = stm_get_ref(o, 199)
+        assert not is_in_nursery(pn)
+        assert pn != p
+
+        assert not stm_was_written(o)
+        stm_write_card(o, 2)
+        assert stm_was_written_card(o)
+
+        # card cleared after last collection,
+        # so no retrace of index 199:
+
+        # without a write-barrier:
+        lib._set_ptr(o, 199, ffi.cast("object_t*", -1))
+        self.push_root(o)
+        stm_minor_collect()
+        o = self.pop_root()
+
+    def test_nursery2(self):
+        o = stm_allocate_old_refs(200)
+        self.start_transaction()
+        p = stm_allocate(64)
+        d = stm_allocate(64)
+        e = stm_allocate(64)
+        stm_set_ref(o, 199, p, True)
+        stm_set_ref(o, 1, d, False)
+        lib._set_ptr(o, 100, e) # no barrier
+
+        self.push_root(o)
+        stm_minor_collect()
+        o = self.pop_root()
+
+        # stm_write in stm_set_ref made it trace everything
+        assert not is_in_nursery(stm_get_ref(o, 199))
+        assert not is_in_nursery(stm_get_ref(o, 1))
+        assert not is_in_nursery(stm_get_ref(o, 100))
+
+    def test_nursery3(self):
+        o = stm_allocate_old_refs(2000)
+        self.start_transaction()
+        stm_minor_collect()
+
+        p = stm_allocate(64)
+        d = stm_allocate(64)
+        stm_set_ref(o, 1999, p, True)
+        stm_set_ref(o, 1, d, True)
+
+        lib._set_ptr(o, 1000, ffi.cast("object_t*", -1))
+
+        assert not stm_was_written(o)
+        assert stm_was_written_card(o)
+
+        self.push_root(o)
+        stm_minor_collect()
+        o = self.pop_root()
+
+        assert not is_in_nursery(stm_get_ref(o, 1999))
+        assert not is_in_nursery(stm_get_ref(o, 1))
+
+
+    def test_abort_cleanup(self):
+        o = stm_allocate_old_refs(200)
+        self.start_transaction()
+        stm_minor_collect()
+
+        p = stm_allocate_refs(64)
+        d = stm_allocate(64)
+        e = stm_allocate(64)
+        stm_set_ref(o, 199, p, True)
+        stm_set_ref(o, 1, d, True)
+        stm_set_ref(p, 1, e)
+
+        self.abort_transaction()
+
+        assert not modified_old_objects()
+        assert not objects_pointing_to_nursery()
+        assert not old_objects_with_cards()
+
+        self.start_transaction()
+        d = stm_allocate(64)
+        e = stm_allocate(64)
+        lib._set_ptr(o, 199, d) # no barrier
+        stm_set_ref(o, 1, e, True) # card barrier
+
+        self.push_root(o)
+        stm_minor_collect()
+        o = self.pop_root()
+
+        assert not is_in_nursery(stm_get_ref(o, 1))
+        assert is_in_nursery(stm_get_ref(o, 199)) # not traced
+
+    @py.test.mark.parametrize("k", range(3))
+    def test_major_gc(self, k):
+        o = stm_allocate_old_refs(200)
+        self.start_transaction()
+        p = stm_allocate(64)
+        stm_set_ref(o, 0, p, True)
+
+        self.push_root(o)
+        stm_major_collect()
+        o = self.pop_root()
+
+        stm_set_ref(o, 1, ffi.NULL, True)
+        p = stm_get_ref(o, 0)
+        assert stm_was_written_card(o)
+
+        self.push_root(o)
+        self._collect(k)
+        o = self.pop_root()
+
+        assert not stm_was_written_card(o)
+        assert stm_get_ref(o, 0) == p
+        self.commit_transaction()
+
+    def test_synchronize_objs(self):
+        o = stm_allocate_old(1000+20*CARD_SIZE)
+
+        self.start_transaction()
+        stm_set_char(o, 'a', 1000, False)
+        self.commit_transaction()
+
+        self.switch(1)
+
+        self.start_transaction()
+        stm_set_char(o, 'b', 1001, False)
+        assert stm_get_char(o, 1000) == 'a'
+        self.commit_transaction()
+
+        self.switch(0)
+
+        self.start_transaction()
+        assert stm_get_char(o, 1001) == 'b'
+
+        stm_set_char(o, 'c', 1000, True)
+        stm_set_char(o, 'c', 1000+CARD_SIZE, True)
+        stm_set_char(o, 'c', 1000+CARD_SIZE*2, True)
+        stm_set_char(o, 'c', 1000+CARD_SIZE*3, True)
+
+        stm_set_char(o, 'd', 1000+CARD_SIZE*10, True)
+
+        stm_set_char(o, 'e', 1000+CARD_SIZE*12, True)
+        self.commit_transaction()
+
+        self.switch(1)
+
+        self.start_transaction()
+        assert stm_get_char(o, 1000) == 'c'
+        assert stm_get_char(o, 1000+CARD_SIZE) == 'c'
+        assert stm_get_char(o, 1000+CARD_SIZE*2) == 'c'
+        assert stm_get_char(o, 1000+CARD_SIZE*3) == 'c'
+
+        assert stm_get_char(o, 1000+CARD_SIZE*10) == 'd'
+
+        assert stm_get_char(o, 1000+CARD_SIZE*12) == 'e'
+
+        self.commit_transaction()

From noreply at buildbot.pypy.org  Thu Feb 26 21:46:47 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu, 26 Feb 2015 21:46:47 +0100 (CET)
Subject: [pypy-commit] stmgc c8-card-marking: try using read markers instead
 of write_locks for cards
Message-ID: <20150226204647.6ABBA1C01CD@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-card-marking
Changeset: r1670:9e3927eeccb3
Date: 2015-02-26 20:23 +0100
http://bitbucket.org/pypy/stmgc/changeset/9e3927eeccb3/

Log:	try using read markers instead of write_locks for cards

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -576,21 +576,67 @@
 }
 
 
-void _stm_write_slowpath_card(object_t *obj, uintptr_t index)
+static bool obj_should_use_cards(object_t *obj)
 {
-    stm_write(obj);
+    struct object_s *realobj = (struct object_s *)
+        REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
+    long supports = stmcb_obj_supports_cards(realobj);
+    if (!supports)
+        return false;
+
+    /* check also if it makes sense: */
+    size_t size = stmcb_size_rounded_up(realobj);
+    return (size >= _STM_MIN_CARD_OBJ_SIZE);
 }
 
-void _stm_write_slowpath(object_t *obj)
+__attribute__((always_inline))
+static void write_gc_only_path(object_t *obj, bool mark_card)
+{
+    assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
+    assert(obj->stm_flags & GCFLAG_WB_EXECUTED);
+    dprintf(("write_slowpath-fast(%p)\n", obj));
+
+    if (!mark_card) {
+        /* The basic case, with no card marking.  We append the object
+           into 'objects_pointing_to_nursery', and remove the flag so
+           that the write_slowpath will not be called again until the
+           next minor collection. */
+        if (obj->stm_flags & GCFLAG_CARDS_SET) {
+            /* if we clear this flag, we also need to clear the cards */
+            _reset_object_cards(get_priv_segment(STM_SEGMENT->segment_num),
+                                obj, CARD_CLEAR, false);
+        }
+        obj->stm_flags &= ~(GCFLAG_WRITE_BARRIER | GCFLAG_CARDS_SET);
+        LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj);
+    } else {
+        /* Card marking.  Don't remove GCFLAG_WRITE_BARRIER because we
+           need to come back to _stm_write_slowpath_card() for every
+           card to mark.  Add GCFLAG_CARDS_SET. */
+        assert(!(obj->stm_flags & GCFLAG_CARDS_SET));
+        obj->stm_flags |= GCFLAG_CARDS_SET;
+        LIST_APPEND(STM_PSEGMENT->old_objects_with_cards_set, obj);
+    }
+}
+
+
+__attribute__((always_inline))
+static void write_slowpath_common(object_t *obj, bool mark_card)
 {
     assert(_seems_to_be_running_transaction());
     assert(!_is_in_nursery(obj));
     assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
 
+    if (obj->stm_flags & GCFLAG_WB_EXECUTED) {
+        /* already executed WB once in this transaction. do GC
+           part again: */
+        write_gc_only_path(obj, mark_card);
+        return;
+    }
+
+    char *realobj;
+    size_t obj_size;
     int my_segnum = STM_SEGMENT->segment_num;
     uintptr_t end_page, first_page = ((uintptr_t)obj) / 4096UL;
-    char *realobj;
-    size_t obj_size;
 
     realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
     obj_size = stmcb_size_rounded_up((struct object_s *)realobj);
@@ -604,15 +650,6 @@
     /* add to read set: */
     stm_read(obj);
 
-    if (obj->stm_flags & GCFLAG_WB_EXECUTED) {
-        /* already executed WB once in this transaction. do GC
-           part again: */
-        dprintf(("write_slowpath-fast(%p)\n", obj));
-        obj->stm_flags &= ~GCFLAG_WRITE_BARRIER;
-        LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj);
-        return;
-    }
-
     assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED));
     dprintf(("write_slowpath(%p): sz=%lu\n", obj, obj_size));
 
@@ -679,20 +716,105 @@
     }
     OPT_ASSERT(remaining_obj_sz == 0);
 
-    /* remove the WRITE_BARRIER flag and add WB_EXECUTED */
-    obj->stm_flags &= ~GCFLAG_WRITE_BARRIER;
-    obj->stm_flags |= GCFLAG_WB_EXECUTED;
+    if (!mark_card) {
+        /* also add it to the GC list for minor collections */
+        LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj);
+
+        if (obj->stm_flags & GCFLAG_CARDS_SET) {
+            /* if we clear this flag, we have to tell sync_old_objs that
+               everything needs to be synced */
+            _reset_object_cards(get_priv_segment(STM_SEGMENT->segment_num),
+                                obj, CARD_MARKED_OLD, true); /* mark all */
+        }
+
+        /* remove the WRITE_BARRIER flag and add WB_EXECUTED */
+        obj->stm_flags &= ~(GCFLAG_WRITE_BARRIER | GCFLAG_CARDS_SET);
+        obj->stm_flags |= GCFLAG_WB_EXECUTED;
+    } else {
+        /* don't remove WRITE_BARRIER, but add CARDS_SET */
+        obj->stm_flags |= (GCFLAG_CARDS_SET | GCFLAG_WB_EXECUTED);
+        /* XXXXXXXXXXXX maybe not set WB_EXECUTED and make CARDS_SET
+           mean the same thing where necessary */
+        LIST_APPEND(STM_PSEGMENT->old_objects_with_cards_set, obj);
+    }
 
     DEBUG_EXPECT_SEGFAULT(true);
 
     release_modification_lock(STM_SEGMENT->segment_num);
     /* done fiddling with protection and privatization */
     release_all_privatization_locks();
+}
 
-    /* also add it to the GC list for minor collections */
-    LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj);
+
+char _stm_write_slowpath_card_extra(object_t *obj)
+{
+    /* the PyPy JIT calls this function directly if it finds that an
+       array doesn't have the GCFLAG_CARDS_SET */
+    bool mark_card = obj_should_use_cards(obj);
+    write_slowpath_common(obj, mark_card);
+    return mark_card;
 }
 
+long _stm_write_slowpath_card_extra_base(void)
+{
+    /* XXX can go away? */
+    /* for the PyPy JIT: _stm_write_slowpath_card_extra_base[obj >> 4]
+       is the byte that must be set to CARD_MARKED.  The logic below
+       does the same, but more explicitly. */
+    return 0;
+}
+
+void _stm_write_slowpath_card(object_t *obj, uintptr_t index)
+{
+    dprintf_test(("write_slowpath_card(%p, %lu)\n",
+                  obj, index));
+
+    /* If CARDS_SET is not set so far, issue a normal write barrier.
+       If the object is large enough, ask it to set up the object for
+       card marking instead. */
+    if (!(obj->stm_flags & GCFLAG_CARDS_SET)) {
+        char mark_card = _stm_write_slowpath_card_extra(obj);
+        if (!mark_card)
+            return;
+    }
+
+    dprintf_test(("write_slowpath_card %p -> index:%lu\n",
+                  obj, index));
+
+    /* We reach this point if we have to mark the card. */
+    assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
+    assert(obj->stm_flags & GCFLAG_CARDS_SET);
+    assert(!is_small_uniform(obj)); /* not supported/tested */
+
+#ifndef NDEBUG
+    struct object_s *realobj = (struct object_s *)
+        REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
+    size_t size = stmcb_size_rounded_up(realobj);
+    /* we need at least one read marker in addition to the STM-reserved object
+       write-lock */
+    assert(size >= 32);
+    /* the 'index' must be in range(length-of-obj), but we don't have
+       a direct way to know the length.  We know that it is smaller
+       than the size in bytes. */
+    assert(index < size);
+#endif
+
+    /* Write into the card's lock.  This is used by the next minor
+       collection to know what parts of the big object may have changed.
+       We already own the object here or it is an overflow obj. */
+    struct stm_read_marker_s *cards = get_read_marker(STM_SEGMENT->segment_base,
+                                                      (uintptr_t)obj);
+    cards[get_index_to_card_index(index)].rm = CARD_MARKED;
+
+    dprintf(("mark %p index %lu, card:%lu with %d\n",
+             obj, index, get_index_to_card_index(index), CARD_MARKED));
+}
+
+void _stm_write_slowpath(object_t *obj) {
+    write_slowpath_common(obj,  /* mark_card */ false);
+}
+
+
 static void reset_transaction_read_version(void)
 {
     /* force-reset all read markers to 0 */
@@ -831,7 +953,10 @@
 
     STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
     STM_PSEGMENT->transaction_state = TS_NONE;
+
+    _verify_cards_cleared_in_all_lists(get_priv_segment(STM_SEGMENT->segment_num));
     list_clear(STM_PSEGMENT->objects_pointing_to_nursery);
+    list_clear(STM_PSEGMENT->old_objects_with_cards_set);
     list_clear(STM_PSEGMENT->new_objects);
 
     release_thread_segment(tl);
@@ -854,12 +979,15 @@
 
 static void push_new_objects_to_other_segments(void)
 {
+    struct stm_priv_segment_info_s *pseg = get_priv_segment(STM_SEGMENT->segment_num);
     acquire_privatization_lock(STM_SEGMENT->segment_num);
     LIST_FOREACH_R(STM_PSEGMENT->new_objects, object_t *,
         ({
             assert(item->stm_flags & GCFLAG_WB_EXECUTED);
+            _cards_cleared_in_object(pseg, item); /* check for C8 */
+            _reset_object_cards(pseg, item, CARD_CLEAR, false); /* unnecessary, as sync_obj_enq does it already? */
             item->stm_flags &= ~GCFLAG_WB_EXECUTED;
-            synchronize_object_enqueue(item);
+            synchronize_object_enqueue(item, true);
         }));
     synchronize_objects_flush();
     release_privatization_lock(STM_SEGMENT->segment_num);
@@ -910,6 +1038,8 @@
         }
     }
 
+    _verify_cards_cleared_in_all_lists(get_priv_segment(STM_SEGMENT->segment_num));
+
     commit_finalizers();
 
     invoke_and_clear_user_callbacks(0);   /* for commit */
@@ -951,6 +1081,10 @@
                undo->backup,
                SLICE_SIZE(undo->slice));
 
+        if (obj_should_use_cards(obj))
+            _reset_object_cards(pseg, obj, CARD_CLEAR, false);
+        /* XXXXXXXXX: only reset cards of slice!! ^^^^^^^ */
+
         dprintf(("reset_modified_from_backup_copies(%d): obj=%p off=%lu bk=%p\n",
                  segment_num, obj, SLICE_OFFSET(undo->slice), undo->backup));
 
@@ -987,9 +1121,23 @@
 
     long bytes_in_nursery = throw_away_nursery(pseg);
 
+    /* some new objects may have cards when aborting, clear them too */
+    LIST_FOREACH_R(pseg->new_objects, object_t * /*item*/,
+        {
+            struct object_s *realobj = (struct object_s *)
+                REAL_ADDRESS(pseg->pub.segment_base, item);
+
+            if (realobj->stm_flags & GCFLAG_CARDS_SET) {
+                /* CARDS_SET is enough since other HAS_CARDS objs
+                   are already cleared */
+                _reset_object_cards(pseg, item, CARD_CLEAR, false);
+            }
+        });
+
     acquire_modification_lock(segment_num);
     reset_modified_from_backup_copies(segment_num);
     release_modification_lock(segment_num);
+    _verify_cards_cleared_in_all_lists(pseg);
 
     stm_thread_local_t *tl = pseg->pub.running_thread;
 #ifdef STM_NO_AUTOMATIC_SETJMP
@@ -1013,6 +1161,7 @@
     tl->last_abort__bytes_in_nursery = bytes_in_nursery;
 
     list_clear(pseg->objects_pointing_to_nursery);
+    list_clear(pseg->old_objects_with_cards_set);
     list_clear(pseg->new_objects);
     list_clear(pseg->young_weakrefs);
 #pragma pop_macro("STM_SEGMENT")
@@ -1143,7 +1292,133 @@
     ++STM_PSEGMENT->sq_len;
 }
 
-static void synchronize_object_enqueue(object_t *obj)
+
+static void _page_wise_synchronize_object_now(object_t *obj, ssize_t obj_size)
+{
+    uintptr_t start = (uintptr_t)obj;
+    uintptr_t end = start + obj_size;
+
+    do {
+        uintptr_t copy_up_to = (start + 4096) & ~4095;   /* end of page */
+        if (copy_up_to >= end) {
+            copy_up_to = end;        /* this is the last fragment */
+        }
+        uintptr_t copy_size = copy_up_to - start;
+
+        /* double-check that the result fits in one page */
+        assert(copy_size > 0);
+        assert(copy_size + (start & 4095) <= 4096);
+
+        _synchronize_fragment((stm_char *)start, copy_size);
+
+        start = copy_up_to;
+    } while (start != end);
+}
+
+static void _card_wise_synchronize_object_now(object_t *obj, ssize_t obj_size)
+{
+    assert(obj_size >= 32);
+    assert(obj_should_use_cards(obj));
+    assert(!(obj->stm_flags & GCFLAG_CARDS_SET));
+
+    uintptr_t offset_itemsize[2];
+    struct object_s *realobj = (struct object_s *)REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
+    stmcb_get_card_base_itemsize(realobj, offset_itemsize);
+    size_t real_idx_count = (obj_size - offset_itemsize[0]) / offset_itemsize[1];
+
+    struct stm_read_marker_s *cards = get_read_marker(STM_SEGMENT->segment_base, (uintptr_t)obj);
+    uintptr_t card_index = 1;
+    uintptr_t last_card_index = get_index_to_card_index(real_idx_count - 1); /* max valid index */
+    assert(cards->rm == STM_SEGMENT->transaction_read_version); /* stm_read() */
+
+    /* simple heuristic to check if probably the whole object is
+       marked anyway so we should do page-wise synchronize */
+    if (cards[1].rm == CARD_MARKED_OLD
+        && cards[last_card_index].rm == CARD_MARKED_OLD
+        && cards[(last_card_index >> 1) + 1].rm == CARD_MARKED_OLD) {
+
+        dprintf(("card_wise_sync assumes %p,size:%lu is fully marked\n", obj, obj_size));
+        _reset_object_cards(get_priv_segment(STM_SEGMENT->segment_num),
+                            obj, CARD_CLEAR, false);
+        _page_wise_synchronize_object_now(obj, obj_size);
+        return;
+    }
+
+    dprintf(("card_wise_sync syncs %p,size:%lu card-wise\n", obj, obj_size));
+
+    /* Combine multiple marked cards and do a memcpy for them. We don't
+       try yet to use page_copy() or otherwise take into account privatization
+       of pages (except _has_private_page_in_range) */
+    bool all_cards_were_cleared = true;
+
+    uintptr_t start_card_index = -1;
+    while (card_index <= last_card_index) {
+        uint8_t card_value = cards[card_index].rm;
+
+        if (card_value == CARD_MARKED_OLD) {
+            cards[card_index].rm = CARD_CLEAR;
+
+            if (start_card_index == -1) {   /* first marked card */
+                start_card_index = card_index;
+                /* start = (uintptr_t)obj + stmcb_index_to_byte_offset( */
+                /*     realobj, get_card_index_to_index(card_index)); */
+                if (all_cards_were_cleared) {
+                    all_cards_were_cleared = false;
+                }
+            }
+        }
+        else {
+            OPT_ASSERT(card_value == CARD_CLEAR);
+        }
+
+        if (start_card_index != -1                    /* something to copy */
+            && (card_value != CARD_MARKED_OLD         /* found non-marked card */
+                || card_index == last_card_index)) {  /* this is the last card */
+            /* do the copying: */
+            uintptr_t start, copy_size;
+            uintptr_t next_card_offset;
+            uintptr_t start_card_offset;
+            uintptr_t next_card_index = card_index;
+
+            if (card_value == CARD_MARKED_OLD) {
+                /* card_index is the last card of the object, but we need
+                   to go one further to get the right offset */
+                next_card_index++;
+            }
+
+            start_card_offset = offset_itemsize[0] +
+                get_card_index_to_index(start_card_index) * offset_itemsize[1];
+
+            next_card_offset = offset_itemsize[0] +
+                get_card_index_to_index(next_card_index) * offset_itemsize[1];
+
+            if (next_card_offset > obj_size)
+                next_card_offset = obj_size;
+
+            start = (uintptr_t)obj + start_card_offset;
+            copy_size = next_card_offset - start_card_offset;
+            OPT_ASSERT(copy_size > 0);
+
+            /* push to seg0 and enqueue for synchronization */
+            _synchronize_fragment((stm_char *)start, copy_size);
+
+            start_card_index = -1;
+        }
+
+        card_index++;
+    }
+
+    if (all_cards_were_cleared) {
+        /* well, seems like we never called stm_write_card() on it, so actually
+           we need to fall back to synchronize the whole object */
+        _page_wise_synchronize_object_now(obj, obj_size);
+        return;
+    }
+
+}
+
+
+static void synchronize_object_enqueue(object_t *obj, bool ignore_cards)
 {
     assert(!_is_young(obj));
     assert(STM_PSEGMENT->privatization_lock);
@@ -1155,28 +1430,18 @@
     OPT_ASSERT(obj_size >= 16);
 
     if (LIKELY(is_small_uniform(obj))) {
+        assert(!(obj->stm_flags & GCFLAG_CARDS_SET));
         OPT_ASSERT(obj_size <= GC_LAST_SMALL_SIZE);
         _synchronize_fragment((stm_char *)obj, obj_size);
         return;
+    } else if (ignore_cards || !obj_should_use_cards(obj)) {
+        /* else, a more complicated case for large objects, to copy
+           around data only within the needed pages */
+        _page_wise_synchronize_object_now(obj, obj_size);
+    } else {
+        /* ... or even only cards that need to be updated */
+        _card_wise_synchronize_object_now(obj, obj_size);
     }
-
-    /* else, a more complicated case for large objects, to copy
-       around data only within the needed pages
-    */
-    uintptr_t start = (uintptr_t)obj;
-    uintptr_t end = start + obj_size;
-
-    do {
-        uintptr_t copy_up_to = (start + 4096) & ~4095;   /* end of page */
-        if (copy_up_to >= end) {
-            copy_up_to = end;        /* this is the last fragment */
-        }
-        uintptr_t copy_size = copy_up_to - start;
-
-        _synchronize_fragment((stm_char *)start, copy_size);
-
-        start = copy_up_to;
-    } while (start != end);
 }
 
 static void synchronize_objects_flush(void)
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -34,19 +34,26 @@
 #define FIRST_OLD_RM_PAGE     (OLD_RM_START / 4096UL)
 #define NB_READMARKER_PAGES   (FIRST_OBJECT_PAGE - FIRST_READMARKER_PAGE)
 
+#define CARD_SIZE   _STM_CARD_SIZE
+
 enum /* stm_flags */ {
     GCFLAG_WRITE_BARRIER = _STM_GCFLAG_WRITE_BARRIER,
     GCFLAG_HAS_SHADOW = 0x02,
     GCFLAG_WB_EXECUTED = 0x04,
-    GCFLAG_HAS_CARDS = 0x08,
     GCFLAG_CARDS_SET = _STM_GCFLAG_CARDS_SET,
-    GCFLAG_VISITED = 0x20,
-    GCFLAG_FINALIZATION_ORDERING = 0x40,
+    GCFLAG_VISITED = 0x10,
+    GCFLAG_FINALIZATION_ORDERING = 0x20,
 };
 
+#define SYNC_QUEUE_SIZE    31
 
+enum /* card values in read markers */ {
+    CARD_CLEAR = 0,                 /* card not used at all */
+    CARD_MARKED = _STM_CARD_MARKED, /* card marked for tracing in the next gc */
+    CARD_MARKED_OLD = _STM_CARD_MARKED+1,
+    /* card was marked before, but cleared in a GC */
+};
 
-#define SYNC_QUEUE_SIZE    31
 
 
 /************************************************************/
@@ -196,6 +203,18 @@
 
 #define REAL_ADDRESS(segment_base, src)   ((segment_base) + (uintptr_t)(src))
 
+static inline uintptr_t get_index_to_card_index(uintptr_t index) {
+    return (index / CARD_SIZE) + 1;
+}
+
+static inline uintptr_t get_card_index_to_index(uintptr_t card_index) {
+    return (card_index - 1) * CARD_SIZE;
+}
+
+static inline struct stm_read_marker_s *get_read_marker(char *segment_base, uintptr_t obj)
+{
+   return (struct stm_read_marker_s *)(segment_base + (obj >> 4));
+}
 
 static inline char *get_segment_base(long segment_num) {
     return stm_object_pages + segment_num * (NB_PAGES * 4096UL);
@@ -226,7 +245,7 @@
 static stm_thread_local_t *abort_with_mutex_no_longjmp(void);
 static void abort_data_structures_from_segment_num(int segment_num);
 
-static void synchronize_object_enqueue(object_t *obj);
+static void synchronize_object_enqueue(object_t *obj, bool ignore_cards);
 static void synchronize_objects_flush(void);
 
 static void _signal_handler(int sig, siginfo_t *siginfo, void *context);
diff --git a/c8/stm/gcpage.h b/c8/stm/gcpage.h
--- a/c8/stm/gcpage.h
+++ b/c8/stm/gcpage.h
@@ -7,7 +7,6 @@
 #define GC_MIN                 (NB_NURSERY_PAGES * 4096 * 8)
 #define GC_MAJOR_COLLECT       1.82
 
-
 static struct list_s *testing_prebuilt_objs;
 static char *uninitialized_page_start;   /* within segment 0 */
 static char *uninitialized_page_stop;
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -142,8 +142,138 @@
 
     /* Must trace the object later */
     LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, nobj_sync_now);
+    _cards_cleared_in_object(get_priv_segment(STM_SEGMENT->segment_num), nobj);
 }
 
+static void _cards_cleared_in_object(struct stm_priv_segment_info_s *pseg, object_t *obj)
+{
+#ifndef NDEBUG
+    struct object_s *realobj = (struct object_s *)REAL_ADDRESS(pseg->pub.segment_base, obj);
+    size_t size = stmcb_size_rounded_up(realobj);
+
+    if (size < _STM_MIN_CARD_OBJ_SIZE)
+        return;                 /* too small for cards */
+
+    struct stm_read_marker_s *cards = get_read_marker(pseg->pub.segment_base, (uintptr_t)obj);
+    uintptr_t card_index = 1;
+    uintptr_t last_card_index = get_index_to_card_index(size - 1); /* max valid index */
+
+    while (card_index <= last_card_index) {
+        assert(cards[card_index].rm == CARD_CLEAR);
+        card_index++;
+    }
+
+    assert(!(realobj->stm_flags & GCFLAG_CARDS_SET));
+#endif
+}
+
+static void _verify_cards_cleared_in_all_lists(struct stm_priv_segment_info_s *pseg)
+{
+#ifndef NDEBUG
+    struct list_s *list = pseg->modified_old_objects;
+    struct stm_undo_s *undo = (struct stm_undo_s *)list->items;
+    struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count);
+
+    for (; undo < end; undo++) {
+        _cards_cleared_in_object(pseg, undo->object);
+    }
+    LIST_FOREACH_R(
+        pseg->new_objects, object_t * /*item*/,
+        _cards_cleared_in_object(pseg, item));
+    LIST_FOREACH_R(
+        pseg->objects_pointing_to_nursery, object_t * /*item*/,
+        _cards_cleared_in_object(pseg, item));
+    LIST_FOREACH_R(
+        pseg->old_objects_with_cards_set, object_t * /*item*/,
+        _cards_cleared_in_object(pseg, item));
+#endif
+}
+
+static void _reset_object_cards(struct stm_priv_segment_info_s *pseg,
+                                object_t *obj, uint8_t mark_value,
+                                bool mark_all)
+{
+#pragma push_macro("STM_PSEGMENT")
+#pragma push_macro("STM_SEGMENT")
+#undef STM_PSEGMENT
+#undef STM_SEGMENT
+    struct object_s *realobj = (struct object_s *)REAL_ADDRESS(pseg->pub.segment_base, obj);
+    size_t size = stmcb_size_rounded_up(realobj);
+    OPT_ASSERT(size >= _STM_MIN_CARD_OBJ_SIZE);
+
+    uintptr_t offset_itemsize[2];
+    stmcb_get_card_base_itemsize(realobj, offset_itemsize);
+    size = (size - offset_itemsize[0]) / offset_itemsize[1];
+
+    assert(IMPLY(mark_value == CARD_CLEAR, !mark_all)); /* not necessary */
+    assert(IMPLY(mark_all, mark_value == CARD_MARKED_OLD)); /* set *all* to OLD */
+    assert(IMPLY(realobj->stm_flags & GCFLAG_WB_EXECUTED,
+                 mark_value == CARD_CLEAR)); /* overflows are always CLEARed */
+
+
+    struct stm_read_marker_s *cards = get_read_marker(pseg->pub.segment_base, (uintptr_t)obj);
+    uintptr_t card_index = 1;
+    uintptr_t last_card_index = get_index_to_card_index(size - 1); /* max valid index */
+
+    /* dprintf(("mark cards of %p, size %lu with %d, all: %d\n",
+                obj, size, mark_value, mark_all));
+       dprintf(("obj has %lu cards\n", last_card_index));*/
+    while (card_index <= last_card_index) {
+        if (mark_all || cards[card_index].rm != CARD_CLEAR) {
+            /* dprintf(("mark card %lu,wl:%lu of %p with %d\n", */
+            /*          card_index, card_lock_idx, obj, mark_value)); */
+            cards[card_index].rm = mark_value;
+        }
+        card_index++;
+    }
+
+    realobj->stm_flags &= ~GCFLAG_CARDS_SET;
+
+#pragma pop_macro("STM_SEGMENT")
+#pragma pop_macro("STM_PSEGMENT")
+}
+
+
+static void _trace_card_object(object_t *obj)
+{
+    assert(!_is_in_nursery(obj));
+    assert(obj->stm_flags & GCFLAG_CARDS_SET);
+    assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
+
+    dprintf(("_trace_card_object(%p)\n", obj));
+    uint8_t mark_value = CARD_MARKED_OLD;
+
+    struct object_s *realobj = (struct object_s *)REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
+    size_t size = stmcb_size_rounded_up(realobj);
+    uintptr_t offset_itemsize[2];
+    stmcb_get_card_base_itemsize(realobj, offset_itemsize);
+    size = (size - offset_itemsize[0]) / offset_itemsize[1];
+
+    struct stm_read_marker_s *cards = get_read_marker(STM_SEGMENT->segment_base, (uintptr_t)obj);
+    uintptr_t card_index = 1;
+    uintptr_t last_card_index = get_index_to_card_index(size - 1); /* max valid index */
+
+    assert(cards->rm == STM_SEGMENT->transaction_read_version); /* stm_read() */
+
+    /* XXX: merge ranges */
+    while (card_index <= last_card_index) {
+        if (cards[card_index].rm == CARD_MARKED) {
+            /* clear or set to old: */
+            cards[card_index].rm = mark_value;
+
+            uintptr_t start = get_card_index_to_index(card_index);
+            uintptr_t stop = get_card_index_to_index(card_index + 1);
+
+            dprintf(("trace_cards on %p with start:%lu stop:%lu\n",
+                     obj, start, stop));
+            stmcb_trace_cards(realobj, &minor_trace_if_young,
+                              start, stop);
+        }
+
+        card_index++;
+    }
+    obj->stm_flags &= ~GCFLAG_CARDS_SET;
+}
 
 static void collect_roots_in_nursery(void)
 {
@@ -177,15 +307,20 @@
 static inline void _collect_now(object_t *obj)
 {
     assert(!_is_young(obj));
+    assert(!(obj->stm_flags & GCFLAG_CARDS_SET));
 
     //dprintf(("_collect_now: %p\n", obj));
 
-    assert(!(obj->stm_flags & GCFLAG_WRITE_BARRIER));
+    if (!(obj->stm_flags & GCFLAG_WRITE_BARRIER)) {
+        /* Trace the 'obj' to replace pointers to nursery with pointers
+           outside the nursery, possibly forcing nursery objects out and
+           adding them to 'objects_pointing_to_nursery' as well. */
+        char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
+        stmcb_trace((struct object_s *)realobj, &minor_trace_if_young);
 
-    char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
-    stmcb_trace((struct object_s *)realobj, &minor_trace_if_young);
-
-    obj->stm_flags |= GCFLAG_WRITE_BARRIER;
+        obj->stm_flags |= GCFLAG_WRITE_BARRIER;
+    }
+    /* else traced in collect_cardrefs_to_nursery if necessary */
 }
 
 
@@ -201,18 +336,21 @@
         assert(!_is_in_nursery(obj));
 
         _collect_now(obj);
+        assert(!(obj->stm_flags & GCFLAG_CARDS_SET));
 
         if (obj_sync_now & FLAG_SYNC_LARGE) {
             /* this is a newly allocated obj in this transaction. We must
                either synchronize the object to other segments now, or
                add the object to new_objects list */
-            if (STM_PSEGMENT->minor_collect_will_commit_now) {
-                acquire_privatization_lock(STM_SEGMENT->segment_num);
-                synchronize_object_enqueue(obj);
-                release_privatization_lock(STM_SEGMENT->segment_num);
+            struct stm_priv_segment_info_s *pseg = get_priv_segment(STM_SEGMENT->segment_num);
+            if (pseg->minor_collect_will_commit_now) {
+                acquire_privatization_lock(pseg->pub.segment_num);
+                synchronize_object_enqueue(obj, true); /* ignore cards! */
+                release_privatization_lock(pseg->pub.segment_num);
             } else {
-                LIST_APPEND(STM_PSEGMENT->new_objects, obj);
+                LIST_APPEND(pseg->new_objects, obj);
             }
+            _cards_cleared_in_object(pseg, obj);
         }
 
         /* the list could have moved while appending */
@@ -230,6 +368,30 @@
     }
 }
 
+
+static void collect_cardrefs_to_nursery(void)
+{
+    dprintf(("collect_cardrefs_to_nursery\n"));
+    struct list_s *lst = STM_PSEGMENT->old_objects_with_cards_set;
+
+    while (!list_is_empty(lst)) {
+        object_t *obj = (object_t*)list_pop_item(lst);
+
+        assert(!_is_young(obj));
+
+        if (!(obj->stm_flags & GCFLAG_CARDS_SET)) {
+            /* sometimes we remove the CARDS_SET in the WB slowpath, see core.c */
+            continue;
+        }
+
+        /* traces cards, clears marked cards or marks them old if necessary */
+        _trace_card_object(obj);
+
+        assert(!(obj->stm_flags & GCFLAG_CARDS_SET));
+    }
+}
+
+
 static void collect_objs_still_young_but_with_finalizers(void)
 {
     struct list_s *lst = STM_PSEGMENT->finalizers->objects_with_finalizers;
@@ -314,12 +476,15 @@
 
     STM_PSEGMENT->minor_collect_will_commit_now = commit;
 
+    collect_cardrefs_to_nursery();
+
     collect_roots_in_nursery();
 
     if (STM_PSEGMENT->finalizers != NULL)
         collect_objs_still_young_but_with_finalizers();
 
     collect_oldrefs_to_nursery();
+    assert(list_is_empty(STM_PSEGMENT->old_objects_with_cards_set));
 
     /* now all surviving nursery objects have been moved out */
     acquire_privatization_lock(STM_SEGMENT->segment_num);
diff --git a/c8/stm/nursery.h b/c8/stm/nursery.h
--- a/c8/stm/nursery.h
+++ b/c8/stm/nursery.h
@@ -2,6 +2,11 @@
 #define NSE_SIGPAUSE   _STM_NSE_SIGNAL_MAX
 #define NSE_SIGABORT   _STM_NSE_SIGNAL_ABORT
 
+static void _cards_cleared_in_object(struct stm_priv_segment_info_s *pseg, object_t *obj);
+static void _reset_object_cards(struct stm_priv_segment_info_s *pseg,
+                                object_t *obj, uint8_t mark_value,
+                                bool mark_all);
+
 static void minor_collection(bool commit);
 static void check_nursery_at_transaction_start(void);
 static size_t throw_away_nursery(struct stm_priv_segment_info_s *pseg);
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -73,11 +73,16 @@
 } stm_thread_local_t;
 
 #define _STM_GCFLAG_WRITE_BARRIER      0x01
-#define _STM_GCFLAG_CARDS_SET          0x10
 #define _STM_FAST_ALLOC           (66*1024)
 #define _STM_NSE_SIGNAL_ABORT             1
 #define _STM_NSE_SIGNAL_MAX               2
 
+#define _STM_CARD_MARKED 1
+#define _STM_GCFLAG_CARDS_SET          0x8
+#define _STM_CARD_SIZE                 32     /* must be >= 32 */
+#define _STM_MIN_CARD_COUNT            17
+#define _STM_MIN_CARD_OBJ_SIZE         (_STM_CARD_SIZE * _STM_MIN_CARD_COUNT)
+
 void _stm_write_slowpath(object_t *);
 void _stm_write_slowpath_card(object_t *, uintptr_t);
 object_t *_stm_allocate_slowpath(ssize_t);
@@ -163,6 +168,16 @@
    ranges of indices (using stm_write_card(o, index)) */
 extern void stmcb_trace_cards(struct object_s *, void (object_t **),
                               uintptr_t start, uintptr_t stop);
+/* this function will be called on objects that support cards.
+   It returns the base_offset (in bytes) inside the object from
+   where the indices start, and item_size (in bytes) for the size of
+   one item */
+extern void stmcb_get_card_base_itemsize(struct object_s *,
+                                         uintptr_t offset_itemsize[2]);
+/* returns whether this object supports cards. we will only call
+   stmcb_get_card_base_itemsize on objs that do so. */
+extern long stmcb_obj_supports_cards(struct object_s *);
+
 
 
 
diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -10,6 +10,7 @@
 #define STM_NB_SEGMENTS ...
 #define _STM_GCFLAG_WRITE_BARRIER ...
 #define _STM_FAST_ALLOC ...
+#define _STM_CARD_SIZE ...
 
 typedef struct {
 ...;
@@ -332,6 +333,11 @@
     }
 }
 
+long stmcb_obj_supports_cards(struct object_s *obj)
+{
+    return 1;
+}
+
 void stmcb_trace_cards(struct object_s *obj, void visit(object_t **),
                        uintptr_t start, uintptr_t stop)
 {
@@ -350,6 +356,19 @@
     }
 }
 
+void stmcb_get_card_base_itemsize(struct object_s *obj,
+                                  uintptr_t offset_itemsize[2])
+{
+    struct myobj_s *myobj = (struct myobj_s*)obj;
+    if (myobj->type_id < 421420) {
+        offset_itemsize[0] = SIZEOF_MYOBJ;
+        offset_itemsize[1] = 1;
+    }
+    else {
+        offset_itemsize[0] = sizeof(struct myobj_s);
+        offset_itemsize[1] = sizeof(object_t *);
+    }
+}
 
 long current_segment_num(void)
 {
@@ -376,6 +395,7 @@
 GCFLAG_WRITE_BARRIER = lib._STM_GCFLAG_WRITE_BARRIER
 NB_SEGMENTS = lib.STM_NB_SEGMENTS
 FAST_ALLOC = lib._STM_FAST_ALLOC
+CARD_SIZE = lib._STM_CARD_SIZE # 16b at least
 
 class Conflict(Exception):
     pass
diff --git a/c8/test/test_card_marking.py b/c8/test/test_card_marking.py
--- a/c8/test/test_card_marking.py
+++ b/c8/test/test_card_marking.py
@@ -42,7 +42,7 @@
 
         stm_write_card(o, 5)
 
-        assert o in old_objects_with_cards()
+        assert o in old_objects_with_cards_set()
         assert o not in modified_old_objects() # overflow object
         assert o not in objects_pointing_to_nursery()
         # don't remove GCFLAG_WB
@@ -141,7 +141,7 @@
 
         assert not modified_old_objects()
         assert not objects_pointing_to_nursery()
-        assert not old_objects_with_cards()
+        assert not old_objects_with_cards_set()
 
         self.start_transaction()
         d = stm_allocate(64)

From noreply at buildbot.pypy.org  Thu Feb 26 21:46:48 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu, 26 Feb 2015 21:46:48 +0100 (CET)
Subject: [pypy-commit] stmgc c8-card-marking: progress
Message-ID: <20150226204648.6CAB91C01CD@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-card-marking
Changeset: r1671:aaa47dab157c
Date: 2015-02-26 20:35 +0100
http://bitbucket.org/pypy/stmgc/changeset/aaa47dab157c/

Log:	progress

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -985,7 +985,7 @@
         ({
             assert(item->stm_flags & GCFLAG_WB_EXECUTED);
             _cards_cleared_in_object(pseg, item); /* check for C8 */
-            _reset_object_cards(pseg, item, CARD_CLEAR, false); /* unnecessary, as sync_obj_enq does it already? */
+
             item->stm_flags &= ~GCFLAG_WB_EXECUTED;
             synchronize_object_enqueue(item, true);
         }));
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -499,6 +499,13 @@
                     assert(!(realobj->stm_flags & GCFLAG_WRITE_BARRIER));
 
                     realobj->stm_flags |= GCFLAG_WRITE_BARRIER;
+
+                    OPT_ASSERT(!(realobj->stm_flags & GCFLAG_CARDS_SET));
+                    if (realobj->stm_flags & GCFLAG_CARDS_SET) {
+                        /* we called a normal WB on this object, so all cards
+                           need to be marked OLD */
+                        _reset_object_cards(pseg, item, CARD_MARKED_OLD, true); /* mark all */
+                    }
                 }));
             list_clear(lst);
         } else {
@@ -507,6 +514,21 @@
                modified_old_objs. */
         }
 
+        lst = pseg->old_objects_with_cards_set;
+        LIST_FOREACH_R(lst, object_t* /*item*/,
+            ({
+                struct object_s *realobj = (struct object_s *)
+                    REAL_ADDRESS(pseg->pub.segment_base, item);
+                OPT_ASSERT(realobj->stm_flags & GCFLAG_CARDS_SET);
+                OPT_ASSERT(realobj->stm_flags & GCFLAG_WRITE_BARRIER);
+
+                /* mark marked cards as old otherwise */
+                uint8_t mark_value = CARD_MARKED_OLD;
+                _reset_object_cards(pseg, item, mark_value, false);
+            }));
+        list_clear(lst);
+
+
         /* remove from new_objects all objects that die */
         lst = pseg->new_objects;
         uintptr_t n = list_count(lst);
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -207,9 +207,6 @@
 
     assert(IMPLY(mark_value == CARD_CLEAR, !mark_all)); /* not necessary */
     assert(IMPLY(mark_all, mark_value == CARD_MARKED_OLD)); /* set *all* to OLD */
-    assert(IMPLY(realobj->stm_flags & GCFLAG_WB_EXECUTED,
-                 mark_value == CARD_CLEAR)); /* overflows are always CLEARed */
-
 
     struct stm_read_marker_s *cards = get_read_marker(pseg->pub.segment_base, (uintptr_t)obj);
     uintptr_t card_index = 1;

From noreply at buildbot.pypy.org  Thu Feb 26 21:46:49 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu, 26 Feb 2015 21:46:49 +0100 (CET)
Subject: [pypy-commit] stmgc c8-card-marking: progress
Message-ID: <20150226204649.718281C01CD@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-card-marking
Changeset: r1672:023ed269165f
Date: 2015-02-26 21:22 +0100
http://bitbucket.org/pypy/stmgc/changeset/023ed269165f/

Log:	progress

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -445,6 +445,7 @@
 }
 
 
+static void reset_cards_from_modified_objects(void);
 static void reset_wb_executed_flags(void);
 static void readd_wb_executed_flags(void);
 static void check_all_write_barrier_flags(char *segbase, struct list_s *list);
@@ -527,6 +528,8 @@
         STM_PSEGMENT->transaction_state = TS_NONE;
         STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
 
+        reset_cards_from_modified_objects();
+
         list_clear(STM_PSEGMENT->modified_old_objects);
         STM_PSEGMENT->last_commit_log_entry = new;
         release_modification_lock(STM_SEGMENT->segment_num);
@@ -553,6 +556,8 @@
         check_all_write_barrier_flags(STM_SEGMENT->segment_base,
                                       STM_PSEGMENT->modified_old_objects);
 
+        reset_cards_from_modified_objects();
+
         /* compare with _validate_and_attach: */
         STM_PSEGMENT->transaction_state = TS_NONE;
         STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
@@ -576,10 +581,13 @@
 }
 
 
-static bool obj_should_use_cards(object_t *obj)
+static bool obj_should_use_cards(char *seg_base, object_t *obj)
 {
+    if (is_small_uniform(obj))
+        return false;
+
     struct object_s *realobj = (struct object_s *)
-        REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
+        REAL_ADDRESS(seg_base, obj);
     long supports = stmcb_obj_supports_cards(realobj);
     if (!supports)
         return false;
@@ -750,7 +758,7 @@
 {
     /* the PyPy JIT calls this function directly if it finds that an
        array doesn't have the GCFLAG_CARDS_SET */
-    bool mark_card = obj_should_use_cards(obj);
+    bool mark_card = obj_should_use_cards(STM_SEGMENT->segment_base, obj);
     write_slowpath_common(obj, mark_card);
     return mark_card;
 }
@@ -835,6 +843,20 @@
     STM_SEGMENT->transaction_read_version = 1;
 }
 
+static void reset_cards_from_modified_objects(void)
+{
+    struct list_s *list = STM_PSEGMENT->modified_old_objects;
+    struct stm_undo_s *undo = (struct stm_undo_s *)list->items;
+    struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count);
+
+    for (; undo < end; undo++) {
+        object_t *obj = undo->object;
+        if (obj_should_use_cards(STM_SEGMENT->segment_base, obj))
+            _reset_object_cards(get_priv_segment(STM_SEGMENT->segment_num),
+                                obj, CARD_CLEAR, false);
+    }
+}
+
 static void reset_wb_executed_flags(void)
 {
     dprintf(("reset_wb_executed_flags()\n"));
@@ -984,9 +1006,9 @@
     LIST_FOREACH_R(STM_PSEGMENT->new_objects, object_t *,
         ({
             assert(item->stm_flags & GCFLAG_WB_EXECUTED);
-            _cards_cleared_in_object(pseg, item); /* check for C8 */
-
             item->stm_flags &= ~GCFLAG_WB_EXECUTED;
+            if (obj_should_use_cards(pseg->pub.segment_base, item))
+                _reset_object_cards(pseg, item, CARD_CLEAR, false);
             synchronize_object_enqueue(item, true);
         }));
     synchronize_objects_flush();
@@ -1020,7 +1042,6 @@
     bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE;
     _validate_and_add_to_commit_log();
 
-
     /* XXX do we still need a s_mutex_lock() section here? */
     s_mutex_lock();
 
@@ -1081,7 +1102,7 @@
                undo->backup,
                SLICE_SIZE(undo->slice));
 
-        if (obj_should_use_cards(obj))
+        if (obj_should_use_cards(pseg->pub.segment_base, obj))
             _reset_object_cards(pseg, obj, CARD_CLEAR, false);
         /* XXXXXXXXX: only reset cards of slice!! ^^^^^^^ */
 
@@ -1124,14 +1145,8 @@
     /* some new objects may have cards when aborting, clear them too */
     LIST_FOREACH_R(pseg->new_objects, object_t * /*item*/,
         {
-            struct object_s *realobj = (struct object_s *)
-                REAL_ADDRESS(pseg->pub.segment_base, item);
-
-            if (realobj->stm_flags & GCFLAG_CARDS_SET) {
-                /* CARDS_SET is enough since other HAS_CARDS objs
-                   are already cleared */
+            if (obj_should_use_cards(pseg->pub.segment_base, item))
                 _reset_object_cards(pseg, item, CARD_CLEAR, false);
-            }
         });
 
     acquire_modification_lock(segment_num);
@@ -1318,7 +1333,7 @@
 static void _card_wise_synchronize_object_now(object_t *obj, ssize_t obj_size)
 {
     assert(obj_size >= 32);
-    assert(obj_should_use_cards(obj));
+    assert(obj_should_use_cards(STM_SEGMENT->segment_base, obj));
     assert(!(obj->stm_flags & GCFLAG_CARDS_SET));
 
     uintptr_t offset_itemsize[2];
@@ -1434,7 +1449,7 @@
         OPT_ASSERT(obj_size <= GC_LAST_SMALL_SIZE);
         _synchronize_fragment((stm_char *)obj, obj_size);
         return;
-    } else if (ignore_cards || !obj_should_use_cards(obj)) {
+    } else if (ignore_cards || !obj_should_use_cards(STM_SEGMENT->segment_base, obj)) {
         /* else, a more complicated case for large objects, to copy
            around data only within the needed pages */
         _page_wise_synchronize_object_now(obj, obj_size);
diff --git a/c8/stm/misc.c b/c8/stm/misc.c
--- a/c8/stm/misc.c
+++ b/c8/stm/misc.c
@@ -111,6 +111,15 @@
 }
 
 
+uint8_t _stm_get_card_value(object_t *obj, long idx)
+{
+    struct stm_read_marker_s *cards = get_read_marker(STM_SEGMENT->segment_base,
+                                                      (uintptr_t)obj);
+    return cards[get_index_to_card_index(idx)].rm;
+}
+
+
+
 static struct stm_commit_log_entry_s *_last_cl_entry;
 static long _last_cl_entry_index;
 void _stm_start_enum_last_cl_entry()
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -250,8 +250,6 @@
     uintptr_t card_index = 1;
     uintptr_t last_card_index = get_index_to_card_index(size - 1); /* max valid index */
 
-    assert(cards->rm == STM_SEGMENT->transaction_read_version); /* stm_read() */
-
     /* XXX: merge ranges */
     while (card_index <= last_card_index) {
         if (cards[card_index].rm == CARD_MARKED) {
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -94,6 +94,7 @@
 char *_stm_real_address(object_t *o);
 #ifdef STM_TESTS
 #include 
+uint8_t _stm_get_card_value(object_t *obj, long idx);
 bool _stm_was_read(object_t *obj);
 bool _stm_was_written(object_t *obj);
 bool _stm_was_written_card(object_t *obj);
diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -11,6 +11,7 @@
 #define _STM_GCFLAG_WRITE_BARRIER ...
 #define _STM_FAST_ALLOC ...
 #define _STM_CARD_SIZE ...
+#define _STM_CARD_MARKED ...
 
 typedef struct {
 ...;
@@ -46,6 +47,8 @@
 
 /*void stm_write_card(); use _checked_stm_write_card() instead */
 
+uint8_t _stm_get_card_value(object_t *obj, long idx);
+
 void stm_setup(void);
 void stm_teardown(void);
 void stm_register_thread_local(stm_thread_local_t *tl);
@@ -396,6 +399,10 @@
 NB_SEGMENTS = lib.STM_NB_SEGMENTS
 FAST_ALLOC = lib._STM_FAST_ALLOC
 CARD_SIZE = lib._STM_CARD_SIZE # 16b at least
+CARD_CLEAR = 0
+CARD_MARKED = lib._STM_CARD_MARKED
+CARD_MARKED_OLD = CARD_MARKED + 1
+
 
 class Conflict(Exception):
     pass
diff --git a/c8/test/test_card_marking.py b/c8/test/test_card_marking.py
--- a/c8/test/test_card_marking.py
+++ b/c8/test/test_card_marking.py
@@ -2,6 +2,8 @@
 import py
 
 
+get_card_value = lib._stm_get_card_value
+
 class TestBasic(BaseTest):
 
     def _collect(self, kind):
@@ -221,3 +223,24 @@
         assert stm_get_char(o, 1000+CARD_SIZE*12) == 'e'
 
         self.commit_transaction()
+
+
+    def test_clear_cards(self):
+        o = stm_allocate_old(1000+20*CARD_SIZE)
+
+        self.start_transaction()
+        assert get_card_value(o, 1000) == CARD_CLEAR
+        stm_set_char(o, 'a', 1000, True)
+        assert get_card_value(o, 1000) == CARD_MARKED
+        assert o in old_objects_with_cards_set()
+
+        stm_minor_collect()
+        assert get_card_value(o, 1000) == CARD_MARKED_OLD
+        self.commit_transaction()
+
+        self.start_transaction()
+        assert get_card_value(o, 1000) == CARD_CLEAR
+        stm_set_char(o, 'b', 1000, True)
+        assert get_card_value(o, 1000) == CARD_MARKED
+        assert o in old_objects_with_cards_set()
+        self.commit_transaction()
diff --git a/c8/test/test_random.py b/c8/test/test_random.py
--- a/c8/test/test_random.py
+++ b/c8/test/test_random.py
@@ -428,7 +428,7 @@
     r = thread_state.get_random_root()
     trs = thread_state.transaction_state
     is_ref = global_state.has_ref_type(r)
-    try_cards = global_state.rnd.randrange(1, 100) > 5 and False
+    try_cards = global_state.rnd.randrange(1, 100) > 5 # and False
     #
     # decide on a value to write
     if is_ref:

From noreply at buildbot.pypy.org  Thu Feb 26 21:46:50 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu, 26 Feb 2015 21:46:50 +0100 (CET)
Subject: [pypy-commit] stmgc c8-card-marking: basic card-marking seems to
	work
Message-ID: <20150226204650.835E31C01CD@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-card-marking
Changeset: r1673:a7b28578ad96
Date: 2015-02-26 21:47 +0100
http://bitbucket.org/pypy/stmgc/changeset/a7b28578ad96/

Log:	basic card-marking seems to work

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -581,7 +581,7 @@
 }
 
 
-static bool obj_should_use_cards(char *seg_base, object_t *obj)
+bool obj_should_use_cards(char *seg_base, object_t *obj)
 {
     if (is_small_uniform(obj))
         return false;
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -237,6 +237,7 @@
     return (addr - stm_object_pages) / (NB_PAGES * 4096UL);
 }
 
+bool obj_should_use_cards(char *seg_base, object_t *obj);
 
 static bool _is_tl_registered(stm_thread_local_t *tl);
 static bool _seems_to_be_running_transaction(void);
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -501,11 +501,6 @@
                     realobj->stm_flags |= GCFLAG_WRITE_BARRIER;
 
                     OPT_ASSERT(!(realobj->stm_flags & GCFLAG_CARDS_SET));
-                    if (realobj->stm_flags & GCFLAG_CARDS_SET) {
-                        /* we called a normal WB on this object, so all cards
-                           need to be marked OLD */
-                        _reset_object_cards(pseg, item, CARD_MARKED_OLD, true); /* mark all */
-                    }
                 }));
             list_clear(lst);
         } else {
@@ -519,11 +514,11 @@
             ({
                 struct object_s *realobj = (struct object_s *)
                     REAL_ADDRESS(pseg->pub.segment_base, item);
-                OPT_ASSERT(realobj->stm_flags & GCFLAG_CARDS_SET);
                 OPT_ASSERT(realobj->stm_flags & GCFLAG_WRITE_BARRIER);
 
-                /* mark marked cards as old otherwise */
-                uint8_t mark_value = CARD_MARKED_OLD;
+                /* mark marked cards as old if it survives */
+                uint8_t mark_value = mark_visited_test(item) ?
+                    CARD_MARKED_OLD : CARD_CLEAR;
                 _reset_object_cards(pseg, item, mark_value, false);
             }));
         list_clear(lst);
@@ -535,6 +530,8 @@
         while (n-- > 0) {
             object_t *obj = (object_t *)list_item(lst, n);
             if (!mark_visited_test(obj)) {
+                if (obj_should_use_cards(pseg->pub.segment_base, obj))
+                    _reset_object_cards(pseg, obj, CARD_CLEAR, false);
                 list_set_item(lst, n, list_pop_item(lst));
             }
         }
diff --git a/c8/test/test_card_marking.py b/c8/test/test_card_marking.py
--- a/c8/test/test_card_marking.py
+++ b/c8/test/test_card_marking.py
@@ -244,3 +244,47 @@
         assert get_card_value(o, 1000) == CARD_MARKED
         assert o in old_objects_with_cards_set()
         self.commit_transaction()
+
+    def test_clear_cards2(self):
+        self.start_transaction()
+        o = stm_allocate(1000+20*CARD_SIZE)
+        assert get_card_value(o, 1000) == CARD_CLEAR
+        stm_set_char(o, 'a', 1000, True)
+        assert get_card_value(o, 1000) == CARD_CLEAR
+        assert o not in old_objects_with_cards_set()
+
+        self.push_root(o)
+        stm_minor_collect()
+        o = self.pop_root()
+
+        assert get_card_value(o, 1000) == CARD_CLEAR
+        stm_set_char(o, 'b', 1000, True)
+        assert get_card_value(o, 1000) == CARD_MARKED
+        assert o in old_objects_with_cards_set()
+        self.commit_transaction()
+
+        self.start_transaction()
+        assert get_card_value(o, 1000) == CARD_CLEAR
+        stm_set_char(o, 'b', 1000, True)
+        assert get_card_value(o, 1000) == CARD_MARKED
+        assert o in old_objects_with_cards_set()
+        self.commit_transaction()
+
+    def test_clear_cards3(self):
+        self.start_transaction()
+        o = stm_allocate(1000+20*CARD_SIZE)
+
+        self.push_root(o)
+        stm_minor_collect()
+        o = self.pop_root()
+
+        assert get_card_value(o, 1000) == CARD_CLEAR
+        stm_set_char(o, 'b', 1000, True)
+        assert get_card_value(o, 1000) == CARD_MARKED
+        assert o in old_objects_with_cards_set()
+
+        stm_major_collect()
+        assert get_card_value(o, 1000) == CARD_CLEAR
+        assert o not in old_objects_with_cards_set()
+
+        self.commit_transaction()

From noreply at buildbot.pypy.org  Fri Feb 27 10:24:46 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 27 Feb 2015 10:24:46 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: Put the guy in the no-show
 category, just in case he doesn't apologize
Message-ID: <20150227092446.4A15A1C0F05@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: extradoc
Changeset: r5510:37f6801d6b11
Date: 2015-02-27 10:25 +0100
http://bitbucket.org/pypy/extradoc/changeset/37f6801d6b11/

Log:	Put the guy in the no-show category, just in case he doesn't
	apologize and tries to do it again

diff --git a/sprintinfo/leysin-winter-2015/people.txt b/sprintinfo/leysin-winter-2015/people.txt
--- a/sprintinfo/leysin-winter-2015/people.txt
+++ b/sprintinfo/leysin-winter-2015/people.txt
@@ -17,7 +17,6 @@
 Manuel Jacob         21-28          Ermina
 Joan Massich         20-?           Ermina
 Quim Sanchez         20-?           Ermina
-Francisco Fernandez  20-28          Ermina
 Alexander Schremmer  21-23          Ermina
 ==================== ============== =======================
 
@@ -27,6 +26,7 @@
 ==================== ============== =====================
        Name          Arrive/Depart  Accomodation 
 ==================== ============== =====================
+Francisco Fernandez  no-show in     Leysin Winter 2015
 Romain Guillebert    ?              ?
 Christian Clauss     ?              ?
 Johan Råde           ?              ?

From noreply at buildbot.pypy.org  Fri Feb 27 11:23:31 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 27 Feb 2015 11:23:31 +0100 (CET)
Subject: [pypy-commit] pypy default: issue #1442: protect against the
 little-sensical reload(sys) that
Message-ID: <20150227102331.8F7371C0EEE@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76158:78cf84d6dd0e
Date: 2015-02-27 11:22 +0100
http://bitbucket.org/pypy/pypy/changeset/78cf84d6dd0e/

Log:	issue #1442: protect against the little-sensical reload(sys) that
	people seem to do by forcing a copy of the saved state after the
	program state was set up.

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -517,6 +517,10 @@
     elif not sys.stdout.isatty():
         set_fully_buffered_io()
 
+    if we_are_translated():
+        import __pypy__
+        __pypy__.save_module_content_for_future_reload(sys)
+
     mainmodule = type(sys)('__main__')
     sys.modules['__main__'] = mainmodule
 
diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py
--- a/pypy/interpreter/mixedmodule.py
+++ b/pypy/interpreter/mixedmodule.py
@@ -55,7 +55,10 @@
         if self.w_initialdict is None:
             Module.init(self, space)
             if not self.lazy and self.w_initialdict is None:
-                self.w_initialdict = space.call_method(self.w_dict, 'items')
+                self.save_module_content_for_future_reload()
+
+    def save_module_content_for_future_reload(self):
+        self.w_initialdict = self.space.call_method(self.w_dict, 'items')
 
 
     def get_applevel_name(cls):
@@ -119,7 +122,7 @@
                 w_value = self.get(name)
                 space.setitem(self.w_dict, space.new_interned_str(name), w_value)
             self.lazy = False
-            self.w_initialdict = space.call_method(self.w_dict, 'items')
+            self.save_module_content_for_future_reload()
         return self.w_dict
 
     def _cleanup_(self):
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -82,6 +82,8 @@
         'strategy'                  : 'interp_magic.strategy',  # dict,set,list
         'set_debug'                 : 'interp_magic.set_debug',
         'locals_to_fast'            : 'interp_magic.locals_to_fast',
+        'save_module_content_for_future_reload':
+                          'interp_magic.save_module_content_for_future_reload',
     }
     if sys.platform == 'win32':
         interpleveldefs['get_console_cp'] = 'interp_magic.get_console_cp'
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -1,6 +1,7 @@
 from pypy.interpreter.error import OperationError, wrap_oserror
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.pyframe import PyFrame
+from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib.objectmodel import we_are_translated
 from pypy.objspace.std.dictmultiobject import W_DictMultiObject
 from pypy.objspace.std.listobject import W_ListObject
@@ -130,3 +131,7 @@
 def locals_to_fast(space, w_frame):
     assert isinstance(w_frame, PyFrame)
     w_frame.locals2fast()
+
+ at unwrap_spec(w_module=MixedModule)
+def save_module_content_for_future_reload(space, w_module):
+    w_module.save_module_content_for_future_reload()
diff --git a/pypy/module/__pypy__/test/test_magic.py b/pypy/module/__pypy__/test/test_magic.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__pypy__/test/test_magic.py
@@ -0,0 +1,15 @@
+
+class AppTestMagic:
+    spaceconfig = dict(usemodules=['__pypy__'])
+
+    def test_save_module_content_for_future_reload(self):
+        import sys, __pypy__
+        d = sys.dont_write_bytecode
+        sys.dont_write_bytecode = "hello world"
+        __pypy__.save_module_content_for_future_reload(sys)
+        sys.dont_write_bytecode = d
+        reload(sys)
+        assert sys.dont_write_bytecode == "hello world"
+        #
+        sys.dont_write_bytecode = d
+        __pypy__.save_module_content_for_future_reload(sys)

From noreply at buildbot.pypy.org  Fri Feb 27 11:44:07 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 27 Feb 2015 11:44:07 +0100 (CET)
Subject: [pypy-commit] pypy default: Make the finalizer of array.array()
 light. Should mostly fix issue #1989.
Message-ID: <20150227104407.8CBA31C009F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76159:8e3b467d1413
Date: 2015-02-27 11:43 +0100
http://bitbucket.org/pypy/pypy/changeset/8e3b467d1413/

Log:	Make the finalizer of array.array() light. Should mostly fix issue
	#1989.

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -1,4 +1,4 @@
-from rpython.rlib import jit
+from rpython.rlib import jit, rgc
 from rpython.rlib.buffer import Buffer
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rarithmetic import ovfcheck, widen
@@ -698,11 +698,9 @@
                                          self.space.wrap(msg))
             return result
 
+        @rgc.must_be_light_finalizer
         def __del__(self):
-            # note that we don't call clear_all_weakrefs here because
-            # an array with freed buffer is ok to see - it's just empty with 0
-            # length
-            self.setlen(0)
+            lltype.free(self.buffer, flavor='raw')
 
         def setlen(self, size, zero=False, overallocate=True):
             if size > 0:

From noreply at buildbot.pypy.org  Fri Feb 27 15:40:44 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 27 Feb 2015 15:40:44 +0100 (CET)
Subject: [pypy-commit] pypy optresult: slow progress
Message-ID: <20150227144044.D16041C0096@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r76160:37ac39dd0b1b
Date: 2015-02-27 14:14 +0200
http://bitbucket.org/pypy/pypy/changeset/37ac39dd0b1b/

Log:	slow progress

diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -136,7 +136,7 @@
     vtable = llmemory.cast_ptr_to_adr(vtable)
     return adr2int(vtable)
 
-def gc_fielddescrs(gccache, STRUCT, res=None):
+def fielddescrs_from_struct(gccache, STRUCT, only_gc=False, res=None):
     from rpython.jit.backend.llsupport import descr
 
     if res is None:
@@ -147,7 +147,10 @@
         if FIELD is lltype.Void:
             continue
         elif isinstance(FIELD, lltype.Struct):
-            gc_fielddescrs(gccache, FIELD, res)
-        elif isinstance(FIELD, lltype.Ptr) and FIELD._needsgc():
+            fielddescrs_from_struct(gccache, FIELD, only_gc, res)
+        elif (not only_gc) or (isinstance(FIELD, lltype.Ptr) and FIELD._needsgc()):
             res.append(descr.get_field_descr(gccache, STRUCT, name))
     return res
+
+def gc_fielddescrs(gccache, STRUCT):
+    return fielddescrs_from_struct(gccache, STRUCT, True)
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -13,31 +13,25 @@
 from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.jit.metainterp.optimizeopt.intutils import IntUnbounded
 
-class AbstractVirtualValue:  #XXX (optimizer.PtrOptInfo):
-    _attrs_ = ('keybox', 'source_op', '_cached_vinfo')
-    box = None
+class AbstractVirtualInfo(optimizer.PtrOptInfo):
+    _attrs_ = ('_cached_vinfo',)
     _tag = optimizer.LEVEL_NONNULL
     is_about_raw = False
     _cached_vinfo = None
 
-    def __init__(self, source_op):
-        self.source_op = source_op
-
     def is_forced_virtual(self):
+        xxx
         return self.box is not None
 
-    def get_key_box(self):
-        if self.box is None:
-            return self.source_op
-        return self.box
-
     def force_box(self, optforce):
+        xxxx
         if self.box is None:
             optforce.forget_numberings(self.source_op)
             self._really_force(optforce)
         return self.box
 
     def force_at_end_of_preamble(self, already_forced, optforce):
+        xxxx
         value = already_forced.get(self, None)
         if value:
             return value
@@ -70,11 +64,11 @@
     return cpu._optimizeopt_fielddescrlist_cache
 get_fielddescrlist_cache._annspecialcase_ = "specialize:memo"
 
-class AbstractVirtualStructValue(AbstractVirtualValue):
-    _attrs_ = ('_fields', 'cpu', '_cached_sorted_fields')
+class AbstractVirtualStructInfo(AbstractVirtualInfo):
+    _attrs_ = ('_fields', '_cached_sorted_fields')
 
-    def __init__(self, cpu, source_op):
-        AbstractVirtualValue.__init__(self, source_op)
+    def __init__(self, cpu):
+        AbstractVirtualInfo.__init__(self)
         self.cpu = cpu
         self._fields = {}
         self._cached_sorted_fields = None
@@ -195,11 +189,11 @@
             fieldvalue = self._fields[ofs]
             fieldvalue.visitor_walk_recursive(visitor)
 
-class VirtualValue(AbstractVirtualStructValue):
+class VirtualInfo(AbstractVirtualStructInfo):
     _tag = optimizer.LEVEL_KNOWNCLASS
 
-    def __init__(self, cpu, known_class, source_op):
-        AbstractVirtualStructValue.__init__(self, cpu, source_op)
+    def __init__(self, known_class):
+        AbstractVirtualStructInfo.__init__(self)
         assert isinstance(known_class, Const)
         self.known_class = known_class
 
@@ -218,7 +212,7 @@
         field_names = [field.name for field in self._fields]
         return "" % (cls_name, field_names)
 
-class VStructValue(AbstractVirtualStructValue):
+class VStructInfo(AbstractVirtualStructInfo):
 
     def __init__(self, cpu, structdescr, source_op):
         AbstractVirtualStructValue.__init__(self, cpu, source_op)
@@ -232,7 +226,7 @@
     def _get_descr(self):
         return self.structdescr
 
-class AbstractVArrayValue(AbstractVirtualValue):
+class AbstractVArrayInfo(AbstractVirtualInfo):
     """
     Base class for VArrayValue (for normal GC arrays) and VRawBufferValue (for
     malloc()ed memory)
@@ -263,7 +257,7 @@
                 itemvalue.visitor_walk_recursive(visitor)
 
 
-class VArrayValue(AbstractVArrayValue):
+class VArrayInfo(AbstractVArrayInfo):
 
     def __init__(self, arraydescr, constvalue, size, source_op,
                  clear=False):
@@ -352,7 +346,7 @@
         return visitor.visit_varray(self.arraydescr, self.clear)
 
 
-class VArrayStructValue(AbstractVirtualValue):
+class VArrayStructInfo(AbstractVirtualInfo):
     def __init__(self, arraydescr, size, source_op):
         AbstractVirtualValue.__init__(self, source_op)
         self.arraydescr = arraydescr
@@ -421,7 +415,7 @@
         return visitor.visit_varraystruct(self.arraydescr, self._get_list_of_descrs())
 
 
-class VRawBufferValue(AbstractVArrayValue):
+class VRawBufferInfo(AbstractVArrayInfo):
     is_about_raw = True
 
     def __init__(self, cpu, logops, size, source_op):
@@ -487,7 +481,7 @@
                                         self.buffer.descrs[:])
 
 
-class VRawSliceValue(AbstractVirtualValue):
+class VRawSliceInfo(AbstractVirtualInfo):
     is_about_raw = True
 
     def __init__(self, rawbuffer_value, offset, source_op):
@@ -529,6 +523,7 @@
     _last_guard_not_forced_2 = None
 
     def make_virtual(self, known_class, source_op):
+        xxx
         vvalue = VirtualValue(self.optimizer.cpu, known_class, source_op)
         self.make_equal_to(source_op, vvalue)
         return vvalue
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -87,7 +87,7 @@
 #                                 CONST_0, offsetbox, lengthbox, mode)
 
 
-class VAbstractStringValue(virtualize.AbstractVirtualValue):
+class VAbstractStringInfo(virtualize.AbstractVirtualInfo):
     _attrs_ = ('mode',)
 
     def __init__(self, source_op, mode):
@@ -122,7 +122,7 @@
                                       offsetbox, mode)
 
 
-class VStringPlainValue(VAbstractStringValue):
+class VStringPlainInfo(VAbstractStringInfo):
     """A string built with newstr(const)."""
     _lengthbox = None     # cache only
 
@@ -216,7 +216,7 @@
         return visitor.visit_vstrplain(self.mode is mode_unicode)
 
 
-class VStringConcatValue(VAbstractStringValue):
+class VStringConcatInfo(VAbstractStringInfo):
     """The concatenation of two other strings."""
     _attrs_ = ('left', 'right', 'lengthbox')
 
@@ -269,7 +269,7 @@
         return visitor.visit_vstrconcat(self.mode is mode_unicode)
 
 
-class VStringSliceValue(VAbstractStringValue):
+class VStringSliceInfo(VAbstractStringInfo):
     """A slice."""
     _attrs_ = ('vstr', 'vstart', 'vlength')
 

From noreply at buildbot.pypy.org  Fri Feb 27 15:40:46 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 27 Feb 2015 15:40:46 +0100 (CET)
Subject: [pypy-commit] pypy optresult: merge default
Message-ID: <20150227144046.CFB491C0096@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r76161:49b01bee468c
Date: 2015-02-27 14:15 +0200
http://bitbucket.org/pypy/pypy/changeset/49b01bee468c/

Log:	merge default

diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -36,7 +36,7 @@
 def pytest_addoption(parser):
     from rpython.conftest import pytest_addoption
     pytest_addoption(parser)
-    
+
     group = parser.getgroup("pypy options")
     group.addoption('-A', '--runappdirect', action="store_true",
            default=False, dest="runappdirect",
@@ -44,6 +44,9 @@
     group.addoption('--direct', action="store_true",
            default=False, dest="rundirect",
            help="run pexpect tests directly")
+    group.addoption('--raise-operr', action="store_true",
+            default=False, dest="raise_operr",
+            help="Show the interp-level OperationError in app-level tests")
 
 def pytest_funcarg__space(request):
     from pypy.tool.pytest.objspace import gettestobjspace
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -36,7 +36,8 @@
    "PyPy home directory".  The arguments are:
 
    * ``home``: NULL terminated path to an executable inside the pypy directory
-     (can be a .so name, can be made up)
+     (can be a .so name, can be made up).  Used to look up the standard
+     library, and is also set as ``sys.executable``.
 
    * ``verbose``: if non-zero, it will print error messages to stderr
 
diff --git a/pypy/doc/jit-hooks.rst b/pypy/doc/jit-hooks.rst
--- a/pypy/doc/jit-hooks.rst
+++ b/pypy/doc/jit-hooks.rst
@@ -39,3 +39,30 @@
     Reason is a string, the meaning of other arguments is the same
     as attributes on JitLoopInfo object
 
+.. function:: enable_debug()
+
+    Start recording debugging counters for ``get_stats_snapshot``
+
+.. function:: disable_debug()
+
+    Stop recording debugging counters for ``get_stats_snapshot``
+
+.. function:: get_stats_snapshot()
+
+    Get the jit status in the specific moment in time. Note that this
+    is eager - the attribute access is not lazy, if you need new stats
+    you need to call this function again. You might want to call
+    ``enable_debug`` to get more information. It returns an instance
+    of ``JitInfoSnapshot``
+
+.. class:: JitInfoSnapshot
+
+    A class describing current snapshot. Usable attributes:
+
+    * ``counters`` - internal JIT integer counters
+
+    * ``counter_times`` - internal JIT float counters, notably time spent
+      TRACING and in the JIT BACKEND
+
+    * ``loop_run_times`` - counters for number of times loops are run, only
+      works when ``enable_debug`` is called.
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -106,6 +106,9 @@
         space.call_function(w_pathsetter, w_path)
         # import site
         try:
+            space.setattr(space.getbuiltinmodule('sys'),
+                          space.wrap('executable'),
+                          space.wrap(home))
             import_ = space.getattr(space.getbuiltinmodule('__builtin__'),
                                     space.wrap('__import__'))
             space.call_function(import_, space.wrap('site'))
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -1,5 +1,5 @@
 #! /usr/bin/env python
-# App-level version of py.py.
+# This is pure Python code that handles the main entry point into "pypy".
 # See test/test_app_main.
 
 # Missing vs CPython: -d, -t, -v, -x, -3
@@ -157,10 +157,13 @@
             current = group
     raise SystemExit
 
+def get_sys_executable():
+    return getattr(sys, 'executable', 'pypy')
+
 def print_help(*args):
     import os
     print 'usage: %s [option] ... [-c cmd | -m mod | file | -] [arg] ...' % (
-        sys.executable,)
+        get_sys_executable(),)
     print USAGE1,
     if 'pypyjit' in sys.builtin_module_names:
         print "--jit options: advanced JIT options: try 'off' or 'help'"
@@ -171,7 +174,7 @@
     try:
         import pypyjit
     except ImportError:
-        print >> sys.stderr, "No jit support in %s" % (sys.executable,)
+        print >> sys.stderr, "No jit support in %s" % (get_sys_executable(),)
         return
     items = sorted(pypyjit.defaults.items())
     print 'Advanced JIT options: a comma-separated list of OPTION=VALUE:'
@@ -209,7 +212,7 @@
         raise SystemExit
     if 'pypyjit' not in sys.builtin_module_names:
         print >> sys.stderr, ("Warning: No jit support in %s" %
-                              (sys.executable,))
+                              (get_sys_executable(),))
     else:
         import pypyjit
         pypyjit.set_param(jitparam)
@@ -219,8 +222,8 @@
 
 def print_error(msg):
     print >> sys.stderr, msg
-    print >> sys.stderr, 'usage: %s [options]' % (sys.executable,)
-    print >> sys.stderr, 'Try `%s -h` for more information.' % (sys.executable,)
+    print >> sys.stderr, 'usage: %s [options]' % (get_sys_executable(),)
+    print >> sys.stderr, 'Try `%s -h` for more information.' % (get_sys_executable(),)
 
 def fdopen(fd, mode, bufsize=-1):
     try:
@@ -514,6 +517,10 @@
     elif not sys.stdout.isatty():
         set_fully_buffered_io()
 
+    if we_are_translated():
+        import __pypy__
+        __pypy__.save_module_content_for_future_reload(sys)
+
     mainmodule = type(sys)('__main__')
     sys.modules['__main__'] = mainmodule
 
diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py
--- a/pypy/interpreter/mixedmodule.py
+++ b/pypy/interpreter/mixedmodule.py
@@ -55,7 +55,10 @@
         if self.w_initialdict is None:
             Module.init(self, space)
             if not self.lazy and self.w_initialdict is None:
-                self.w_initialdict = space.call_method(self.w_dict, 'items')
+                self.save_module_content_for_future_reload()
+
+    def save_module_content_for_future_reload(self):
+        self.w_initialdict = self.space.call_method(self.w_dict, 'items')
 
 
     def get_applevel_name(cls):
@@ -119,7 +122,7 @@
                 w_value = self.get(name)
                 space.setitem(self.w_dict, space.new_interned_str(name), w_value)
             self.lazy = False
-            self.w_initialdict = space.call_method(self.w_dict, 'items')
+            self.save_module_content_for_future_reload()
         return self.w_dict
 
     def _cleanup_(self):
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -373,7 +373,7 @@
         config = make_config(None)
         space = make_objspace(config)
         w_executable = space.wrap('executable')
-        assert space.str_w(space.getattr(space.sys, w_executable)) == 'py.py'
+        assert space.findattr(space.sys, w_executable) is None
         space.setattr(space.sys, w_executable, space.wrap('foobar'))
         assert space.str_w(space.getattr(space.sys, w_executable)) == 'foobar'
         space.startup()
diff --git a/pypy/interpreter/test/test_targetpypy.py b/pypy/interpreter/test/test_targetpypy.py
--- a/pypy/interpreter/test/test_targetpypy.py
+++ b/pypy/interpreter/test/test_targetpypy.py
@@ -8,7 +8,7 @@
         entry_point = get_entry_point(config)[0]
         entry_point(['pypy-c' , '-S', '-c', 'print 3'])
 
-def test_exeucte_source(space):
+def test_execute_source(space):
     _, d = create_entry_point(space, None)
     execute_source = d['pypy_execute_source']
     lls = rffi.str2charp("import sys; sys.modules['xyz'] = 3")
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -82,6 +82,8 @@
         'strategy'                  : 'interp_magic.strategy',  # dict,set,list
         'set_debug'                 : 'interp_magic.set_debug',
         'locals_to_fast'            : 'interp_magic.locals_to_fast',
+        'save_module_content_for_future_reload':
+                          'interp_magic.save_module_content_for_future_reload',
     }
     if sys.platform == 'win32':
         interpleveldefs['get_console_cp'] = 'interp_magic.get_console_cp'
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -1,6 +1,7 @@
 from pypy.interpreter.error import OperationError, wrap_oserror
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.pyframe import PyFrame
+from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib.objectmodel import we_are_translated
 from pypy.objspace.std.dictmultiobject import W_DictMultiObject
 from pypy.objspace.std.listobject import W_ListObject
@@ -130,3 +131,7 @@
 def locals_to_fast(space, w_frame):
     assert isinstance(w_frame, PyFrame)
     w_frame.locals2fast()
+
+ at unwrap_spec(w_module=MixedModule)
+def save_module_content_for_future_reload(space, w_module):
+    w_module.save_module_content_for_future_reload()
diff --git a/pypy/module/__pypy__/test/test_magic.py b/pypy/module/__pypy__/test/test_magic.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__pypy__/test/test_magic.py
@@ -0,0 +1,15 @@
+
+class AppTestMagic:
+    spaceconfig = dict(usemodules=['__pypy__'])
+
+    def test_save_module_content_for_future_reload(self):
+        import sys, __pypy__
+        d = sys.dont_write_bytecode
+        sys.dont_write_bytecode = "hello world"
+        __pypy__.save_module_content_for_future_reload(sys)
+        sys.dont_write_bytecode = d
+        reload(sys)
+        assert sys.dont_write_bytecode == "hello world"
+        #
+        sys.dont_write_bytecode = d
+        __pypy__.save_module_content_for_future_reload(sys)
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -1,4 +1,4 @@
-from rpython.rlib import jit
+from rpython.rlib import jit, rgc
 from rpython.rlib.buffer import Buffer
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rarithmetic import ovfcheck, widen
@@ -698,11 +698,9 @@
                                          self.space.wrap(msg))
             return result
 
+        @rgc.must_be_light_finalizer
         def __del__(self):
-            # note that we don't call clear_all_weakrefs here because
-            # an array with freed buffer is ok to see - it's just empty with 0
-            # length
-            self.setlen(0)
+            lltype.free(self.buffer, flavor='raw')
 
         def setlen(self, size, zero=False, overallocate=True):
             if size > 0:
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -64,7 +64,6 @@
         'call_tracing'          : 'vm.call_tracing',
         'getsizeof'             : 'vm.getsizeof',
 
-        'executable'            : 'space.wrap("py.py")',
         'api_version'           : 'version.get_api_version(space)',
         'version_info'          : 'version.get_version_info(space)',
         'version'               : 'version.get_version(space)',
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -500,7 +500,7 @@
         assert isinstance(sys.builtin_module_names, tuple)
         assert isinstance(sys.copyright, basestring)
         #assert isinstance(sys.exec_prefix, basestring) -- not present!
-        assert isinstance(sys.executable, basestring)
+        #assert isinstance(sys.executable, basestring) -- not present!
         assert isinstance(sys.hexversion, int)
         assert isinstance(sys.maxint, int)
         assert isinstance(sys.maxsize, int)
@@ -519,6 +519,12 @@
         assert vi[3] in ("alpha", "beta", "candidate", "final")
         assert isinstance(vi[4], int)
 
+    def test_reload_doesnt_override_sys_executable(self):
+        import sys
+        sys.executable = 'from_test_sysmodule'
+        reload(sys)
+        assert sys.executable == 'from_test_sysmodule'
+
     def test_settrace(self):
         import sys
         counts = []
diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py
--- a/pypy/tool/pytest/apptest.py
+++ b/pypy/tool/pytest/apptest.py
@@ -35,7 +35,7 @@
         try:
             target(*args)
         except OperationError, e:
-            if self.config.option.verbose:
+            if self.config.option.raise_operr:
                 raise
             tb = sys.exc_info()[2]
             if e.match(space, space.w_KeyboardInterrupt):
diff --git a/rpython/doc/index.rst b/rpython/doc/index.rst
--- a/rpython/doc/index.rst
+++ b/rpython/doc/index.rst
@@ -36,6 +36,7 @@
    :maxdepth: 1
 
    arm
+   logging
 
 
 Writing your own interpreter in RPython
diff --git a/rpython/doc/logging.rst b/rpython/doc/logging.rst
new file mode 100644
--- /dev/null
+++ b/rpython/doc/logging.rst
@@ -0,0 +1,55 @@
+Logging environment variables
+=============================
+
+PyPy, and all other RPython programs, support some special environment
+variables used to tweak various advanced parameters.
+
+
+Garbage collector
+-----------------
+
+Right now the default GC is (an incremental version of) MiniMark__.
+It has :ref:`a number of environment variables
+` that can be tweaked.  Their default
+value should be ok for most usages.
+
+.. __: garbage_collection.html#minimark-gc
+
+
+PYPYLOG
+-------
+
+The ``PYPYLOG`` environment variable enables debugging output.  For
+example::
+
+   PYPYLOG=jit:log
+
+means enabling all debugging output from the JIT, and writing to a
+file called ``log``.  More precisely, the condition ``jit`` means
+enabling output of all sections whose name start with ``jit``; other
+interesting names to use here are ``gc`` to get output from the GC, or
+``jit-backend`` to get only output from the JIT's machine code
+backend.  It is possible to use several prefixes, like in the
+following example::
+
+   PYPYLOG=jit-log-opt,jit-backend:log
+
+which outputs sections containing to the optimized loops plus anything
+produced from the JIT backend.  The above example is what you need for
+jitviewer_.
+
+.. _jitviewer: https://bitbucket.org/pypy/jitviewer
+
+The filename can be given as ``-`` to dump the log to stderr.
+
+As a special case, the value ``PYPYLOG=+filename`` means that only
+the section markers are written (for any section).  This is mostly
+only useful for ``rpython/tool/logparser.py``.
+
+
+PYPYSTM
+-------
+
+Only available in ``pypy-stm``.  Names a log file into which the
+PyPy-STM will output contention information.  Can be read with
+``pypy/stm/print_stm_log.py``.
diff --git a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -178,7 +178,11 @@
         assert p[3] == 'y'
         assert p[4] == 'Z'
         assert p[5] == 'z'
-        assert allblocks == [(rawstart, rawstart + 6)]
+        # 'allblocks' should be one block of length 6 + 15
+        # (15 = alignment - 1) containing the range(rawstart, rawstart + 6)
+        [(blockstart, blockend)] = allblocks
+        assert blockend == blockstart + 6 + (mc.ALIGN_MATERIALIZE - 1)
+        assert blockstart <= rawstart < rawstart + 6 <= blockend
         assert puts == [(rawstart + 2, ['a', 'b', 'c', 'd']),
                         (rawstart + 4, ['e', 'f', 'g'])]
 
diff --git a/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py b/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
--- a/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
+++ b/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py
@@ -317,7 +317,9 @@
                 # CALL_j is actually relative, so tricky to test
                 (instrname == 'CALL' and argmodes == 'j') or
                 # SET_ir must be tested manually
-                (instrname == 'SET' and argmodes == 'ir')
+                (instrname == 'SET' and argmodes == 'ir') or
+                # MULTIBYTE_NOPs can't easily be tested the same way
+                (instrname == 'MULTIBYTE')
         )
 
 
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -696,6 +696,9 @@
 class ResumeGuardNotInvalidated(ResumeGuardDescr):
     guard_opnum = rop.GUARD_NOT_INVALIDATED
 
+    def must_compile(self, deadframe, metainterp_sd, jitdriver_sd):
+        return False
+
 class ResumeAtPositionDescr(ResumeGuardDescr):
     guard_opnum = rop.GUARD_FUTURE_CONDITION
 
diff --git a/rpython/rlib/listsort.py b/rpython/rlib/listsort.py
--- a/rpython/rlib/listsort.py
+++ b/rpython/rlib/listsort.py
@@ -496,6 +496,12 @@
         # 1. len[-3] > len[-2] + len[-1]
         # 2. len[-2] > len[-1]
         #
+        # Note these invariants will not hold for the entire pending array even
+        # after this function completes. [1] This does not affect the
+        # correctness of the overall algorithm.
+        #
+        # [1] http://envisage-project.eu/proving-android-java-and-python-sorting-algorithm-is-broken-and-how-to-fix-it/
+        #
         # See listsort.txt for more info.
 
         def merge_collapse(self):
diff --git a/rpython/translator/goal/order.py b/rpython/translator/goal/order.py
deleted file mode 100644
--- a/rpython/translator/goal/order.py
+++ /dev/null
@@ -1,56 +0,0 @@
-import sys
-import os
-
-RTYPERORDER = os.getenv('RTYPERORDER').split(',')
-if len(RTYPERORDER) == 2:
-    module_list = RTYPERORDER[1]
-else:
-    module_list = 'module-list'
-
-lst = open(module_list, 'r')
-try:
-    print "reading module-list: %s" % module_list
-    prefixes = lst.readlines()
-finally:
-    lst.close()
-
-prefixes = [line.strip() for line in prefixes]
-prefixes = [line for line in prefixes if line and not line.startswith('#')]
-
-NOMATCH = sys.maxint
-
-def order(annotator, pending):
-    cache = {}
-    annotated = annotator.annotated
-    def indx(block):
-        func = annotated[block]
-        module = func.__module__
-        if module is None:
-            module = 'None'
-        tag = "%s:%s" % (module, func.__name__)
-        try:
-            return cache[tag]
-        except KeyError:
-            match = NOMATCH
-            i = 0
-            for pfx in prefixes:
-                if tag.startswith(pfx):
-                    if match == NOMATCH:
-                        match = i
-                    else:
-                        if len(pfx) > len(prefixes[match]):
-                            match = i
-                i += 1
-            cache[tag] = match, module
-            return match
-
-    pending.sort(lambda blk1, blk2: cmp(indx(blk1), indx(blk2)))
-
-    cur_module = ['$']
-
-    def track(block):
-        module = annotated[block].__module__
-        if module != cur_module[0]:
-            print "--- Specializing blocks in module: %s" % module
-            cur_module[0] = module
-    return track

From noreply at buildbot.pypy.org  Fri Feb 27 15:40:48 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 27 Feb 2015 15:40:48 +0100 (CET)
Subject: [pypy-commit] pypy optresult: (fijal,
 arigo) hack enough on VirtualInfo to make the first test allocating
Message-ID: <20150227144048.143FE1C0096@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r76162:7430103dc792
Date: 2015-02-27 16:39 +0200
http://bitbucket.org/pypy/pypy/changeset/7430103dc792/

Log:	(fijal, arigo) hack enough on VirtualInfo to make the first test
	allocating NEW_WITH_VTABLE pass

diff --git a/rpython/jit/metainterp/optimizeopt/generalize.py b/rpython/jit/metainterp/optimizeopt/generalize.py
--- a/rpython/jit/metainterp/optimizeopt/generalize.py
+++ b/rpython/jit/metainterp/optimizeopt/generalize.py
@@ -1,4 +1,4 @@
-from rpython.jit.metainterp.optimizeopt.optimizer import MININT, MAXINT
+#from rpython.jit.metainterp.optimizeopt.optimizer import MININT, MAXINT
 
 
 class GeneralizationStrategy(object):
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -4,8 +4,9 @@
 from rpython.jit.metainterp.optimizeopt.util import args_dict
 from rpython.jit.metainterp.history import Const, ConstInt
 from rpython.jit.metainterp.jitexc import JitException
-from rpython.jit.metainterp.optimizeopt.optimizer import Optimization,\
-     MODE_ARRAY, LEVEL_KNOWNCLASS, REMOVED
+from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, REMOVED
+from rpython.jit.metainterp.optimizeopt.info import MODE_ARRAY,\
+     LEVEL_KNOWNCLASS
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound
 from rpython.jit.metainterp.optimize import InvalidLoop
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -0,0 +1,139 @@
+
+from rpython.jit.metainterp.resoperation import AbstractValue
+
+""" The tag field on PtrOptInfo has a following meaning:
+
+lower two bits are LEVEL
+"""
+
+LEVEL_UNKNOWN    = 0
+LEVEL_NONNULL    = 1
+LEVEL_KNOWNCLASS = 2     # might also mean KNOWNARRAYDESCR, for arrays
+LEVEL_CONSTANT   = 3
+
+MODE_ARRAY   = '\x00'
+MODE_STR     = '\x01'
+MODE_UNICODE = '\x02'
+
+class AbstractInfo(AbstractValue):
+    is_info_class = True
+
+    def force_box(self, op, optforce):
+        return op
+
+class PtrOptInfo(AbstractInfo):
+    _attrs_ = ('_tag', 'known_class', 'last_guard_pos', 'lenbound')
+    is_info_class = True
+
+    _tag = 0
+    known_class = None
+    last_guard_pos = -1
+    lenbound = None
+
+    #def __init__(self, level=None, known_class=None, intbound=None):
+    #    OptValue.__init__(self, box, level, None, intbound)
+    #    if not isinstance(box, Const):
+    #        self.known_class = known_class
+
+    def getlevel(self):
+        return self._tag & 0x3
+
+    def setlevel(self, level):
+        self._tag = (self._tag & (~0x3)) | level
+
+    def __repr__(self):
+        level = {LEVEL_UNKNOWN: 'UNKNOWN',
+                 LEVEL_NONNULL: 'NONNULL',
+                 LEVEL_KNOWNCLASS: 'KNOWNCLASS',
+                 LEVEL_CONSTANT: 'CONSTANT'}.get(self.getlevel(),
+                                                 self.getlevel())
+        return '<%s %s %s>' % (
+            self.__class__.__name__,
+            level,
+            self.box)
+
+    def make_len_gt(self, mode, descr, val):
+        if self.lenbound:
+            if self.lenbound.mode != mode or self.lenbound.descr != descr:
+                # XXX a rare case?  it seems to occur sometimes when
+                # running lib-python's test_io.py in PyPy on Linux 32...
+                from rpython.jit.metainterp.optimize import InvalidLoop
+                raise InvalidLoop("bad mode/descr")
+            self.lenbound.bound.make_gt(IntBound(val, val))
+        else:
+            self.lenbound = LenBound(mode, descr, IntLowerBound(val + 1))
+
+    def make_nonnull(self, optimizer):
+        assert self.getlevel() < LEVEL_NONNULL
+        self.setlevel(LEVEL_NONNULL)
+        if optimizer is not None:
+            self.last_guard_pos = len(optimizer._newoperations) - 1
+            assert self.get_last_guard(optimizer).is_guard()
+
+    def make_constant_class(self, optimizer, classbox):
+        assert self.getlevel() < LEVEL_KNOWNCLASS
+        self.known_class = classbox
+        self.setlevel(LEVEL_KNOWNCLASS)
+        if optimizer is not None:
+            self.last_guard_pos = len(optimizer._newoperations) - 1
+            assert self.get_last_guard(optimizer).is_guard()
+
+    def import_from(self, other, optimizer):
+        OptValue.import_from(self, other, optimizer)
+        if self.getlevel() != LEVEL_CONSTANT:
+            if other.getlenbound():
+                if self.lenbound:
+                    assert other.getlenbound().mode == self.lenbound.mode
+                    assert other.getlenbound().descr == self.lenbound.descr
+                    self.lenbound.bound.intersect(other.getlenbound().bound)
+                else:
+                    self.lenbound = other.getlenbound().clone()
+
+    def make_guards(self, box):
+        guards = []
+        level = self.getlevel()
+        if level == LEVEL_CONSTANT:
+            op = ResOperation(rop.GUARD_VALUE, [box, self.box], None)
+            guards.append(op)
+        elif level == LEVEL_KNOWNCLASS:
+            op = ResOperation(rop.GUARD_NONNULL_CLASS,
+                              [box, self.known_class], None)
+            guards.append(op)
+        else:
+            if level == LEVEL_NONNULL:
+                op = ResOperation(rop.GUARD_NONNULL, [box], None)
+                guards.append(op)
+            if self.lenbound:
+                lenbox = BoxInt()
+                if self.lenbound.mode == MODE_ARRAY:
+                    op = ResOperation(rop.ARRAYLEN_GC, [box], lenbox, self.lenbound.descr)
+                elif self.lenbound.mode == MODE_STR:
+                    op = ResOperation(rop.STRLEN, [box], lenbox, self.lenbound.descr)
+                elif self.lenbound.mode == MODE_UNICODE:
+                    op = ResOperation(rop.UNICODELEN, [box], lenbox, self.lenbound.descr)
+                else:
+                    debug_print("Unknown lenbound mode")
+                    assert False
+                guards.append(op)
+                self.lenbound.bound.make_guards(lenbox, guards)
+        return guards
+
+    def get_constant_class(self, cpu):
+        level = self.getlevel()
+        if level == LEVEL_KNOWNCLASS:
+            return self.known_class
+        elif level == LEVEL_CONSTANT and not self.is_null():
+            return cpu.ts.cls_of_box(self.box)
+        else:
+            return None
+
+    def getlenbound(self):
+        return self.lenbound
+
+    def get_last_guard(self, optimizer):
+        if self.last_guard_pos == -1:
+            return None
+        return optimizer._newoperations[self.last_guard_pos]
+
+    def get_known_class(self):
+        return self.known_class
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -1,13 +1,14 @@
 import sys
 from rpython.jit.metainterp.history import ConstInt
 from rpython.jit.metainterp.optimize import InvalidLoop
-from rpython.jit.metainterp.optimizeopt.intutils import (IntBound, IntLowerBound,
-    IntUpperBound)
+from rpython.jit.metainterp.optimizeopt.intutils import (IntBound,
+    IntLowerBound, IntUpperBound)
 from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, CONST_1,
-    CONST_0, MODE_ARRAY, MODE_STR, MODE_UNICODE)
+    CONST_0)
+from rpython.jit.metainterp.optimizeopt.info import MODE_ARRAY, MODE_STR,\
+     MODE_UNICODE
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from rpython.jit.metainterp.resoperation import rop, AbstractResOp
-from rpython.jit.backend.llsupport import symbolic
 
 
 def get_integer_min(is_unsigned, byte_size):
diff --git a/rpython/jit/metainterp/optimizeopt/intutils.py b/rpython/jit/metainterp/optimizeopt/intutils.py
--- a/rpython/jit/metainterp/optimizeopt/intutils.py
+++ b/rpython/jit/metainterp/optimizeopt/intutils.py
@@ -1,15 +1,16 @@
 from rpython.rlib.rarithmetic import ovfcheck, LONG_BIT, maxint, is_valid_int
 from rpython.rlib.objectmodel import we_are_translated
-from rpython.jit.metainterp.resoperation import rop, ResOperation, AbstractValue
+from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.jit.metainterp.optimizeopt.info import AbstractInfo
 from rpython.jit.metainterp.history import ConstInt
 
+
 MAXINT = maxint
 MININT = -maxint - 1
 
 
-class IntBound(AbstractValue):
+class IntBound(AbstractInfo):
     _attrs_ = ('has_upper', 'has_lower', 'upper', 'lower')
-    is_info_class = True
 
     def __init__(self, lower, upper):
         self.has_upper = True
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -3,30 +3,13 @@
 from rpython.jit.metainterp.logger import LogOperations
 from rpython.jit.metainterp.history import Const, ConstInt, REF, ConstPtr
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound,\
-     IntLowerBound, MININT, MAXINT, IntUnbounded, ConstIntBound
+     IntUnbounded, ConstIntBound
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
-from rpython.jit.metainterp.resoperation import rop, ResOperation,\
-     AbstractResOp, AbstractInputArg, GuardResOp, AbstractValue
+from rpython.jit.metainterp.resoperation import rop, AbstractResOp, GuardResOp
+from rpython.jit.metainterp.optimizeopt.info import PtrOptInfo
 from rpython.jit.metainterp.typesystem import llhelper
-from rpython.tool.pairtype import extendabletype
-from rpython.rlib.debug import debug_print
 from rpython.rlib.objectmodel import specialize, we_are_translated
 
-""" The tag field on OptValue has a following meaning:
-
-lower two bits are LEVEL
-next 16 bits is the position in the original list, 0 if unknown or a constant
-"""
-
-LEVEL_UNKNOWN    = 0
-LEVEL_NONNULL    = 1
-LEVEL_KNOWNCLASS = 2     # might also mean KNOWNARRAYDESCR, for arrays
-LEVEL_CONSTANT   = 3
-
-MODE_ARRAY   = '\x00'
-MODE_STR     = '\x01'
-MODE_UNICODE = '\x02'
-
 
 class LenBound(object):
     def __init__(self, mode, descr, bound):
@@ -196,131 +179,6 @@
 ##         return None
 
 
-class PtrOptInfo(AbstractValue):
-    _attrs_ = ('_tag', 'known_class', 'last_guard_pos', 'lenbound')
-    is_info_class = True
-
-    _tag = 0
-    known_class = None
-    last_guard_pos = -1
-    lenbound = None
-
-    #def __init__(self, level=None, known_class=None, intbound=None):
-    #    OptValue.__init__(self, box, level, None, intbound)
-    #    if not isinstance(box, Const):
-    #        self.known_class = known_class
-
-    def getlevel(self):
-        return self._tag & 0x3
-
-    def setlevel(self, level):
-        self._tag = (self._tag & (~0x3)) | level
-
-    def __repr__(self):
-        level = {LEVEL_UNKNOWN: 'UNKNOWN',
-                 LEVEL_NONNULL: 'NONNULL',
-                 LEVEL_KNOWNCLASS: 'KNOWNCLASS',
-                 LEVEL_CONSTANT: 'CONSTANT'}.get(self.getlevel(),
-                                                 self.getlevel())
-        return '<%s %s %s>' % (
-            self.__class__.__name__,
-            level,
-            self.box)
-
-    def copy_from(self, other_value):
-        assert isinstance(other_value, PtrOptValue)
-        self.box = other_value.box
-        self.known_class = other_value.known_class
-        self._tag = other_value._tag
-        self.last_guard_pos = other_value.last_guard_pos
-        self.lenbound = other_value.lenbound
-
-    def make_len_gt(self, mode, descr, val):
-        if self.lenbound:
-            if self.lenbound.mode != mode or self.lenbound.descr != descr:
-                # XXX a rare case?  it seems to occur sometimes when
-                # running lib-python's test_io.py in PyPy on Linux 32...
-                from rpython.jit.metainterp.optimize import InvalidLoop
-                raise InvalidLoop("bad mode/descr")
-            self.lenbound.bound.make_gt(IntBound(val, val))
-        else:
-            self.lenbound = LenBound(mode, descr, IntLowerBound(val + 1))
-
-    def make_nonnull(self, optimizer):
-        assert self.getlevel() < LEVEL_NONNULL
-        self.setlevel(LEVEL_NONNULL)
-        if optimizer is not None:
-            self.last_guard_pos = len(optimizer._newoperations) - 1
-            assert self.get_last_guard(optimizer).is_guard()
-
-    def make_constant_class(self, optimizer, classbox):
-        assert self.getlevel() < LEVEL_KNOWNCLASS
-        self.known_class = classbox
-        self.setlevel(LEVEL_KNOWNCLASS)
-        if optimizer is not None:
-            self.last_guard_pos = len(optimizer._newoperations) - 1
-            assert self.get_last_guard(optimizer).is_guard()
-
-    def import_from(self, other, optimizer):
-        OptValue.import_from(self, other, optimizer)
-        if self.getlevel() != LEVEL_CONSTANT:
-            if other.getlenbound():
-                if self.lenbound:
-                    assert other.getlenbound().mode == self.lenbound.mode
-                    assert other.getlenbound().descr == self.lenbound.descr
-                    self.lenbound.bound.intersect(other.getlenbound().bound)
-                else:
-                    self.lenbound = other.getlenbound().clone()
-
-    def make_guards(self, box):
-        guards = []
-        level = self.getlevel()
-        if level == LEVEL_CONSTANT:
-            op = ResOperation(rop.GUARD_VALUE, [box, self.box], None)
-            guards.append(op)
-        elif level == LEVEL_KNOWNCLASS:
-            op = ResOperation(rop.GUARD_NONNULL_CLASS,
-                              [box, self.known_class], None)
-            guards.append(op)
-        else:
-            if level == LEVEL_NONNULL:
-                op = ResOperation(rop.GUARD_NONNULL, [box], None)
-                guards.append(op)
-            if self.lenbound:
-                lenbox = BoxInt()
-                if self.lenbound.mode == MODE_ARRAY:
-                    op = ResOperation(rop.ARRAYLEN_GC, [box], lenbox, self.lenbound.descr)
-                elif self.lenbound.mode == MODE_STR:
-                    op = ResOperation(rop.STRLEN, [box], lenbox, self.lenbound.descr)
-                elif self.lenbound.mode == MODE_UNICODE:
-                    op = ResOperation(rop.UNICODELEN, [box], lenbox, self.lenbound.descr)
-                else:
-                    debug_print("Unknown lenbound mode")
-                    assert False
-                guards.append(op)
-                self.lenbound.bound.make_guards(lenbox, guards)
-        return guards
-
-    def get_constant_class(self, cpu):
-        level = self.getlevel()
-        if level == LEVEL_KNOWNCLASS:
-            return self.known_class
-        elif level == LEVEL_CONSTANT and not self.is_null():
-            return cpu.ts.cls_of_box(self.box)
-        else:
-            return None
-
-    def getlenbound(self):
-        return self.lenbound
-
-    def get_last_guard(self, optimizer):
-        if self.last_guard_pos == -1:
-            return None
-        return optimizer._newoperations[self.last_guard_pos]
-
-    def get_known_class(self):
-        return self.known_class
-
 ## class IntOptInfo(OptInfo):
 ##     _attrs_ = ('intbound',)
 
@@ -641,7 +499,11 @@
         return op
 
     def force_box(self, op):
-        return self.get_box_replacement(op)
+        op = self.get_box_replacement(op)
+        info = op.get_forwarded()
+        if info is not None:
+            return info.force_box(op, self)
+        return op
 
     def ensure_imported(self, value):
         pass
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -6,7 +6,8 @@
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound
 from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, REMOVED,
-    CONST_0, CONST_1, PtrOptInfo, INFO_NONNULL, INFO_NULL)
+    CONST_0, CONST_1, INFO_NONNULL, INFO_NULL)
+from rpython.jit.metainterp.optimizeopt.info import PtrOptInfo
 from rpython.jit.metainterp.optimizeopt.util import _findall, make_dispatcher_method
 from rpython.jit.metainterp.resoperation import rop, ResOperation, opclasses,\
      OpHelpers
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -3,7 +3,7 @@
 from rpython.jit.codewriter.heaptracker import vtable2descr
 from rpython.jit.metainterp.history import Const, ConstInt, BoxInt
 from rpython.jit.metainterp.history import CONST_NULL, BoxPtr
-from rpython.jit.metainterp.optimizeopt import optimizer
+from rpython.jit.metainterp.optimizeopt import info, optimizer
 from rpython.jit.metainterp.optimizeopt.optimizer import REMOVED
 from rpython.jit.metainterp.optimizeopt.util import (make_dispatcher_method,
                                                      descrlist_dict, sort_descrs)
@@ -13,9 +13,9 @@
 from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.jit.metainterp.optimizeopt.intutils import IntUnbounded
 
-class AbstractVirtualInfo(optimizer.PtrOptInfo):
+class AbstractVirtualInfo(info.PtrOptInfo):
     _attrs_ = ('_cached_vinfo',)
-    _tag = optimizer.LEVEL_NONNULL
+    _tag = info.LEVEL_NONNULL
     is_about_raw = False
     _cached_vinfo = None
 
@@ -23,12 +23,20 @@
         xxx
         return self.box is not None
 
-    def force_box(self, optforce):
-        xxxx
-        if self.box is None:
-            optforce.forget_numberings(self.source_op)
-            self._really_force(optforce)
-        return self.box
+    def force_box(self, op, optforce):
+        op.set_forwarded(None)
+        optforce.emit_operation(op)
+        newop = optforce.getlastop()
+        op.set_forwarded(newop)
+        optforce.getptrinfo(newop).make_constant_class(None, self.known_class)
+        return newop
+    
+    #def force_box(self, optforce):
+    #    xxxx
+    #    if self.box is None:
+    #        optforce.forget_numberings(self.source_op)
+    #        self._really_force(optforce)
+    #    return self.box
 
     def force_at_end_of_preamble(self, already_forced, optforce):
         xxxx
@@ -65,13 +73,11 @@
 get_fielddescrlist_cache._annspecialcase_ = "specialize:memo"
 
 class AbstractVirtualStructInfo(AbstractVirtualInfo):
-    _attrs_ = ('_fields', '_cached_sorted_fields')
+    _attrs_ = ('_fields',)
 
-    def __init__(self, cpu):
+    def __init__(self):
         AbstractVirtualInfo.__init__(self)
-        self.cpu = cpu
-        self._fields = {}
-        self._cached_sorted_fields = None
+        #self._fields = {}
 
     def getfield(self, ofs, default):
         return self._fields.get(ofs, default)
@@ -190,12 +196,13 @@
             fieldvalue.visitor_walk_recursive(visitor)
 
 class VirtualInfo(AbstractVirtualStructInfo):
-    _tag = optimizer.LEVEL_KNOWNCLASS
+    _tag = info.LEVEL_KNOWNCLASS
 
-    def __init__(self, known_class):
+    def __init__(self, known_class, descr):
         AbstractVirtualStructInfo.__init__(self)
         assert isinstance(known_class, Const)
         self.known_class = known_class
+        self.descr = descr
 
     @specialize.argtype(1)
     def _visitor_dispatch_virtual_type(self, visitor):
@@ -215,6 +222,7 @@
 class VStructInfo(AbstractVirtualStructInfo):
 
     def __init__(self, cpu, structdescr, source_op):
+        xxx
         AbstractVirtualStructValue.__init__(self, cpu, source_op)
         self.structdescr = structdescr
 
@@ -522,11 +530,10 @@
 
     _last_guard_not_forced_2 = None
 
-    def make_virtual(self, known_class, source_op):
-        xxx
-        vvalue = VirtualValue(self.optimizer.cpu, known_class, source_op)
-        self.make_equal_to(source_op, vvalue)
-        return vvalue
+    def make_virtual(self, known_class, source_op, descr):
+        info = VirtualInfo(known_class, descr)
+        source_op.set_forwarded(info)
+        return info
 
     def make_varray(self, arraydescr, size, source_op, clear=False):
         if arraydescr.is_array_of_structs():
@@ -703,7 +710,7 @@
             self.emit_operation(op)
 
     def optimize_NEW_WITH_VTABLE(self, op):
-        self.make_virtual(op.getarg(0), op)
+        self.make_virtual(op.getarg(0), op, op.getdescr())
 
     def optimize_NEW(self, op):
         self.make_vstruct(op.getdescr(), op)
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -3,7 +3,7 @@
         ConstPtr, ConstFloat)
 from rpython.jit.metainterp.optimizeopt import virtualize
 from rpython.jit.metainterp.optimizeopt.intutils import IntUnbounded
-from rpython.jit.metainterp.optimizeopt.optimizer import (LEVEL_CONSTANT,
+from rpython.jit.metainterp.optimizeopt.info import (LEVEL_CONSTANT,
     LEVEL_KNOWNCLASS, LEVEL_NONNULL, LEVEL_UNKNOWN)
 from rpython.jit.metainterp.resoperation import rop, ResOperation,\
      AbstractInputArg

From noreply at buildbot.pypy.org  Fri Feb 27 17:14:31 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 27 Feb 2015 17:14:31 +0100 (CET)
Subject: [pypy-commit] pypy default: issue #1986: auto-indent support in
 "backspace" and "enter"
Message-ID: <20150227161431.660F21C009F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r76163:b37ef838fa61
Date: 2015-02-27 17:14 +0100
http://bitbucket.org/pypy/pypy/changeset/b37ef838fa61/

Log:	issue #1986: auto-indent support in "backspace" and "enter"

diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py
--- a/lib_pypy/pyrepl/readline.py
+++ b/lib_pypy/pyrepl/readline.py
@@ -73,7 +73,6 @@
     assume_immutable_completions = False
     use_brackets = False
     sort_in_column = True
-    tab_insert_spaces_if_stem_is_empty = False
 
     def error(self, msg="none"):
         pass    # don't show error messages by default
@@ -87,7 +86,7 @@
         return ''.join(b[p+1:self.pos])
 
     def get_completions(self, stem):
-        if len(stem) == 0 and self.tab_insert_spaces_if_stem_is_empty:
+        if len(stem) == 0 and self.more_lines is not None:
             b = self.buffer
             p = self.pos
             while p > 0 and b[p - 1] != '\n':
@@ -141,12 +140,16 @@
 
     def collect_keymap(self):
         return super(ReadlineAlikeReader, self).collect_keymap() + (
-            (r'\n', 'maybe-accept'),)
+            (r'\n', 'maybe-accept'),
+            (r'\', 'backspace-dedent'),
+            )
 
     def __init__(self, console):
         super(ReadlineAlikeReader, self).__init__(console)
         self.commands['maybe_accept'] = maybe_accept
         self.commands['maybe-accept'] = maybe_accept
+        self.commands['backspace_dedent'] = backspace_dedent
+        self.commands['backspace-dedent'] = backspace_dedent
 
     def after_command(self, cmd):
         super(ReadlineAlikeReader, self).after_command(cmd)
@@ -164,6 +167,28 @@
                 if self.pos > len(self.buffer):
                     self.pos = len(self.buffer)
 
+def _get_this_line_indent(buffer, pos):
+    indent = 0
+    while pos > 0 and buffer[pos - 1] in " \t":
+        indent += 1
+        pos -= 1
+    if pos > 0 and buffer[pos - 1] == "\n":
+        return indent
+    return 0
+
+def _get_previous_line_indent(buffer, pos):
+    prevlinestart = pos
+    while prevlinestart > 0 and buffer[prevlinestart - 1] != "\n":
+        prevlinestart -= 1
+    prevlinetext = prevlinestart
+    while prevlinetext < pos and buffer[prevlinetext] in " \t":
+        prevlinetext += 1
+    if prevlinetext == pos:
+        indent = None
+    else:
+        indent = prevlinetext - prevlinestart
+    return prevlinestart, indent
+
 class maybe_accept(commands.Command):
     def do(self):
         r = self.reader
@@ -172,13 +197,39 @@
         # if there are already several lines and the cursor
         # is not on the last one, always insert a new \n.
         text = r.get_unicode()
-        if "\n" in r.buffer[r.pos:]:
+        if ("\n" in r.buffer[r.pos:] or
+            (r.more_lines is not None and r.more_lines(text))):
+            #
+            # auto-indent the next line like the previous line
+            prevlinestart, indent = _get_previous_line_indent(r.buffer, r.pos)
             r.insert("\n")
-        elif r.more_lines is not None and r.more_lines(text):
-            r.insert("\n")
+            if indent:
+                for i in range(prevlinestart, prevlinestart + indent):
+                    r.insert(r.buffer[i])
         else:
             self.finish = 1
 
+class backspace_dedent(commands.Command):
+    def do(self):
+        r = self.reader
+        b = r.buffer
+        if r.pos > 0:
+            repeat = 1
+            if b[r.pos - 1] != "\n":
+                indent = _get_this_line_indent(b, r.pos)
+                if indent > 0:
+                    ls = r.pos - indent
+                    while ls > 0:
+                        ls, pi = _get_previous_line_indent(b, ls - 1)
+                        if pi is not None and pi < indent:
+                            repeat = indent - pi
+                            break
+            r.pos -= repeat
+            del b[r.pos:r.pos + repeat]
+            r.dirty = 1
+        else:
+            self.reader.error("can't backspace at start")
+
 # ____________________________________________________________
 
 class _ReadlineWrapper(object):
@@ -212,15 +263,14 @@
         boolean value is true.
         """
         reader = self.get_reader()
-        saved = reader.more_lines, reader.tab_insert_spaces_if_stem_is_empty
+        saved = reader.more_lines
         try:
             reader.more_lines = more_lines
             reader.ps1 = reader.ps2 = ps1
             reader.ps3 = reader.ps4 = ps2
-            reader.tab_insert_spaces_if_stem_is_empty = True
             return reader.readline(returns_unicode=returns_unicode)
         finally:
-            reader.more_lines, reader.tab_insert_spaces_if_stem_is_empty = saved
+            reader.more_lines = saved
 
     def parse_and_bind(self, string):
         pass  # XXX we don't support parsing GNU-readline-style init files

From noreply at buildbot.pypy.org  Fri Feb 27 17:32:23 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 27 Feb 2015 17:32:23 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Implement mmap.read(None),
 which is the same as mmap.read() or mmap.read(-1).
Message-ID: <20150227163223.42D451C0399@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76164:31755e06a2e3
Date: 2015-02-27 10:48 +0100
http://bitbucket.org/pypy/pypy/changeset/31755e06a2e3/

Log:	Implement mmap.read(None), which is the same as mmap.read() or
	mmap.read(-1).

diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -1,7 +1,7 @@
 from pypy.interpreter.error import OperationError, wrap_oserror
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
-from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
 from rpython.rlib import rmmap, rarithmetic
 from rpython.rlib.buffer import Buffer
 from rpython.rlib.rmmap import RValueError, RTypeError, RMMapError
@@ -37,9 +37,13 @@
         self.check_valid()
         return self.space.wrapbytes(self.mmap.readline())
 
-    @unwrap_spec(num=int)
-    def read(self, num=-1):
+    @unwrap_spec(w_num=WrappedDefault(None))
+    def read(self, w_num):
         self.check_valid()
+        if self.space.is_none(w_num):
+            num = -1
+        else:
+            num = self.space.int_w(w_num)
         return self.space.wrapbytes(self.mmap.read(num))
 
     def find(self, w_tofind, w_start=None, w_end=None):
diff --git a/pypy/module/mmap/test/test_mmap.py b/pypy/module/mmap/test/test_mmap.py
--- a/pypy/module/mmap/test/test_mmap.py
+++ b/pypy/module/mmap/test/test_mmap.py
@@ -823,3 +823,12 @@
                 assert str(e) == "cannot mmap an empty file"
             except BaseException as e:
                 assert False, "unexpected exception: " + str(e)
+
+    def test_read_all(self):
+        from mmap import mmap
+        f = open(self.tmpname + "f", "wb+")
+        f.write(b"foobar")
+        f.flush()
+
+        m = mmap(f.fileno(), 6)
+        assert m.read(None) == b"foobar"

From noreply at buildbot.pypy.org  Fri Feb 27 17:32:24 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 27 Feb 2015 17:32:24 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Fix mmap memoryview test.
Message-ID: <20150227163224.A07281C0399@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76165:90616977c977
Date: 2015-02-27 10:49 +0100
http://bitbucket.org/pypy/pypy/changeset/90616977c977/

Log:	Fix mmap memoryview test.

diff --git a/pypy/module/mmap/test/test_mmap.py b/pypy/module/mmap/test/test_mmap.py
--- a/pypy/module/mmap/test/test_mmap.py
+++ b/pypy/module/mmap/test/test_mmap.py
@@ -535,7 +535,7 @@
         b = memoryview(m)
         assert len(b) == 6
         assert b.readonly is False
-        assert b[3] == b"b"
+        assert b[3] == ord(b"b")
         assert b[:] == b"foobar"
         del b  # For CPython: "exported pointers exist"
         m.close()

From noreply at buildbot.pypy.org  Fri Feb 27 17:32:25 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 27 Feb 2015 17:32:25 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Fix strange syntax error in lib-python
	test.
Message-ID: <20150227163225.C84531C0399@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76166:e3f7ada75b17
Date: 2015-02-27 11:08 +0100
http://bitbucket.org/pypy/pypy/changeset/e3f7ada75b17/

Log:	Fix strange syntax error in lib-python test.

diff --git a/lib-python/3/test/test_xml_etree.py b/lib-python/3/test/test_xml_etree.py
--- a/lib-python/3/test/test_xml_etree.py
+++ b/lib-python/3/test/test_xml_etree.py
@@ -273,7 +273,7 @@
         self.serialize_check(element, '') # 5
         with self.assertRaises(ValueError) as cm:
             element.remove(subelement)
-        self.assertTrue(str(cm.exception).startswith('list.remove(')
+        self.assertTrue(str(cm.exception).startswith('list.remove('))
         self.serialize_check(element, '') # 6
         element[0:0] = [subelement, subelement, subelement]
         self.serialize_check(element[1], '')

From noreply at buildbot.pypy.org  Fri Feb 27 17:32:26 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 27 Feb 2015 17:32:26 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Adapt lib-python test_xml_etree to PyPy.
Message-ID: <20150227163226.EAD2F1C0399@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76167:c3bca8f186f6
Date: 2015-02-27 11:10 +0100
http://bitbucket.org/pypy/pypy/changeset/c3bca8f186f6/

Log:	Adapt lib-python test_xml_etree to PyPy.

diff --git a/lib-python/3/test/test_xml_etree.py b/lib-python/3/test/test_xml_etree.py
--- a/lib-python/3/test/test_xml_etree.py
+++ b/lib-python/3/test/test_xml_etree.py
@@ -15,7 +15,7 @@
 
 from itertools import product
 from test import support
-from test.support import TESTFN, findfile, import_fresh_module, gc_collect
+from test.support import TESTFN, findfile, import_fresh_module, gc_collect, impl_detail
 
 # pyET is the pure-Python implementation.
 #
@@ -1318,6 +1318,7 @@
         self.assertEqual(t.find('.//paragraph').text,
             'A new cultivar of Begonia plant named \u2018BCT9801BEG\u2019.')
 
+    @impl_detail
     def test_bug_xmltoolkit63(self):
         # Check reference leak.
         def xmltoolkit63():
@@ -1522,6 +1523,7 @@
         wref = weakref.ref(e, wref_cb)
         self.assertEqual(wref().tag, 'e')
         del e
+        gc_collect()
         self.assertEqual(flag, True)
         self.assertEqual(wref(), None)
 

From noreply at buildbot.pypy.org  Fri Feb 27 17:32:28 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 27 Feb 2015 17:32:28 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Remove dummy _elementtree module which
 isn't needed for lib-python test_xml_etree_c anymore. Actually there is a
 test that depends on that there is no such dummy module.
Message-ID: <20150227163228.2188A1C0399@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76168:d7432ee8463c
Date: 2015-02-27 11:17 +0100
http://bitbucket.org/pypy/pypy/changeset/d7432ee8463c/

Log:	Remove dummy _elementtree module which isn't needed for lib-python
	test_xml_etree_c anymore. Actually there is a test that depends on
	that there is no such dummy module.

diff --git a/lib_pypy/_elementtree.py b/lib_pypy/_elementtree.py
deleted file mode 100644
--- a/lib_pypy/_elementtree.py
+++ /dev/null
@@ -1,6 +0,0 @@
-# Just use ElementTree.
-
-from xml.etree import ElementTree
-
-globals().update(ElementTree.__dict__)
-del __all__

From noreply at buildbot.pypy.org  Fri Feb 27 17:32:29 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 27 Feb 2015 17:32:29 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Skip this test on PyPy.
Message-ID: <20150227163229.506F51C0399@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76169:98e3fb9e4d01
Date: 2015-02-27 11:40 +0100
http://bitbucket.org/pypy/pypy/changeset/98e3fb9e4d01/

Log:	Skip this test on PyPy.

diff --git a/lib-python/3/test/test_traceback.py b/lib-python/3/test/test_traceback.py
--- a/lib-python/3/test/test_traceback.py
+++ b/lib-python/3/test/test_traceback.py
@@ -63,6 +63,7 @@
         self.assertEqual(len(err), 3)
         self.assertEqual(err[1].strip(), "bad syntax")
 
+    @cpython_only
     def test_bad_indentation(self):
         err = self.get_exception_format(self.syntax_error_bad_indentation,
                                         IndentationError)

From noreply at buildbot.pypy.org  Fri Feb 27 17:32:30 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 27 Feb 2015 17:32:30 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Implement a return value of None in
 getinterpevalloader, which means that no such attribute exists.
Message-ID: <20150227163230.7DACA1C0399@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76170:78b221682191
Date: 2015-02-27 12:33 +0100
http://bitbucket.org/pypy/pypy/changeset/78b221682191/

Log:	Implement a return value of None in getinterpevalloader, which means
	that no such attribute exists.

diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py
--- a/pypy/interpreter/mixedmodule.py
+++ b/pypy/interpreter/mixedmodule.py
@@ -190,6 +190,8 @@
                         # clarity
                         raise etype, evalue, etb
             else:
+                if value is None:
+                    return value
                 #print spec, "->", value
                 if hasattr(value, 'func_code'):  # semi-evil
                     return space.wrap(gateway.interp2app(value))

From noreply at buildbot.pypy.org  Fri Feb 27 17:32:31 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 27 Feb 2015 17:32:31 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Partially implement sys.thread_info by
 returning None (which means unknown) for each field.
Message-ID: <20150227163231.A5A021C0399@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76171:7dd999f6e56a
Date: 2015-02-27 12:34 +0100
http://bitbucket.org/pypy/pypy/changeset/7dd999f6e56a/

Log:	Partially implement sys.thread_info by returning None (which means
	unknown) for each field.

diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -77,7 +77,8 @@
         'float_info'            : 'system.get_float_info(space)',
         'int_info'              : 'system.get_int_info(space)',
         'hash_info'             : 'system.get_hash_info(space)',
-        'float_repr_style'      : 'system.get_float_repr_style(space)'
+        'float_repr_style'      : 'system.get_float_repr_style(space)',
+        'thread_info'           : 'system.get_thread_info(space)'
         }
 
     if sys.platform == 'win32':
diff --git a/pypy/module/sys/system.py b/pypy/module/sys/system.py
--- a/pypy/module/sys/system.py
+++ b/pypy/module/sys/system.py
@@ -37,6 +37,11 @@
     inf = structseqfield(2)
     nan = structseqfield(3)
     imag = structseqfield(4)
+
+class thread_info(metaclass=structseqtype):
+    name = structseqfield(0)
+    lock = structseqfield(1)
+    version = structseqfield(2)
 """)
 
 
@@ -80,3 +85,16 @@
 
 def get_float_repr_style(space):
     return space.wrap("short")
+
+def get_thread_info(space):
+    # TODO: implement this instead of returning None (which means unknown) for
+    # every field
+    if not space.config.objspace.usemodules.thread:
+        return None
+    info_w = [
+        space.wrap(space.w_None),
+        space.wrap(space.w_None),
+        space.wrap(space.w_None),
+    ]
+    w_thread_info = app.wget(space, "thread_info")
+    return space.call_function(w_thread_info, space.newtuple(info_w))

From noreply at buildbot.pypy.org  Fri Feb 27 17:32:32 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 27 Feb 2015 17:32:32 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Fix math.log2(2**1023) by implementing
 the base-2 special case in rbigint.
Message-ID: <20150227163232.C90011C0399@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76172:f38ee37c81fa
Date: 2015-02-27 14:05 +0100
http://bitbucket.org/pypy/pypy/changeset/f38ee37c81fa/

Log:	Fix math.log2(2**1023) by implementing the base-2 special case in
	rbigint.

	This commit changes the rpython subdirectory, which it shouldn't
	because this is the py3.3 branch. But there already are differences
	in the rpython subdirectory between default and py3.3, which are
	related to this change. Unfortunately, all these changes can't just
	be applied to default, because it would change sematics of the
	Python 2.7 implementation. There should be a discussion about how
	to refactor this.

diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -1218,6 +1218,9 @@
         # base is supposed to be positive or 0.0, which means we use e
         if base == 10.0:
             return _loghelper(math.log10, self)
+        if base == 2.0:
+            from rpython.rlib import rfloat
+            return _loghelper(rfloat.log2, self)
         ret = _loghelper(math.log, self)
         if base != 0.0:
             ret /= math.log(base)
diff --git a/rpython/rlib/test/test_rbigint.py b/rpython/rlib/test/test_rbigint.py
--- a/rpython/rlib/test/test_rbigint.py
+++ b/rpython/rlib/test/test_rbigint.py
@@ -672,6 +672,11 @@
                 else:
                     assert ulps_check(l, math.log(op)) is None
 
+    def test_log2(self):
+        assert rbigint.fromlong(1).log(2.0) == 0.0
+        assert rbigint.fromlong(2).log(2.0) == 1.0
+        assert rbigint.fromlong(2**1023).log(2.0) == 1023.0
+
 class TestInternalFunctions(object):
     def test__inplace_divrem1(self):
         # signs are not handled in the helpers!

From noreply at buildbot.pypy.org  Fri Feb 27 17:32:33 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 27 Feb 2015 17:32:33 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Implement 'x' flag on io.open() /
	io.FileIO.
Message-ID: <20150227163233.F25861C0399@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76173:7f15f2e2ab69
Date: 2015-02-27 16:15 +0100
http://bitbucket.org/pypy/pypy/changeset/7f15f2e2ab69/

Log:	Implement 'x' flag on io.open() / io.FileIO.

diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py
--- a/pypy/module/_io/interp_fileio.py
+++ b/pypy/module/_io/interp_fileio.py
@@ -4,7 +4,7 @@
 from pypy.interpreter.error import wrap_oserror, wrap_oserror2
 from rpython.rlib.rarithmetic import r_longlong
 from rpython.rlib.rstring import StringBuilder
-from os import O_RDONLY, O_WRONLY, O_RDWR, O_CREAT, O_TRUNC
+from os import O_RDONLY, O_WRONLY, O_RDWR, O_CREAT, O_TRUNC, O_EXCL
 import sys, os, stat, errno
 from pypy.module._io.interp_iobase import W_RawIOBase, convert_size
 
@@ -34,13 +34,14 @@
 
 def _bad_mode(space):
     raise OperationError(space.w_ValueError, space.wrap(
-        "Must have exactly one of read/write/append mode"))
+        "Must have exactly one of read/write/create/append mode"))
 
 def decode_mode(space, mode):
     flags = 0
     rwa = False
     readable = False
     writable = False
+    created = False
     append = False
     plus = False
 
@@ -56,6 +57,13 @@
             rwa = True
             writable = True
             flags |= O_CREAT | O_TRUNC
+        elif s == 'x':
+            if rwa:
+                _bad_mode(space)
+            rwa = True
+            created = True
+            writable = True
+            flags |= O_EXCL | O_CREAT
         elif s == 'a':
             if rwa:
                 _bad_mode(space)
@@ -86,7 +94,7 @@
 
     flags |= O_BINARY
 
-    return readable, writable, append, flags
+    return readable, writable, created, append, flags
 
 SMALLCHUNK = 8 * 1024
 BIGCHUNK = 512 * 1024
@@ -121,6 +129,7 @@
         self.fd = -1
         self.readable = False
         self.writable = False
+        self.created = False
         self.appending = False
         self.seekable = -1
         self.closefd = True
@@ -147,7 +156,7 @@
                 raise OperationError(space.w_ValueError, space.wrap(
                     "negative file descriptor"))
 
-        self.readable, self.writable, self.appending, flags = decode_mode(space, mode)
+        self.readable, self.writable, self.created, self.appending, flags = decode_mode(space, mode)
 
         fd_is_own = False
         try:
@@ -204,6 +213,11 @@
             raise
 
     def _mode(self):
+        if self.created:
+            if self.readable:
+                return 'xb+'
+            else:
+                return 'xb'
         if self.appending:
             if self.readable:
                 return 'ab+'
diff --git a/pypy/module/_io/interp_io.py b/pypy/module/_io/interp_io.py
--- a/pypy/module/_io/interp_io.py
+++ b/pypy/module/_io/interp_io.py
@@ -30,7 +30,7 @@
             space.isinstance_w(w_file, space.w_int)):
         raise oefmt(space.w_TypeError, "invalid file: %R", w_file)
 
-    reading = writing = appending = updating = text = binary = universal = False
+    reading = writing = creating = appending = updating = text = binary = universal = False
 
     uniq_mode = {}
     for flag in mode:
@@ -42,6 +42,8 @@
             reading = True
         elif flag == "w":
             writing = True
+        elif flag == "x":
+            creating = True
         elif flag == "a":
             appending = True
         elif flag == "+":
@@ -61,6 +63,8 @@
         rawmode += "r"
     if writing:
         rawmode += "w"
+    if creating:
+        rawmode += "x"
     if appending:
         rawmode += "a"
     if updating:
@@ -74,9 +78,9 @@
         raise OperationError(space.w_ValueError,
             space.wrap("can't have text and binary mode at once")
         )
-    if reading + writing + appending > 1:
+    if reading + writing + creating + appending > 1:
         raise OperationError(space.w_ValueError,
-            space.wrap("must have exactly one of read/write/append mode")
+            space.wrap("must have exactly one of read/write/create/append mode")
         )
     if binary and encoding is not None:
         raise OperationError(space.w_ValueError,
@@ -123,7 +127,7 @@
 
     if updating:
         buffer_cls = W_BufferedRandom
-    elif writing or appending:
+    elif writing or creating or appending:
         buffer_cls = W_BufferedWriter
     elif reading:
         buffer_cls = W_BufferedReader
diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py
--- a/pypy/module/_io/test/test_fileio.py
+++ b/pypy/module/_io/test/test_fileio.py
@@ -222,6 +222,17 @@
             if os.path.exists(self.tmpfile):
                 os.unlink(self.tmpfile)
 
+    def test_open_exclusive(self):
+        # XXX: should raise FileExistsError
+        FileExistsError = OSError
+
+        import _io
+        filename = self.tmpfile + '_x'
+        raises(ValueError, _io.FileIO, filename, 'xw')
+        with _io.FileIO(filename, 'x') as f:
+            assert f.mode == 'xb'
+        raises(FileExistsError, _io.FileIO, filename, 'x')
+
 
 def test_flush_at_exit():
     from pypy import conftest
diff --git a/pypy/module/_io/test/test_io.py b/pypy/module/_io/test/test_io.py
--- a/pypy/module/_io/test/test_io.py
+++ b/pypy/module/_io/test/test_io.py
@@ -439,3 +439,14 @@
             f.seek(1, 0)
             f.read(buffer_size * 2)
             assert f.tell() == 1 + buffer_size * 2
+
+    def test_open_exclusive(self):
+        # XXX: should raise FileExistsError
+        FileExistsError = OSError
+
+        import _io
+        filename = self.tmpfile + '_x'
+        raises(ValueError, _io.open, filename, 'xw')
+        with _io.open(filename, 'x') as f:
+            assert f.mode == 'x'
+        raises(FileExistsError, _io.open, filename, 'x')

From noreply at buildbot.pypy.org  Fri Feb 27 17:32:35 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 27 Feb 2015 17:32:35 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Mark parts of this test as an
 implementation detail. As it turned out after a short discussion on the
 issue tracker, this test deliberately checks exactly the wrong behaviour.
Message-ID: <20150227163235.32BE21C0399@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76174:7be778a7332a
Date: 2015-02-27 16:21 +0100
http://bitbucket.org/pypy/pypy/changeset/7be778a7332a/

Log:	Mark parts of this test as an implementation detail. As it turned
	out after a short discussion on the issue tracker, this test
	deliberately checks exactly the wrong behaviour.

diff --git a/lib-python/3/test/test_index.py b/lib-python/3/test/test_index.py
--- a/lib-python/3/test/test_index.py
+++ b/lib-python/3/test/test_index.py
@@ -66,7 +66,10 @@
         direct_index = my_int.__index__()
         operator_index = operator.index(my_int)
         self.assertEqual(direct_index, 8)
-        self.assertEqual(operator_index, 7)
+        if support.check_impl_detail():
+            self.assertEqual(operator_index, 7)
+        else:
+            self.assertEqual(operator_index, 8)
         # Both results should be of exact type int.
         self.assertIs(type(direct_index), int)
         #self.assertIs(type(operator_index), int)
@@ -86,7 +89,10 @@
 
         bad_int = BadInt2()
         n = operator.index(bad_int)
-        self.assertEqual(n, 0)
+        if support.check_impl_detail():
+            self.assertEqual(n, 0)
+        else:
+            self.assertEqual(n, 1)
 
 
 class SeqTestCase:

From noreply at buildbot.pypy.org  Fri Feb 27 17:32:36 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 27 Feb 2015 17:32:36 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Adapt this test to the new memoryview API.
Message-ID: <20150227163236.657B91C0399@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76175:64f039c98dcf
Date: 2015-02-27 16:29 +0100
http://bitbucket.org/pypy/pypy/changeset/64f039c98dcf/

Log:	Adapt this test to the new memoryview API.

diff --git a/pypy/module/__pypy__/test/test_bytebuffer.py b/pypy/module/__pypy__/test/test_bytebuffer.py
--- a/pypy/module/__pypy__/test/test_bytebuffer.py
+++ b/pypy/module/__pypy__/test/test_bytebuffer.py
@@ -5,15 +5,13 @@
         from __pypy__ import bytebuffer
         b = bytebuffer(12)
         assert len(b) == 12
-        b[3] = b'!'
-        b[5] = b'?'
+        b[3] = ord(b'!')
+        b[5] = ord(b'?')
         assert b[2:7] == b'\x00!\x00?\x00'
         b[9:] = b'+-*'
-        assert b[-1] == b'*'
-        assert b[-2] == b'-'
-        assert b[-3] == b'+'
-        exc = raises(ValueError, "b[3] = b'abc'")
-        assert str(exc.value) == "cannot modify size of memoryview object"
+        assert b[-1] == ord(b'*')
+        assert b[-2] == ord(b'-')
+        assert b[-3] == ord(b'+')
         exc = raises(ValueError, "b[3:5] = b'abc'")
         assert str(exc.value) == "cannot modify size of memoryview object"
         raises(NotImplementedError, "b[3:7:2] = b'abc'")

From noreply at buildbot.pypy.org  Fri Feb 27 17:32:37 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 27 Feb 2015 17:32:37 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Adapt this test to the new memoryview API.
Message-ID: <20150227163237.93C321C0399@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76176:06f89631273f
Date: 2015-02-27 16:31 +0100
http://bitbucket.org/pypy/pypy/changeset/06f89631273f/

Log:	Adapt this test to the new memoryview API.

diff --git a/pypy/module/_io/test/test_bytesio.py b/pypy/module/_io/test/test_bytesio.py
--- a/pypy/module/_io/test/test_bytesio.py
+++ b/pypy/module/_io/test/test_bytesio.py
@@ -124,7 +124,7 @@
         memio.seek(5)
         buf = memio.getbuffer()
         assert bytes(buf) == b"1234567890"
-        assert buf[5] == b"6"
+        assert buf[5] == ord(b"6")
         # Mutating the buffer updates the BytesIO
         buf[3:6] = b"abc"
         assert bytes(buf) == b"123abc7890"

From noreply at buildbot.pypy.org  Fri Feb 27 17:32:38 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 27 Feb 2015 17:32:38 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Let _rawffi.Array make use of the
 extended internal buffer API.
Message-ID: <20150227163238.B85181C0399@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76177:00ae8705f679
Date: 2015-02-27 16:59 +0100
http://bitbucket.org/pypy/pypy/changeset/00ae8705f679/

Log:	Let _rawffi.Array make use of the extended internal buffer API.

diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py
--- a/pypy/module/_rawffi/array.py
+++ b/pypy/module/_rawffi/array.py
@@ -198,6 +198,10 @@
         for i in range(len(value)):
             ll_buffer[start + i] = value[i]
 
+    def buffer_w_ex(self, space, flags):
+        return self.buffer_w(space, flags), self.shape.itemcode, self.shape.size
+
+
 W_ArrayInstance.typedef = TypeDef(
     'ArrayInstance',
     __repr__    = interp2app(W_ArrayInstance.descr_repr),
diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -1115,7 +1115,7 @@
         s = S(autofree=True)
         b = memoryview(s)
         assert len(b) == 40
-        b[4] = b'X'
+        b[4] = ord(b'X')
         b[:3] = b'ABC'
         assert b[:6] == b'ABC\x00X\x00'
 

From noreply at buildbot.pypy.org  Fri Feb 27 17:32:39 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 27 Feb 2015 17:32:39 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: _abcoll was renamed to collections.abc.
Message-ID: <20150227163239.E8BDF1C0399@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76178:ebdd56200c8a
Date: 2015-02-27 17:08 +0100
http://bitbucket.org/pypy/pypy/changeset/ebdd56200c8a/

Log:	_abcoll was renamed to collections.abc.

diff --git a/pypy/objspace/std/test/test_iterobject.py b/pypy/objspace/std/test/test_iterobject.py
--- a/pypy/objspace/std/test/test_iterobject.py
+++ b/pypy/objspace/std/test/test_iterobject.py
@@ -91,8 +91,8 @@
         raises(TypeError, len, iter(iterable))
 
     def test_no_len_on_UserList_iter_reversed(self):
-        import sys, _abcoll
-        sys.modules['collections'] = _abcoll
+        import sys, collections.abc
+        sys.modules['collections'] = collections.abc
         class UserList(object):
             def __init__(self, i):
                 self.i = i

From noreply at buildbot.pypy.org  Fri Feb 27 17:32:41 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 27 Feb 2015 17:32:41 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: The dictimplementation attribute was
 renamed to w_dict in 3a37dfe596eb.
Message-ID: <20150227163241.1A89F1C0399@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76179:f43c61438796
Date: 2015-02-27 17:16 +0100
http://bitbucket.org/pypy/pypy/changeset/f43c61438796/

Log:	The dictimplementation attribute was renamed to w_dict in
	3a37dfe596eb.

diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -735,14 +735,14 @@
     _become(obj, new_obj)
 
 class MapDictIteratorKeys(BaseKeyIterator):
-    def __init__(self, space, strategy, dictimplementation):
-        BaseKeyIterator.__init__(self, space, strategy, dictimplementation)
-        w_obj = strategy.unerase(dictimplementation.dstorage)
+    def __init__(self, space, strategy, w_dict):
+        BaseKeyIterator.__init__(self, space, strategy, w_dict)
+        w_obj = strategy.unerase(w_dict.dstorage)
         self.w_obj = w_obj
         self.orig_map = self.curr_map = w_obj._get_mapdict_map()
 
     def next_key_entry(self):
-        implementation = self.dictimplementation
+        implementation = self.w_dict
         assert isinstance(implementation.strategy, MapDictStrategy)
         if self.orig_map is not self.w_obj._get_mapdict_map():
             return None
@@ -757,14 +757,14 @@
 
 
 class MapDictIteratorValues(BaseValueIterator):
-    def __init__(self, space, strategy, dictimplementation):
-        BaseValueIterator.__init__(self, space, strategy, dictimplementation)
-        w_obj = strategy.unerase(dictimplementation.dstorage)
+    def __init__(self, space, strategy, w_dict):
+        BaseValueIterator.__init__(self, space, strategy, w_dict)
+        w_obj = strategy.unerase(w_dict.dstorage)
         self.w_obj = w_obj
         self.orig_map = self.curr_map = w_obj._get_mapdict_map()
 
     def next_value_entry(self):
-        implementation = self.dictimplementation
+        implementation = self.w_dict
         assert isinstance(implementation.strategy, MapDictStrategy)
         if self.orig_map is not self.w_obj._get_mapdict_map():
             return None
@@ -778,14 +778,14 @@
 
 
 class MapDictIteratorItems(BaseItemIterator):
-    def __init__(self, space, strategy, dictimplementation):
-        BaseItemIterator.__init__(self, space, strategy, dictimplementation)
-        w_obj = strategy.unerase(dictimplementation.dstorage)
+    def __init__(self, space, strategy, w_dict):
+        BaseItemIterator.__init__(self, space, strategy, w_dict)
+        w_obj = strategy.unerase(w_dict.dstorage)
         self.w_obj = w_obj
         self.orig_map = self.curr_map = w_obj._get_mapdict_map()
 
     def next_item_entry(self):
-        implementation = self.dictimplementation
+        implementation = self.w_dict
         assert isinstance(implementation.strategy, MapDictStrategy)
         if self.orig_map is not self.w_obj._get_mapdict_map():
             return None, None

From noreply at buildbot.pypy.org  Fri Feb 27 17:33:06 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 27 Feb 2015 17:33:06 +0100 (CET)
Subject: [pypy-commit] pypy optresult: (fijal,
 arigo) start rewriting the class hierarchy for XxxInfo
Message-ID: <20150227163306.1C9C71C0399@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r76180:694e29ab1d9a
Date: 2015-02-27 18:32 +0200
http://bitbucket.org/pypy/pypy/changeset/694e29ab1d9a/

Log:	(fijal, arigo) start rewriting the class hierarchy for XxxInfo

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -21,7 +21,49 @@
     def force_box(self, op, optforce):
         return op
 
-class PtrOptInfo(AbstractInfo):
+    
+class PtrInfo(AbstractInfo):
+    _attrs_ = ()
+
+    
+class NonNullPtrInfo(PtrInfo):
+    _attrs_ = ()
+
+    
+class InstancePtrInfo(NonNullPtrInfo):
+    _attrs_ = ('_known_class', '_is_virtual', '_fields')
+
+    def __init__(self, known_class=None, is_virtual=False):
+        self._known_class = known_class
+        self._is_virtual = is_virtual
+
+    def force_box(self, op, optforce):
+        if self._is_virtual:
+            op.set_forwarded(None)
+            optforce.emit_operation(op)
+            newop = optforce.getlastop()
+            op.set_forwarded(newop)
+            newop.set_forwarded(self)
+            self._is_virtual = False
+            return newop
+        return op
+
+    def get_known_class(self):
+        return self._known_class
+    
+class StructPtrInfo(NonNullPtrInfo):
+    _attrs_ = ('is_virtual', '_fields')
+
+    
+class ArrayPtrInfo(NonNullPtrInfo):
+    _attrs_ = ('is_virtual', 'length', '_items')
+
+    
+class StrPtrInfo(NonNullPtrInfo):
+    _attrs_ = ()
+
+    
+class XPtrOptInfo(AbstractInfo):
     _attrs_ = ('_tag', 'known_class', 'last_guard_pos', 'lenbound')
     is_info_class = True
 
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -6,7 +6,7 @@
      IntUnbounded, ConstIntBound
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from rpython.jit.metainterp.resoperation import rop, AbstractResOp, GuardResOp
-from rpython.jit.metainterp.optimizeopt.info import PtrOptInfo
+from rpython.jit.metainterp.optimizeopt import info
 from rpython.jit.metainterp.typesystem import llhelper
 from rpython.rlib.objectmodel import specialize, we_are_translated
 
@@ -293,6 +293,15 @@
             return self.getintbound(op).getnullness()
         xxxx
 
+    def make_constant_class(self, op, class_const):
+        op = self.get_box_replacement(op)
+        opinfo = op.get_forwarded()
+        if opinfo is not None:
+            return opinfo
+        opinfo = info.InstancePtrInfo(class_const)
+        op.set_forwarded(opinfo)
+        return opinfo
+
     def getptrinfo(self, op):
         assert op.type == 'r'
         op = self.get_box_replacement(op)
@@ -301,11 +310,9 @@
             xxx
         fw = op.get_forwarded()
         if fw is not None:
-            assert isinstance(fw, PtrOptInfo)
+            assert isinstance(fw, info.PtrInfo)
             return fw
-        ptrinfo = PtrOptInfo()
-        op.set_forwarded(ptrinfo)
-        return ptrinfo
+        return None
 
     def get_box_replacement(self, op):
         return self.optimizer.get_box_replacement(op)
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -7,7 +7,6 @@
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound
 from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, REMOVED,
     CONST_0, CONST_1, INFO_NONNULL, INFO_NULL)
-from rpython.jit.metainterp.optimizeopt.info import PtrOptInfo
 from rpython.jit.metainterp.optimizeopt.util import _findall, make_dispatcher_method
 from rpython.jit.metainterp.resoperation import rop, ResOperation, opclasses,\
      OpHelpers
@@ -347,41 +346,43 @@
         value.make_constant_class(None, expectedclassbox)
 
     def optimize_GUARD_CLASS(self, op):
-        value = self.getptrinfo(op.getarg(0))
         expectedclassbox = op.getarg(1)
+        info = self.getptrinfo(op.getarg(0))
         assert isinstance(expectedclassbox, Const)
-        realclassbox = value.get_constant_class(self.optimizer.cpu)
-        if realclassbox is not None:
-            if realclassbox.same_constant(expectedclassbox):
-                return
-            r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op)
-            raise InvalidLoop('A GUARD_CLASS (%s) was proven to always fail'
-                              % r)
-        old_guard_op = value.get_last_guard(self.optimizer)
-        if old_guard_op and not isinstance(old_guard_op.getdescr(),
-                                           compile.ResumeAtPositionDescr):
-            # there already has been a guard_nonnull or guard_class or
-            # guard_nonnull_class on this value.
-            if old_guard_op.getopnum() == rop.GUARD_NONNULL:
-                # it was a guard_nonnull, which we replace with a
-                # guard_nonnull_class.
-                descr = compile.ResumeGuardNonnullClassDescr()
-                op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
-                            args = [old_guard_op.getarg(0), op.getarg(1)],
-                            descr=descr)
-                # Note: we give explicitly a new descr for 'op'; this is why the
-                # old descr must not be ResumeAtPositionDescr (checked above).
-                # Better-safe-than-sorry but it should never occur: we should
-                # not put in short preambles guard_nonnull and guard_class
-                # on the same box.
-                self.optimizer.replace_guard(op, value)
-                # not emitting the guard, so we have to pass None to
-                # make_constant_class, so last_guard_pos is not updated
-                self.emit_operation(op)
-                value.make_constant_class(None, expectedclassbox)
-                return
+        if info is not None:
+            realclassbox = info.get_known_class()
+            if realclassbox is not None:
+                if realclassbox.same_constant(expectedclassbox):
+                    return
+                r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op)
+                raise InvalidLoop('A GUARD_CLASS (%s) was proven to always fail'
+                                  % r)
+            old_guard_op = info.get_last_guard(self.optimizer)
+            if old_guard_op and not isinstance(old_guard_op.getdescr(),
+                                               compile.ResumeAtPositionDescr):
+                xxx
+                # there already has been a guard_nonnull or guard_class or
+                # guard_nonnull_class on this value.
+                if old_guard_op.getopnum() == rop.GUARD_NONNULL:
+                    # it was a guard_nonnull, which we replace with a
+                    # guard_nonnull_class.
+                    descr = compile.ResumeGuardNonnullClassDescr()
+                    op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
+                                args = [old_guard_op.getarg(0), op.getarg(1)],
+                                descr=descr)
+                    # Note: we give explicitly a new descr for 'op'; this is why the
+                    # old descr must not be ResumeAtPositionDescr (checked above).
+                    # Better-safe-than-sorry but it should never occur: we should
+                    # not put in short preambles guard_nonnull and guard_class
+                    # on the same box.
+                    self.optimizer.replace_guard(op, value)
+                    # not emitting the guard, so we have to pass None to
+                    # make_constant_class, so last_guard_pos is not updated
+                    self.emit_operation(op)
+                    value.make_constant_class(None, expectedclassbox)
+                    return
+        self.make_constant_class(op.getarg(0), expectedclassbox)
         self.emit_operation(op)
-        value.make_constant_class(self.optimizer, expectedclassbox)
 
     def optimize_GUARD_NONNULL_CLASS(self, op):
         value = self.getvalue(op.getarg(0))
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -13,7 +13,7 @@
 from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.jit.metainterp.optimizeopt.intutils import IntUnbounded
 
-class AbstractVirtualInfo(info.PtrOptInfo):
+class AbstractVirtualInfo(info.PtrInfo):
     _attrs_ = ('_cached_vinfo',)
     _tag = info.LEVEL_NONNULL
     is_about_raw = False
@@ -22,14 +22,6 @@
     def is_forced_virtual(self):
         xxx
         return self.box is not None
-
-    def force_box(self, op, optforce):
-        op.set_forwarded(None)
-        optforce.emit_operation(op)
-        newop = optforce.getlastop()
-        op.set_forwarded(newop)
-        optforce.getptrinfo(newop).make_constant_class(None, self.known_class)
-        return newop
     
     #def force_box(self, optforce):
     #    xxxx
@@ -531,9 +523,9 @@
     _last_guard_not_forced_2 = None
 
     def make_virtual(self, known_class, source_op, descr):
-        info = VirtualInfo(known_class, descr)
-        source_op.set_forwarded(info)
-        return info
+        opinfo = info.InstancePtrInfo(known_class, is_virtual=True)
+        source_op.set_forwarded(opinfo)
+        return opinfo
 
     def make_varray(self, arraydescr, size, source_op, clear=False):
         if arraydescr.is_array_of_structs():

From noreply at buildbot.pypy.org  Fri Feb 27 17:40:40 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 27 Feb 2015 17:40:40 +0100 (CET)
Subject: [pypy-commit] pypy optresult: (fijal, arigo) hack hack hack,
	next test passes
Message-ID: <20150227164040.4BE401C02BD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r76181:623b19f3f2ec
Date: 2015-02-27 18:40 +0200
http://bitbucket.org/pypy/pypy/changeset/623b19f3f2ec/

Log:	(fijal, arigo) hack hack hack, next test passes

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -48,7 +48,7 @@
             return newop
         return op
 
-    def get_known_class(self):
+    def get_known_class(self, cpu):
         return self._known_class
     
 class StructPtrInfo(NonNullPtrInfo):
@@ -62,6 +62,15 @@
 class StrPtrInfo(NonNullPtrInfo):
     _attrs_ = ()
 
+
+class ConstPtrInfo(PtrInfo):
+    _attrs_ = ('_const',)
+    
+    def __init__(self, const):
+        self._const = const
+
+    def get_known_class(self, cpu):
+        return cpu.ts.cls_of_box(self._const)
     
 class XPtrOptInfo(AbstractInfo):
     _attrs_ = ('_tag', 'known_class', 'last_guard_pos', 'lenbound')
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -307,7 +307,7 @@
         op = self.get_box_replacement(op)
         assert op.type == 'r'
         if isinstance(op, ConstPtr):
-            xxx
+            return info.ConstPtrInfo(op)
         fw = op.get_forwarded()
         if fw is not None:
             assert isinstance(fw, info.PtrInfo)
@@ -528,18 +528,9 @@
     def clear_newoperations(self):
         self._newoperations = []
 
-    def make_equal_to(self, box, value):
-        assert isinstance(value, OptValue)
-        try:
-            cur_value = self.values[box]
-        except KeyError:
-            pass
-        else:
-            assert cur_value.getlevel() != LEVEL_CONSTANT
-            # replacing with a different box
-            cur_value.copy_from(value)
-            return
-        self.values[box] = value
+    def make_equal_to(self, op, oldop):
+        assert op.get_forwarded() is None
+        op.set_forwarded(oldop)
 
     def replace_op_with(self, op, newopnum, args=None, descr=None):
         newop = op.copy_and_change(newopnum, args, descr)
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -350,7 +350,7 @@
         info = self.getptrinfo(op.getarg(0))
         assert isinstance(expectedclassbox, Const)
         if info is not None:
-            realclassbox = info.get_known_class()
+            realclassbox = info.get_known_class(self.optimizer.cpu)
             if realclassbox is not None:
                 if realclassbox.same_constant(expectedclassbox):
                     return
@@ -595,7 +595,7 @@
         self.emit_operation(op)
 
     def optimize_SAME_AS_I(self, op):
-        self.make_equal_to(op, self.getvalue(op.getarg(0)))
+        self.make_equal_to(op, op.getarg(0))
     optimize_SAME_AS_R = optimize_SAME_AS_I
     optimize_SAME_AS_F = optimize_SAME_AS_I
 

From noreply at buildbot.pypy.org  Fri Feb 27 18:00:50 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 27 Feb 2015 18:00:50 +0100 (CET)
Subject: [pypy-commit] pypy optresult: minor fix
Message-ID: <20150227170050.AE8921C02BD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r76182:3a7e61ab764a
Date: 2015-02-27 18:41 +0200
http://bitbucket.org/pypy/pypy/changeset/3a7e61ab764a/

Log:	minor fix

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -70,6 +70,8 @@
         self._const = const
 
     def get_known_class(self, cpu):
+        if not self._const.nonnull():
+            return None
         return cpu.ts.cls_of_box(self._const)
     
 class XPtrOptInfo(AbstractInfo):

From noreply at buildbot.pypy.org  Fri Feb 27 18:00:51 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 27 Feb 2015 18:00:51 +0100 (CET)
Subject: [pypy-commit] pypy optresult: (fijal, arigo) whack whack type type
Message-ID: <20150227170051.EE2331C02BD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r76183:a9318d7e2e4d
Date: 2015-02-27 18:51 +0200
http://bitbucket.org/pypy/pypy/changeset/a9318d7e2e4d/

Log:	(fijal, arigo) whack whack type type

diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -272,11 +272,13 @@
         r.getintbound().intersect(resbound)
 
     def optimize_INT_LT(self, op):
-        v1 = self.getvalue(op.getarg(0))
-        v2 = self.getvalue(op.getarg(1))
-        if v1.getintbound().known_lt(v2.getintbound()):
+        arg1 = self.get_box_replacement(op.getarg(0))
+        arg2 = self.get_box_replacement(op.getarg(1))
+        b1 = self.getintbound(arg1)
+        b2 = self.getintbound(arg2)
+        if b1.known_lt(b2):
             self.make_constant_int(op, 1)
-        elif v1.getintbound().known_ge(v2.getintbound()) or v1.box is v2.box:
+        elif b1.known_ge(b2) or arg1 is arg2:
             self.make_constant_int(op, 0)
         else:
             self.emit_operation(op)
@@ -302,11 +304,13 @@
             self.emit_operation(op)
 
     def optimize_INT_GE(self, op):
-        v1 = self.getvalue(op.getarg(0))
-        v2 = self.getvalue(op.getarg(1))
-        if v1.getintbound().known_ge(v2.getintbound()) or v1.box is v2.box:
+        arg1 = self.get_box_replacement(op.getarg(0))
+        arg2 = self.get_box_replacement(op.getarg(1))
+        b1 = self.getintbound(arg1)
+        b2 = self.getintbound(arg2)
+        if b1.known_ge(b2) or arg1 is arg2:
             self.make_constant_int(op, 1)
-        elif v1.getintbound().known_lt(v2.getintbound()):
+        elif b1.known_lt(b2):
             self.make_constant_int(op, 0)
         else:
             self.emit_operation(op)
@@ -424,12 +428,12 @@
         v1.getintbound().make_ge(IntLowerBound(0))
 
     def make_int_lt(self, box1, box2):
-        v1 = self.getvalue(box1)
-        v2 = self.getvalue(box2)
-        if v1.getintbound().make_lt(v2.getintbound()):
-            self.propagate_bounds_backward(box1, v1)
-        if v2.getintbound().make_gt(v1.getintbound()):
-            self.propagate_bounds_backward(box2, v2)
+        b1 = self.getintbound(box1)
+        b2 = self.getintbound(box2)
+        if b1.make_lt(b2):
+            self.propagate_bounds_backward(box1)
+        if b2.make_gt(b1):
+            self.propagate_bounds_backward(box2)
 
     def make_int_le(self, box1, box2):
         v1 = self.getvalue(box1)
@@ -446,11 +450,12 @@
         self.make_int_le(box2, box1)
 
     def propagate_bounds_INT_LT(self, op):
-        r = self.getvalue(op)
+        r = self.get_box_replacement(op)
         if r.is_constant():
-            if r.box.same_constant(CONST_1):
+            if r.getint() == 1:
                 self.make_int_lt(op.getarg(0), op.getarg(1))
             else:
+                assert r.getint() == 0
                 self.make_int_ge(op.getarg(0), op.getarg(1))
 
     def propagate_bounds_INT_GT(self, op):
@@ -470,11 +475,12 @@
                 self.make_int_gt(op.getarg(0), op.getarg(1))
 
     def propagate_bounds_INT_GE(self, op):
-        r = self.getvalue(op)
+        r = self.get_box_replacement(op)
         if r.is_constant():
-            if r.box.same_constant(CONST_1):
+            if r.getint() == 1:
                 self.make_int_ge(op.getarg(0), op.getarg(1))
             else:
+                assert r.getint() == 0
                 self.make_int_lt(op.getarg(0), op.getarg(1))
 
     def propagate_bounds_INT_EQ(self, op):
diff --git a/rpython/jit/metainterp/optimizeopt/intutils.py b/rpython/jit/metainterp/optimizeopt/intutils.py
--- a/rpython/jit/metainterp/optimizeopt/intutils.py
+++ b/rpython/jit/metainterp/optimizeopt/intutils.py
@@ -257,6 +257,9 @@
         return (self.bounded() and self.known_ge(ConstIntBound(0)) and
                 self.known_le(ConstIntBound(1)))
 
+    def make_bool(self):
+        self.intersect(IntBound(0, 1))
+
     def getnullness(self):
         from rpython.jit.metainterp.optimizeopt import optimizer
 
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -535,7 +535,10 @@
     def replace_op_with(self, op, newopnum, args=None, descr=None):
         newop = op.copy_and_change(newopnum, args, descr)
         if newop.type != 'v':
-            assert op.get_forwarded() is None  #XXX
+            op = self.get_box_replacement(op)
+            opinfo = op.get_forwarded()
+            if opinfo is not None:
+                newop.set_forwarded(opinfo)
             op.set_forwarded(newop)
         return newop
 
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -50,7 +50,7 @@
         # otherwise, the operation remains
         self.emit_operation(op)
         if op.returns_bool_result():
-            self.optimizer.bool_boxes[self.getvalue(op)] = None
+            self.getintbound(op).make_bool()
         if nextop:
             self.emit_operation(nextop)
         #if args is not None:

From noreply at buildbot.pypy.org  Fri Feb 27 18:00:53 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 27 Feb 2015 18:00:53 +0100 (CET)
Subject: [pypy-commit] pypy optresult: (fijal, arigo) progress on intbounds
Message-ID: <20150227170053.127CB1C02BD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r76184:58042fcb433b
Date: 2015-02-27 18:53 +0200
http://bitbucket.org/pypy/pypy/changeset/58042fcb433b/

Log:	(fijal, arigo) progress on intbounds

diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -284,21 +284,25 @@
             self.emit_operation(op)
 
     def optimize_INT_GT(self, op):
-        v1 = self.getvalue(op.getarg(0))
-        v2 = self.getvalue(op.getarg(1))
-        if v1.getintbound().known_gt(v2.getintbound()):
+        arg1 = self.get_box_replacement(op.getarg(0))
+        arg2 = self.get_box_replacement(op.getarg(1))
+        b1 = self.getintbound(arg1)
+        b2 = self.getintbound(arg2)
+        if b1.known_gt(b2):
             self.make_constant_int(op, 1)
-        elif v1.getintbound().known_le(v2.getintbound()) or v1.box is v2.box:
+        elif b1.known_le(b2) or arg1 is arg2:
             self.make_constant_int(op, 0)
         else:
             self.emit_operation(op)
 
     def optimize_INT_LE(self, op):
-        v1 = self.getvalue(op.getarg(0))
-        v2 = self.getvalue(op.getarg(1))
-        if v1.getintbound().known_le(v2.getintbound()) or v1.box is v2.box:
+        arg1 = self.get_box_replacement(op.getarg(0))
+        arg2 = self.get_box_replacement(op.getarg(1))
+        b1 = self.getintbound(arg1)
+        b2 = self.getintbound(arg2)
+        if b1.known_le(b2) or arg1 is arg2:
             self.make_constant_int(op, 1)
-        elif v1.getintbound().known_gt(v2.getintbound()):
+        elif b1.known_gt(b2):
             self.make_constant_int(op, 0)
         else:
             self.emit_operation(op)
@@ -459,19 +463,21 @@
                 self.make_int_ge(op.getarg(0), op.getarg(1))
 
     def propagate_bounds_INT_GT(self, op):
-        r = self.getvalue(op)
+        r = self.get_box_replacement(op)
         if r.is_constant():
-            if r.box.same_constant(CONST_1):
+            if r.getint() == 1:
                 self.make_int_gt(op.getarg(0), op.getarg(1))
             else:
+                assert r.getint() == 0
                 self.make_int_le(op.getarg(0), op.getarg(1))
 
     def propagate_bounds_INT_LE(self, op):
-        r = self.getvalue(op)
+        r = self.get_box_replacement(op)
         if r.is_constant():
-            if r.box.same_constant(CONST_1):
+            if r.getint() == 1:
                 self.make_int_le(op.getarg(0), op.getarg(1))
             else:
+                assert r.getint() == 0
                 self.make_int_gt(op.getarg(0), op.getarg(1))
 
     def propagate_bounds_INT_GE(self, op):

From noreply at buildbot.pypy.org  Fri Feb 27 18:00:54 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 27 Feb 2015 18:00:54 +0100 (CET)
Subject: [pypy-commit] pypy optresult: (fijal, arigo) make nonnull tests pass
Message-ID: <20150227170054.296931C02BD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r76185:4011499ac0a2
Date: 2015-02-27 19:00 +0200
http://bitbucket.org/pypy/pypy/changeset/4011499ac0a2/

Log:	(fijal, arigo) make nonnull tests pass

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -25,10 +25,19 @@
 class PtrInfo(AbstractInfo):
     _attrs_ = ()
 
+    def is_nonnull(self):
+        return False
+
+    def is_null(self):
+        return False
+
     
 class NonNullPtrInfo(PtrInfo):
     _attrs_ = ()
 
+    def is_nonnull(self):
+        return True
+
     
 class InstancePtrInfo(NonNullPtrInfo):
     _attrs_ = ('_known_class', '_is_virtual', '_fields')
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -332,6 +332,9 @@
     def make_equal_to(self, box, value):
         return self.optimizer.make_equal_to(box, value)
 
+    def make_nonnull(self, op):
+        return self.optimizer.make_nonnull(op)
+
     def get_constant_box(self, box):
         return self.optimizer.get_constant_box(box)
 
@@ -555,6 +558,14 @@
     def make_constant_int(self, box, intvalue):
         self.make_constant(box, ConstInt(intvalue))
 
+    def make_nonnull(self, op):
+        op = self.get_box_replacement(op)
+        opinfo = op.get_forwarded()
+        if opinfo is not None:
+            assert opinfo.is_nonnull()
+            return
+        op.set_forwarded(info.NonNullPtrInfo())
+
     def new_ptr_box(self):
         xxx
         return self.cpu.ts.BoxRef()
@@ -642,24 +653,6 @@
         self._last_emitted_op = op
         self._newoperations.append(op)
 
-    def get_op_replacement(self, op):
-        # XXX this is wrong
-        xxx
-        changed = False
-        for i, arg in enumerate(op.getarglist()):
-            try:
-                v = self.values[arg]
-            except KeyError:
-                pass
-            else:
-                box = v.get_key_box()
-                if box is not arg:
-                    if not changed:
-                        changed = True
-                        op = op.copy_and_change(op.getopnum())
-                    op.setarg(i, box)
-        return op
-
     def replace_guard_op(self, old_op_pos, new_op):
         old_op = self._newoperations[old_op_pos]
         assert old_op.is_guard()
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -129,9 +129,9 @@
 
         # If one side of the op is 0 the result is the other side.
         if arg1.is_constant() and arg1.getint() == 0:
-            self.make_equal_to(op, v2)
+            self.make_equal_to(op, arg2)
         elif arg2.is_constant() and arg2.getint() == 0:
-            self.make_equal_to(op, v1)
+            self.make_equal_to(op, arg1)
         else:
             self.emit_operation(op)
             self.optimizer.pure_reverse(op)
@@ -274,15 +274,15 @@
         value.make_constant(self.optimizer.cpu.ts.CONST_NULL)
 
     def optimize_GUARD_NONNULL(self, op):
-        value = self.getvalue(op.getarg(0))
-        if value.is_nonnull():
+        opinfo = self.getptrinfo(op.getarg(0))
+        if opinfo.is_nonnull():
             return
-        elif value.is_null():
+        elif opinfo.is_null():
             r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop(op)
             raise InvalidLoop('A GUARD_NONNULL (%s) was proven to always fail'
                               % r)
         self.emit_operation(op)
-        value.make_nonnull(self.optimizer)
+        self.make_nonnull(op.getarg(0))
 
     def optimize_GUARD_VALUE(self, op):
         ## value = self.getvalue(op.getarg(0))

From noreply at buildbot.pypy.org  Fri Feb 27 21:34:21 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 27 Feb 2015 21:34:21 +0100 (CET)
Subject: [pypy-commit] pypy optresult: small progress
Message-ID: <20150227203421.572F01C009F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult
Changeset: r76186:f087cd1a3f06
Date: 2015-02-27 22:34 +0200
http://bitbucket.org/pypy/pypy/changeset/f087cd1a3f06/

Log:	small progress

diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -63,7 +63,8 @@
 
     def _optimize_guard_true_false_value(self, op):
         self.emit_operation(op)
-        self.propagate_bounds_backward(op.getarg(0))
+        if op.getarg(0).type == 'i':
+            self.propagate_bounds_backward(op.getarg(0))
 
     optimize_GUARD_TRUE = _optimize_guard_true_false_value
     optimize_GUARD_FALSE = _optimize_guard_true_false_value
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -546,10 +546,11 @@
         return newop
 
     def make_constant(self, box, constbox):
-        assert isinstance(constbox, ConstInt)
+        assert isinstance(constbox, Const)
         box = self.get_box_replacement(box)
         if not we_are_translated():    # safety-check
-            if box.get_forwarded() is not None:
+            if (box.get_forwarded() is not None and
+                isinstance(constbox, ConstInt)):
                 assert box.get_forwarded().contains(constbox.getint())
         if box.is_constant():
             return
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -317,14 +317,15 @@
         ops = """
         [p1]
         guard_value(p1, ConstPtr(myptr)) []
+        guard_value(p1, ConstPtr(myptr)) []
         jump(ConstPtr(myptr))
         """
         expected = """
-        []
-        jump()
-        """
-        py.test.skip("XXX")
-        self.optimize_loop(ops, 'Constant(myptr)', expected)
+        [p1]
+        guard_value(p1, ConstPtr(myptr)) []
+        jump(ConstPtr(myptr))
+        """
+        self.optimize_loop(ops, expected)
 
     def test_ooisnull_oononnull_1(self):
         ops = """

From noreply at buildbot.pypy.org  Fri Feb 27 22:34:45 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 27 Feb 2015 22:34:45 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Set recursion limit higher in conftest.py
 because otherwise many app-level tests fail (this is probably because the
 import system is now written in pure python).
Message-ID: <20150227213445.D2D6A1C08C1@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76187:a535e6deaf20
Date: 2015-02-27 22:34 +0100
http://bitbucket.org/pypy/pypy/changeset/a535e6deaf20/

Log:	Set recursion limit higher in conftest.py because otherwise many
	app-level tests fail (this is probably because the import system is
	now written in pure python).

diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -1,6 +1,10 @@
 import py, pytest, sys, os, textwrap
 from inspect import isclass
 
+if hasattr(sys, 'setrecursionlimit'):
+    # some tests fail otherwise
+    sys.setrecursionlimit(2000)
+
 PYTHON3 = os.getenv('PYTHON3') or py.path.local.sysfind('python3')
 if PYTHON3 is not None:
     PYTHON3 = str(PYTHON3)

From noreply at buildbot.pypy.org  Fri Feb 27 22:43:59 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 27 Feb 2015 22:43:59 +0100 (CET)
Subject: [pypy-commit] stmgc c8-card-marking: in-progress state that needs
 overflow objs to continue
Message-ID: <20150227214359.E024D1C02BB@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-card-marking
Changeset: r1674:11b04ac95b6c
Date: 2015-02-27 11:22 +0100
http://bitbucket.org/pypy/stmgc/changeset/11b04ac95b6c/

Log:	in-progress state that needs overflow objs to continue

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -610,9 +610,11 @@
            that the write_slowpath will not be called again until the
            next minor collection. */
         if (obj->stm_flags & GCFLAG_CARDS_SET) {
-            /* if we clear this flag, we also need to clear the cards */
+            /* if we clear this flag, we also need to clear the cards.
+               bk_slices are not needed as this is a new object */
+            /* XXX: add_missing_bk_slices_and_"clear"_cards */
             _reset_object_cards(get_priv_segment(STM_SEGMENT->segment_num),
-                                obj, CARD_CLEAR, false);
+                                obj, CARD_CLEAR, false, false);
         }
         obj->stm_flags &= ~(GCFLAG_WRITE_BARRIER | GCFLAG_CARDS_SET);
         LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj);
@@ -731,8 +733,12 @@
         if (obj->stm_flags & GCFLAG_CARDS_SET) {
             /* if we clear this flag, we have to tell sync_old_objs that
                everything needs to be synced */
+            /* if we clear this flag, we have to tell later write barriers
+               that we already did all backup slices: */
+            /* XXX: do_other_bk_slices_and_"clear"_cards */
             _reset_object_cards(get_priv_segment(STM_SEGMENT->segment_num),
-                                obj, CARD_MARKED_OLD, true); /* mark all */
+                                obj, STM_SEGMENT->transaction_read_version,
+                                true, false); /* mark all */
         }
 
         /* remove the WRITE_BARRIER flag and add WB_EXECUTED */
@@ -840,7 +846,8 @@
 #endif
         memset(readmarkers, 0, NB_READMARKER_PAGES * 4096UL);
     }
-    STM_SEGMENT->transaction_read_version = 1;
+    STM_SEGMENT->transaction_read_version = 2;
+    assert(STM_SEGMENT->transaction_read_version > _STM_CARD_MARKED);
 }
 
 static void reset_cards_from_modified_objects(void)
@@ -1308,8 +1315,27 @@
 }
 
 
-static void _page_wise_synchronize_object_now(object_t *obj, ssize_t obj_size)
+
+static void synchronize_object_enqueue(object_t *obj)
 {
+    assert(!_is_young(obj));
+    assert(STM_PSEGMENT->privatization_lock);
+    assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
+    assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED));
+
+    ssize_t obj_size = stmcb_size_rounded_up(
+        (struct object_s *)REAL_ADDRESS(STM_SEGMENT->segment_base, obj));
+    OPT_ASSERT(obj_size >= 16);
+
+    if (LIKELY(is_small_uniform(obj))) {
+        assert(!(obj->stm_flags & GCFLAG_CARDS_SET));
+        OPT_ASSERT(obj_size <= GC_LAST_SMALL_SIZE);
+        _synchronize_fragment((stm_char *)obj, obj_size);
+        return;
+    }
+
+    /* else, a more complicated case for large objects, to copy
+       around data only within the needed pages */
     uintptr_t start = (uintptr_t)obj;
     uintptr_t end = start + obj_size;
 
@@ -1330,135 +1356,6 @@
     } while (start != end);
 }
 
-static void _card_wise_synchronize_object_now(object_t *obj, ssize_t obj_size)
-{
-    assert(obj_size >= 32);
-    assert(obj_should_use_cards(STM_SEGMENT->segment_base, obj));
-    assert(!(obj->stm_flags & GCFLAG_CARDS_SET));
-
-    uintptr_t offset_itemsize[2];
-    struct object_s *realobj = (struct object_s *)REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
-    stmcb_get_card_base_itemsize(realobj, offset_itemsize);
-    size_t real_idx_count = (obj_size - offset_itemsize[0]) / offset_itemsize[1];
-
-    struct stm_read_marker_s *cards = get_read_marker(STM_SEGMENT->segment_base, (uintptr_t)obj);
-    uintptr_t card_index = 1;
-    uintptr_t last_card_index = get_index_to_card_index(real_idx_count - 1); /* max valid index */
-    assert(cards->rm == STM_SEGMENT->transaction_read_version); /* stm_read() */
-
-    /* simple heuristic to check if probably the whole object is
-       marked anyway so we should do page-wise synchronize */
-    if (cards[1].rm == CARD_MARKED_OLD
-        && cards[last_card_index].rm == CARD_MARKED_OLD
-        && cards[(last_card_index >> 1) + 1].rm == CARD_MARKED_OLD) {
-
-        dprintf(("card_wise_sync assumes %p,size:%lu is fully marked\n", obj, obj_size));
-        _reset_object_cards(get_priv_segment(STM_SEGMENT->segment_num),
-                            obj, CARD_CLEAR, false);
-        _page_wise_synchronize_object_now(obj, obj_size);
-        return;
-    }
-
-    dprintf(("card_wise_sync syncs %p,size:%lu card-wise\n", obj, obj_size));
-
-    /* Combine multiple marked cards and do a memcpy for them. We don't
-       try yet to use page_copy() or otherwise take into account privatization
-       of pages (except _has_private_page_in_range) */
-    bool all_cards_were_cleared = true;
-
-    uintptr_t start_card_index = -1;
-    while (card_index <= last_card_index) {
-        uint8_t card_value = cards[card_index].rm;
-
-        if (card_value == CARD_MARKED_OLD) {
-            cards[card_index].rm = CARD_CLEAR;
-
-            if (start_card_index == -1) {   /* first marked card */
-                start_card_index = card_index;
-                /* start = (uintptr_t)obj + stmcb_index_to_byte_offset( */
-                /*     realobj, get_card_index_to_index(card_index)); */
-                if (all_cards_were_cleared) {
-                    all_cards_were_cleared = false;
-                }
-            }
-        }
-        else {
-            OPT_ASSERT(card_value == CARD_CLEAR);
-        }
-
-        if (start_card_index != -1                    /* something to copy */
-            && (card_value != CARD_MARKED_OLD         /* found non-marked card */
-                || card_index == last_card_index)) {  /* this is the last card */
-            /* do the copying: */
-            uintptr_t start, copy_size;
-            uintptr_t next_card_offset;
-            uintptr_t start_card_offset;
-            uintptr_t next_card_index = card_index;
-
-            if (card_value == CARD_MARKED_OLD) {
-                /* card_index is the last card of the object, but we need
-                   to go one further to get the right offset */
-                next_card_index++;
-            }
-
-            start_card_offset = offset_itemsize[0] +
-                get_card_index_to_index(start_card_index) * offset_itemsize[1];
-
-            next_card_offset = offset_itemsize[0] +
-                get_card_index_to_index(next_card_index) * offset_itemsize[1];
-
-            if (next_card_offset > obj_size)
-                next_card_offset = obj_size;
-
-            start = (uintptr_t)obj + start_card_offset;
-            copy_size = next_card_offset - start_card_offset;
-            OPT_ASSERT(copy_size > 0);
-
-            /* push to seg0 and enqueue for synchronization */
-            _synchronize_fragment((stm_char *)start, copy_size);
-
-            start_card_index = -1;
-        }
-
-        card_index++;
-    }
-
-    if (all_cards_were_cleared) {
-        /* well, seems like we never called stm_write_card() on it, so actually
-           we need to fall back to synchronize the whole object */
-        _page_wise_synchronize_object_now(obj, obj_size);
-        return;
-    }
-
-}
-
-
-static void synchronize_object_enqueue(object_t *obj, bool ignore_cards)
-{
-    assert(!_is_young(obj));
-    assert(STM_PSEGMENT->privatization_lock);
-    assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
-    assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED));
-
-    ssize_t obj_size = stmcb_size_rounded_up(
-        (struct object_s *)REAL_ADDRESS(STM_SEGMENT->segment_base, obj));
-    OPT_ASSERT(obj_size >= 16);
-
-    if (LIKELY(is_small_uniform(obj))) {
-        assert(!(obj->stm_flags & GCFLAG_CARDS_SET));
-        OPT_ASSERT(obj_size <= GC_LAST_SMALL_SIZE);
-        _synchronize_fragment((stm_char *)obj, obj_size);
-        return;
-    } else if (ignore_cards || !obj_should_use_cards(STM_SEGMENT->segment_base, obj)) {
-        /* else, a more complicated case for large objects, to copy
-           around data only within the needed pages */
-        _page_wise_synchronize_object_now(obj, obj_size);
-    } else {
-        /* ... or even only cards that need to be updated */
-        _card_wise_synchronize_object_now(obj, obj_size);
-    }
-}
-
 static void synchronize_objects_flush(void)
 {
     long j = STM_PSEGMENT->sq_len;
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -50,7 +50,7 @@
 enum /* card values in read markers */ {
     CARD_CLEAR = 0,                 /* card not used at all */
     CARD_MARKED = _STM_CARD_MARKED, /* card marked for tracing in the next gc */
-    CARD_MARKED_OLD = _STM_CARD_MARKED+1,
+    /* CARD_MARKED_OLD = STM_PSEGMENT->transaction_read_version, */
     /* card was marked before, but cleared in a GC */
 };
 
@@ -246,7 +246,7 @@
 static stm_thread_local_t *abort_with_mutex_no_longjmp(void);
 static void abort_data_structures_from_segment_num(int segment_num);
 
-static void synchronize_object_enqueue(object_t *obj, bool ignore_cards);
+static void synchronize_object_enqueue(object_t *obj);
 static void synchronize_objects_flush(void);
 
 static void _signal_handler(int sig, siginfo_t *siginfo, void *context);
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -518,8 +518,9 @@
 
                 /* mark marked cards as old if it survives */
                 uint8_t mark_value = mark_visited_test(item) ?
-                    CARD_MARKED_OLD : CARD_CLEAR;
-                _reset_object_cards(pseg, item, mark_value, false);
+                    pseg->pub.transaction_read_version : CARD_CLEAR;
+                _reset_object_cards(pseg, item, mark_value, false,
+                                    mark_value == CARD_CLEAR);
             }));
         list_clear(lst);
 
@@ -531,7 +532,7 @@
             object_t *obj = (object_t *)list_item(lst, n);
             if (!mark_visited_test(obj)) {
                 if (obj_should_use_cards(pseg->pub.segment_base, obj))
-                    _reset_object_cards(pseg, obj, CARD_CLEAR, false);
+                    _reset_object_cards(pseg, obj, CARD_CLEAR, false, true);
                 list_set_item(lst, n, list_pop_item(lst));
             }
         }
diff --git a/c8/stm/misc.c b/c8/stm/misc.c
--- a/c8/stm/misc.c
+++ b/c8/stm/misc.c
@@ -118,6 +118,11 @@
     return cards[get_index_to_card_index(idx)].rm;
 }
 
+uint8_t _stm_get_transaction_read_version()
+{
+    return STM_SEGMENT->transaction_read_version;
+}
+
 
 
 static struct stm_commit_log_entry_s *_last_cl_entry;
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -159,7 +159,8 @@
     uintptr_t last_card_index = get_index_to_card_index(size - 1); /* max valid index */
 
     while (card_index <= last_card_index) {
-        assert(cards[card_index].rm == CARD_CLEAR);
+        assert(cards[card_index].rm == CARD_CLEAR
+               || cards[card_index].rm < pseg->pub.transaction_read_version);
         card_index++;
     }
 
@@ -191,7 +192,7 @@
 
 static void _reset_object_cards(struct stm_priv_segment_info_s *pseg,
                                 object_t *obj, uint8_t mark_value,
-                                bool mark_all)
+                                bool mark_all, bool really_clear)
 {
 #pragma push_macro("STM_PSEGMENT")
 #pragma push_macro("STM_SEGMENT")
@@ -205,8 +206,12 @@
     stmcb_get_card_base_itemsize(realobj, offset_itemsize);
     size = (size - offset_itemsize[0]) / offset_itemsize[1];
 
+    /* really_clear only used for freed new objs in minor collections, as
+       they need to clear cards even if they are set to transaction_read_version */
+    assert(IMPLY(really_clear, mark_value == CARD_CLEAR && !mark_all));
     assert(IMPLY(mark_value == CARD_CLEAR, !mark_all)); /* not necessary */
-    assert(IMPLY(mark_all, mark_value == CARD_MARKED_OLD)); /* set *all* to OLD */
+    assert(IMPLY(mark_all,
+                 mark_value == pseg->pub.transaction_read_version)); /* set *all* to OLD */
 
     struct stm_read_marker_s *cards = get_read_marker(pseg->pub.segment_base, (uintptr_t)obj);
     uintptr_t card_index = 1;
@@ -216,7 +221,8 @@
                 obj, size, mark_value, mark_all));
        dprintf(("obj has %lu cards\n", last_card_index));*/
     while (card_index <= last_card_index) {
-        if (mark_all || cards[card_index].rm != CARD_CLEAR) {
+        if (mark_all || cards[card_index].rm == CARD_MARKED
+            || (really_clear && cards[card_index].rm != CARD_CLEAR)) {
             /* dprintf(("mark card %lu,wl:%lu of %p with %d\n", */
             /*          card_index, card_lock_idx, obj, mark_value)); */
             cards[card_index].rm = mark_value;
@@ -238,7 +244,6 @@
     assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
 
     dprintf(("_trace_card_object(%p)\n", obj));
-    uint8_t mark_value = CARD_MARKED_OLD;
 
     struct object_s *realobj = (struct object_s *)REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
     size_t size = stmcb_size_rounded_up(realobj);
@@ -254,7 +259,7 @@
     while (card_index <= last_card_index) {
         if (cards[card_index].rm == CARD_MARKED) {
             /* clear or set to old: */
-            cards[card_index].rm = mark_value;
+            cards[card_index].rm = STM_SEGMENT->transaction_read_version;
 
             uintptr_t start = get_card_index_to_index(card_index);
             uintptr_t stop = get_card_index_to_index(card_index + 1);
@@ -340,7 +345,7 @@
             struct stm_priv_segment_info_s *pseg = get_priv_segment(STM_SEGMENT->segment_num);
             if (pseg->minor_collect_will_commit_now) {
                 acquire_privatization_lock(pseg->pub.segment_num);
-                synchronize_object_enqueue(obj, true); /* ignore cards! */
+                synchronize_object_enqueue(obj);
                 release_privatization_lock(pseg->pub.segment_num);
             } else {
                 LIST_APPEND(pseg->new_objects, obj);
diff --git a/c8/stm/nursery.h b/c8/stm/nursery.h
--- a/c8/stm/nursery.h
+++ b/c8/stm/nursery.h
@@ -5,7 +5,7 @@
 static void _cards_cleared_in_object(struct stm_priv_segment_info_s *pseg, object_t *obj);
 static void _reset_object_cards(struct stm_priv_segment_info_s *pseg,
                                 object_t *obj, uint8_t mark_value,
-                                bool mark_all);
+                                bool mark_all, bool really_clear);
 
 static void minor_collection(bool commit);
 static void check_nursery_at_transaction_start(void);
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -77,7 +77,7 @@
 #define _STM_NSE_SIGNAL_ABORT             1
 #define _STM_NSE_SIGNAL_MAX               2
 
-#define _STM_CARD_MARKED 1
+#define _STM_CARD_MARKED 1      /* should always be 1... */
 #define _STM_GCFLAG_CARDS_SET          0x8
 #define _STM_CARD_SIZE                 32     /* must be >= 32 */
 #define _STM_MIN_CARD_COUNT            17
@@ -94,6 +94,7 @@
 char *_stm_real_address(object_t *o);
 #ifdef STM_TESTS
 #include 
+uint8_t _stm_get_transaction_read_version();
 uint8_t _stm_get_card_value(object_t *obj, long idx);
 bool _stm_was_read(object_t *obj);
 bool _stm_was_written(object_t *obj);
diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -48,6 +48,7 @@
 /*void stm_write_card(); use _checked_stm_write_card() instead */
 
 uint8_t _stm_get_card_value(object_t *obj, long idx);
+uint8_t _stm_get_transaction_read_version();
 
 void stm_setup(void);
 void stm_teardown(void);
@@ -401,7 +402,7 @@
 CARD_SIZE = lib._STM_CARD_SIZE # 16b at least
 CARD_CLEAR = 0
 CARD_MARKED = lib._STM_CARD_MARKED
-CARD_MARKED_OLD = CARD_MARKED + 1
+CARD_MARKED_OLD = lib._stm_get_transaction_read_version
 
 
 class Conflict(Exception):
diff --git a/c8/test/test_card_marking.py b/c8/test/test_card_marking.py
--- a/c8/test/test_card_marking.py
+++ b/c8/test/test_card_marking.py
@@ -235,7 +235,7 @@
         assert o in old_objects_with_cards_set()
 
         stm_minor_collect()
-        assert get_card_value(o, 1000) == CARD_MARKED_OLD
+        assert get_card_value(o, 1000) == CARD_MARKED_OLD()
         self.commit_transaction()
 
         self.start_transaction()
@@ -288,3 +288,43 @@
         assert o not in old_objects_with_cards_set()
 
         self.commit_transaction()
+
+    def test_clear_cards4(self):
+        self.start_transaction()
+        o = stm_allocate(1000+20*CARD_SIZE)
+        p = stm_allocate(1000+20*CARD_SIZE)
+        assert get_card_value(o, 1000) == CARD_CLEAR
+        assert get_card_value(p, 1000) == CARD_CLEAR
+
+        self.push_root(o)
+        self.push_root(p)
+        stm_minor_collect()
+        p = self.pop_root()
+        o = self.pop_root()
+
+        assert get_card_value(o, 1000) == CARD_CLEAR
+        assert get_card_value(p, 1000) == CARD_CLEAR
+        stm_set_char(o, 'b', 1000, True)
+        stm_set_char(p, 'b', 1000, True)
+        assert get_card_value(o, 1000) == CARD_MARKED
+        assert get_card_value(p, 1000) == CARD_MARKED
+
+        self.push_root(o)
+        stm_minor_collect()
+        o = self.pop_root()
+
+        assert get_card_value(o, 1000) == CARD_MARKED_OLD()
+        assert get_card_value(p, 1000) == CARD_CLEAR
+
+        self.push_root(o)
+        self.commit_transaction()
+
+        self.start_transaction()
+        o = self.pop_root()
+
+        assert get_card_value(o, 1000) != CARD_CLEAR
+        assert get_card_value(o, 1000) < CARD_MARKED_OLD()
+        stm_set_char(o, 'b', 1000, True)
+        assert get_card_value(o, 1000) == CARD_MARKED
+
+        self.commit_transaction()

From noreply at buildbot.pypy.org  Fri Feb 27 22:44:01 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 27 Feb 2015 22:44:01 +0100 (CET)
Subject: [pypy-commit] stmgc c8-overflow-objs: rename new_objects to
	overflow objs
Message-ID: <20150227214401.0085F1C02BB@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-overflow-objs
Changeset: r1675:c41ca067e953
Date: 2015-02-27 11:35 +0100
http://bitbucket.org/pypy/stmgc/changeset/c41ca067e953/

Log:	rename new_objects to overflow objs

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -766,7 +766,7 @@
     }
 
     assert(list_is_empty(STM_PSEGMENT->modified_old_objects));
-    assert(list_is_empty(STM_PSEGMENT->new_objects));
+    assert(list_is_empty(STM_PSEGMENT->large_overflow_objects));
     assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery));
     assert(list_is_empty(STM_PSEGMENT->young_weakrefs));
     assert(tree_is_cleared(STM_PSEGMENT->young_outside_nursery));
@@ -827,7 +827,7 @@
     STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
     STM_PSEGMENT->transaction_state = TS_NONE;
     list_clear(STM_PSEGMENT->objects_pointing_to_nursery);
-    list_clear(STM_PSEGMENT->new_objects);
+    list_clear(STM_PSEGMENT->large_overflow_objects);
 
     release_thread_segment(tl);
     /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */
@@ -847,10 +847,11 @@
 #endif
 }
 
-static void push_new_objects_to_other_segments(void)
+static void push_large_overflow_objects_to_other_segments(void)
 {
+    /* XXX: also pushes small ones right now */
     acquire_privatization_lock(STM_SEGMENT->segment_num);
-    LIST_FOREACH_R(STM_PSEGMENT->new_objects, object_t *,
+    LIST_FOREACH_R(STM_PSEGMENT->large_overflow_objects, object_t *,
         ({
             assert(item->stm_flags & GCFLAG_WB_EXECUTED);
             item->stm_flags &= ~GCFLAG_WB_EXECUTED;
@@ -867,7 +868,7 @@
        in handle_segfault_in_page() that also copies
        unknown-to-the-segment/uncommitted things.
     */
-    list_clear(STM_PSEGMENT->new_objects);
+    list_clear(STM_PSEGMENT->large_overflow_objects);
 }
 
 
@@ -882,7 +883,7 @@
     dprintf(("> stm_commit_transaction()\n"));
     minor_collection(1);
 
-    push_new_objects_to_other_segments();
+    push_large_overflow_objects_to_other_segments();
     /* push before validate. otherwise they are reachable too early */
     bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE;
     _validate_and_add_to_commit_log();
@@ -1008,7 +1009,7 @@
     tl->last_abort__bytes_in_nursery = bytes_in_nursery;
 
     list_clear(pseg->objects_pointing_to_nursery);
-    list_clear(pseg->new_objects);
+    list_clear(pseg->large_overflow_objects);
     list_clear(pseg->young_weakrefs);
 #pragma pop_macro("STM_SEGMENT")
 #pragma pop_macro("STM_PSEGMENT")
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -88,8 +88,9 @@
     /* list of objects created in the current transaction and
        that survived at least one minor collection. They need
        to be synchronized to other segments on commit, but they
-       do not need to be in the commit log entry. */
-    struct list_s *new_objects;
+       do not need to be in the commit log entry.
+       XXX: for now it also contains small overflow objs */
+    struct list_s *large_overflow_objects;
 
     uint8_t privatization_lock;  // XXX KILL
 
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -242,7 +242,7 @@
     stmcb_trace(realobj, &mark_record_trace);
 
     /* trace all references found in sharing seg0 (should always be
-       up-to-date and not cause segfaults, except for new objs) */
+       up-to-date and not cause segfaults, except for overflow objs) */
     while (!list_is_empty(marked_objects_to_trace)) {
         obj = (object_t *)list_pop_item(marked_objects_to_trace);
 
@@ -380,7 +380,7 @@
         /* look at all objs on the shadow stack (they are old but may
            be uncommitted so far, so only exist in the associated_segment_num).
 
-           IF they are uncommitted new objs, trace in the actual segment,
+           IF they are uncommitted overflow objs, trace in the actual segment,
            otherwise, since we just executed a minor collection, they were
            all synced to the sharing seg0. Thus we can trace them there.
 
@@ -423,13 +423,13 @@
     }
 }
 
-static void ready_new_objects(void)
+static void ready_large_overflow_objects(void)
 {
 #pragma push_macro("STM_PSEGMENT")
 #pragma push_macro("STM_SEGMENT")
 #undef STM_PSEGMENT
 #undef STM_SEGMENT
-    /* objs in new_objects only have garbage in the sharing seg0,
+    /* objs in large_overflow only have garbage in the sharing seg0,
        since it is used to mark objs as visited, we must make
        sure the flag is cleared at the start of a major collection.
        (XXX: ^^^ may be optional if we have the part below)
@@ -437,14 +437,14 @@
        Also, we need to be able to recognize these objects in order
        to only trace them in the segment they are valid in. So we
        also make sure to set WB_EXECUTED in the sharing seg0. No
-       other objs than new_objects have WB_EXECUTED in seg0 (since
+       other objs than large_overflow_objects have WB_EXECUTED in seg0 (since
        there can only be committed versions there).
     */
 
     long i;
     for (i = 1; i < NB_SEGMENTS; i++) {
         struct stm_priv_segment_info_s *pseg = get_priv_segment(i);
-        struct list_s *lst = pseg->new_objects;
+        struct list_s *lst = pseg->large_overflow_objects;
 
         LIST_FOREACH_R(lst, object_t* /*item*/,
             ({
@@ -507,8 +507,8 @@
                modified_old_objs. */
         }
 
-        /* remove from new_objects all objects that die */
-        lst = pseg->new_objects;
+        /* remove from large_overflow_objects all objects that die */
+        lst = pseg->large_overflow_objects;
         uintptr_t n = list_count(lst);
         while (n-- > 0) {
             object_t *obj = (object_t *)list_item(lst, n);
@@ -683,7 +683,7 @@
 
     DEBUG_EXPECT_SEGFAULT(false);
 
-    ready_new_objects();
+    ready_large_overflow_objects();
 
     /* marking */
     LIST_CREATE(marked_objects_to_trace);
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -132,9 +132,8 @@
         nobj_sync_now = ((uintptr_t)nobj) | FLAG_SYNC_LARGE;
     }
 
-    /* if this is not during commit, we will add them to the new_objects
-       list and push them to other segments on commit. Thus we can add
-       the WB_EXECUTED flag so that they don't end up in modified_old_objects */
+    /* if this is not during commit, we make them overflow objects
+       and push them to other segments on commit. */
     assert(!(nobj->stm_flags & GCFLAG_WB_EXECUTED));
     if (!STM_PSEGMENT->minor_collect_will_commit_now) {
         nobj->stm_flags |= GCFLAG_WB_EXECUTED;
@@ -203,15 +202,16 @@
         _collect_now(obj);
 
         if (obj_sync_now & FLAG_SYNC_LARGE) {
+            /* XXX: SYNC_LARGE even set for small objs right now */
             /* this is a newly allocated obj in this transaction. We must
                either synchronize the object to other segments now, or
-               add the object to new_objects list */
+               add the object to large_overflow_objects list */
             if (STM_PSEGMENT->minor_collect_will_commit_now) {
                 acquire_privatization_lock(STM_SEGMENT->segment_num);
                 synchronize_object_enqueue(obj);
                 release_privatization_lock(STM_SEGMENT->segment_num);
             } else {
-                LIST_APPEND(STM_PSEGMENT->new_objects, obj);
+                LIST_APPEND(STM_PSEGMENT->large_overflow_objects, obj);
             }
         }
 
@@ -219,7 +219,7 @@
         lst = STM_PSEGMENT->objects_pointing_to_nursery;
     }
 
-    /* flush all new objects to other segments now */
+    /* flush all overflow objects to other segments now */
     if (STM_PSEGMENT->minor_collect_will_commit_now) {
         acquire_privatization_lock(STM_SEGMENT->segment_num);
         synchronize_objects_flush();
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -100,7 +100,7 @@
         pr->pub.segment_num = i;
         pr->pub.segment_base = segment_base;
         pr->modified_old_objects = list_create();
-        pr->new_objects = list_create();
+        pr->large_overflow_objects = list_create();
         pr->young_weakrefs = list_create();
         pr->old_weakrefs = list_create();
         pr->objects_pointing_to_nursery = list_create();
@@ -148,8 +148,8 @@
         assert(list_is_empty(pr->objects_pointing_to_nursery));
         list_free(pr->objects_pointing_to_nursery);
         list_free(pr->modified_old_objects);
-        assert(list_is_empty(pr->new_objects));
-        list_free(pr->new_objects);
+        assert(list_is_empty(pr->large_overflow_objects));
+        list_free(pr->large_overflow_objects);
         list_free(pr->young_weakrefs);
         list_free(pr->old_weakrefs);
         tree_free(pr->young_outside_nursery);

From noreply at buildbot.pypy.org  Fri Feb 27 22:44:02 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 27 Feb 2015 22:44:02 +0100 (CET)
Subject: [pypy-commit] stmgc c8-overflow-objs: implement overflow objs
Message-ID: <20150227214402.181BA1C02BB@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-overflow-objs
Changeset: r1676:20294d4006bc
Date: 2015-02-27 13:51 +0100
http://bitbucket.org/pypy/stmgc/changeset/20294d4006bc/

Log:	implement overflow objs

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -582,6 +582,16 @@
     assert(!_is_in_nursery(obj));
     assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
 
+    if (obj->stm_flags & GCFLAG_WB_EXECUTED
+        || IS_OVERFLOW_OBJ(STM_PSEGMENT, obj)) {
+        /* already executed WB once in this transaction or is
+           overflow object -> do only GC part: */
+        dprintf(("write_slowpath-fast(%p)\n", obj));
+        obj->stm_flags &= ~GCFLAG_WRITE_BARRIER;
+        LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj);
+        return;
+    }
+
     int my_segnum = STM_SEGMENT->segment_num;
     uintptr_t end_page, first_page = ((uintptr_t)obj) / 4096UL;
     char *realobj;
@@ -599,15 +609,6 @@
     /* add to read set: */
     stm_read(obj);
 
-    if (obj->stm_flags & GCFLAG_WB_EXECUTED) {
-        /* already executed WB once in this transaction. do GC
-           part again: */
-        dprintf(("write_slowpath-fast(%p)\n", obj));
-        obj->stm_flags &= ~GCFLAG_WRITE_BARRIER;
-        LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj);
-        return;
-    }
-
     assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED));
     dprintf(("write_slowpath(%p): sz=%lu\n", obj, obj_size));
 
@@ -849,12 +850,14 @@
 
 static void push_large_overflow_objects_to_other_segments(void)
 {
+    if (list_is_empty(STM_PSEGMENT->large_overflow_objects))
+        return;
+
     /* XXX: also pushes small ones right now */
     acquire_privatization_lock(STM_SEGMENT->segment_num);
     LIST_FOREACH_R(STM_PSEGMENT->large_overflow_objects, object_t *,
         ({
-            assert(item->stm_flags & GCFLAG_WB_EXECUTED);
-            item->stm_flags &= ~GCFLAG_WB_EXECUTED;
+            assert(!(item->stm_flags & GCFLAG_WB_EXECUTED));
             synchronize_object_enqueue(item);
         }));
     synchronize_objects_flush();
@@ -908,6 +911,15 @@
 
     commit_finalizers();
 
+    /* update 'overflow_number' if needed */
+    if (STM_PSEGMENT->overflow_number_has_been_used) {
+        highest_overflow_number += GCFLAG_OVERFLOW_NUMBER_bit0;
+        assert(highest_overflow_number !=        /* XXX else, overflow! */
+               (uint32_t)-GCFLAG_OVERFLOW_NUMBER_bit0);
+        STM_PSEGMENT->overflow_number = highest_overflow_number;
+        STM_PSEGMENT->overflow_number_has_been_used = false;
+    }
+
     invoke_and_clear_user_callbacks(0);   /* for commit */
 
     if (globally_unique_transaction && was_inev) {
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -40,6 +40,15 @@
     GCFLAG_WB_EXECUTED = 0x04,
     GCFLAG_VISITED = 0x08,
     GCFLAG_FINALIZATION_ORDERING = 0x10,
+
+    /* All remaining bits of the 32-bit 'stm_flags' field are taken by
+       the "overflow number".  This is a number that identifies the
+       "overflow objects" from the current transaction among all old
+       objects.  More precisely, overflow objects are objects from the
+       current transaction that have been flushed out of the nursery,
+       which occurs if the same transaction allocates too many objects.
+    */
+    GCFLAG_OVERFLOW_NUMBER_bit0 = 0x20   /* must be last */
 };
 
 
@@ -102,6 +111,14 @@
 
     struct tree_s *callbacks_on_commit_and_abort[2];
 
+    /* This is the number stored in the overflowed objects (a multiple of
+       GCFLAG_OVERFLOW_NUMBER_bit0).  It is incremented when the
+       transaction is done, but only if we actually overflowed any
+       object; otherwise, no object has got this number. */
+    uint32_t overflow_number;
+    bool overflow_number_has_been_used;
+
+
     struct stm_commit_log_entry_s *last_commit_log_entry;
 
     struct stm_shadowentry_s *shadowstack_at_start_of_transaction;
@@ -194,6 +211,9 @@
 
 #define REAL_ADDRESS(segment_base, src)   ((segment_base) + (uintptr_t)(src))
 
+#define IS_OVERFLOW_OBJ(pseg, obj) (((obj)->stm_flags & -GCFLAG_OVERFLOW_NUMBER_bit0) \
+                                    == (pseg)->overflow_number)
+
 
 static inline char *get_segment_base(long segment_num) {
     return stm_object_pages + segment_num * (NB_PAGES * 4096UL);
diff --git a/c8/stm/finalizer.c b/c8/stm/finalizer.c
--- a/c8/stm/finalizer.c
+++ b/c8/stm/finalizer.c
@@ -98,14 +98,14 @@
         list_clear(lst);
     }
 
-    /* also deals with newly created objects: they are at the tail of
+    /* also deals with overflow objects: they are at the tail of
        old_objects_with_light_finalizers (this list is kept in order
        and we cannot add any already-committed object) */
     lst = pseg->old_objects_with_light_finalizers;
     count = list_count(lst);
     while (count > 0) {
         object_t *obj = (object_t *)list_item(lst, --count);
-        if (!(obj->stm_flags & GCFLAG_WB_EXECUTED))
+        if (!IS_OVERFLOW_OBJ(pseg, obj))
             break;
         lst->count = count;
         if (must_fix_gs) {
@@ -264,11 +264,14 @@
         LIST_APPEND(_finalizer_tmpstack, obj);
 }
 
-static inline struct list_s *finalizer_trace(char *base, object_t *obj,
-                                             struct list_s *lst)
+static inline struct list_s *finalizer_trace(
+    struct stm_priv_segment_info_s *pseg, object_t *obj, struct list_s *lst)
 {
-    if (!is_new_object(obj))
+    char *base;
+    if (!is_overflow_obj_safe(pseg, obj))
         base = stm_object_pages;
+    else
+        base = pseg->pub.segment_base;
 
     struct object_s *realobj = (struct object_s *)REAL_ADDRESS(base, obj);
     _finalizer_tmpstack = lst;
@@ -277,7 +280,8 @@
 }
 
 
-static void _recursively_bump_finalization_state_from_2_to_3(char *base, object_t *obj)
+static void _recursively_bump_finalization_state_from_2_to_3(
+    struct stm_priv_segment_info_s *pseg, object_t *obj)
 {
     assert(_finalization_state(obj) == 2);
     struct list_s *tmpstack = _finalizer_emptystack;
@@ -289,7 +293,7 @@
             realobj->stm_flags &= ~GCFLAG_FINALIZATION_ORDERING;
 
             /* trace */
-            tmpstack = finalizer_trace(base, obj, tmpstack);
+            tmpstack = finalizer_trace(pseg, obj, tmpstack);
         }
 
         if (list_is_empty(tmpstack))
@@ -300,14 +304,16 @@
     _finalizer_emptystack = tmpstack;
 }
 
-static void _recursively_bump_finalization_state_from_1_to_2(char *base, object_t *obj)
+static void _recursively_bump_finalization_state_from_1_to_2(
+    struct stm_priv_segment_info_s *pseg, object_t *obj)
 {
     assert(_finalization_state(obj) == 1);
     /* The call will add GCFLAG_VISITED recursively, thus bump state 1->2 */
-    mark_visit_possibly_new_object(base, obj);
+    mark_visit_possibly_new_object(obj, pseg);
 }
 
-static struct list_s *mark_finalize_step1(char *base, struct finalizers_s *f)
+static struct list_s *mark_finalize_step1(
+    struct stm_priv_segment_info_s *pseg, struct finalizers_s *f)
 {
     if (f == NULL)
         return NULL;
@@ -336,21 +342,22 @@
             int state = _finalization_state(y);
             if (state <= 0) {
                 _bump_finalization_state_from_0_to_1(y);
-                pending = finalizer_trace(base, y, pending);
+                pending = finalizer_trace(pseg, y, pending);
             }
             else if (state == 2) {
-                _recursively_bump_finalization_state_from_2_to_3(base, y);
+                _recursively_bump_finalization_state_from_2_to_3(pseg, y);
             }
         }
         _finalizer_pending = pending;
         assert(_finalization_state(x) == 1);
-        _recursively_bump_finalization_state_from_1_to_2(base, x);
+        _recursively_bump_finalization_state_from_1_to_2(pseg, x);
     }
     return marked;
 }
 
-static void mark_finalize_step2(char *base, struct finalizers_s *f,
-                                struct list_s *marked)
+static void mark_finalize_step2(
+    struct stm_priv_segment_info_s *pseg, struct finalizers_s *f,
+    struct list_s *marked)
 {
     if (f == NULL)
         return;
@@ -367,7 +374,7 @@
             if (run_finalizers == NULL)
                 run_finalizers = list_create();
             LIST_APPEND(run_finalizers, x);
-            _recursively_bump_finalization_state_from_2_to_3(base, x);
+            _recursively_bump_finalization_state_from_2_to_3(pseg, x);
         }
         else {
             struct list_s *lst = f->objects_with_finalizers;
@@ -403,29 +410,28 @@
     long j;
     for (j = 1; j < NB_SEGMENTS; j++) {
         struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
-        marked_seg[j] = mark_finalize_step1(pseg->pub.segment_base,
-                                            pseg->finalizers);
+        marked_seg[j] = mark_finalize_step1(pseg, pseg->finalizers);
     }
-    marked_seg[0] = mark_finalize_step1(stm_object_pages, &g_finalizers);
+    marked_seg[0] = mark_finalize_step1(get_priv_segment(0), &g_finalizers);
 
     LIST_FREE(_finalizer_pending);
 
     for (j = 1; j < NB_SEGMENTS; j++) {
         struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
-        mark_finalize_step2(pseg->pub.segment_base, pseg->finalizers,
-                            marked_seg[j]);
+        mark_finalize_step2(pseg, pseg->finalizers, marked_seg[j]);
     }
-    mark_finalize_step2(stm_object_pages, &g_finalizers, marked_seg[0]);
+    mark_finalize_step2(get_priv_segment(0), &g_finalizers, marked_seg[0]);
 
     LIST_FREE(_finalizer_emptystack);
 }
 
-static void mark_visit_from_finalizer1(char *base, struct finalizers_s *f)
+static void mark_visit_from_finalizer1(
+    struct stm_priv_segment_info_s *pseg, struct finalizers_s *f)
 {
     if (f != NULL && f->run_finalizers != NULL) {
         LIST_FOREACH_R(f->run_finalizers, object_t * /*item*/,
                        ({
-                           mark_visit_possibly_new_object(base, item);
+                           mark_visit_possibly_new_object(item, pseg);
                        }));
     }
 }
@@ -435,9 +441,9 @@
     long j;
     for (j = 1; j < NB_SEGMENTS; j++) {
         struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
-        mark_visit_from_finalizer1(pseg->pub.segment_base, pseg->finalizers);
+        mark_visit_from_finalizer1(pseg, pseg->finalizers);
     }
-    mark_visit_from_finalizer1(stm_object_pages, &g_finalizers);
+    mark_visit_from_finalizer1(get_priv_segment(0), &g_finalizers);
 }
 
 static void _execute_finalizers(struct finalizers_s *f)
diff --git a/c8/stm/forksupport.c b/c8/stm/forksupport.c
--- a/c8/stm/forksupport.c
+++ b/c8/stm/forksupport.c
@@ -84,7 +84,7 @@
     stm_thread_local_t *tl = pr->pub.running_thread;
     dprintf(("forksupport_child: abort in seg%ld\n", i));
     assert(tl->associated_segment_num == i);
-    assert(pr->transaction_state == TS_REGULAR);
+    assert(pr->transaction_state != TS_INEVITABLE);
     set_gs_register(get_segment_base(i));
     assert(STM_SEGMENT->segment_num == i);
 
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -200,14 +200,18 @@
 
 /************************************************************/
 
+static bool is_overflow_obj_safe(struct stm_priv_segment_info_s *pseg, object_t *obj)
+{
+    /* this function first also checks if the page is accessible in order
+       to not cause segfaults during major gc (it does exactly the same
+       as IS_OVERFLOW_OBJ otherwise) */
+    if (get_page_status_in(pseg->pub.segment_num, (uintptr_t)obj / 4096UL) == PAGE_NO_ACCESS)
+        return false;
 
-static bool is_new_object(object_t *obj)
-{
-    struct object_s *realobj = (struct object_s*)REAL_ADDRESS(stm_object_pages, obj); /* seg0 */
-    return realobj->stm_flags & GCFLAG_WB_EXECUTED;
+    struct object_s *realobj = (struct object_s*)REAL_ADDRESS(pseg->pub.segment_base, obj);
+    return IS_OVERFLOW_OBJ(pseg, realobj);
 }
 
-
 static inline void mark_record_trace(object_t **pobj)
 {
     /* takes a normal pointer to a thread-local pointer to an object */
@@ -230,7 +234,10 @@
 }
 
 
-static void mark_and_trace(object_t *obj, char *segment_base)
+static void mark_and_trace(
+    object_t *obj,
+    char *segment_base, /* to trace obj in */
+    struct stm_priv_segment_info_s *pseg) /* to trace children in */
 {
     /* mark the obj and trace all reachable objs from it */
 
@@ -243,35 +250,39 @@
 
     /* trace all references found in sharing seg0 (should always be
        up-to-date and not cause segfaults, except for overflow objs) */
+    segment_base = pseg->pub.segment_base;
     while (!list_is_empty(marked_objects_to_trace)) {
         obj = (object_t *)list_pop_item(marked_objects_to_trace);
 
-        char *base = is_new_object(obj) ? segment_base : stm_object_pages;
+        char *base = is_overflow_obj_safe(pseg, obj) ? segment_base : stm_object_pages;
         realobj = (struct object_s *)REAL_ADDRESS(base, obj);
         stmcb_trace(realobj, &mark_record_trace);
     }
 }
 
-static inline void mark_visit_object(object_t *obj, char *segment_base)
+static inline void mark_visit_object(
+    object_t *obj,
+    char *segment_base, /* to trace ojb in */
+    struct stm_priv_segment_info_s *pseg) /* to trace children in */
 {
     /* if already visited, don't trace */
     if (obj == NULL || mark_visited_test_and_set(obj))
         return;
-    mark_and_trace(obj, segment_base);
+    mark_and_trace(obj, segment_base, pseg);
 }
 
 
-static void mark_visit_possibly_new_object(char *segment_base, object_t *obj)
+static void mark_visit_possibly_new_object(object_t *obj, struct stm_priv_segment_info_s *pseg)
 {
     /* if newly allocated object, we trace in segment_base, otherwise in
        the sharing seg0 */
     if (obj == NULL)
         return;
 
-    if (is_new_object(obj)) {
-        mark_visit_object(obj, segment_base);
+    if (is_overflow_obj_safe(pseg, obj)) {
+        mark_visit_object(obj, pseg->pub.segment_base, pseg);
     } else {
-        mark_visit_object(obj, stm_object_pages);
+        mark_visit_object(obj, stm_object_pages, pseg);
     }
 }
 
@@ -282,8 +293,10 @@
     end = (const struct stm_shadowentry_s *)(slice + size);
     for (; p < end; p++)
         if ((((uintptr_t)p->ss) & 3) == 0) {
-            assert(!is_new_object(p->ss));
-            mark_visit_object(p->ss, stm_object_pages); // seg0
+            mark_visit_object(p->ss, stm_object_pages, // seg0
+                              /* there should be no overflow objs not already
+                                 visited, so any pseg is fine really: */
+                              get_priv_segment(STM_SEGMENT->segment_num));
         }
     return NULL;
 }
@@ -350,7 +363,7 @@
                   and thus make all pages accessible. */
                assert_obj_accessible_in(i, item);
 
-               assert(!is_new_object(item)); /* should never be in that list */
+               assert(!is_overflow_obj_safe(get_priv_segment(i), item)); /* should never be in that list */
 
                if (!mark_visited_test_and_set(item)) {
                    /* trace shared, committed version: only do this if we didn't
@@ -358,9 +371,9 @@
                       objs before mark_visit_from_modified_objects AND if we
                       do mark_and_trace on an obj that is modified in >1 segment,
                       the tracing always happens in seg0 (see mark_and_trace). */
-                   mark_and_trace(item, stm_object_pages);
+                   mark_and_trace(item, stm_object_pages, get_priv_segment(i));
                }
-               mark_and_trace(item, base);   /* private, modified version */
+               mark_and_trace(item, base, get_priv_segment(i));   /* private, modified version */
            }));
 
         list_clear(uniques);
@@ -372,7 +385,11 @@
 {
     if (testing_prebuilt_objs != NULL) {
         LIST_FOREACH_R(testing_prebuilt_objs, object_t * /*item*/,
-                       mark_visit_object(item, stm_object_pages)); // seg0
+                   mark_visit_object(item, stm_object_pages, // seg0
+                                     /* any pseg is fine, as we already traced modified
+                                        objs and thus covered all overflow objs reachable
+                                        from here */
+                                     get_priv_segment(STM_SEGMENT->segment_num)));
     }
 
     stm_thread_local_t *tl = stm_all_thread_locals;
@@ -392,17 +409,17 @@
            If 'tl' is currently running, its 'last_associated_segment_num'
            field is the segment number that contains the correct
            version of its overflowed objects. */
-        char *segment_base = get_segment_base(tl->last_associated_segment_num);
+        struct stm_priv_segment_info_s *pseg = get_priv_segment(tl->last_associated_segment_num);
 
         struct stm_shadowentry_s *current = tl->shadowstack;
         struct stm_shadowentry_s *base = tl->shadowstack_base;
         while (current-- != base) {
             if ((((uintptr_t)current->ss) & 3) == 0) {
-                mark_visit_possibly_new_object(segment_base, current->ss);
+                mark_visit_possibly_new_object(current->ss, pseg);
             }
         }
 
-        mark_visit_possibly_new_object(segment_base, tl->thread_local_obj);
+        mark_visit_possibly_new_object(tl->thread_local_obj, pseg);
 
         tl = tl->next;
     } while (tl != stm_all_thread_locals);
@@ -413,8 +430,8 @@
     for (i = 1; i < NB_SEGMENTS; i++) {
         if (get_priv_segment(i)->transaction_state != TS_NONE) {
             mark_visit_possibly_new_object(
-                get_segment_base(i),
-                get_priv_segment(i)->threadlocal_at_start_of_transaction);
+                get_priv_segment(i)->threadlocal_at_start_of_transaction,
+                get_priv_segment(i));
 
             stm_rewind_jmp_enum_shadowstack(
                 get_segment(i)->running_thread,
@@ -423,49 +440,6 @@
     }
 }
 
-static void ready_large_overflow_objects(void)
-{
-#pragma push_macro("STM_PSEGMENT")
-#pragma push_macro("STM_SEGMENT")
-#undef STM_PSEGMENT
-#undef STM_SEGMENT
-    /* objs in large_overflow only have garbage in the sharing seg0,
-       since it is used to mark objs as visited, we must make
-       sure the flag is cleared at the start of a major collection.
-       (XXX: ^^^ may be optional if we have the part below)
-
-       Also, we need to be able to recognize these objects in order
-       to only trace them in the segment they are valid in. So we
-       also make sure to set WB_EXECUTED in the sharing seg0. No
-       other objs than large_overflow_objects have WB_EXECUTED in seg0 (since
-       there can only be committed versions there).
-    */
-
-    long i;
-    for (i = 1; i < NB_SEGMENTS; i++) {
-        struct stm_priv_segment_info_s *pseg = get_priv_segment(i);
-        struct list_s *lst = pseg->large_overflow_objects;
-
-        LIST_FOREACH_R(lst, object_t* /*item*/,
-            ({
-                struct object_s *realobj;
-                /* WB_EXECUTED always set in this segment */
-                assert(realobj = (struct object_s*)REAL_ADDRESS(pseg->pub.segment_base, item));
-                assert(realobj->stm_flags & GCFLAG_WB_EXECUTED);
-
-                /* clear VISITED (garbage) and ensure WB_EXECUTED in seg0 */
-                mark_visited_test_and_clear(item);
-                realobj = (struct object_s*)REAL_ADDRESS(stm_object_pages, item);
-                realobj->stm_flags |= GCFLAG_WB_EXECUTED;
-
-                /* make sure this flag is cleared as well */
-                realobj->stm_flags &= ~GCFLAG_FINALIZATION_ORDERING;
-            }));
-    }
-#pragma pop_macro("STM_SEGMENT")
-#pragma pop_macro("STM_PSEGMENT")
-}
-
 
 static void clean_up_segment_lists(void)
 {
@@ -494,10 +468,7 @@
                 ({
                     struct object_s *realobj = (struct object_s *)
                         REAL_ADDRESS(pseg->pub.segment_base, (uintptr_t)item);
-
-                    assert(realobj->stm_flags & GCFLAG_WB_EXECUTED);
                     assert(!(realobj->stm_flags & GCFLAG_WRITE_BARRIER));
-
                     realobj->stm_flags |= GCFLAG_WRITE_BARRIER;
                 }));
             list_clear(lst);
@@ -683,8 +654,6 @@
 
     DEBUG_EXPECT_SEGFAULT(false);
 
-    ready_large_overflow_objects();
-
     /* marking */
     LIST_CREATE(marked_objects_to_trace);
     mark_visit_from_modified_objects();
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -38,7 +38,7 @@
 }
 
 static inline bool _is_from_same_transaction(object_t *obj) {
-    return _is_young(obj) || (obj->stm_flags & GCFLAG_WB_EXECUTED);
+    return _is_young(obj) || IS_OVERFLOW_OBJ(STM_PSEGMENT, obj);
 }
 
 long stm_can_move(object_t *obj)
@@ -135,8 +135,9 @@
     /* if this is not during commit, we make them overflow objects
        and push them to other segments on commit. */
     assert(!(nobj->stm_flags & GCFLAG_WB_EXECUTED));
+    assert((nobj->stm_flags & -GCFLAG_OVERFLOW_NUMBER_bit0) == 0);
     if (!STM_PSEGMENT->minor_collect_will_commit_now) {
-        nobj->stm_flags |= GCFLAG_WB_EXECUTED;
+        nobj->stm_flags |= STM_PSEGMENT->overflow_number;
     }
 
     /* Must trace the object later */
@@ -313,6 +314,11 @@
     dprintf(("minor_collection commit=%d\n", (int)commit));
 
     STM_PSEGMENT->minor_collect_will_commit_now = commit;
+    if (!commit) {
+        /* 'STM_PSEGMENT->overflow_number' is used now by this collection,
+           in the sense that it's copied to the overflow objects */
+        STM_PSEGMENT->overflow_number_has_been_used = true;
+    }
 
     collect_roots_in_nursery();
 
diff --git a/c8/stm/nursery.h b/c8/stm/nursery.h
--- a/c8/stm/nursery.h
+++ b/c8/stm/nursery.h
@@ -2,6 +2,9 @@
 #define NSE_SIGPAUSE   _STM_NSE_SIGNAL_MAX
 #define NSE_SIGABORT   _STM_NSE_SIGNAL_ABORT
 
+static uint32_t highest_overflow_number;
+
+
 static void minor_collection(bool commit);
 static void check_nursery_at_transaction_start(void);
 static size_t throw_away_nursery(struct stm_priv_segment_info_s *pseg);
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -112,6 +112,8 @@
         pr->old_objects_with_light_finalizers = list_create();
 
         pr->last_commit_log_entry = &commit_log_root;
+        pr->overflow_number = GCFLAG_OVERFLOW_NUMBER_bit0 * i;
+        highest_overflow_number = pr->overflow_number;
         pr->pub.transaction_read_version = 0xff;
     }
 
diff --git a/c8/test/test_gcpage.py b/c8/test/test_gcpage.py
--- a/c8/test/test_gcpage.py
+++ b/c8/test/test_gcpage.py
@@ -473,3 +473,27 @@
         stm_major_collect()
         assert stm_get_char(s) == '\0'
         self.commit_transaction()
+
+
+    def test_overflow_on_ss_in_major_gc(self):
+        self.start_transaction()
+        o = stm_allocate_refs(100)
+        p = stm_allocate(16)
+        stm_set_ref(o, 0, p)
+        self.push_root(o)
+        stm_minor_collect()
+        o = self.pop_root()
+        p = stm_get_ref(o, 0)
+        assert stm_get_char(p) == '\0'
+        self.push_root(o)
+
+        self.switch(1)
+
+        self.start_transaction()
+        stm_major_collect()
+        self.commit_transaction()
+
+        self.switch(0)
+        # p not freed
+        assert stm_get_char(p) == '\0'
+        self.commit_transaction()

From noreply at buildbot.pypy.org  Fri Feb 27 22:44:03 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 27 Feb 2015 22:44:03 +0100 (CET)
Subject: [pypy-commit] stmgc c8-overflow-objs: maybe fix a fork issue
Message-ID: <20150227214403.220791C02BB@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-overflow-objs
Changeset: r1677:4133909b27dd
Date: 2015-02-27 14:09 +0100
http://bitbucket.org/pypy/stmgc/changeset/4133909b27dd/

Log:	maybe fix a fork issue

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -891,24 +891,10 @@
     bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE;
     _validate_and_add_to_commit_log();
 
+    stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
 
     /* XXX do we still need a s_mutex_lock() section here? */
     s_mutex_lock();
-
-    enter_safe_point_if_requested();
-    assert(STM_SEGMENT->nursery_end == NURSERY_END);
-
-    stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
-
-    /* if a major collection is required, do it here */
-    if (is_major_collection_requested()) {
-        synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK);
-
-        if (is_major_collection_requested()) {   /* if *still* true */
-            major_collection_now_at_safe_point();
-        }
-    }
-
     commit_finalizers();
 
     /* update 'overflow_number' if needed */
@@ -922,6 +908,19 @@
 
     invoke_and_clear_user_callbacks(0);   /* for commit */
 
+    /* >>>>> there may be a FORK() happening in the safepoint below <<<<<*/
+    enter_safe_point_if_requested();
+    assert(STM_SEGMENT->nursery_end == NURSERY_END);
+
+    /* if a major collection is required, do it here */
+    if (is_major_collection_requested()) {
+        synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK);
+
+        if (is_major_collection_requested()) {   /* if *still* true */
+            major_collection_now_at_safe_point();
+        }
+    }
+
     if (globally_unique_transaction && was_inev) {
         committed_globally_unique_transaction();
     }
diff --git a/c8/stm/forksupport.c b/c8/stm/forksupport.c
--- a/c8/stm/forksupport.c
+++ b/c8/stm/forksupport.c
@@ -89,6 +89,18 @@
     assert(STM_SEGMENT->segment_num == i);
 
     s_mutex_lock();
+    if (pr->transaction_state == TS_NONE) {
+        /* just committed, TS_NONE but still has running_thread */
+
+        /* do _finish_transaction() */
+        STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
+        list_clear(STM_PSEGMENT->objects_pointing_to_nursery);
+        list_clear(STM_PSEGMENT->large_overflow_objects);
+
+        s_mutex_unlock();
+        return;
+    }
+
 #ifndef NDEBUG
     pr->running_pthread = pthread_self();
 #endif

From noreply at buildbot.pypy.org  Fri Feb 27 22:44:04 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 27 Feb 2015 22:44:04 +0100 (CET)
Subject: [pypy-commit] stmgc c8-card-marking: Merge with c8-overflow-objs
	(not compiling)
Message-ID: <20150227214404.22EDD1C02BB@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-card-marking
Changeset: r1678:96f5911028e7
Date: 2015-02-27 14:30 +0100
http://bitbucket.org/pypy/stmgc/changeset/96f5911028e7/

Log:	Merge with c8-overflow-objs (not compiling)

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -636,7 +636,8 @@
     assert(!_is_in_nursery(obj));
     assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
 
-    if (obj->stm_flags & GCFLAG_WB_EXECUTED) {
+    if (obj->stm_flags & GCFLAG_WB_EXECUTED
+|| isoverflow) {
         /* already executed WB once in this transaction. do GC
            part again: */
         write_gc_only_path(obj, mark_card);
@@ -922,7 +923,7 @@
     }
 
     assert(list_is_empty(STM_PSEGMENT->modified_old_objects));
-    assert(list_is_empty(STM_PSEGMENT->new_objects));
+    assert(list_is_empty(STM_PSEGMENT->large_overflow_objects));
     assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery));
     assert(list_is_empty(STM_PSEGMENT->young_weakrefs));
     assert(tree_is_cleared(STM_PSEGMENT->young_outside_nursery));
@@ -986,7 +987,7 @@
     _verify_cards_cleared_in_all_lists(get_priv_segment(STM_SEGMENT->segment_num));
     list_clear(STM_PSEGMENT->objects_pointing_to_nursery);
     list_clear(STM_PSEGMENT->old_objects_with_cards_set);
-    list_clear(STM_PSEGMENT->new_objects);
+    list_clear(STM_PSEGMENT->large_overflow_objects);
 
     release_thread_segment(tl);
     /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */
@@ -1006,15 +1007,17 @@
 #endif
 }
 
-static void push_new_objects_to_other_segments(void)
+static void push_large_overflow_objects_to_other_segments(void)
 {
+    if (list_is_empty(STM_PSEGMENT->large_overflow_objects))
+        return;
+
+    /* XXX: also pushes small ones right now */
     struct stm_priv_segment_info_s *pseg = get_priv_segment(STM_SEGMENT->segment_num);
     acquire_privatization_lock(STM_SEGMENT->segment_num);
-    LIST_FOREACH_R(STM_PSEGMENT->new_objects, object_t *,
+    LIST_FOREACH_R(STM_PSEGMENT->large_overflow_objects, object_t *,
         ({
-            assert(item->stm_flags & GCFLAG_WB_EXECUTED);
-            item->stm_flags &= ~GCFLAG_WB_EXECUTED;
-            if (obj_should_use_cards(pseg->pub.segment_base, item))
+            assert(!(item->stm_flags & GCFLAG_WB_EXECUTED));            if (obj_should_use_cards(pseg->pub.segment_base, item))
                 _reset_object_cards(pseg, item, CARD_CLEAR, false);
             synchronize_object_enqueue(item, true);
         }));
@@ -1029,7 +1032,7 @@
        in handle_segfault_in_page() that also copies
        unknown-to-the-segment/uncommitted things.
     */
-    list_clear(STM_PSEGMENT->new_objects);
+    list_clear(STM_PSEGMENT->large_overflow_objects);
 }
 
 
@@ -1044,19 +1047,32 @@
     dprintf(("> stm_commit_transaction()\n"));
     minor_collection(1);
 
-    push_new_objects_to_other_segments();
+    push_large_overflow_objects_to_other_segments();
     /* push before validate. otherwise they are reachable too early */
     bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE;
     _validate_and_add_to_commit_log();
 
+    stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
+
     /* XXX do we still need a s_mutex_lock() section here? */
     s_mutex_lock();
+    commit_finalizers();
 
+    /* update 'overflow_number' if needed */
+    if (STM_PSEGMENT->overflow_number_has_been_used) {
+        highest_overflow_number += GCFLAG_OVERFLOW_NUMBER_bit0;
+        assert(highest_overflow_number !=        /* XXX else, overflow! */
+               (uint32_t)-GCFLAG_OVERFLOW_NUMBER_bit0);
+        STM_PSEGMENT->overflow_number = highest_overflow_number;
+        STM_PSEGMENT->overflow_number_has_been_used = false;
+    }
+
+    invoke_and_clear_user_callbacks(0);   /* for commit */
+
+    /* >>>>> there may be a FORK() happening in the safepoint below <<<<<*/
     enter_safe_point_if_requested();
     assert(STM_SEGMENT->nursery_end == NURSERY_END);
 
-    stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
-
     /* if a major collection is required, do it here */
     if (is_major_collection_requested()) {
         synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK);
@@ -1068,10 +1084,6 @@
 
     _verify_cards_cleared_in_all_lists(get_priv_segment(STM_SEGMENT->segment_num));
 
-    commit_finalizers();
-
-    invoke_and_clear_user_callbacks(0);   /* for commit */
-
     if (globally_unique_transaction && was_inev) {
         committed_globally_unique_transaction();
     }
@@ -1184,7 +1196,7 @@
 
     list_clear(pseg->objects_pointing_to_nursery);
     list_clear(pseg->old_objects_with_cards_set);
-    list_clear(pseg->new_objects);
+    list_clear(pseg->large_overflow_objects);
     list_clear(pseg->young_weakrefs);
 #pragma pop_macro("STM_SEGMENT")
 #pragma pop_macro("STM_PSEGMENT")
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -43,6 +43,14 @@
     GCFLAG_CARDS_SET = _STM_GCFLAG_CARDS_SET,
     GCFLAG_VISITED = 0x10,
     GCFLAG_FINALIZATION_ORDERING = 0x20,
+    /* All remaining bits of the 32-bit 'stm_flags' field are taken by
+       the "overflow number".  This is a number that identifies the
+       "overflow objects" from the current transaction among all old
+       objects.  More precisely, overflow objects are objects from the
+       current transaction that have been flushed out of the nursery,
+       which occurs if the same transaction allocates too many objects.
+    */
+    GCFLAG_OVERFLOW_NUMBER_bit0 = 0x40   /* must be last */
 };
 
 #define SYNC_QUEUE_SIZE    31
@@ -98,8 +106,9 @@
     /* list of objects created in the current transaction and
        that survived at least one minor collection. They need
        to be synchronized to other segments on commit, but they
-       do not need to be in the commit log entry. */
-    struct list_s *new_objects;
+       do not need to be in the commit log entry.
+       XXX: for now it also contains small overflow objs */
+    struct list_s *large_overflow_objects;
 
     uint8_t privatization_lock;  // XXX KILL
 
@@ -111,6 +120,14 @@
 
     struct tree_s *callbacks_on_commit_and_abort[2];
 
+    /* This is the number stored in the overflowed objects (a multiple of
+       GCFLAG_OVERFLOW_NUMBER_bit0).  It is incremented when the
+       transaction is done, but only if we actually overflowed any
+       object; otherwise, no object has got this number. */
+    uint32_t overflow_number;
+    bool overflow_number_has_been_used;
+
+
     struct stm_commit_log_entry_s *last_commit_log_entry;
 
     struct stm_shadowentry_s *shadowstack_at_start_of_transaction;
@@ -203,6 +220,9 @@
 
 #define REAL_ADDRESS(segment_base, src)   ((segment_base) + (uintptr_t)(src))
 
+#define IS_OVERFLOW_OBJ(pseg, obj) (((obj)->stm_flags & -GCFLAG_OVERFLOW_NUMBER_bit0) \
+                                    == (pseg)->overflow_number)
+
 static inline uintptr_t get_index_to_card_index(uintptr_t index) {
     return (index / CARD_SIZE) + 1;
 }
diff --git a/c8/stm/finalizer.c b/c8/stm/finalizer.c
--- a/c8/stm/finalizer.c
+++ b/c8/stm/finalizer.c
@@ -98,14 +98,14 @@
         list_clear(lst);
     }
 
-    /* also deals with newly created objects: they are at the tail of
+    /* also deals with overflow objects: they are at the tail of
        old_objects_with_light_finalizers (this list is kept in order
        and we cannot add any already-committed object) */
     lst = pseg->old_objects_with_light_finalizers;
     count = list_count(lst);
     while (count > 0) {
         object_t *obj = (object_t *)list_item(lst, --count);
-        if (!(obj->stm_flags & GCFLAG_WB_EXECUTED))
+        if (!IS_OVERFLOW_OBJ(pseg, obj))
             break;
         lst->count = count;
         if (must_fix_gs) {
@@ -264,11 +264,14 @@
         LIST_APPEND(_finalizer_tmpstack, obj);
 }
 
-static inline struct list_s *finalizer_trace(char *base, object_t *obj,
-                                             struct list_s *lst)
+static inline struct list_s *finalizer_trace(
+    struct stm_priv_segment_info_s *pseg, object_t *obj, struct list_s *lst)
 {
-    if (!is_new_object(obj))
+    char *base;
+    if (!is_overflow_obj_safe(pseg, obj))
         base = stm_object_pages;
+    else
+        base = pseg->pub.segment_base;
 
     struct object_s *realobj = (struct object_s *)REAL_ADDRESS(base, obj);
     _finalizer_tmpstack = lst;
@@ -277,7 +280,8 @@
 }
 
 
-static void _recursively_bump_finalization_state_from_2_to_3(char *base, object_t *obj)
+static void _recursively_bump_finalization_state_from_2_to_3(
+    struct stm_priv_segment_info_s *pseg, object_t *obj)
 {
     assert(_finalization_state(obj) == 2);
     struct list_s *tmpstack = _finalizer_emptystack;
@@ -289,7 +293,7 @@
             realobj->stm_flags &= ~GCFLAG_FINALIZATION_ORDERING;
 
             /* trace */
-            tmpstack = finalizer_trace(base, obj, tmpstack);
+            tmpstack = finalizer_trace(pseg, obj, tmpstack);
         }
 
         if (list_is_empty(tmpstack))
@@ -300,14 +304,16 @@
     _finalizer_emptystack = tmpstack;
 }
 
-static void _recursively_bump_finalization_state_from_1_to_2(char *base, object_t *obj)
+static void _recursively_bump_finalization_state_from_1_to_2(
+    struct stm_priv_segment_info_s *pseg, object_t *obj)
 {
     assert(_finalization_state(obj) == 1);
     /* The call will add GCFLAG_VISITED recursively, thus bump state 1->2 */
-    mark_visit_possibly_new_object(base, obj);
+    mark_visit_possibly_new_object(obj, pseg);
 }
 
-static struct list_s *mark_finalize_step1(char *base, struct finalizers_s *f)
+static struct list_s *mark_finalize_step1(
+    struct stm_priv_segment_info_s *pseg, struct finalizers_s *f)
 {
     if (f == NULL)
         return NULL;
@@ -336,21 +342,22 @@
             int state = _finalization_state(y);
             if (state <= 0) {
                 _bump_finalization_state_from_0_to_1(y);
-                pending = finalizer_trace(base, y, pending);
+                pending = finalizer_trace(pseg, y, pending);
             }
             else if (state == 2) {
-                _recursively_bump_finalization_state_from_2_to_3(base, y);
+                _recursively_bump_finalization_state_from_2_to_3(pseg, y);
             }
         }
         _finalizer_pending = pending;
         assert(_finalization_state(x) == 1);
-        _recursively_bump_finalization_state_from_1_to_2(base, x);
+        _recursively_bump_finalization_state_from_1_to_2(pseg, x);
     }
     return marked;
 }
 
-static void mark_finalize_step2(char *base, struct finalizers_s *f,
-                                struct list_s *marked)
+static void mark_finalize_step2(
+    struct stm_priv_segment_info_s *pseg, struct finalizers_s *f,
+    struct list_s *marked)
 {
     if (f == NULL)
         return;
@@ -367,7 +374,7 @@
             if (run_finalizers == NULL)
                 run_finalizers = list_create();
             LIST_APPEND(run_finalizers, x);
-            _recursively_bump_finalization_state_from_2_to_3(base, x);
+            _recursively_bump_finalization_state_from_2_to_3(pseg, x);
         }
         else {
             struct list_s *lst = f->objects_with_finalizers;
@@ -403,29 +410,28 @@
     long j;
     for (j = 1; j < NB_SEGMENTS; j++) {
         struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
-        marked_seg[j] = mark_finalize_step1(pseg->pub.segment_base,
-                                            pseg->finalizers);
+        marked_seg[j] = mark_finalize_step1(pseg, pseg->finalizers);
     }
-    marked_seg[0] = mark_finalize_step1(stm_object_pages, &g_finalizers);
+    marked_seg[0] = mark_finalize_step1(get_priv_segment(0), &g_finalizers);
 
     LIST_FREE(_finalizer_pending);
 
     for (j = 1; j < NB_SEGMENTS; j++) {
         struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
-        mark_finalize_step2(pseg->pub.segment_base, pseg->finalizers,
-                            marked_seg[j]);
+        mark_finalize_step2(pseg, pseg->finalizers, marked_seg[j]);
     }
-    mark_finalize_step2(stm_object_pages, &g_finalizers, marked_seg[0]);
+    mark_finalize_step2(get_priv_segment(0), &g_finalizers, marked_seg[0]);
 
     LIST_FREE(_finalizer_emptystack);
 }
 
-static void mark_visit_from_finalizer1(char *base, struct finalizers_s *f)
+static void mark_visit_from_finalizer1(
+    struct stm_priv_segment_info_s *pseg, struct finalizers_s *f)
 {
     if (f != NULL && f->run_finalizers != NULL) {
         LIST_FOREACH_R(f->run_finalizers, object_t * /*item*/,
                        ({
-                           mark_visit_possibly_new_object(base, item);
+                           mark_visit_possibly_new_object(item, pseg);
                        }));
     }
 }
@@ -435,9 +441,9 @@
     long j;
     for (j = 1; j < NB_SEGMENTS; j++) {
         struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
-        mark_visit_from_finalizer1(pseg->pub.segment_base, pseg->finalizers);
+        mark_visit_from_finalizer1(pseg, pseg->finalizers);
     }
-    mark_visit_from_finalizer1(stm_object_pages, &g_finalizers);
+    mark_visit_from_finalizer1(get_priv_segment(0), &g_finalizers);
 }
 
 static void _execute_finalizers(struct finalizers_s *f)
diff --git a/c8/stm/forksupport.c b/c8/stm/forksupport.c
--- a/c8/stm/forksupport.c
+++ b/c8/stm/forksupport.c
@@ -84,11 +84,23 @@
     stm_thread_local_t *tl = pr->pub.running_thread;
     dprintf(("forksupport_child: abort in seg%ld\n", i));
     assert(tl->associated_segment_num == i);
-    assert(pr->transaction_state == TS_REGULAR);
+    assert(pr->transaction_state != TS_INEVITABLE);
     set_gs_register(get_segment_base(i));
     assert(STM_SEGMENT->segment_num == i);
 
     s_mutex_lock();
+    if (pr->transaction_state == TS_NONE) {
+        /* just committed, TS_NONE but still has running_thread */
+
+        /* do _finish_transaction() */
+        STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
+        list_clear(STM_PSEGMENT->objects_pointing_to_nursery);
+        list_clear(STM_PSEGMENT->large_overflow_objects);
+
+        s_mutex_unlock();
+        return;
+    }
+
 #ifndef NDEBUG
     pr->running_pthread = pthread_self();
 #endif
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -200,11 +200,16 @@
 
 /************************************************************/
 
+static bool is_overflow_obj_safe(struct stm_priv_segment_info_s *pseg, object_t *obj)
+{
+    /* this function first also checks if the page is accessible in order
+       to not cause segfaults during major gc (it does exactly the same
+       as IS_OVERFLOW_OBJ otherwise) */
+    if (get_page_status_in(pseg->pub.segment_num, (uintptr_t)obj / 4096UL) == PAGE_NO_ACCESS)
+        return false;
 
-static bool is_new_object(object_t *obj)
-{
-    struct object_s *realobj = (struct object_s*)REAL_ADDRESS(stm_object_pages, obj); /* seg0 */
-    return realobj->stm_flags & GCFLAG_WB_EXECUTED;
+    struct object_s *realobj = (struct object_s*)REAL_ADDRESS(pseg->pub.segment_base, obj);
+    return IS_OVERFLOW_OBJ(pseg, realobj);
 }
 
 
@@ -230,7 +235,10 @@
 }
 
 
-static void mark_and_trace(object_t *obj, char *segment_base)
+static void mark_and_trace(
+    object_t *obj,
+    char *segment_base, /* to trace obj in */
+    struct stm_priv_segment_info_s *pseg) /* to trace children in */
 {
     /* mark the obj and trace all reachable objs from it */
 
@@ -242,36 +250,40 @@
     stmcb_trace(realobj, &mark_record_trace);
 
     /* trace all references found in sharing seg0 (should always be
-       up-to-date and not cause segfaults, except for new objs) */
+       up-to-date and not cause segfaults, except for overflow objs) */
+    segment_base = pseg->pub.segment_base;
     while (!list_is_empty(marked_objects_to_trace)) {
         obj = (object_t *)list_pop_item(marked_objects_to_trace);
 
-        char *base = is_new_object(obj) ? segment_base : stm_object_pages;
+        char *base = is_overflow_obj_safe(pseg, obj) ? segment_base : stm_object_pages;
         realobj = (struct object_s *)REAL_ADDRESS(base, obj);
         stmcb_trace(realobj, &mark_record_trace);
     }
 }
 
-static inline void mark_visit_object(object_t *obj, char *segment_base)
+static inline void mark_visit_object(
+    object_t *obj,
+    char *segment_base, /* to trace ojb in */
+    struct stm_priv_segment_info_s *pseg) /* to trace children in */
 {
     /* if already visited, don't trace */
     if (obj == NULL || mark_visited_test_and_set(obj))
         return;
-    mark_and_trace(obj, segment_base);
+    mark_and_trace(obj, segment_base, pseg);
 }
 
 
-static void mark_visit_possibly_new_object(char *segment_base, object_t *obj)
+static void mark_visit_possibly_new_object(object_t *obj, struct stm_priv_segment_info_s *pseg)
 {
     /* if newly allocated object, we trace in segment_base, otherwise in
        the sharing seg0 */
     if (obj == NULL)
         return;
 
-    if (is_new_object(obj)) {
-        mark_visit_object(obj, segment_base);
+    if (is_overflow_obj_safe(pseg, obj)) {
+        mark_visit_object(obj, pseg->pub.segment_base, pseg);
     } else {
-        mark_visit_object(obj, stm_object_pages);
+        mark_visit_object(obj, stm_object_pages, pseg);
     }
 }
 
@@ -282,8 +294,10 @@
     end = (const struct stm_shadowentry_s *)(slice + size);
     for (; p < end; p++)
         if ((((uintptr_t)p->ss) & 3) == 0) {
-            assert(!is_new_object(p->ss));
-            mark_visit_object(p->ss, stm_object_pages); // seg0
+            mark_visit_object(p->ss, stm_object_pages, // seg0
+                              /* there should be no overflow objs not already
+                                 visited, so any pseg is fine really: */
+                              get_priv_segment(STM_SEGMENT->segment_num));
         }
     return NULL;
 }
@@ -350,7 +364,7 @@
                   and thus make all pages accessible. */
                assert_obj_accessible_in(i, item);
 
-               assert(!is_new_object(item)); /* should never be in that list */
+               assert(!is_overflow_obj_safe(get_priv_segment(i), item)); /* should never be in that list */
 
                if (!mark_visited_test_and_set(item)) {
                    /* trace shared, committed version: only do this if we didn't
@@ -358,9 +372,9 @@
                       objs before mark_visit_from_modified_objects AND if we
                       do mark_and_trace on an obj that is modified in >1 segment,
                       the tracing always happens in seg0 (see mark_and_trace). */
-                   mark_and_trace(item, stm_object_pages);
+                   mark_and_trace(item, stm_object_pages, get_priv_segment(i));
                }
-               mark_and_trace(item, base);   /* private, modified version */
+               mark_and_trace(item, base, get_priv_segment(i));   /* private, modified version */
            }));
 
         list_clear(uniques);
@@ -372,7 +386,11 @@
 {
     if (testing_prebuilt_objs != NULL) {
         LIST_FOREACH_R(testing_prebuilt_objs, object_t * /*item*/,
-                       mark_visit_object(item, stm_object_pages)); // seg0
+                   mark_visit_object(item, stm_object_pages, // seg0
+                                     /* any pseg is fine, as we already traced modified
+                                        objs and thus covered all overflow objs reachable
+                                        from here */
+                                     get_priv_segment(STM_SEGMENT->segment_num)));
     }
 
     stm_thread_local_t *tl = stm_all_thread_locals;
@@ -380,7 +398,7 @@
         /* look at all objs on the shadow stack (they are old but may
            be uncommitted so far, so only exist in the associated_segment_num).
 
-           IF they are uncommitted new objs, trace in the actual segment,
+           IF they are uncommitted overflow objs, trace in the actual segment,
            otherwise, since we just executed a minor collection, they were
            all synced to the sharing seg0. Thus we can trace them there.
 
@@ -392,17 +410,17 @@
            If 'tl' is currently running, its 'last_associated_segment_num'
            field is the segment number that contains the correct
            version of its overflowed objects. */
-        char *segment_base = get_segment_base(tl->last_associated_segment_num);
+        struct stm_priv_segment_info_s *pseg = get_priv_segment(tl->last_associated_segment_num);
 
         struct stm_shadowentry_s *current = tl->shadowstack;
         struct stm_shadowentry_s *base = tl->shadowstack_base;
         while (current-- != base) {
             if ((((uintptr_t)current->ss) & 3) == 0) {
-                mark_visit_possibly_new_object(segment_base, current->ss);
+                mark_visit_possibly_new_object(current->ss, pseg);
             }
         }
 
-        mark_visit_possibly_new_object(segment_base, tl->thread_local_obj);
+        mark_visit_possibly_new_object(tl->thread_local_obj, pseg);
 
         tl = tl->next;
     } while (tl != stm_all_thread_locals);
@@ -413,8 +431,8 @@
     for (i = 1; i < NB_SEGMENTS; i++) {
         if (get_priv_segment(i)->transaction_state != TS_NONE) {
             mark_visit_possibly_new_object(
-                get_segment_base(i),
-                get_priv_segment(i)->threadlocal_at_start_of_transaction);
+                get_priv_segment(i)->threadlocal_at_start_of_transaction,
+                get_priv_segment(i));
 
             stm_rewind_jmp_enum_shadowstack(
                 get_segment(i)->running_thread,
@@ -423,49 +441,6 @@
     }
 }
 
-static void ready_new_objects(void)
-{
-#pragma push_macro("STM_PSEGMENT")
-#pragma push_macro("STM_SEGMENT")
-#undef STM_PSEGMENT
-#undef STM_SEGMENT
-    /* objs in new_objects only have garbage in the sharing seg0,
-       since it is used to mark objs as visited, we must make
-       sure the flag is cleared at the start of a major collection.
-       (XXX: ^^^ may be optional if we have the part below)
-
-       Also, we need to be able to recognize these objects in order
-       to only trace them in the segment they are valid in. So we
-       also make sure to set WB_EXECUTED in the sharing seg0. No
-       other objs than new_objects have WB_EXECUTED in seg0 (since
-       there can only be committed versions there).
-    */
-
-    long i;
-    for (i = 1; i < NB_SEGMENTS; i++) {
-        struct stm_priv_segment_info_s *pseg = get_priv_segment(i);
-        struct list_s *lst = pseg->new_objects;
-
-        LIST_FOREACH_R(lst, object_t* /*item*/,
-            ({
-                struct object_s *realobj;
-                /* WB_EXECUTED always set in this segment */
-                assert(realobj = (struct object_s*)REAL_ADDRESS(pseg->pub.segment_base, item));
-                assert(realobj->stm_flags & GCFLAG_WB_EXECUTED);
-
-                /* clear VISITED (garbage) and ensure WB_EXECUTED in seg0 */
-                mark_visited_test_and_clear(item);
-                realobj = (struct object_s*)REAL_ADDRESS(stm_object_pages, item);
-                realobj->stm_flags |= GCFLAG_WB_EXECUTED;
-
-                /* make sure this flag is cleared as well */
-                realobj->stm_flags &= ~GCFLAG_FINALIZATION_ORDERING;
-            }));
-    }
-#pragma pop_macro("STM_SEGMENT")
-#pragma pop_macro("STM_PSEGMENT")
-}
-
 
 static void clean_up_segment_lists(void)
 {
@@ -494,10 +469,7 @@
                 ({
                     struct object_s *realobj = (struct object_s *)
                         REAL_ADDRESS(pseg->pub.segment_base, (uintptr_t)item);
-
-                    assert(realobj->stm_flags & GCFLAG_WB_EXECUTED);
                     assert(!(realobj->stm_flags & GCFLAG_WRITE_BARRIER));
-
                     realobj->stm_flags |= GCFLAG_WRITE_BARRIER;
 
                     OPT_ASSERT(!(realobj->stm_flags & GCFLAG_CARDS_SET));
@@ -525,8 +497,8 @@
         list_clear(lst);
 
 
-        /* remove from new_objects all objects that die */
-        lst = pseg->new_objects;
+        /* remove from large_overflow_objects all objects that die */
+        lst = pseg->large_overflow_objects;
         uintptr_t n = list_count(lst);
         while (n-- > 0) {
             object_t *obj = (object_t *)list_item(lst, n);
@@ -703,8 +675,6 @@
 
     DEBUG_EXPECT_SEGFAULT(false);
 
-    ready_new_objects();
-
     /* marking */
     LIST_CREATE(marked_objects_to_trace);
     mark_visit_from_modified_objects();
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -38,7 +38,7 @@
 }
 
 static inline bool _is_from_same_transaction(object_t *obj) {
-    return _is_young(obj) || (obj->stm_flags & GCFLAG_WB_EXECUTED);
+    return _is_young(obj) || IS_OVERFLOW_OBJ(STM_PSEGMENT, obj);
 }
 
 long stm_can_move(object_t *obj)
@@ -132,12 +132,12 @@
         nobj_sync_now = ((uintptr_t)nobj) | FLAG_SYNC_LARGE;
     }
 
-    /* if this is not during commit, we will add them to the new_objects
-       list and push them to other segments on commit. Thus we can add
-       the WB_EXECUTED flag so that they don't end up in modified_old_objects */
+    /* if this is not during commit, we make them overflow objects
+       and push them to other segments on commit. */
     assert(!(nobj->stm_flags & GCFLAG_WB_EXECUTED));
+    assert((nobj->stm_flags & -GCFLAG_OVERFLOW_NUMBER_bit0) == 0);
     if (!STM_PSEGMENT->minor_collect_will_commit_now) {
-        nobj->stm_flags |= GCFLAG_WB_EXECUTED;
+        nobj->stm_flags |= STM_PSEGMENT->overflow_number;
     }
 
     /* Must trace the object later */
@@ -339,16 +339,17 @@
         assert(!(obj->stm_flags & GCFLAG_CARDS_SET));
 
         if (obj_sync_now & FLAG_SYNC_LARGE) {
+            /* XXX: SYNC_LARGE even set for small objs right now */
             /* this is a newly allocated obj in this transaction. We must
                either synchronize the object to other segments now, or
-               add the object to new_objects list */
+               add the object to large_overflow_objects list */
             struct stm_priv_segment_info_s *pseg = get_priv_segment(STM_SEGMENT->segment_num);
             if (pseg->minor_collect_will_commit_now) {
                 acquire_privatization_lock(pseg->pub.segment_num);
                 synchronize_object_enqueue(obj);
                 release_privatization_lock(pseg->pub.segment_num);
             } else {
-                LIST_APPEND(pseg->new_objects, obj);
+                LIST_APPEND(STM_PSEGMENT->large_overflow_objects, obj);
             }
             _cards_cleared_in_object(pseg, obj);
         }
@@ -357,7 +358,7 @@
         lst = STM_PSEGMENT->objects_pointing_to_nursery;
     }
 
-    /* flush all new objects to other segments now */
+    /* flush all overflow objects to other segments now */
     if (STM_PSEGMENT->minor_collect_will_commit_now) {
         acquire_privatization_lock(STM_SEGMENT->segment_num);
         synchronize_objects_flush();
@@ -475,6 +476,11 @@
     dprintf(("minor_collection commit=%d\n", (int)commit));
 
     STM_PSEGMENT->minor_collect_will_commit_now = commit;
+    if (!commit) {
+        /* 'STM_PSEGMENT->overflow_number' is used now by this collection,
+           in the sense that it's copied to the overflow objects */
+        STM_PSEGMENT->overflow_number_has_been_used = true;
+    }
 
     collect_cardrefs_to_nursery();
 
diff --git a/c8/stm/nursery.h b/c8/stm/nursery.h
--- a/c8/stm/nursery.h
+++ b/c8/stm/nursery.h
@@ -2,6 +2,8 @@
 #define NSE_SIGPAUSE   _STM_NSE_SIGNAL_MAX
 #define NSE_SIGABORT   _STM_NSE_SIGNAL_ABORT
 
+static uint32_t highest_overflow_number;
+
 static void _cards_cleared_in_object(struct stm_priv_segment_info_s *pseg, object_t *obj);
 static void _reset_object_cards(struct stm_priv_segment_info_s *pseg,
                                 object_t *obj, uint8_t mark_value,
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -100,7 +100,7 @@
         pr->pub.segment_num = i;
         pr->pub.segment_base = segment_base;
         pr->modified_old_objects = list_create();
-        pr->new_objects = list_create();
+        pr->large_overflow_objects = list_create();
         pr->young_weakrefs = list_create();
         pr->old_weakrefs = list_create();
         pr->objects_pointing_to_nursery = list_create();
@@ -113,6 +113,8 @@
         pr->old_objects_with_light_finalizers = list_create();
 
         pr->last_commit_log_entry = &commit_log_root;
+        pr->overflow_number = GCFLAG_OVERFLOW_NUMBER_bit0 * i;
+        highest_overflow_number = pr->overflow_number;
         pr->pub.transaction_read_version = 0xff;
     }
 
@@ -150,8 +152,8 @@
         list_free(pr->objects_pointing_to_nursery);
         list_free(pr->old_objects_with_cards_set);
         list_free(pr->modified_old_objects);
-        assert(list_is_empty(pr->new_objects));
-        list_free(pr->new_objects);
+        assert(list_is_empty(pr->large_overflow_objects));
+        list_free(pr->large_overflow_objects);
         list_free(pr->young_weakrefs);
         list_free(pr->old_weakrefs);
         tree_free(pr->young_outside_nursery);
diff --git a/c8/test/test_gcpage.py b/c8/test/test_gcpage.py
--- a/c8/test/test_gcpage.py
+++ b/c8/test/test_gcpage.py
@@ -473,3 +473,27 @@
         stm_major_collect()
         assert stm_get_char(s) == '\0'
         self.commit_transaction()
+
+
+    def test_overflow_on_ss_in_major_gc(self):
+        self.start_transaction()
+        o = stm_allocate_refs(100)
+        p = stm_allocate(16)
+        stm_set_ref(o, 0, p)
+        self.push_root(o)
+        stm_minor_collect()
+        o = self.pop_root()
+        p = stm_get_ref(o, 0)
+        assert stm_get_char(p) == '\0'
+        self.push_root(o)
+
+        self.switch(1)
+
+        self.start_transaction()
+        stm_major_collect()
+        self.commit_transaction()
+
+        self.switch(0)
+        # p not freed
+        assert stm_get_char(p) == '\0'
+        self.commit_transaction()

From noreply at buildbot.pypy.org  Fri Feb 27 22:44:05 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 27 Feb 2015 22:44:05 +0100 (CET)
Subject: [pypy-commit] stmgc c8-card-marking: progress
Message-ID: <20150227214405.1F2551C02BB@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-card-marking
Changeset: r1679:4fdb8abbec3a
Date: 2015-02-27 17:30 +0100
http://bitbucket.org/pypy/stmgc/changeset/4fdb8abbec3a/

Log:	progress

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -445,7 +445,6 @@
 }
 
 
-static void reset_cards_from_modified_objects(void);
 static void reset_wb_executed_flags(void);
 static void readd_wb_executed_flags(void);
 static void check_all_write_barrier_flags(char *segbase, struct list_s *list);
@@ -528,8 +527,6 @@
         STM_PSEGMENT->transaction_state = TS_NONE;
         STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
 
-        reset_cards_from_modified_objects();
-
         list_clear(STM_PSEGMENT->modified_old_objects);
         STM_PSEGMENT->last_commit_log_entry = new;
         release_modification_lock(STM_SEGMENT->segment_num);
@@ -556,8 +553,6 @@
         check_all_write_barrier_flags(STM_SEGMENT->segment_base,
                                       STM_PSEGMENT->modified_old_objects);
 
-        reset_cards_from_modified_objects();
-
         /* compare with _validate_and_attach: */
         STM_PSEGMENT->transaction_state = TS_NONE;
         STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
@@ -597,11 +592,197 @@
     return (size >= _STM_MIN_CARD_OBJ_SIZE);
 }
 
+
+static void make_bk_slices_for_range(
+    object_t *obj,
+    stm_char *start, stm_char *end) /* [start, end[ */
+{
+    dprintf(("make_bk_slices_for_range(%p, %lu, %lu)\n",
+             obj, start - (stm_char*)obj, end - start));
+    char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
+    size_t obj_size = stmcb_size_rounded_up((struct object_s*)realobj);
+    uintptr_t first_page = ((uintptr_t)start) / 4096UL;
+    uintptr_t end_page = ((uintptr_t)end) / 4096UL;
+
+    uintptr_t page;
+    uintptr_t slice_sz;
+    uintptr_t in_page_offset = (uintptr_t)start % 4096UL;
+    uintptr_t remaining_obj_sz = end - start;
+    for (page = first_page; page <= end_page; page++) {
+        OPT_ASSERT(remaining_obj_sz);
+
+        slice_sz = remaining_obj_sz;
+        if (in_page_offset + slice_sz > 4096UL) {
+            /* not over page boundaries */
+            slice_sz = 4096UL - in_page_offset;
+        }
+
+        size_t slice_off = obj_size - remaining_obj_sz;
+        remaining_obj_sz -= slice_sz;
+        in_page_offset = (in_page_offset + slice_sz) % 4096UL; /* mostly 0 */
+
+        /* make backup slice: */
+        char *bk_slice = malloc(slice_sz);
+        increment_total_allocated(slice_sz);
+        memcpy(bk_slice, realobj + slice_off, slice_sz);
+
+        acquire_modification_lock(STM_SEGMENT->segment_num);
+        /* !! follows layout of "struct stm_undo_s" !! */
+        STM_PSEGMENT->modified_old_objects = list_append3(
+            STM_PSEGMENT->modified_old_objects,
+            (uintptr_t)obj,     /* obj */
+            (uintptr_t)bk_slice,  /* bk_addr */
+            NEW_SLICE(slice_off, slice_sz));
+        release_modification_lock(STM_SEGMENT->segment_num);
+    }
+
+}
+
+static void make_bk_slices(object_t *obj,
+                           bool first_call, /* tells us if we also need to make a bk
+                                               of the non-array part of the object */
+                           uintptr_t index,  /* index == -1: all cards, index == -2: no cards */
+                           bool do_missing_cards /* only bk the cards that don't have a bk */
+                           )
+{
+    dprintf(("make_bk_slices(%p, %d, %ld, %d)\n", obj, first_call, index, do_missing_cards));
+    /* do_missing_cards also implies that all cards are cleared at the end */
+    /* index == -1 but not do_missing_cards: bk whole obj */
+    assert(IMPLY(index == -2, first_call && !do_missing_cards));
+    assert(IMPLY(index == -1 && !do_missing_cards, first_call));
+    assert(IMPLY(do_missing_cards, index == -1));
+    assert(IMPLY(is_small_uniform(obj), index == -1 && !do_missing_cards && first_call));
+    assert(IMPLY(first_call, !do_missing_cards));
+    assert(IMPLY(index != -1, obj_should_use_cards(STM_SEGMENT->segment_base, obj)));
+
+    /* get whole card range */
+    struct object_s *realobj = (struct object_s*)REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
+    size_t obj_size = stmcb_size_rounded_up(realobj);
+
+    uintptr_t offset_itemsize[2];
+    stmcb_get_card_base_itemsize(realobj, offset_itemsize);
+    size_t real_idx_count = (obj_size - offset_itemsize[0]) / offset_itemsize[1];
+    assert(IMPLY(index != -1 && index != -2, index >= 0 && index < real_idx_count));
+
+    struct stm_read_marker_s *cards = get_read_marker(STM_SEGMENT->segment_base, (uintptr_t)obj);
+    uintptr_t last_card_index = get_index_to_card_index(real_idx_count - 1); /* max valid index */
+
+    /* decide where to start copying: */
+    size_t start_offset;
+    if (first_call) {
+        start_offset = 0;
+    } else {
+        start_offset = -1;
+    }
+
+    /* decide if we don't want to look at cards at all: */
+    if ((index == -1 || index == -2) && !do_missing_cards) {
+        assert(first_call);
+        if (index == -1) {
+            /* whole obj */
+            make_bk_slices_for_range(obj, (stm_char*)obj + start_offset,
+                                     (stm_char*)obj + obj_size);
+        } else {
+            /* only fixed part */
+            make_bk_slices_for_range(obj, (stm_char*)obj + start_offset,
+                                     (stm_char*)obj + offset_itemsize[0]);
+        }
+        return;
+    }
+
+    /* decide if we want only a specific card: */
+    if (index != -1) {
+        if (start_offset != -1) {
+            /* bk fixed part separately: */
+            make_bk_slices_for_range(obj, (stm_char*)obj + start_offset,
+                                     (stm_char*)obj + offset_itemsize[0]);
+        }
+
+        size_t after_card_offset = offset_itemsize[0] + (index + 1) * offset_itemsize[1];
+        if (after_card_offset > obj_size)
+            after_card_offset = obj_size;
+
+        make_bk_slices_for_range(
+            obj, (stm_char*)obj + offset_itemsize[0] + index * offset_itemsize[1],
+            (stm_char*)obj + after_card_offset);
+
+        return;
+    }
+
+    /* look for CARD_CLEAR or some non-transaction_read_version cards
+       and make bk slices for them */
+    assert(do_missing_cards && index == -1 && start_offset == -1);
+    uintptr_t card_index = 1;
+    uintptr_t start_card_index = -1;
+    while (card_index <= last_card_index) {
+        uint8_t card_value = cards[card_index].rm;
+
+        if (card_value == CARD_CLEAR
+            || (card_value != CARD_MARKED
+                && card_value < STM_SEGMENT->transaction_read_version)) {
+            /* we need a backup of this card */
+            if (start_card_index == -1) {   /* first unmarked card */
+                start_card_index = card_index;
+            }
+        }
+        else {
+            /* "CARD_MARKED_OLD" or CARD_MARKED */
+            OPT_ASSERT(card_value == STM_SEGMENT->transaction_read_version
+                       || card_value == CARD_MARKED);
+            /* we don't need the cards anymore after executing this function */
+            cards[card_index].rm = CARD_CLEAR;
+        }
+
+        if (start_card_index != -1                    /* something to copy */
+            && (card_value == CARD_MARKED             /* found marked card */
+                || card_value == STM_SEGMENT->transaction_read_version/* old marked */
+                || card_index == last_card_index)) {  /* this is the last card */
+
+            /* do the bk slice: */
+            uintptr_t copy_size;
+            uintptr_t next_card_offset;
+            uintptr_t start_card_offset;
+            uintptr_t next_card_index = card_index;
+
+            if (card_value == CARD_CLEAR
+                || (card_value != CARD_MARKED
+                    && card_value < STM_SEGMENT->transaction_read_version)) {
+                /* this was actually the last card which wasn't set, but we
+                   need to go one further to get the right offset */
+                next_card_index++;
+            }
+
+            start_card_offset = offset_itemsize[0] +
+                get_card_index_to_index(start_card_index) * offset_itemsize[1];
+
+            next_card_offset = offset_itemsize[0] +
+                get_card_index_to_index(next_card_index) * offset_itemsize[1];
+
+            if (next_card_offset > obj_size)
+                next_card_offset = obj_size;
+
+            copy_size = next_card_offset - start_card_offset;
+            OPT_ASSERT(copy_size > 0);
+
+            /* add the slices: */
+            make_bk_slices_for_range(
+                obj, (stm_char*)obj + start_card_offset,
+                (stm_char*)obj + next_card_offset);
+
+            start_card_index = -1;
+        }
+
+        card_index++;
+    }
+
+    _cards_cleared_in_object(get_priv_segment(STM_SEGMENT->segment_num), obj);
+}
+
 __attribute__((always_inline))
-static void write_gc_only_path(object_t *obj, bool mark_card)
+static void write_slowpath_overflow_obj(object_t *obj, bool mark_card)
 {
     assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
-    assert(obj->stm_flags & GCFLAG_WB_EXECUTED);
+    assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED));
     dprintf(("write_slowpath-fast(%p)\n", obj));
 
     if (!mark_card) {
@@ -611,8 +792,7 @@
            next minor collection. */
         if (obj->stm_flags & GCFLAG_CARDS_SET) {
             /* if we clear this flag, we also need to clear the cards.
-               bk_slices are not needed as this is a new object */
-            /* XXX: add_missing_bk_slices_and_"clear"_cards */
+               bk_slices are not needed as this is an overflow object */
             _reset_object_cards(get_priv_segment(STM_SEGMENT->segment_num),
                                 obj, CARD_CLEAR, false, false);
         }
@@ -621,7 +801,8 @@
     } else {
         /* Card marking.  Don't remove GCFLAG_WRITE_BARRIER because we
            need to come back to _stm_write_slowpath_card() for every
-           card to mark.  Add GCFLAG_CARDS_SET. */
+           card to mark.  Add GCFLAG_CARDS_SET.
+           again, we don't need bk_slices as this is an overflow obj */
         assert(!(obj->stm_flags & GCFLAG_CARDS_SET));
         obj->stm_flags |= GCFLAG_CARDS_SET;
         LIST_APPEND(STM_PSEGMENT->old_objects_with_cards_set, obj);
@@ -636,128 +817,63 @@
     assert(!_is_in_nursery(obj));
     assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
 
-    if (obj->stm_flags & GCFLAG_WB_EXECUTED
-|| isoverflow) {
+    if (IS_OVERFLOW_OBJ(STM_PSEGMENT, obj)) {
         /* already executed WB once in this transaction. do GC
            part again: */
-        write_gc_only_path(obj, mark_card);
+        assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED));
+        write_slowpath_overflow_obj(obj, mark_card);
         return;
     }
 
-    char *realobj;
-    size_t obj_size;
-    int my_segnum = STM_SEGMENT->segment_num;
-    uintptr_t end_page, first_page = ((uintptr_t)obj) / 4096UL;
-
-    realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
-    obj_size = stmcb_size_rounded_up((struct object_s *)realobj);
-    /* get the last page containing data from the object */
-    if (LIKELY(is_small_uniform(obj))) {
-        end_page = first_page;
-    } else {
-        end_page = (((uintptr_t)obj) + obj_size - 1) / 4096UL;
-    }
+    dprintf(("write_slowpath(%p)\n", obj));
 
     /* add to read set: */
     stm_read(obj);
 
-    assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED));
-    dprintf(("write_slowpath(%p): sz=%lu\n", obj, obj_size));
-
- retry:
-    /* privatize pages: */
-    /* XXX don't always acquire all locks... */
-    acquire_all_privatization_locks();
-
-    uintptr_t page;
-    for (page = first_page; page <= end_page; page++) {
-        if (get_page_status_in(my_segnum, page) == PAGE_NO_ACCESS) {
-            /* XXX: slow? */
-            release_all_privatization_locks();
-
-            volatile char *dummy = REAL_ADDRESS(STM_SEGMENT->segment_base, page * 4096UL);
-            *dummy;            /* force segfault */
-
-            goto retry;
-        }
-    }
-    /* all pages are private to us and we hold the privatization_locks so
-       we are allowed to modify them */
-
-    /* phew, now add the obj to the write-set and register the
-       backup copy. */
-    /* XXX: we should not be here at all fiddling with page status
-       if 'obj' is merely an overflow object.  FIX ME, likely by copying
-       the overflow number logic from c7. */
-
     DEBUG_EXPECT_SEGFAULT(false);
 
-    acquire_modification_lock(STM_SEGMENT->segment_num);
-    uintptr_t slice_sz;
-    uintptr_t in_page_offset = (uintptr_t)obj % 4096UL;
-    uintptr_t remaining_obj_sz = obj_size;
-    for (page = first_page; page <= end_page; page++) {
-        /* XXX Maybe also use mprotect() again to mark pages of the object as read-only, and
-           only stick it into modified_old_objects page-by-page?  Maybe it's
-           possible to do card-marking that way, too. */
-        OPT_ASSERT(remaining_obj_sz);
+    if (mark_card) {
+        make_bk_slices(obj,
+                       true,        /* first_call */
+                       -2,          /* index: backup only fixed part */
+                       false);      /* do_missing_cards */
 
-        slice_sz = remaining_obj_sz;
-        if (in_page_offset + slice_sz > 4096UL) {
-            /* not over page boundaries */
-            slice_sz = 4096UL - in_page_offset;
-        }
+        /* don't remove WRITE_BARRIER, but add CARDS_SET */
+        obj->stm_flags |= (GCFLAG_CARDS_SET | GCFLAG_WB_EXECUTED);
+        LIST_APPEND(STM_PSEGMENT->old_objects_with_cards_set, obj);
+    } else {
+        /* called if WB_EXECUTED is set or this is the first time
+           for this obj: */
 
-        size_t slice_off = obj_size - remaining_obj_sz;
-
-        /* make backup slice: */
-        char *bk_slice = malloc(slice_sz);
-        increment_total_allocated(slice_sz);
-        memcpy(bk_slice, realobj + slice_off, slice_sz);
-
-        /* !! follows layout of "struct stm_undo_s" !! */
-        STM_PSEGMENT->modified_old_objects = list_append3(
-            STM_PSEGMENT->modified_old_objects,
-            (uintptr_t)obj,     /* obj */
-            (uintptr_t)bk_slice,  /* bk_addr */
-            NEW_SLICE(slice_off, slice_sz));
-
-        remaining_obj_sz -= slice_sz;
-        in_page_offset = (in_page_offset + slice_sz) % 4096UL; /* mostly 0 */
-    }
-    OPT_ASSERT(remaining_obj_sz == 0);
-
-    if (!mark_card) {
-        /* also add it to the GC list for minor collections */
+        /* add it to the GC list for minor collections */
         LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj);
 
         if (obj->stm_flags & GCFLAG_CARDS_SET) {
-            /* if we clear this flag, we have to tell sync_old_objs that
-               everything needs to be synced */
-            /* if we clear this flag, we have to tell later write barriers
-               that we already did all backup slices: */
-            /* XXX: do_other_bk_slices_and_"clear"_cards */
-            _reset_object_cards(get_priv_segment(STM_SEGMENT->segment_num),
-                                obj, STM_SEGMENT->transaction_read_version,
-                                true, false); /* mark all */
+            assert(obj->stm_flags & GCFLAG_WB_EXECUTED);
+
+            /* this is not the first_call to the WB for this obj,
+               we executed the above then-part before.
+               if we clear this flag, we have to add all the other
+               bk slices we didn't add yet */
+            make_bk_slices(obj,
+                           false,       /* first_call */
+                           -1,          /* index: whole obj */
+                           true);       /* do_missing_cards */
+
+        } else if (!(obj->stm_flags & GCFLAG_WB_EXECUTED)) {
+            /* first and only time we enter here: */
+            make_bk_slices(obj,
+                           true,        /* first_call */
+                           -1,          /* index: whole obj */
+                           false);      /* do_missing_cards */
         }
 
         /* remove the WRITE_BARRIER flag and add WB_EXECUTED */
         obj->stm_flags &= ~(GCFLAG_WRITE_BARRIER | GCFLAG_CARDS_SET);
         obj->stm_flags |= GCFLAG_WB_EXECUTED;
-    } else {
-        /* don't remove WRITE_BARRIER, but add CARDS_SET */
-        obj->stm_flags |= (GCFLAG_CARDS_SET | GCFLAG_WB_EXECUTED);
-        /* XXXXXXXXXXXX maybe not set WB_EXECUTED and make CARDS_SET
-           mean the same thing where necessary */
-        LIST_APPEND(STM_PSEGMENT->old_objects_with_cards_set, obj);
     }
 
     DEBUG_EXPECT_SEGFAULT(true);
-
-    release_modification_lock(STM_SEGMENT->segment_num);
-    /* done fiddling with protection and privatization */
-    release_all_privatization_locks();
 }
 
 
@@ -819,7 +935,16 @@
        We already own the object here or it is an overflow obj. */
     struct stm_read_marker_s *cards = get_read_marker(STM_SEGMENT->segment_base,
                                                       (uintptr_t)obj);
-    cards[get_index_to_card_index(index)].rm = CARD_MARKED;
+    uintptr_t card_index = get_index_to_card_index(index);
+    if (!(cards[card_index].rm == CARD_MARKED
+          || cards[card_index].rm == STM_SEGMENT->transaction_read_version)) {
+        /* need to do the backup slice of the card */
+        make_bk_slices(obj,
+                       false,       /* first_call */
+                       index,       /* index: only 1 card */
+                       false);      /* do_missing_cards */
+    }
+    cards[card_index].rm = CARD_MARKED;
 
     dprintf(("mark %p index %lu, card:%lu with %d\n",
              obj, index, get_index_to_card_index(index), CARD_MARKED));
@@ -851,20 +976,6 @@
     assert(STM_SEGMENT->transaction_read_version > _STM_CARD_MARKED);
 }
 
-static void reset_cards_from_modified_objects(void)
-{
-    struct list_s *list = STM_PSEGMENT->modified_old_objects;
-    struct stm_undo_s *undo = (struct stm_undo_s *)list->items;
-    struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count);
-
-    for (; undo < end; undo++) {
-        object_t *obj = undo->object;
-        if (obj_should_use_cards(STM_SEGMENT->segment_base, obj))
-            _reset_object_cards(get_priv_segment(STM_SEGMENT->segment_num),
-                                obj, CARD_CLEAR, false);
-    }
-}
-
 static void reset_wb_executed_flags(void)
 {
     dprintf(("reset_wb_executed_flags()\n"));
@@ -1013,13 +1124,11 @@
         return;
 
     /* XXX: also pushes small ones right now */
-    struct stm_priv_segment_info_s *pseg = get_priv_segment(STM_SEGMENT->segment_num);
     acquire_privatization_lock(STM_SEGMENT->segment_num);
     LIST_FOREACH_R(STM_PSEGMENT->large_overflow_objects, object_t *,
         ({
-            assert(!(item->stm_flags & GCFLAG_WB_EXECUTED));            if (obj_should_use_cards(pseg->pub.segment_base, item))
-                _reset_object_cards(pseg, item, CARD_CLEAR, false);
-            synchronize_object_enqueue(item, true);
+            assert(!(item->stm_flags & GCFLAG_WB_EXECUTED));
+            synchronize_object_enqueue(item);
         }));
     synchronize_objects_flush();
     release_privatization_lock(STM_SEGMENT->segment_num);
@@ -1121,10 +1230,6 @@
                undo->backup,
                SLICE_SIZE(undo->slice));
 
-        if (obj_should_use_cards(pseg->pub.segment_base, obj))
-            _reset_object_cards(pseg, obj, CARD_CLEAR, false);
-        /* XXXXXXXXX: only reset cards of slice!! ^^^^^^^ */
-
         dprintf(("reset_modified_from_backup_copies(%d): obj=%p off=%lu bk=%p\n",
                  segment_num, obj, SLICE_OFFSET(undo->slice), undo->backup));
 
@@ -1161,11 +1266,13 @@
 
     long bytes_in_nursery = throw_away_nursery(pseg);
 
-    /* some new objects may have cards when aborting, clear them too */
-    LIST_FOREACH_R(pseg->new_objects, object_t * /*item*/,
+    /* clear CARD_MARKED on objs (don't care about CARD_MARKED_OLD) */
+    LIST_FOREACH_R(pseg->old_objects_with_cards_set, object_t * /*item*/,
         {
-            if (obj_should_use_cards(pseg->pub.segment_base, item))
-                _reset_object_cards(pseg, item, CARD_CLEAR, false);
+            struct object_s *realobj = (struct object_s *)
+                REAL_ADDRESS(pseg->pub.segment_base, item);
+            if (realobj->stm_flags & GCFLAG_CARDS_SET)
+                _reset_object_cards(pseg, item, CARD_CLEAR, false, false);
         });
 
     acquire_modification_lock(segment_num);
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -179,7 +179,7 @@
         _cards_cleared_in_object(pseg, undo->object);
     }
     LIST_FOREACH_R(
-        pseg->new_objects, object_t * /*item*/,
+        pseg->large_overflow_objects, object_t * /*item*/,
         _cards_cleared_in_object(pseg, item));
     LIST_FOREACH_R(
         pseg->objects_pointing_to_nursery, object_t * /*item*/,

From noreply at buildbot.pypy.org  Fri Feb 27 22:44:06 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 27 Feb 2015 22:44:06 +0100 (CET)
Subject: [pypy-commit] stmgc c8-card-marking: fix
Message-ID: <20150227214406.1A3881C02BB@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-card-marking
Changeset: r1680:3a1d24bc19e8
Date: 2015-02-27 17:59 +0100
http://bitbucket.org/pypy/stmgc/changeset/3a1d24bc19e8/

Log:	fix

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -600,12 +600,12 @@
     dprintf(("make_bk_slices_for_range(%p, %lu, %lu)\n",
              obj, start - (stm_char*)obj, end - start));
     char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
-    size_t obj_size = stmcb_size_rounded_up((struct object_s*)realobj);
     uintptr_t first_page = ((uintptr_t)start) / 4096UL;
     uintptr_t end_page = ((uintptr_t)end) / 4096UL;
 
     uintptr_t page;
     uintptr_t slice_sz;
+    uintptr_t slice_off = start - (stm_char*)obj;
     uintptr_t in_page_offset = (uintptr_t)start % 4096UL;
     uintptr_t remaining_obj_sz = end - start;
     for (page = first_page; page <= end_page; page++) {
@@ -617,7 +617,6 @@
             slice_sz = 4096UL - in_page_offset;
         }
 
-        size_t slice_off = obj_size - remaining_obj_sz;
         remaining_obj_sz -= slice_sz;
         in_page_offset = (in_page_offset + slice_sz) % 4096UL; /* mostly 0 */
 
@@ -633,7 +632,10 @@
             (uintptr_t)obj,     /* obj */
             (uintptr_t)bk_slice,  /* bk_addr */
             NEW_SLICE(slice_off, slice_sz));
+        dprintf(("> append slice %p, off=%lu, sz=%lu\n", bk_slice, slice_off, slice_sz));
         release_modification_lock(STM_SEGMENT->segment_num);
+
+        slice_off += slice_sz;
     }
 
 }
@@ -775,6 +777,7 @@
         card_index++;
     }
 
+    obj->stm_flags &= ~GCFLAG_CARDS_SET;
     _cards_cleared_in_object(get_priv_segment(STM_SEGMENT->segment_num), obj);
 }
 
@@ -783,7 +786,7 @@
 {
     assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
     assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED));
-    dprintf(("write_slowpath-fast(%p)\n", obj));
+    dprintf(("write_slowpath_overflow_obj(%p)\n", obj));
 
     if (!mark_card) {
         /* The basic case, with no card marking.  We append the object
@@ -833,10 +836,12 @@
     DEBUG_EXPECT_SEGFAULT(false);
 
     if (mark_card) {
-        make_bk_slices(obj,
-                       true,        /* first_call */
-                       -2,          /* index: backup only fixed part */
-                       false);      /* do_missing_cards */
+        if (!(obj->stm_flags & GCFLAG_WB_EXECUTED)) {
+            make_bk_slices(obj,
+                           true,        /* first_call */
+                           -2,          /* index: backup only fixed part */
+                           false);      /* do_missing_cards */
+        }
 
         /* don't remove WRITE_BARRIER, but add CARDS_SET */
         obj->stm_flags |= (GCFLAG_CARDS_SET | GCFLAG_WB_EXECUTED);
@@ -936,8 +941,9 @@
     struct stm_read_marker_s *cards = get_read_marker(STM_SEGMENT->segment_base,
                                                       (uintptr_t)obj);
     uintptr_t card_index = get_index_to_card_index(index);
-    if (!(cards[card_index].rm == CARD_MARKED
-          || cards[card_index].rm == STM_SEGMENT->transaction_read_version)) {
+    if (!IS_OVERFLOW_OBJ(STM_PSEGMENT, obj)
+        && !(cards[card_index].rm == CARD_MARKED
+             || cards[card_index].rm == STM_SEGMENT->transaction_read_version)) {
         /* need to do the backup slice of the card */
         make_bk_slices(obj,
                        false,       /* first_call */
diff --git a/c8/test/test_card_marking.py b/c8/test/test_card_marking.py
--- a/c8/test/test_card_marking.py
+++ b/c8/test/test_card_marking.py
@@ -239,7 +239,7 @@
         self.commit_transaction()
 
         self.start_transaction()
-        assert get_card_value(o, 1000) == CARD_CLEAR
+        assert get_card_value(o, 1000) not in (CARD_CLEAR, CARD_MARKED_OLD())
         stm_set_char(o, 'b', 1000, True)
         assert get_card_value(o, 1000) == CARD_MARKED
         assert o in old_objects_with_cards_set()
@@ -264,7 +264,7 @@
         self.commit_transaction()
 
         self.start_transaction()
-        assert get_card_value(o, 1000) == CARD_CLEAR
+        assert get_card_value(o, 1000) not in (CARD_CLEAR, CARD_MARKED_OLD())
         stm_set_char(o, 'b', 1000, True)
         assert get_card_value(o, 1000) == CARD_MARKED
         assert o in old_objects_with_cards_set()
@@ -310,8 +310,9 @@
         assert get_card_value(p, 1000) == CARD_MARKED
 
         self.push_root(o)
-        stm_minor_collect()
+        stm_major_collect()
         o = self.pop_root()
+        # p dies and gets cards CLEARed
 
         assert get_card_value(o, 1000) == CARD_MARKED_OLD()
         assert get_card_value(p, 1000) == CARD_CLEAR

From noreply at buildbot.pypy.org  Fri Feb 27 22:44:07 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 27 Feb 2015 22:44:07 +0100 (CET)
Subject: [pypy-commit] stmgc c8-card-marking: fixes
Message-ID: <20150227214407.133D11C02BB@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-card-marking
Changeset: r1681:f03a170af5c5
Date: 2015-02-27 18:04 +0100
http://bitbucket.org/pypy/stmgc/changeset/f03a170af5c5/

Log:	fixes

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -608,9 +608,7 @@
     uintptr_t slice_off = start - (stm_char*)obj;
     uintptr_t in_page_offset = (uintptr_t)start % 4096UL;
     uintptr_t remaining_obj_sz = end - start;
-    for (page = first_page; page <= end_page; page++) {
-        OPT_ASSERT(remaining_obj_sz);
-
+    for (page = first_page; page <= end_page && remaining_obj_sz; page++) {
         slice_sz = remaining_obj_sz;
         if (in_page_offset + slice_sz > 4096UL) {
             /* not over page boundaries */
@@ -662,12 +660,10 @@
     size_t obj_size = stmcb_size_rounded_up(realobj);
 
     uintptr_t offset_itemsize[2];
-    stmcb_get_card_base_itemsize(realobj, offset_itemsize);
-    size_t real_idx_count = (obj_size - offset_itemsize[0]) / offset_itemsize[1];
-    assert(IMPLY(index != -1 && index != -2, index >= 0 && index < real_idx_count));
-
-    struct stm_read_marker_s *cards = get_read_marker(STM_SEGMENT->segment_base, (uintptr_t)obj);
-    uintptr_t last_card_index = get_index_to_card_index(real_idx_count - 1); /* max valid index */
+    if (index != -1) {
+        /* don't call if index==-1 as callback may not be supported at all */
+        stmcb_get_card_base_itemsize(realobj, offset_itemsize);
+    }
 
     /* decide where to start copying: */
     size_t start_offset;
@@ -692,6 +688,12 @@
         return;
     }
 
+    size_t real_idx_count = (obj_size - offset_itemsize[0]) / offset_itemsize[1];
+    assert(IMPLY(index != -1 && index != -2, index >= 0 && index < real_idx_count));
+    struct stm_read_marker_s *cards = get_read_marker(STM_SEGMENT->segment_base, (uintptr_t)obj);
+    uintptr_t last_card_index = get_index_to_card_index(real_idx_count - 1); /* max valid index */
+
+
     /* decide if we want only a specific card: */
     if (index != -1) {
         if (start_offset != -1) {
@@ -833,8 +835,6 @@
     /* add to read set: */
     stm_read(obj);
 
-    DEBUG_EXPECT_SEGFAULT(false);
-
     if (mark_card) {
         if (!(obj->stm_flags & GCFLAG_WB_EXECUTED)) {
             make_bk_slices(obj,
@@ -843,6 +843,8 @@
                            false);      /* do_missing_cards */
         }
 
+        DEBUG_EXPECT_SEGFAULT(false);
+
         /* don't remove WRITE_BARRIER, but add CARDS_SET */
         obj->stm_flags |= (GCFLAG_CARDS_SET | GCFLAG_WB_EXECUTED);
         LIST_APPEND(STM_PSEGMENT->old_objects_with_cards_set, obj);
@@ -873,6 +875,7 @@
                            false);      /* do_missing_cards */
         }
 
+        DEBUG_EXPECT_SEGFAULT(false);
         /* remove the WRITE_BARRIER flag and add WB_EXECUTED */
         obj->stm_flags &= ~(GCFLAG_WRITE_BARRIER | GCFLAG_CARDS_SET);
         obj->stm_flags |= GCFLAG_WB_EXECUTED;

From noreply at buildbot.pypy.org  Fri Feb 27 22:44:08 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 27 Feb 2015 22:44:08 +0100 (CET)
Subject: [pypy-commit] stmgc c8-card-marking: more fixes
Message-ID: <20150227214408.0991A1C02BB@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-card-marking
Changeset: r1682:133dcfae5ec0
Date: 2015-02-27 21:51 +0100
http://bitbucket.org/pypy/stmgc/changeset/133dcfae5ec0/

Log:	more fixes

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -658,12 +658,7 @@
     /* get whole card range */
     struct object_s *realobj = (struct object_s*)REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
     size_t obj_size = stmcb_size_rounded_up(realobj);
-
-    uintptr_t offset_itemsize[2];
-    if (index != -1) {
-        /* don't call if index==-1 as callback may not be supported at all */
-        stmcb_get_card_base_itemsize(realobj, offset_itemsize);
-    }
+    uintptr_t offset_itemsize[2] = {-1, -1};
 
     /* decide where to start copying: */
     size_t start_offset;
@@ -682,17 +677,20 @@
                                      (stm_char*)obj + obj_size);
         } else {
             /* only fixed part */
+            stmcb_get_card_base_itemsize(realobj, offset_itemsize);
             make_bk_slices_for_range(obj, (stm_char*)obj + start_offset,
                                      (stm_char*)obj + offset_itemsize[0]);
         }
         return;
     }
 
+    stmcb_get_card_base_itemsize(realobj, offset_itemsize);
+
     size_t real_idx_count = (obj_size - offset_itemsize[0]) / offset_itemsize[1];
     assert(IMPLY(index != -1 && index != -2, index >= 0 && index < real_idx_count));
     struct stm_read_marker_s *cards = get_read_marker(STM_SEGMENT->segment_base, (uintptr_t)obj);
     uintptr_t last_card_index = get_index_to_card_index(real_idx_count - 1); /* max valid index */
-
+    uintptr_t card_index;
 
     /* decide if we want only a specific card: */
     if (index != -1) {
@@ -702,13 +700,16 @@
                                      (stm_char*)obj + offset_itemsize[0]);
         }
 
-        size_t after_card_offset = offset_itemsize[0] + (index + 1) * offset_itemsize[1];
+        card_index = get_index_to_card_index(index);
+        size_t card_offset = offset_itemsize[0]
+            + get_card_index_to_index(card_index) * offset_itemsize[1];
+        size_t after_card_offset = offset_itemsize[0]
+            + get_card_index_to_index(card_index + 1) * offset_itemsize[1];
         if (after_card_offset > obj_size)
             after_card_offset = obj_size;
 
         make_bk_slices_for_range(
-            obj, (stm_char*)obj + offset_itemsize[0] + index * offset_itemsize[1],
-            (stm_char*)obj + after_card_offset);
+            obj, (stm_char*)obj + card_offset, (stm_char*)obj + after_card_offset);
 
         return;
     }
@@ -716,7 +717,7 @@
     /* look for CARD_CLEAR or some non-transaction_read_version cards
        and make bk slices for them */
     assert(do_missing_cards && index == -1 && start_offset == -1);
-    uintptr_t card_index = 1;
+    card_index = 1;
     uintptr_t start_card_index = -1;
     while (card_index <= last_card_index) {
         uint8_t card_value = cards[card_index].rm;
@@ -728,13 +729,13 @@
             if (start_card_index == -1) {   /* first unmarked card */
                 start_card_index = card_index;
             }
-        }
-        else {
+        } else {
             /* "CARD_MARKED_OLD" or CARD_MARKED */
             OPT_ASSERT(card_value == STM_SEGMENT->transaction_read_version
                        || card_value == CARD_MARKED);
-            /* we don't need the cards anymore after executing this function */
-            cards[card_index].rm = CARD_CLEAR;
+            /* we don't need CARD_MARKED anymore after executing this function,
+               but we may still need to avoid creating bk copies in the future: */
+            cards[card_index].rm = STM_SEGMENT->transaction_read_version;
         }
 
         if (start_card_index != -1                    /* something to copy */
@@ -780,7 +781,7 @@
     }
 
     obj->stm_flags &= ~GCFLAG_CARDS_SET;
-    _cards_cleared_in_object(get_priv_segment(STM_SEGMENT->segment_num), obj);
+    _cards_cleared_in_object(get_priv_segment(STM_SEGMENT->segment_num), obj, false);
 }
 
 __attribute__((always_inline))
@@ -835,6 +836,37 @@
     /* add to read set: */
     stm_read(obj);
 
+    if (!(obj->stm_flags & GCFLAG_WB_EXECUTED)) {
+        /* the first time we write this obj, make sure it is fully
+           accessible, as major gc may depend on being able to trace
+           the full obj in this segment (XXX) */
+        char *realobj;
+        size_t obj_size;
+        int my_segnum = STM_SEGMENT->segment_num;
+        uintptr_t end_page, first_page = ((uintptr_t)obj) / 4096UL;
+
+        realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
+        obj_size = stmcb_size_rounded_up((struct object_s *)realobj);
+        /* get the last page containing data from the object */
+        if (LIKELY(is_small_uniform(obj))) {
+            end_page = first_page;
+        } else {
+            end_page = (((uintptr_t)obj) + obj_size - 1) / 4096UL;
+        }
+
+        acquire_privatization_lock(STM_SEGMENT->segment_num);
+        uintptr_t page;
+        for (page = first_page; page <= end_page; page++) {
+            if (get_page_status_in(my_segnum, page) == PAGE_NO_ACCESS) {
+                release_privatization_lock(STM_SEGMENT->segment_num);
+                volatile char *dummy = REAL_ADDRESS(STM_SEGMENT->segment_base, page * 4096UL);
+                *dummy;            /* force segfault */
+                acquire_privatization_lock(STM_SEGMENT->segment_num);
+            }
+        }
+        release_privatization_lock(STM_SEGMENT->segment_num);
+    }
+
     if (mark_card) {
         if (!(obj->stm_flags & GCFLAG_WB_EXECUTED)) {
             make_bk_slices(obj,
@@ -894,14 +926,6 @@
     return mark_card;
 }
 
-long _stm_write_slowpath_card_extra_base(void)
-{
-    /* XXX can go away? */
-    /* for the PyPy JIT: _stm_write_slowpath_card_extra_base[obj >> 4]
-       is the byte that must be set to CARD_MARKED.  The logic below
-       does the same, but more explicitly. */
-    return 0;
-}
 
 void _stm_write_slowpath_card(object_t *obj, uintptr_t index)
 {
@@ -1278,10 +1302,9 @@
     /* clear CARD_MARKED on objs (don't care about CARD_MARKED_OLD) */
     LIST_FOREACH_R(pseg->old_objects_with_cards_set, object_t * /*item*/,
         {
-            struct object_s *realobj = (struct object_s *)
-                REAL_ADDRESS(pseg->pub.segment_base, item);
-            if (realobj->stm_flags & GCFLAG_CARDS_SET)
-                _reset_object_cards(pseg, item, CARD_CLEAR, false, false);
+            /* CARDS_SET may have already been lost because stm_validate()
+               may call reset_modified_from_backup_copies() */
+            _reset_object_cards(pseg, item, CARD_CLEAR, false, false);
         });
 
     acquire_modification_lock(segment_num);
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -142,10 +142,11 @@
 
     /* Must trace the object later */
     LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, nobj_sync_now);
-    _cards_cleared_in_object(get_priv_segment(STM_SEGMENT->segment_num), nobj);
+    _cards_cleared_in_object(get_priv_segment(STM_SEGMENT->segment_num), nobj, true);
 }
 
-static void _cards_cleared_in_object(struct stm_priv_segment_info_s *pseg, object_t *obj)
+static void _cards_cleared_in_object(struct stm_priv_segment_info_s *pseg, object_t *obj,
+                                     bool strict) /* strict = MARKED_OLD not allowed */
 {
 #ifndef NDEBUG
     struct object_s *realobj = (struct object_s *)REAL_ADDRESS(pseg->pub.segment_base, obj);
@@ -154,17 +155,25 @@
     if (size < _STM_MIN_CARD_OBJ_SIZE)
         return;                 /* too small for cards */
 
+    assert(!(realobj->stm_flags & GCFLAG_CARDS_SET));
+
+    if (!stmcb_obj_supports_cards(realobj))
+        return;
+
+    uintptr_t offset_itemsize[2] = {0, 0};
+    stmcb_get_card_base_itemsize(realobj, offset_itemsize);
     struct stm_read_marker_s *cards = get_read_marker(pseg->pub.segment_base, (uintptr_t)obj);
     uintptr_t card_index = 1;
-    uintptr_t last_card_index = get_index_to_card_index(size - 1); /* max valid index */
+    size_t real_idx_count = (size - offset_itemsize[0]) / offset_itemsize[1];
+    uintptr_t last_card_index = get_index_to_card_index(real_idx_count - 1); /* max valid index */
 
     while (card_index <= last_card_index) {
         assert(cards[card_index].rm == CARD_CLEAR
-               || cards[card_index].rm < pseg->pub.transaction_read_version);
+               || (cards[card_index].rm != CARD_MARKED
+                   && cards[card_index].rm < pseg->pub.transaction_read_version)
+               || (!strict && cards[card_index].rm != CARD_MARKED));
         card_index++;
     }
-
-    assert(!(realobj->stm_flags & GCFLAG_CARDS_SET));
 #endif
 }
 
@@ -176,17 +185,17 @@
     struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count);
 
     for (; undo < end; undo++) {
-        _cards_cleared_in_object(pseg, undo->object);
+        _cards_cleared_in_object(pseg, undo->object, false);
     }
     LIST_FOREACH_R(
         pseg->large_overflow_objects, object_t * /*item*/,
-        _cards_cleared_in_object(pseg, item));
+        _cards_cleared_in_object(pseg, item, false));
     LIST_FOREACH_R(
         pseg->objects_pointing_to_nursery, object_t * /*item*/,
-        _cards_cleared_in_object(pseg, item));
+        _cards_cleared_in_object(pseg, item, false));
     LIST_FOREACH_R(
         pseg->old_objects_with_cards_set, object_t * /*item*/,
-        _cards_cleared_in_object(pseg, item));
+        _cards_cleared_in_object(pseg, item, false));
 #endif
 }
 
@@ -198,6 +207,8 @@
 #pragma push_macro("STM_SEGMENT")
 #undef STM_PSEGMENT
 #undef STM_SEGMENT
+    dprintf(("_reset_object_cards(%p, mark=%d, mark_all=%d, really_clear=%d)\n",
+             obj, mark_value, mark_all, really_clear));
     struct object_s *realobj = (struct object_s *)REAL_ADDRESS(pseg->pub.segment_base, obj);
     size_t size = stmcb_size_rounded_up(realobj);
     OPT_ASSERT(size >= _STM_MIN_CARD_OBJ_SIZE);
@@ -351,7 +362,7 @@
             } else {
                 LIST_APPEND(STM_PSEGMENT->large_overflow_objects, obj);
             }
-            _cards_cleared_in_object(pseg, obj);
+            _cards_cleared_in_object(pseg, obj, false);
         }
 
         /* the list could have moved while appending */
diff --git a/c8/stm/nursery.h b/c8/stm/nursery.h
--- a/c8/stm/nursery.h
+++ b/c8/stm/nursery.h
@@ -4,7 +4,8 @@
 
 static uint32_t highest_overflow_number;
 
-static void _cards_cleared_in_object(struct stm_priv_segment_info_s *pseg, object_t *obj);
+static void _cards_cleared_in_object(struct stm_priv_segment_info_s *pseg, object_t *obj,
+                                     bool strict);
 static void _reset_object_cards(struct stm_priv_segment_info_s *pseg,
                                 object_t *obj, uint8_t mark_value,
                                 bool mark_all, bool really_clear);
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -85,6 +85,7 @@
 
 void _stm_write_slowpath(object_t *);
 void _stm_write_slowpath_card(object_t *, uintptr_t);
+char _stm_write_slowpath_card_extra(object_t *);
 object_t *_stm_allocate_slowpath(ssize_t);
 object_t *_stm_allocate_external(ssize_t);
 void _stm_become_inevitable(const char*);
diff --git a/c8/test/test_card_marking.py b/c8/test/test_card_marking.py
--- a/c8/test/test_card_marking.py
+++ b/c8/test/test_card_marking.py
@@ -329,3 +329,15 @@
         assert get_card_value(o, 1000) == CARD_MARKED
 
         self.commit_transaction()
+
+    def test_card_marked_old(self):
+        o = stm_allocate_old(1000+20*CARD_SIZE)
+        self.start_transaction()
+        stm_set_char(o, 'x', HDR, True)
+        stm_set_char(o, 'u', HDR, False)
+        stm_minor_collect()
+        stm_set_char(o, 'y', HDR, True)
+        self.abort_transaction()
+
+        self.start_transaction()
+        assert stm_get_char(o, HDR) == '\0'

From noreply at buildbot.pypy.org  Fri Feb 27 22:44:09 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 27 Feb 2015 22:44:09 +0100 (CET)
Subject: [pypy-commit] stmgc c8-card-marking: even more fixes
Message-ID: <20150227214409.01F311C02BB@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-card-marking
Changeset: r1683:b551018deaf1
Date: 2015-02-27 22:15 +0100
http://bitbucket.org/pypy/stmgc/changeset/b551018deaf1/

Log:	even more fixes

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -701,10 +701,12 @@
         }
 
         card_index = get_index_to_card_index(index);
+
         size_t card_offset = offset_itemsize[0]
             + get_card_index_to_index(card_index) * offset_itemsize[1];
         size_t after_card_offset = offset_itemsize[0]
             + get_card_index_to_index(card_index + 1) * offset_itemsize[1];
+
         if (after_card_offset > obj_size)
             after_card_offset = obj_size;
 
@@ -733,10 +735,11 @@
             /* "CARD_MARKED_OLD" or CARD_MARKED */
             OPT_ASSERT(card_value == STM_SEGMENT->transaction_read_version
                        || card_value == CARD_MARKED);
-            /* we don't need CARD_MARKED anymore after executing this function,
-               but we may still need to avoid creating bk copies in the future: */
-            cards[card_index].rm = STM_SEGMENT->transaction_read_version;
         }
+        /* in any case, remember that we already made a bk slice for this
+           card, so set to "MARKED_OLD": */
+        cards[card_index].rm = STM_SEGMENT->transaction_read_version;
+
 
         if (start_card_index != -1                    /* something to copy */
             && (card_value == CARD_MARKED             /* found marked card */
diff --git a/c8/test/test_card_marking.py b/c8/test/test_card_marking.py
--- a/c8/test/test_card_marking.py
+++ b/c8/test/test_card_marking.py
@@ -229,44 +229,44 @@
         o = stm_allocate_old(1000+20*CARD_SIZE)
 
         self.start_transaction()
-        assert get_card_value(o, 1000) == CARD_CLEAR
+        assert get_card_value(o, 1000-HDR) == CARD_CLEAR
         stm_set_char(o, 'a', 1000, True)
-        assert get_card_value(o, 1000) == CARD_MARKED
+        assert get_card_value(o, 1000-HDR) == CARD_MARKED
         assert o in old_objects_with_cards_set()
 
         stm_minor_collect()
-        assert get_card_value(o, 1000) == CARD_MARKED_OLD()
+        assert get_card_value(o, 1000-HDR) == CARD_MARKED_OLD()
         self.commit_transaction()
 
         self.start_transaction()
-        assert get_card_value(o, 1000) not in (CARD_CLEAR, CARD_MARKED_OLD())
+        assert get_card_value(o, 1000-HDR) not in (CARD_CLEAR, CARD_MARKED_OLD())
         stm_set_char(o, 'b', 1000, True)
-        assert get_card_value(o, 1000) == CARD_MARKED
+        assert get_card_value(o, 1000-HDR) == CARD_MARKED
         assert o in old_objects_with_cards_set()
         self.commit_transaction()
 
     def test_clear_cards2(self):
         self.start_transaction()
         o = stm_allocate(1000+20*CARD_SIZE)
-        assert get_card_value(o, 1000) == CARD_CLEAR
+        assert get_card_value(o, 1000-HDR) == CARD_CLEAR
         stm_set_char(o, 'a', 1000, True)
-        assert get_card_value(o, 1000) == CARD_CLEAR
+        assert get_card_value(o, 1000-HDR) == CARD_CLEAR
         assert o not in old_objects_with_cards_set()
 
         self.push_root(o)
         stm_minor_collect()
         o = self.pop_root()
 
-        assert get_card_value(o, 1000) == CARD_CLEAR
+        assert get_card_value(o, 1000-HDR) == CARD_CLEAR
         stm_set_char(o, 'b', 1000, True)
-        assert get_card_value(o, 1000) == CARD_MARKED
+        assert get_card_value(o, 1000-HDR) == CARD_MARKED
         assert o in old_objects_with_cards_set()
         self.commit_transaction()
 
         self.start_transaction()
-        assert get_card_value(o, 1000) not in (CARD_CLEAR, CARD_MARKED_OLD())
+        assert get_card_value(o, 1000-HDR) not in (CARD_CLEAR, CARD_MARKED_OLD())
         stm_set_char(o, 'b', 1000, True)
-        assert get_card_value(o, 1000) == CARD_MARKED
+        assert get_card_value(o, 1000-HDR) == CARD_MARKED
         assert o in old_objects_with_cards_set()
         self.commit_transaction()
 
@@ -278,13 +278,13 @@
         stm_minor_collect()
         o = self.pop_root()
 
-        assert get_card_value(o, 1000) == CARD_CLEAR
+        assert get_card_value(o, 1000-HDR) == CARD_CLEAR
         stm_set_char(o, 'b', 1000, True)
-        assert get_card_value(o, 1000) == CARD_MARKED
+        assert get_card_value(o, 1000-HDR) == CARD_MARKED
         assert o in old_objects_with_cards_set()
 
         stm_major_collect()
-        assert get_card_value(o, 1000) == CARD_CLEAR
+        assert get_card_value(o, 1000-HDR) == CARD_CLEAR
         assert o not in old_objects_with_cards_set()
 
         self.commit_transaction()
@@ -293,8 +293,8 @@
         self.start_transaction()
         o = stm_allocate(1000+20*CARD_SIZE)
         p = stm_allocate(1000+20*CARD_SIZE)
-        assert get_card_value(o, 1000) == CARD_CLEAR
-        assert get_card_value(p, 1000) == CARD_CLEAR
+        assert get_card_value(o, 1000-HDR) == CARD_CLEAR
+        assert get_card_value(p, 1000-HDR) == CARD_CLEAR
 
         self.push_root(o)
         self.push_root(p)
@@ -302,20 +302,20 @@
         p = self.pop_root()
         o = self.pop_root()
 
-        assert get_card_value(o, 1000) == CARD_CLEAR
-        assert get_card_value(p, 1000) == CARD_CLEAR
+        assert get_card_value(o, 1000-HDR) == CARD_CLEAR
+        assert get_card_value(p, 1000-HDR) == CARD_CLEAR
         stm_set_char(o, 'b', 1000, True)
         stm_set_char(p, 'b', 1000, True)
-        assert get_card_value(o, 1000) == CARD_MARKED
-        assert get_card_value(p, 1000) == CARD_MARKED
+        assert get_card_value(o, 1000-HDR) == CARD_MARKED
+        assert get_card_value(p, 1000-HDR) == CARD_MARKED
 
         self.push_root(o)
         stm_major_collect()
         o = self.pop_root()
         # p dies and gets cards CLEARed
 
-        assert get_card_value(o, 1000) == CARD_MARKED_OLD()
-        assert get_card_value(p, 1000) == CARD_CLEAR
+        assert get_card_value(o, 1000-HDR) == CARD_MARKED_OLD()
+        assert get_card_value(p, 1000-HDR) == CARD_CLEAR
 
         self.push_root(o)
         self.commit_transaction()
@@ -323,10 +323,10 @@
         self.start_transaction()
         o = self.pop_root()
 
-        assert get_card_value(o, 1000) != CARD_CLEAR
-        assert get_card_value(o, 1000) < CARD_MARKED_OLD()
+        assert get_card_value(o, 1000-HDR) != CARD_CLEAR
+        assert get_card_value(o, 1000-HDR) < CARD_MARKED_OLD()
         stm_set_char(o, 'b', 1000, True)
-        assert get_card_value(o, 1000) == CARD_MARKED
+        assert get_card_value(o, 1000-HDR) == CARD_MARKED
 
         self.commit_transaction()
 
@@ -341,3 +341,24 @@
 
         self.start_transaction()
         assert stm_get_char(o, HDR) == '\0'
+
+
+    def test_card_marked_old2(self):
+        o = stm_allocate_old(1000+20*CARD_SIZE)
+
+        self.start_transaction()
+        stm_set_char(o, 'v', HDR, False)
+        self.commit_transaction()
+
+        self.start_transaction()
+        assert get_card_value(o, HDR-HDR) == CARD_CLEAR
+        stm_set_char(o, 'x', 1000, True)
+
+        stm_set_char(o, 'u', HDR, False)
+
+        stm_minor_collect()
+        stm_set_char(o, 'y', HDR, True)
+        self.abort_transaction()
+
+        self.start_transaction()
+        assert stm_get_char(o, HDR) == 'v'

From noreply at buildbot.pypy.org  Fri Feb 27 22:44:09 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Fri, 27 Feb 2015 22:44:09 +0100 (CET)
Subject: [pypy-commit] stmgc c8-card-marking: one more
Message-ID: <20150227214409.F16601C02BB@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-card-marking
Changeset: r1684:17b08c18f7b4
Date: 2015-02-27 22:42 +0100
http://bitbucket.org/pypy/stmgc/changeset/17b08c18f7b4/

Log:	one more

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -675,6 +675,12 @@
             /* whole obj */
             make_bk_slices_for_range(obj, (stm_char*)obj + start_offset,
                                      (stm_char*)obj + obj_size);
+            if (obj_should_use_cards(STM_SEGMENT->segment_base, obj)) {
+                /* mark whole obj as MARKED_OLD so we don't do bk slices anymore */
+                _reset_object_cards(get_priv_segment(STM_SEGMENT->segment_num),
+                                    obj, STM_SEGMENT->transaction_read_version,
+                                    true, false);
+            }
         } else {
             /* only fixed part */
             stmcb_get_card_base_itemsize(realobj, offset_itemsize);
@@ -944,6 +950,7 @@
             return;
     }
 
+    assert(obj_should_use_cards(STM_SEGMENT->segment_base, obj));
     dprintf_test(("write_slowpath_card %p -> index:%lu\n",
                   obj, index));
 
@@ -1266,8 +1273,9 @@
                undo->backup,
                SLICE_SIZE(undo->slice));
 
-        dprintf(("reset_modified_from_backup_copies(%d): obj=%p off=%lu bk=%p\n",
-                 segment_num, obj, SLICE_OFFSET(undo->slice), undo->backup));
+        dprintf(("reset_modified_from_backup_copies(%d): obj=%p off=%lu sz=%d bk=%p\n",
+                 segment_num, obj, SLICE_OFFSET(undo->slice),
+                 SLICE_SIZE(undo->slice), undo->backup));
 
         free_bk(undo);
     }
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -488,7 +488,8 @@
                     REAL_ADDRESS(pseg->pub.segment_base, item);
                 OPT_ASSERT(realobj->stm_flags & GCFLAG_WRITE_BARRIER);
 
-                /* mark marked cards as old if it survives */
+                /* mark marked cards as old if it survives, otherwise
+                   CLEAR, as their spot could get reused */
                 uint8_t mark_value = mark_visited_test(item) ?
                     pseg->pub.transaction_read_version : CARD_CLEAR;
                 _reset_object_cards(pseg, item, mark_value, false,
diff --git a/c8/test/test_card_marking.py b/c8/test/test_card_marking.py
--- a/c8/test/test_card_marking.py
+++ b/c8/test/test_card_marking.py
@@ -351,7 +351,7 @@
         self.commit_transaction()
 
         self.start_transaction()
-        assert get_card_value(o, HDR-HDR) == CARD_CLEAR
+        assert get_card_value(o, HDR-HDR) < CARD_MARKED_OLD()
         stm_set_char(o, 'x', 1000, True)
 
         stm_set_char(o, 'u', HDR, False)
@@ -362,3 +362,15 @@
 
         self.start_transaction()
         assert stm_get_char(o, HDR) == 'v'
+
+    def test_card_marked_old3(self):
+        o = stm_allocate_old(1000+20*CARD_SIZE)
+
+        self.start_transaction()
+        stm_set_char(o, 'v', HDR, False)
+        stm_minor_collect()
+        stm_set_char(o, 'u', HDR, True)
+        self.abort_transaction()
+
+        self.start_transaction()
+        assert stm_get_char(o, HDR) == '\0'

From noreply at buildbot.pypy.org  Fri Feb 27 23:13:25 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 27 Feb 2015 23:13:25 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Adapt tests to changed import error
	messages.
Message-ID: <20150227221325.35B1D1C01CD@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76189:c957500b9b7b
Date: 2015-02-27 23:12 +0100
http://bitbucket.org/pypy/pypy/changeset/c957500b9b7b/

Log:	Adapt tests to changed import error messages.

diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -388,19 +388,19 @@
         def imp():
             from pkg import relative_f
         exc = raises(ImportError, imp)
-        assert exc.value.args[0] == "No module named pkg.imp"
+        assert exc.value.args[0] == "No module named 'pkg.imp'"
 
     def test_no_relative_import_bug(self):
         def imp():
             from pkg import relative_g
         exc = raises(ImportError, imp)
-        assert exc.value.args[0] == "No module named pkg.imp"
+        assert exc.value.args[0] == "No module named 'pkg.imp'"
 
     def test_import_msg(self):
         def imp():
             import pkg.i_am_not_here.neither_am_i
         exc = raises(ImportError, imp)
-        assert exc.value.args[0] == "No module named pkg.i_am_not_here"
+        assert exc.value.args[0] == "No module named 'pkg.i_am_not_here'"
 
     def test_future_relative_import_level_1(self):
         from pkg import relative_c

From noreply at buildbot.pypy.org  Fri Feb 27 23:13:23 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 27 Feb 2015 23:13:23 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Use different filenames for the two
 test_open_exclusive (one is in test_fileio.py and one is in test_io).
Message-ID: <20150227221323.DAE321C01CD@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76188:f4712520c45f
Date: 2015-02-27 22:53 +0100
http://bitbucket.org/pypy/pypy/changeset/f4712520c45f/

Log:	Use different filenames for the two test_open_exclusive (one is in
	test_fileio.py and one is in test_io).

diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py
--- a/pypy/module/_io/test/test_fileio.py
+++ b/pypy/module/_io/test/test_fileio.py
@@ -227,7 +227,7 @@
         FileExistsError = OSError
 
         import _io
-        filename = self.tmpfile + '_x'
+        filename = self.tmpfile + '_x1'
         raises(ValueError, _io.FileIO, filename, 'xw')
         with _io.FileIO(filename, 'x') as f:
             assert f.mode == 'xb'
diff --git a/pypy/module/_io/test/test_io.py b/pypy/module/_io/test/test_io.py
--- a/pypy/module/_io/test/test_io.py
+++ b/pypy/module/_io/test/test_io.py
@@ -445,7 +445,7 @@
         FileExistsError = OSError
 
         import _io
-        filename = self.tmpfile + '_x'
+        filename = self.tmpfile + '_x2'
         raises(ValueError, _io.open, filename, 'xw')
         with _io.open(filename, 'x') as f:
             assert f.mode == 'x'

From noreply at buildbot.pypy.org  Sat Feb 28 11:06:54 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Sat, 28 Feb 2015 11:06:54 +0100 (CET)
Subject: [pypy-commit] pypy py3.3: Adapt test to new function repr (which
 now contains the qualname).
Message-ID: <20150228100655.0088E1C155E@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r76190:4b41587d9970
Date: 2015-02-28 09:48 +0100
http://bitbucket.org/pypy/pypy/changeset/4b41587d9970/

Log:	Adapt test to new function repr (which now contains the qualname).

diff --git a/pypy/objspace/std/test/test_proxy_function.py b/pypy/objspace/std/test/test_proxy_function.py
--- a/pypy/objspace/std/test/test_proxy_function.py
+++ b/pypy/objspace/std/test/test_proxy_function.py
@@ -59,7 +59,7 @@
             pass
         
         fun = self.get_proxy(f)
-        assert repr(fun).startswith(".f")
 
     def test_func_code(self):
         def f():

From noreply at buildbot.pypy.org  Sat Feb 28 12:13:44 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Sat, 28 Feb 2015 12:13:44 +0100 (CET)
Subject: [pypy-commit] stmgc c8-card-marking: add fastpath for card marking
Message-ID: <20150228111344.7DA1B1C010B@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-card-marking
Changeset: r1685:8ca689687bfa
Date: 2015-02-28 12:14 +0100
http://bitbucket.org/pypy/stmgc/changeset/8ca689687bfa/

Log:	add fastpath for card marking

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -978,6 +978,9 @@
     struct stm_read_marker_s *cards = get_read_marker(STM_SEGMENT->segment_base,
                                                       (uintptr_t)obj);
     uintptr_t card_index = get_index_to_card_index(index);
+    if (cards[card_index].rm == CARD_MARKED)
+        return;
+
     if (!IS_OVERFLOW_OBJ(STM_PSEGMENT, obj)
         && !(cards[card_index].rm == CARD_MARKED
              || cards[card_index].rm == STM_SEGMENT->transaction_read_version)) {

From noreply at buildbot.pypy.org  Sat Feb 28 12:14:25 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 28 Feb 2015 12:14:25 +0100 (CET)
Subject: [pypy-commit] pypy recent-pure-ops: Killing one dict
Message-ID: <20150228111425.6ACCB1C010B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: recent-pure-ops
Changeset: r76191:00c3abc7848a
Date: 2015-02-28 12:13 +0100
http://bitbucket.org/pypy/pypy/changeset/00c3abc7848a/

Log:	Killing one dict


From noreply at buildbot.pypy.org  Sat Feb 28 12:14:26 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 28 Feb 2015 12:14:26 +0100 (CET)
Subject: [pypy-commit] pypy recent-pure-ops: (fijal, arigo)  in-progress
Message-ID: <20150228111426.A8D121C010B@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: recent-pure-ops
Changeset: r76192:5954b774370c
Date: 2015-02-28 12:14 +0100
http://bitbucket.org/pypy/pypy/changeset/5954b774370c/

Log:	(fijal, arigo) in-progress

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -721,6 +721,7 @@
         if clear:
             self.clear_newoperations()
         for op in self.loop.operations:
+            self._really_emitted_operation = None
             self.first_optimization.propagate_forward(op)
         self.loop.operations = self.get_newoperations()
         self.loop.quasi_immutable_deps = self.quasi_immutable_deps
@@ -773,9 +774,12 @@
                 op = self.store_final_boxes_in_guard(guard_op, pendingfields)
         elif op.can_raise():
             self.exception_might_have_happened = True
-        self._last_emitted_op = orig_op
+        self._really_emitted_operation = op
         self._newoperations.append(op)
 
+    def getlastop(self):
+        return self._really_emitted_operation
+
     def get_op_replacement(self, op):
         changed = False
         for i, arg in enumerate(op.getarglist()):
diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py
--- a/rpython/jit/metainterp/optimizeopt/pure.py
+++ b/rpython/jit/metainterp/optimizeopt/pure.py
@@ -1,12 +1,54 @@
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, REMOVED
 from rpython.jit.metainterp.resoperation import rop, ResOperation
-from rpython.jit.metainterp.optimizeopt.util import (make_dispatcher_method,
-    args_dict)
+from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
+
+
+class RecentPureOps(object):
+    REMEMBER_LIMIT = 16
+
+    def __init__(self):
+        self.lst = [None] * self.REMEMBER_LIMIT
+        self.next_index = 0
+
+    def add(self, op):
+        next_index = self.next_index
+        self.next_index = (next_index + 1) % self.REMEMBER_LIMIT
+        self.lst[next_index] = op
+
+    def lookup1(self, box0):
+        for i in range(self.REMEMBER_LIMIT):
+            op = self.lst[i]
+            if op is None:
+                break
+            if op.getarg(0).same_box(box0):
+                return op
+        return None
+
+    def lookup2(self, box0, box1):
+        for i in range(self.REMEMBER_LIMIT):
+            op = self.lst[i]
+            if op is None:
+                break
+            if op.getarg(0).same_box(box0) and op.getarg(1).same_box(box1):
+                return op
+        return None
+
+    def lookup(self, optimizer, op):
+        numargs = op.numargs()
+        if numargs == 1:
+            return self.lookup1(optimizer.get_box_replacement(op.getarg(0)))
+        elif numargs == 2:
+            return self.lookup2(optimizer.get_box_replacement(op.getarg(0)),
+                                optimizer.get_box_replacement(op.getarg(1)))
+        else:
+            assert False
+
 
 class OptPure(Optimization):
     def __init__(self):
         self.postponed_op = None
-        self.pure_operations = args_dict()
+        self._pure_operations = [None] * (rop._ALWAYS_PURE_LAST -
+                                          rop._ALWAYS_PURE_FIRST)
         self.call_pure_positions = []
 
     def propagate_forward(self, op):
@@ -39,20 +81,31 @@
                 return
 
             # did we do the exact same operation already?
-            args = self.optimizer.make_args_key(op)
-            oldvalue = self.pure_operations.get(args, None)
-            if oldvalue is not None:
-                self.optimizer.make_equal_to(op.result, oldvalue, True)
+            recentops = self.getrecentops(op.getopnum())
+            oldop = recentops.lookup(self.optimizer, op)
+            if oldop is not None:
+                self.optimizer.make_equal_to(op.result, oldop.result, True)
                 return
 
         # otherwise, the operation remains
         self.emit_operation(op)
         if op.returns_bool_result():
             self.optimizer.bool_boxes[self.getvalue(op.result)] = None
+        if canfold:
+            realop = self.optimizer.getlastop()
+            if realop is not None:
+                recentops = self.getrecentops(realop.getopnum())
+                recentops.add(realop)
         if nextop:
             self.emit_operation(nextop)
-        if args is not None:
-            self.pure_operations[args] = self.getvalue(op.result)
+
+    def getrecentops(self, opnum):
+        opnum = opnum - rop._ALWAYS_PURE_FIRST
+        assert 0 <= opnum < len(self._pure_operations)
+        recentops = self._pure_operations[opnum]
+        if recentops is None:
+            self._pure_operations[opnum] = recentops = RecentPureOps()
+        return recentops
 
     def optimize_CALL_PURE(self, op):
         # Step 1: check if all arguments are constant

From noreply at buildbot.pypy.org  Sat Feb 28 12:19:47 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Sat, 28 Feb 2015 12:19:47 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c8: import stmgc 17b08c18f7b4
Message-ID: <20150228111947.9B9181C009F@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: stmgc-c8
Changeset: r76193:aef00a6d5e2b
Date: 2015-02-28 09:27 +0100
http://bitbucket.org/pypy/pypy/changeset/aef00a6d5e2b/

Log:	import stmgc 17b08c18f7b4

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-e32b94df8ecb
+17b08c18f7b4
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -576,118 +576,428 @@
 }
 
 
-void _stm_write_slowpath(object_t *obj)
+bool obj_should_use_cards(char *seg_base, object_t *obj)
 {
-    assert(_seems_to_be_running_transaction());
-    assert(!_is_in_nursery(obj));
-    assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
+    if (is_small_uniform(obj))
+        return false;
 
-    int my_segnum = STM_SEGMENT->segment_num;
-    uintptr_t end_page, first_page = ((uintptr_t)obj) / 4096UL;
-    char *realobj;
-    size_t obj_size;
+    struct object_s *realobj = (struct object_s *)
+        REAL_ADDRESS(seg_base, obj);
+    long supports = stmcb_obj_supports_cards(realobj);
+    if (!supports)
+        return false;
 
-    realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
-    obj_size = stmcb_size_rounded_up((struct object_s *)realobj);
-    /* get the last page containing data from the object */
-    if (LIKELY(is_small_uniform(obj))) {
-        end_page = first_page;
-    } else {
-        end_page = (((uintptr_t)obj) + obj_size - 1) / 4096UL;
-    }
+    /* check also if it makes sense: */
+    size_t size = stmcb_size_rounded_up(realobj);
+    return (size >= _STM_MIN_CARD_OBJ_SIZE);
+}
 
-    /* add to read set: */
-    stm_read(obj);
 
-    if (obj->stm_flags & GCFLAG_WB_EXECUTED) {
-        /* already executed WB once in this transaction. do GC
-           part again: */
-        dprintf(("write_slowpath-fast(%p)\n", obj));
-        obj->stm_flags &= ~GCFLAG_WRITE_BARRIER;
-        LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj);
-        return;
-    }
-
-    assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED));
-    dprintf(("write_slowpath(%p): sz=%lu\n", obj, obj_size));
-
- retry:
-    /* privatize pages: */
-    /* XXX don't always acquire all locks... */
-    acquire_all_privatization_locks();
+static void make_bk_slices_for_range(
+    object_t *obj,
+    stm_char *start, stm_char *end) /* [start, end[ */
+{
+    dprintf(("make_bk_slices_for_range(%p, %lu, %lu)\n",
+             obj, start - (stm_char*)obj, end - start));
+    char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
+    uintptr_t first_page = ((uintptr_t)start) / 4096UL;
+    uintptr_t end_page = ((uintptr_t)end) / 4096UL;
 
     uintptr_t page;
-    for (page = first_page; page <= end_page; page++) {
-        if (get_page_status_in(my_segnum, page) == PAGE_NO_ACCESS) {
-            /* XXX: slow? */
-            release_all_privatization_locks();
-
-            volatile char *dummy = REAL_ADDRESS(STM_SEGMENT->segment_base, page * 4096UL);
-            *dummy;            /* force segfault */
-
-            goto retry;
-        }
-    }
-    /* all pages are private to us and we hold the privatization_locks so
-       we are allowed to modify them */
-
-    /* phew, now add the obj to the write-set and register the
-       backup copy. */
-    /* XXX: we should not be here at all fiddling with page status
-       if 'obj' is merely an overflow object.  FIX ME, likely by copying
-       the overflow number logic from c7. */
-
-    DEBUG_EXPECT_SEGFAULT(false);
-
-    acquire_modification_lock(STM_SEGMENT->segment_num);
     uintptr_t slice_sz;
-    uintptr_t in_page_offset = (uintptr_t)obj % 4096UL;
-    uintptr_t remaining_obj_sz = obj_size;
-    for (page = first_page; page <= end_page; page++) {
-        /* XXX Maybe also use mprotect() again to mark pages of the object as read-only, and
-           only stick it into modified_old_objects page-by-page?  Maybe it's
-           possible to do card-marking that way, too. */
-        OPT_ASSERT(remaining_obj_sz);
-
+    uintptr_t slice_off = start - (stm_char*)obj;
+    uintptr_t in_page_offset = (uintptr_t)start % 4096UL;
+    uintptr_t remaining_obj_sz = end - start;
+    for (page = first_page; page <= end_page && remaining_obj_sz; page++) {
         slice_sz = remaining_obj_sz;
         if (in_page_offset + slice_sz > 4096UL) {
             /* not over page boundaries */
             slice_sz = 4096UL - in_page_offset;
         }
 
-        size_t slice_off = obj_size - remaining_obj_sz;
+        remaining_obj_sz -= slice_sz;
+        in_page_offset = (in_page_offset + slice_sz) % 4096UL; /* mostly 0 */
 
         /* make backup slice: */
         char *bk_slice = malloc(slice_sz);
         increment_total_allocated(slice_sz);
         memcpy(bk_slice, realobj + slice_off, slice_sz);
 
+        acquire_modification_lock(STM_SEGMENT->segment_num);
         /* !! follows layout of "struct stm_undo_s" !! */
         STM_PSEGMENT->modified_old_objects = list_append3(
             STM_PSEGMENT->modified_old_objects,
             (uintptr_t)obj,     /* obj */
             (uintptr_t)bk_slice,  /* bk_addr */
             NEW_SLICE(slice_off, slice_sz));
+        dprintf(("> append slice %p, off=%lu, sz=%lu\n", bk_slice, slice_off, slice_sz));
+        release_modification_lock(STM_SEGMENT->segment_num);
 
-        remaining_obj_sz -= slice_sz;
-        in_page_offset = (in_page_offset + slice_sz) % 4096UL; /* mostly 0 */
+        slice_off += slice_sz;
     }
-    OPT_ASSERT(remaining_obj_sz == 0);
 
-    /* remove the WRITE_BARRIER flag and add WB_EXECUTED */
-    obj->stm_flags &= ~GCFLAG_WRITE_BARRIER;
-    obj->stm_flags |= GCFLAG_WB_EXECUTED;
+}
+
+static void make_bk_slices(object_t *obj,
+                           bool first_call, /* tells us if we also need to make a bk
+                                               of the non-array part of the object */
+                           uintptr_t index,  /* index == -1: all cards, index == -2: no cards */
+                           bool do_missing_cards /* only bk the cards that don't have a bk */
+                           )
+{
+    dprintf(("make_bk_slices(%p, %d, %ld, %d)\n", obj, first_call, index, do_missing_cards));
+    /* do_missing_cards also implies that all cards are cleared at the end */
+    /* index == -1 but not do_missing_cards: bk whole obj */
+    assert(IMPLY(index == -2, first_call && !do_missing_cards));
+    assert(IMPLY(index == -1 && !do_missing_cards, first_call));
+    assert(IMPLY(do_missing_cards, index == -1));
+    assert(IMPLY(is_small_uniform(obj), index == -1 && !do_missing_cards && first_call));
+    assert(IMPLY(first_call, !do_missing_cards));
+    assert(IMPLY(index != -1, obj_should_use_cards(STM_SEGMENT->segment_base, obj)));
+
+    /* get whole card range */
+    struct object_s *realobj = (struct object_s*)REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
+    size_t obj_size = stmcb_size_rounded_up(realobj);
+    uintptr_t offset_itemsize[2] = {-1, -1};
+
+    /* decide where to start copying: */
+    size_t start_offset;
+    if (first_call) {
+        start_offset = 0;
+    } else {
+        start_offset = -1;
+    }
+
+    /* decide if we don't want to look at cards at all: */
+    if ((index == -1 || index == -2) && !do_missing_cards) {
+        assert(first_call);
+        if (index == -1) {
+            /* whole obj */
+            make_bk_slices_for_range(obj, (stm_char*)obj + start_offset,
+                                     (stm_char*)obj + obj_size);
+            if (obj_should_use_cards(STM_SEGMENT->segment_base, obj)) {
+                /* mark whole obj as MARKED_OLD so we don't do bk slices anymore */
+                _reset_object_cards(get_priv_segment(STM_SEGMENT->segment_num),
+                                    obj, STM_SEGMENT->transaction_read_version,
+                                    true, false);
+            }
+        } else {
+            /* only fixed part */
+            stmcb_get_card_base_itemsize(realobj, offset_itemsize);
+            make_bk_slices_for_range(obj, (stm_char*)obj + start_offset,
+                                     (stm_char*)obj + offset_itemsize[0]);
+        }
+        return;
+    }
+
+    stmcb_get_card_base_itemsize(realobj, offset_itemsize);
+
+    size_t real_idx_count = (obj_size - offset_itemsize[0]) / offset_itemsize[1];
+    assert(IMPLY(index != -1 && index != -2, index >= 0 && index < real_idx_count));
+    struct stm_read_marker_s *cards = get_read_marker(STM_SEGMENT->segment_base, (uintptr_t)obj);
+    uintptr_t last_card_index = get_index_to_card_index(real_idx_count - 1); /* max valid index */
+    uintptr_t card_index;
+
+    /* decide if we want only a specific card: */
+    if (index != -1) {
+        if (start_offset != -1) {
+            /* bk fixed part separately: */
+            make_bk_slices_for_range(obj, (stm_char*)obj + start_offset,
+                                     (stm_char*)obj + offset_itemsize[0]);
+        }
+
+        card_index = get_index_to_card_index(index);
+
+        size_t card_offset = offset_itemsize[0]
+            + get_card_index_to_index(card_index) * offset_itemsize[1];
+        size_t after_card_offset = offset_itemsize[0]
+            + get_card_index_to_index(card_index + 1) * offset_itemsize[1];
+
+        if (after_card_offset > obj_size)
+            after_card_offset = obj_size;
+
+        make_bk_slices_for_range(
+            obj, (stm_char*)obj + card_offset, (stm_char*)obj + after_card_offset);
+
+        return;
+    }
+
+    /* look for CARD_CLEAR or some non-transaction_read_version cards
+       and make bk slices for them */
+    assert(do_missing_cards && index == -1 && start_offset == -1);
+    card_index = 1;
+    uintptr_t start_card_index = -1;
+    while (card_index <= last_card_index) {
+        uint8_t card_value = cards[card_index].rm;
+
+        if (card_value == CARD_CLEAR
+            || (card_value != CARD_MARKED
+                && card_value < STM_SEGMENT->transaction_read_version)) {
+            /* we need a backup of this card */
+            if (start_card_index == -1) {   /* first unmarked card */
+                start_card_index = card_index;
+            }
+        } else {
+            /* "CARD_MARKED_OLD" or CARD_MARKED */
+            OPT_ASSERT(card_value == STM_SEGMENT->transaction_read_version
+                       || card_value == CARD_MARKED);
+        }
+        /* in any case, remember that we already made a bk slice for this
+           card, so set to "MARKED_OLD": */
+        cards[card_index].rm = STM_SEGMENT->transaction_read_version;
+
+
+        if (start_card_index != -1                    /* something to copy */
+            && (card_value == CARD_MARKED             /* found marked card */
+                || card_value == STM_SEGMENT->transaction_read_version/* old marked */
+                || card_index == last_card_index)) {  /* this is the last card */
+
+            /* do the bk slice: */
+            uintptr_t copy_size;
+            uintptr_t next_card_offset;
+            uintptr_t start_card_offset;
+            uintptr_t next_card_index = card_index;
+
+            if (card_value == CARD_CLEAR
+                || (card_value != CARD_MARKED
+                    && card_value < STM_SEGMENT->transaction_read_version)) {
+                /* this was actually the last card which wasn't set, but we
+                   need to go one further to get the right offset */
+                next_card_index++;
+            }
+
+            start_card_offset = offset_itemsize[0] +
+                get_card_index_to_index(start_card_index) * offset_itemsize[1];
+
+            next_card_offset = offset_itemsize[0] +
+                get_card_index_to_index(next_card_index) * offset_itemsize[1];
+
+            if (next_card_offset > obj_size)
+                next_card_offset = obj_size;
+
+            copy_size = next_card_offset - start_card_offset;
+            OPT_ASSERT(copy_size > 0);
+
+            /* add the slices: */
+            make_bk_slices_for_range(
+                obj, (stm_char*)obj + start_card_offset,
+                (stm_char*)obj + next_card_offset);
+
+            start_card_index = -1;
+        }
+
+        card_index++;
+    }
+
+    obj->stm_flags &= ~GCFLAG_CARDS_SET;
+    _cards_cleared_in_object(get_priv_segment(STM_SEGMENT->segment_num), obj, false);
+}
+
+__attribute__((always_inline))
+static void write_slowpath_overflow_obj(object_t *obj, bool mark_card)
+{
+    assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
+    assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED));
+    dprintf(("write_slowpath_overflow_obj(%p)\n", obj));
+
+    if (!mark_card) {
+        /* The basic case, with no card marking.  We append the object
+           into 'objects_pointing_to_nursery', and remove the flag so
+           that the write_slowpath will not be called again until the
+           next minor collection. */
+        if (obj->stm_flags & GCFLAG_CARDS_SET) {
+            /* if we clear this flag, we also need to clear the cards.
+               bk_slices are not needed as this is an overflow object */
+            _reset_object_cards(get_priv_segment(STM_SEGMENT->segment_num),
+                                obj, CARD_CLEAR, false, false);
+        }
+        obj->stm_flags &= ~(GCFLAG_WRITE_BARRIER | GCFLAG_CARDS_SET);
+        LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj);
+    } else {
+        /* Card marking.  Don't remove GCFLAG_WRITE_BARRIER because we
+           need to come back to _stm_write_slowpath_card() for every
+           card to mark.  Add GCFLAG_CARDS_SET.
+           again, we don't need bk_slices as this is an overflow obj */
+        assert(!(obj->stm_flags & GCFLAG_CARDS_SET));
+        obj->stm_flags |= GCFLAG_CARDS_SET;
+        LIST_APPEND(STM_PSEGMENT->old_objects_with_cards_set, obj);
+    }
+}
+
+
+__attribute__((always_inline))
+static void write_slowpath_common(object_t *obj, bool mark_card)
+{
+    assert(_seems_to_be_running_transaction());
+    assert(!_is_in_nursery(obj));
+    assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
+
+    if (IS_OVERFLOW_OBJ(STM_PSEGMENT, obj)) {
+        /* already executed WB once in this transaction. do GC
+           part again: */
+        assert(!(obj->stm_flags & GCFLAG_WB_EXECUTED));
+        write_slowpath_overflow_obj(obj, mark_card);
+        return;
+    }
+
+    dprintf(("write_slowpath(%p)\n", obj));
+
+    /* add to read set: */
+    stm_read(obj);
+
+    if (!(obj->stm_flags & GCFLAG_WB_EXECUTED)) {
+        /* the first time we write this obj, make sure it is fully
+           accessible, as major gc may depend on being able to trace
+           the full obj in this segment (XXX) */
+        char *realobj;
+        size_t obj_size;
+        int my_segnum = STM_SEGMENT->segment_num;
+        uintptr_t end_page, first_page = ((uintptr_t)obj) / 4096UL;
+
+        realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
+        obj_size = stmcb_size_rounded_up((struct object_s *)realobj);
+        /* get the last page containing data from the object */
+        if (LIKELY(is_small_uniform(obj))) {
+            end_page = first_page;
+        } else {
+            end_page = (((uintptr_t)obj) + obj_size - 1) / 4096UL;
+        }
+
+        acquire_privatization_lock(STM_SEGMENT->segment_num);
+        uintptr_t page;
+        for (page = first_page; page <= end_page; page++) {
+            if (get_page_status_in(my_segnum, page) == PAGE_NO_ACCESS) {
+                release_privatization_lock(STM_SEGMENT->segment_num);
+                volatile char *dummy = REAL_ADDRESS(STM_SEGMENT->segment_base, page * 4096UL);
+                *dummy;            /* force segfault */
+                acquire_privatization_lock(STM_SEGMENT->segment_num);
+            }
+        }
+        release_privatization_lock(STM_SEGMENT->segment_num);
+    }
+
+    if (mark_card) {
+        if (!(obj->stm_flags & GCFLAG_WB_EXECUTED)) {
+            make_bk_slices(obj,
+                           true,        /* first_call */
+                           -2,          /* index: backup only fixed part */
+                           false);      /* do_missing_cards */
+        }
+
+        DEBUG_EXPECT_SEGFAULT(false);
+
+        /* don't remove WRITE_BARRIER, but add CARDS_SET */
+        obj->stm_flags |= (GCFLAG_CARDS_SET | GCFLAG_WB_EXECUTED);
+        LIST_APPEND(STM_PSEGMENT->old_objects_with_cards_set, obj);
+    } else {
+        /* called if WB_EXECUTED is set or this is the first time
+           for this obj: */
+
+        /* add it to the GC list for minor collections */
+        LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj);
+
+        if (obj->stm_flags & GCFLAG_CARDS_SET) {
+            assert(obj->stm_flags & GCFLAG_WB_EXECUTED);
+
+            /* this is not the first_call to the WB for this obj,
+               we executed the above then-part before.
+               if we clear this flag, we have to add all the other
+               bk slices we didn't add yet */
+            make_bk_slices(obj,
+                           false,       /* first_call */
+                           -1,          /* index: whole obj */
+                           true);       /* do_missing_cards */
+
+        } else if (!(obj->stm_flags & GCFLAG_WB_EXECUTED)) {
+            /* first and only time we enter here: */
+            make_bk_slices(obj,
+                           true,        /* first_call */
+                           -1,          /* index: whole obj */
+                           false);      /* do_missing_cards */
+        }
+
+        DEBUG_EXPECT_SEGFAULT(false);
+        /* remove the WRITE_BARRIER flag and add WB_EXECUTED */
+        obj->stm_flags &= ~(GCFLAG_WRITE_BARRIER | GCFLAG_CARDS_SET);
+        obj->stm_flags |= GCFLAG_WB_EXECUTED;
+    }
 
     DEBUG_EXPECT_SEGFAULT(true);
+}
 
-    release_modification_lock(STM_SEGMENT->segment_num);
-    /* done fiddling with protection and privatization */
-    release_all_privatization_locks();
 
-    /* also add it to the GC list for minor collections */
-    LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, obj);
+char _stm_write_slowpath_card_extra(object_t *obj)
+{
+    /* the PyPy JIT calls this function directly if it finds that an
+       array doesn't have the GCFLAG_CARDS_SET */
+    bool mark_card = obj_should_use_cards(STM_SEGMENT->segment_base, obj);
+    write_slowpath_common(obj, mark_card);
+    return mark_card;
 }
 
+
+void _stm_write_slowpath_card(object_t *obj, uintptr_t index)
+{
+    dprintf_test(("write_slowpath_card(%p, %lu)\n",
+                  obj, index));
+
+    /* If CARDS_SET is not set so far, issue a normal write barrier.
+       If the object is large enough, ask it to set up the object for
+       card marking instead. */
+    if (!(obj->stm_flags & GCFLAG_CARDS_SET)) {
+        char mark_card = _stm_write_slowpath_card_extra(obj);
+        if (!mark_card)
+            return;
+    }
+
+    assert(obj_should_use_cards(STM_SEGMENT->segment_base, obj));
+    dprintf_test(("write_slowpath_card %p -> index:%lu\n",
+                  obj, index));
+
+    /* We reach this point if we have to mark the card. */
+    assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
+    assert(obj->stm_flags & GCFLAG_CARDS_SET);
+    assert(!is_small_uniform(obj)); /* not supported/tested */
+
+#ifndef NDEBUG
+    struct object_s *realobj = (struct object_s *)
+        REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
+    size_t size = stmcb_size_rounded_up(realobj);
+    /* we need at least one read marker in addition to the STM-reserved object
+       write-lock */
+    assert(size >= 32);
+    /* the 'index' must be in range(length-of-obj), but we don't have
+       a direct way to know the length.  We know that it is smaller
+       than the size in bytes. */
+    assert(index < size);
+#endif
+
+    /* Write into the card's lock.  This is used by the next minor
+       collection to know what parts of the big object may have changed.
+       We already own the object here or it is an overflow obj. */
+    struct stm_read_marker_s *cards = get_read_marker(STM_SEGMENT->segment_base,
+                                                      (uintptr_t)obj);
+    uintptr_t card_index = get_index_to_card_index(index);
+    if (!IS_OVERFLOW_OBJ(STM_PSEGMENT, obj)
+        && !(cards[card_index].rm == CARD_MARKED
+             || cards[card_index].rm == STM_SEGMENT->transaction_read_version)) {
+        /* need to do the backup slice of the card */
+        make_bk_slices(obj,
+                       false,       /* first_call */
+                       index,       /* index: only 1 card */
+                       false);      /* do_missing_cards */
+    }
+    cards[card_index].rm = CARD_MARKED;
+
+    dprintf(("mark %p index %lu, card:%lu with %d\n",
+             obj, index, get_index_to_card_index(index), CARD_MARKED));
+}
+
+void _stm_write_slowpath(object_t *obj) {
+    write_slowpath_common(obj,  /* mark_card */ false);
+}
+
+
 static void reset_transaction_read_version(void)
 {
     /* force-reset all read markers to 0 */
@@ -705,7 +1015,8 @@
 #endif
         memset(readmarkers, 0, NB_READMARKER_PAGES * 4096UL);
     }
-    STM_SEGMENT->transaction_read_version = 1;
+    STM_SEGMENT->transaction_read_version = 2;
+    assert(STM_SEGMENT->transaction_read_version > _STM_CARD_MARKED);
 }
 
 static void reset_wb_executed_flags(void)
@@ -766,7 +1077,7 @@
     }
 
     assert(list_is_empty(STM_PSEGMENT->modified_old_objects));
-    assert(list_is_empty(STM_PSEGMENT->new_objects));
+    assert(list_is_empty(STM_PSEGMENT->large_overflow_objects));
     assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery));
     assert(list_is_empty(STM_PSEGMENT->young_weakrefs));
     assert(tree_is_cleared(STM_PSEGMENT->young_outside_nursery));
@@ -826,8 +1137,11 @@
 
     STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
     STM_PSEGMENT->transaction_state = TS_NONE;
+
+    _verify_cards_cleared_in_all_lists(get_priv_segment(STM_SEGMENT->segment_num));
     list_clear(STM_PSEGMENT->objects_pointing_to_nursery);
-    list_clear(STM_PSEGMENT->new_objects);
+    list_clear(STM_PSEGMENT->old_objects_with_cards_set);
+    list_clear(STM_PSEGMENT->large_overflow_objects);
 
     release_thread_segment(tl);
     /* cannot access STM_SEGMENT or STM_PSEGMENT from here ! */
@@ -847,13 +1161,16 @@
 #endif
 }
 
-static void push_new_objects_to_other_segments(void)
+static void push_large_overflow_objects_to_other_segments(void)
 {
+    if (list_is_empty(STM_PSEGMENT->large_overflow_objects))
+        return;
+
+    /* XXX: also pushes small ones right now */
     acquire_privatization_lock(STM_SEGMENT->segment_num);
-    LIST_FOREACH_R(STM_PSEGMENT->new_objects, object_t *,
+    LIST_FOREACH_R(STM_PSEGMENT->large_overflow_objects, object_t *,
         ({
-            assert(item->stm_flags & GCFLAG_WB_EXECUTED);
-            item->stm_flags &= ~GCFLAG_WB_EXECUTED;
+            assert(!(item->stm_flags & GCFLAG_WB_EXECUTED));
             synchronize_object_enqueue(item);
         }));
     synchronize_objects_flush();
@@ -867,7 +1184,7 @@
        in handle_segfault_in_page() that also copies
        unknown-to-the-segment/uncommitted things.
     */
-    list_clear(STM_PSEGMENT->new_objects);
+    list_clear(STM_PSEGMENT->large_overflow_objects);
 }
 
 
@@ -882,23 +1199,42 @@
     dprintf(("> stm_commit_transaction()\n"));
     minor_collection(1);
 
-    push_new_objects_to_other_segments();
+    push_large_overflow_objects_to_other_segments();
     /* push before validate. otherwise they are reachable too early */
     bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE;
     _validate_and_add_to_commit_log();
 
+    stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
 
     /* XXX do we still need a s_mutex_lock() section here? */
     s_mutex_lock();
+    commit_finalizers();
 
+    /* update 'overflow_number' if needed */
+    if (STM_PSEGMENT->overflow_number_has_been_used) {
+        highest_overflow_number += GCFLAG_OVERFLOW_NUMBER_bit0;
+        assert(highest_overflow_number !=        /* XXX else, overflow! */
+               (uint32_t)-GCFLAG_OVERFLOW_NUMBER_bit0);
+        STM_PSEGMENT->overflow_number = highest_overflow_number;
+        STM_PSEGMENT->overflow_number_has_been_used = false;
+    }
+
+    invoke_and_clear_user_callbacks(0);   /* for commit */
+
+    /* >>>>> there may be a FORK() happening in the safepoint below <<<<<*/
     enter_safe_point_if_requested();
     assert(STM_SEGMENT->nursery_end == NURSERY_END);
 
-    stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
+    /* if a major collection is required, do it here */
+    if (is_major_collection_requested()) {
+        synchronize_all_threads(STOP_OTHERS_UNTIL_MUTEX_UNLOCK);
 
-    commit_finalizers();
+        if (is_major_collection_requested()) {   /* if *still* true */
+            major_collection_now_at_safe_point();
+        }
+    }
 
-    invoke_and_clear_user_callbacks(0);   /* for commit */
+    _verify_cards_cleared_in_all_lists(get_priv_segment(STM_SEGMENT->segment_num));
 
     if (globally_unique_transaction && was_inev) {
         committed_globally_unique_transaction();
@@ -937,8 +1273,9 @@
                undo->backup,
                SLICE_SIZE(undo->slice));
 
-        dprintf(("reset_modified_from_backup_copies(%d): obj=%p off=%lu bk=%p\n",
-                 segment_num, obj, SLICE_OFFSET(undo->slice), undo->backup));
+        dprintf(("reset_modified_from_backup_copies(%d): obj=%p off=%lu sz=%d bk=%p\n",
+                 segment_num, obj, SLICE_OFFSET(undo->slice),
+                 SLICE_SIZE(undo->slice), undo->backup));
 
         free_bk(undo);
     }
@@ -973,9 +1310,18 @@
 
     long bytes_in_nursery = throw_away_nursery(pseg);
 
+    /* clear CARD_MARKED on objs (don't care about CARD_MARKED_OLD) */
+    LIST_FOREACH_R(pseg->old_objects_with_cards_set, object_t * /*item*/,
+        {
+            /* CARDS_SET may have already been lost because stm_validate()
+               may call reset_modified_from_backup_copies() */
+            _reset_object_cards(pseg, item, CARD_CLEAR, false, false);
+        });
+
     acquire_modification_lock(segment_num);
     reset_modified_from_backup_copies(segment_num);
     release_modification_lock(segment_num);
+    _verify_cards_cleared_in_all_lists(pseg);
 
     stm_thread_local_t *tl = pseg->pub.running_thread;
 #ifdef STM_NO_AUTOMATIC_SETJMP
@@ -999,7 +1345,8 @@
     tl->last_abort__bytes_in_nursery = bytes_in_nursery;
 
     list_clear(pseg->objects_pointing_to_nursery);
-    list_clear(pseg->new_objects);
+    list_clear(pseg->old_objects_with_cards_set);
+    list_clear(pseg->large_overflow_objects);
     list_clear(pseg->young_weakrefs);
 #pragma pop_macro("STM_SEGMENT")
 #pragma pop_macro("STM_PSEGMENT")
@@ -1129,6 +1476,8 @@
     ++STM_PSEGMENT->sq_len;
 }
 
+
+
 static void synchronize_object_enqueue(object_t *obj)
 {
     assert(!_is_young(obj));
@@ -1141,14 +1490,14 @@
     OPT_ASSERT(obj_size >= 16);
 
     if (LIKELY(is_small_uniform(obj))) {
+        assert(!(obj->stm_flags & GCFLAG_CARDS_SET));
         OPT_ASSERT(obj_size <= GC_LAST_SMALL_SIZE);
         _synchronize_fragment((stm_char *)obj, obj_size);
         return;
     }
 
     /* else, a more complicated case for large objects, to copy
-       around data only within the needed pages
-    */
+       around data only within the needed pages */
     uintptr_t start = (uintptr_t)obj;
     uintptr_t end = start + obj_size;
 
@@ -1159,6 +1508,10 @@
         }
         uintptr_t copy_size = copy_up_to - start;
 
+        /* double-check that the result fits in one page */
+        assert(copy_size > 0);
+        assert(copy_size + (start & 4095) <= 4096);
+
         _synchronize_fragment((stm_char *)start, copy_size);
 
         start = copy_up_to;
diff --git a/rpython/translator/stm/src_stm/stm/core.h b/rpython/translator/stm/src_stm/stm/core.h
--- a/rpython/translator/stm/src_stm/stm/core.h
+++ b/rpython/translator/stm/src_stm/stm/core.h
@@ -34,17 +34,34 @@
 #define FIRST_OLD_RM_PAGE     (OLD_RM_START / 4096UL)
 #define NB_READMARKER_PAGES   (FIRST_OBJECT_PAGE - FIRST_READMARKER_PAGE)
 
+#define CARD_SIZE   _STM_CARD_SIZE
+
 enum /* stm_flags */ {
     GCFLAG_WRITE_BARRIER = _STM_GCFLAG_WRITE_BARRIER,
     GCFLAG_HAS_SHADOW = 0x02,
     GCFLAG_WB_EXECUTED = 0x04,
-    GCFLAG_VISITED = 0x08,
-    GCFLAG_FINALIZATION_ORDERING = 0x10,
+    GCFLAG_CARDS_SET = _STM_GCFLAG_CARDS_SET,
+    GCFLAG_VISITED = 0x10,
+    GCFLAG_FINALIZATION_ORDERING = 0x20,
+    /* All remaining bits of the 32-bit 'stm_flags' field are taken by
+       the "overflow number".  This is a number that identifies the
+       "overflow objects" from the current transaction among all old
+       objects.  More precisely, overflow objects are objects from the
+       current transaction that have been flushed out of the nursery,
+       which occurs if the same transaction allocates too many objects.
+    */
+    GCFLAG_OVERFLOW_NUMBER_bit0 = 0x40   /* must be last */
 };
 
+#define SYNC_QUEUE_SIZE    31
 
+enum /* card values in read markers */ {
+    CARD_CLEAR = 0,                 /* card not used at all */
+    CARD_MARKED = _STM_CARD_MARKED, /* card marked for tracing in the next gc */
+    /* CARD_MARKED_OLD = STM_PSEGMENT->transaction_read_version, */
+    /* card was marked before, but cleared in a GC */
+};
 
-#define SYNC_QUEUE_SIZE    31
 
 
 /************************************************************/
@@ -72,6 +89,7 @@
     struct list_s *modified_old_objects;
 
     struct list_s *objects_pointing_to_nursery;
+    struct list_s *old_objects_with_cards_set;
     struct tree_s *young_outside_nursery;
     struct tree_s *nursery_objects_shadows;
 
@@ -88,8 +106,9 @@
     /* list of objects created in the current transaction and
        that survived at least one minor collection. They need
        to be synchronized to other segments on commit, but they
-       do not need to be in the commit log entry. */
-    struct list_s *new_objects;
+       do not need to be in the commit log entry.
+       XXX: for now it also contains small overflow objs */
+    struct list_s *large_overflow_objects;
 
     uint8_t privatization_lock;  // XXX KILL
 
@@ -101,6 +120,14 @@
 
     struct tree_s *callbacks_on_commit_and_abort[2];
 
+    /* This is the number stored in the overflowed objects (a multiple of
+       GCFLAG_OVERFLOW_NUMBER_bit0).  It is incremented when the
+       transaction is done, but only if we actually overflowed any
+       object; otherwise, no object has got this number. */
+    uint32_t overflow_number;
+    bool overflow_number_has_been_used;
+
+
     struct stm_commit_log_entry_s *last_commit_log_entry;
 
     struct stm_shadowentry_s *shadowstack_at_start_of_transaction;
@@ -193,6 +220,21 @@
 
 #define REAL_ADDRESS(segment_base, src)   ((segment_base) + (uintptr_t)(src))
 
+#define IS_OVERFLOW_OBJ(pseg, obj) (((obj)->stm_flags & -GCFLAG_OVERFLOW_NUMBER_bit0) \
+                                    == (pseg)->overflow_number)
+
+static inline uintptr_t get_index_to_card_index(uintptr_t index) {
+    return (index / CARD_SIZE) + 1;
+}
+
+static inline uintptr_t get_card_index_to_index(uintptr_t card_index) {
+    return (card_index - 1) * CARD_SIZE;
+}
+
+static inline struct stm_read_marker_s *get_read_marker(char *segment_base, uintptr_t obj)
+{
+   return (struct stm_read_marker_s *)(segment_base + (obj >> 4));
+}
 
 static inline char *get_segment_base(long segment_num) {
     return stm_object_pages + segment_num * (NB_PAGES * 4096UL);
@@ -215,6 +257,7 @@
     return (addr - stm_object_pages) / (NB_PAGES * 4096UL);
 }
 
+bool obj_should_use_cards(char *seg_base, object_t *obj);
 
 static bool _is_tl_registered(stm_thread_local_t *tl);
 static bool _seems_to_be_running_transaction(void);
diff --git a/rpython/translator/stm/src_stm/stm/finalizer.c b/rpython/translator/stm/src_stm/stm/finalizer.c
--- a/rpython/translator/stm/src_stm/stm/finalizer.c
+++ b/rpython/translator/stm/src_stm/stm/finalizer.c
@@ -98,14 +98,14 @@
         list_clear(lst);
     }
 
-    /* also deals with newly created objects: they are at the tail of
+    /* also deals with overflow objects: they are at the tail of
        old_objects_with_light_finalizers (this list is kept in order
        and we cannot add any already-committed object) */
     lst = pseg->old_objects_with_light_finalizers;
     count = list_count(lst);
     while (count > 0) {
         object_t *obj = (object_t *)list_item(lst, --count);
-        if (!(obj->stm_flags & GCFLAG_WB_EXECUTED))
+        if (!IS_OVERFLOW_OBJ(pseg, obj))
             break;
         lst->count = count;
         if (must_fix_gs) {
@@ -264,11 +264,14 @@
         LIST_APPEND(_finalizer_tmpstack, obj);
 }
 
-static inline struct list_s *finalizer_trace(char *base, object_t *obj,
-                                             struct list_s *lst)
+static inline struct list_s *finalizer_trace(
+    struct stm_priv_segment_info_s *pseg, object_t *obj, struct list_s *lst)
 {
-    if (!is_new_object(obj))
+    char *base;
+    if (!is_overflow_obj_safe(pseg, obj))
         base = stm_object_pages;
+    else
+        base = pseg->pub.segment_base;
 
     struct object_s *realobj = (struct object_s *)REAL_ADDRESS(base, obj);
     _finalizer_tmpstack = lst;
@@ -277,7 +280,8 @@
 }
 
 
-static void _recursively_bump_finalization_state_from_2_to_3(char *base, object_t *obj)
+static void _recursively_bump_finalization_state_from_2_to_3(
+    struct stm_priv_segment_info_s *pseg, object_t *obj)
 {
     assert(_finalization_state(obj) == 2);
     struct list_s *tmpstack = _finalizer_emptystack;
@@ -289,7 +293,7 @@
             realobj->stm_flags &= ~GCFLAG_FINALIZATION_ORDERING;
 
             /* trace */
-            tmpstack = finalizer_trace(base, obj, tmpstack);
+            tmpstack = finalizer_trace(pseg, obj, tmpstack);
         }
 
         if (list_is_empty(tmpstack))
@@ -300,14 +304,16 @@
     _finalizer_emptystack = tmpstack;
 }
 
-static void _recursively_bump_finalization_state_from_1_to_2(char *base, object_t *obj)
+static void _recursively_bump_finalization_state_from_1_to_2(
+    struct stm_priv_segment_info_s *pseg, object_t *obj)
 {
     assert(_finalization_state(obj) == 1);
     /* The call will add GCFLAG_VISITED recursively, thus bump state 1->2 */
-    mark_visit_possibly_new_object(base, obj);
+    mark_visit_possibly_new_object(obj, pseg);
 }
 
-static struct list_s *mark_finalize_step1(char *base, struct finalizers_s *f)
+static struct list_s *mark_finalize_step1(
+    struct stm_priv_segment_info_s *pseg, struct finalizers_s *f)
 {
     if (f == NULL)
         return NULL;
@@ -336,21 +342,22 @@
             int state = _finalization_state(y);
             if (state <= 0) {
                 _bump_finalization_state_from_0_to_1(y);
-                pending = finalizer_trace(base, y, pending);
+                pending = finalizer_trace(pseg, y, pending);
             }
             else if (state == 2) {
-                _recursively_bump_finalization_state_from_2_to_3(base, y);
+                _recursively_bump_finalization_state_from_2_to_3(pseg, y);
             }
         }
         _finalizer_pending = pending;
         assert(_finalization_state(x) == 1);
-        _recursively_bump_finalization_state_from_1_to_2(base, x);
+        _recursively_bump_finalization_state_from_1_to_2(pseg, x);
     }
     return marked;
 }
 
-static void mark_finalize_step2(char *base, struct finalizers_s *f,
-                                struct list_s *marked)
+static void mark_finalize_step2(
+    struct stm_priv_segment_info_s *pseg, struct finalizers_s *f,
+    struct list_s *marked)
 {
     if (f == NULL)
         return;
@@ -367,7 +374,7 @@
             if (run_finalizers == NULL)
                 run_finalizers = list_create();
             LIST_APPEND(run_finalizers, x);
-            _recursively_bump_finalization_state_from_2_to_3(base, x);
+            _recursively_bump_finalization_state_from_2_to_3(pseg, x);
         }
         else {
             struct list_s *lst = f->objects_with_finalizers;
@@ -403,29 +410,28 @@
     long j;
     for (j = 1; j < NB_SEGMENTS; j++) {
         struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
-        marked_seg[j] = mark_finalize_step1(pseg->pub.segment_base,
-                                            pseg->finalizers);
+        marked_seg[j] = mark_finalize_step1(pseg, pseg->finalizers);
     }
-    marked_seg[0] = mark_finalize_step1(stm_object_pages, &g_finalizers);
+    marked_seg[0] = mark_finalize_step1(get_priv_segment(0), &g_finalizers);
 
     LIST_FREE(_finalizer_pending);
 
     for (j = 1; j < NB_SEGMENTS; j++) {
         struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
-        mark_finalize_step2(pseg->pub.segment_base, pseg->finalizers,
-                            marked_seg[j]);
+        mark_finalize_step2(pseg, pseg->finalizers, marked_seg[j]);
     }
-    mark_finalize_step2(stm_object_pages, &g_finalizers, marked_seg[0]);
+    mark_finalize_step2(get_priv_segment(0), &g_finalizers, marked_seg[0]);
 
     LIST_FREE(_finalizer_emptystack);
 }
 
-static void mark_visit_from_finalizer1(char *base, struct finalizers_s *f)
+static void mark_visit_from_finalizer1(
+    struct stm_priv_segment_info_s *pseg, struct finalizers_s *f)
 {
     if (f != NULL && f->run_finalizers != NULL) {
         LIST_FOREACH_R(f->run_finalizers, object_t * /*item*/,
                        ({
-                           mark_visit_possibly_new_object(base, item);
+                           mark_visit_possibly_new_object(item, pseg);
                        }));
     }
 }
@@ -435,9 +441,9 @@
     long j;
     for (j = 1; j < NB_SEGMENTS; j++) {
         struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
-        mark_visit_from_finalizer1(pseg->pub.segment_base, pseg->finalizers);
+        mark_visit_from_finalizer1(pseg, pseg->finalizers);
     }
-    mark_visit_from_finalizer1(stm_object_pages, &g_finalizers);
+    mark_visit_from_finalizer1(get_priv_segment(0), &g_finalizers);
 }
 
 static void _execute_finalizers(struct finalizers_s *f)
diff --git a/rpython/translator/stm/src_stm/stm/forksupport.c b/rpython/translator/stm/src_stm/stm/forksupport.c
--- a/rpython/translator/stm/src_stm/stm/forksupport.c
+++ b/rpython/translator/stm/src_stm/stm/forksupport.c
@@ -84,11 +84,23 @@
     stm_thread_local_t *tl = pr->pub.running_thread;
     dprintf(("forksupport_child: abort in seg%ld\n", i));
     assert(tl->associated_segment_num == i);
-    assert(pr->transaction_state == TS_REGULAR);
+    assert(pr->transaction_state != TS_INEVITABLE);
     set_gs_register(get_segment_base(i));
     assert(STM_SEGMENT->segment_num == i);
 
     s_mutex_lock();
+    if (pr->transaction_state == TS_NONE) {
+        /* just committed, TS_NONE but still has running_thread */
+
+        /* do _finish_transaction() */
+        STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
+        list_clear(STM_PSEGMENT->objects_pointing_to_nursery);
+        list_clear(STM_PSEGMENT->large_overflow_objects);
+
+        s_mutex_unlock();
+        return;
+    }
+
 #ifndef NDEBUG
     pr->running_pthread = pthread_self();
 #endif
diff --git a/rpython/translator/stm/src_stm/stm/gcpage.c b/rpython/translator/stm/src_stm/stm/gcpage.c
--- a/rpython/translator/stm/src_stm/stm/gcpage.c
+++ b/rpython/translator/stm/src_stm/stm/gcpage.c
@@ -200,11 +200,16 @@
 
 /************************************************************/
 
+static bool is_overflow_obj_safe(struct stm_priv_segment_info_s *pseg, object_t *obj)
+{
+    /* this function first also checks if the page is accessible in order
+       to not cause segfaults during major gc (it does exactly the same
+       as IS_OVERFLOW_OBJ otherwise) */
+    if (get_page_status_in(pseg->pub.segment_num, (uintptr_t)obj / 4096UL) == PAGE_NO_ACCESS)
+        return false;
 
-static bool is_new_object(object_t *obj)
-{
-    struct object_s *realobj = (struct object_s*)REAL_ADDRESS(stm_object_pages, obj); /* seg0 */
-    return realobj->stm_flags & GCFLAG_WB_EXECUTED;
+    struct object_s *realobj = (struct object_s*)REAL_ADDRESS(pseg->pub.segment_base, obj);
+    return IS_OVERFLOW_OBJ(pseg, realobj);
 }
 
 
@@ -230,7 +235,10 @@
 }
 
 
-static void mark_and_trace(object_t *obj, char *segment_base)
+static void mark_and_trace(
+    object_t *obj,
+    char *segment_base, /* to trace obj in */
+    struct stm_priv_segment_info_s *pseg) /* to trace children in */
 {
     /* mark the obj and trace all reachable objs from it */
 
@@ -242,36 +250,40 @@
     stmcb_trace(realobj, &mark_record_trace);
 
     /* trace all references found in sharing seg0 (should always be
-       up-to-date and not cause segfaults, except for new objs) */
+       up-to-date and not cause segfaults, except for overflow objs) */
+    segment_base = pseg->pub.segment_base;
     while (!list_is_empty(marked_objects_to_trace)) {
         obj = (object_t *)list_pop_item(marked_objects_to_trace);
 
-        char *base = is_new_object(obj) ? segment_base : stm_object_pages;
+        char *base = is_overflow_obj_safe(pseg, obj) ? segment_base : stm_object_pages;
         realobj = (struct object_s *)REAL_ADDRESS(base, obj);
         stmcb_trace(realobj, &mark_record_trace);
     }
 }
 
-static inline void mark_visit_object(object_t *obj, char *segment_base)
+static inline void mark_visit_object(
+    object_t *obj,
+    char *segment_base, /* to trace ojb in */
+    struct stm_priv_segment_info_s *pseg) /* to trace children in */
 {
     /* if already visited, don't trace */
     if (obj == NULL || mark_visited_test_and_set(obj))
         return;
-    mark_and_trace(obj, segment_base);
+    mark_and_trace(obj, segment_base, pseg);
 }
 
 
-static void mark_visit_possibly_new_object(char *segment_base, object_t *obj)
+static void mark_visit_possibly_new_object(object_t *obj, struct stm_priv_segment_info_s *pseg)
 {
     /* if newly allocated object, we trace in segment_base, otherwise in
        the sharing seg0 */
     if (obj == NULL)
         return;
 
-    if (is_new_object(obj)) {
-        mark_visit_object(obj, segment_base);
+    if (is_overflow_obj_safe(pseg, obj)) {
+        mark_visit_object(obj, pseg->pub.segment_base, pseg);
     } else {
-        mark_visit_object(obj, stm_object_pages);
+        mark_visit_object(obj, stm_object_pages, pseg);
     }
 }
 
@@ -282,8 +294,10 @@
     end = (const struct stm_shadowentry_s *)(slice + size);
     for (; p < end; p++)
         if ((((uintptr_t)p->ss) & 3) == 0) {
-            assert(!is_new_object(p->ss));
-            mark_visit_object(p->ss, stm_object_pages); // seg0
+            mark_visit_object(p->ss, stm_object_pages, // seg0
+                              /* there should be no overflow objs not already
+                                 visited, so any pseg is fine really: */
+                              get_priv_segment(STM_SEGMENT->segment_num));
         }
     return NULL;
 }
@@ -350,7 +364,7 @@
                   and thus make all pages accessible. */
                assert_obj_accessible_in(i, item);
 
-               assert(!is_new_object(item)); /* should never be in that list */
+               assert(!is_overflow_obj_safe(get_priv_segment(i), item)); /* should never be in that list */
 
                if (!mark_visited_test_and_set(item)) {
                    /* trace shared, committed version: only do this if we didn't
@@ -358,9 +372,9 @@
                       objs before mark_visit_from_modified_objects AND if we
                       do mark_and_trace on an obj that is modified in >1 segment,
                       the tracing always happens in seg0 (see mark_and_trace). */
-                   mark_and_trace(item, stm_object_pages);
+                   mark_and_trace(item, stm_object_pages, get_priv_segment(i));
                }
-               mark_and_trace(item, base);   /* private, modified version */
+               mark_and_trace(item, base, get_priv_segment(i));   /* private, modified version */
            }));
 
         list_clear(uniques);
@@ -372,7 +386,11 @@
 {
     if (testing_prebuilt_objs != NULL) {
         LIST_FOREACH_R(testing_prebuilt_objs, object_t * /*item*/,
-                       mark_visit_object(item, stm_object_pages)); // seg0
+                   mark_visit_object(item, stm_object_pages, // seg0
+                                     /* any pseg is fine, as we already traced modified
+                                        objs and thus covered all overflow objs reachable
+                                        from here */
+                                     get_priv_segment(STM_SEGMENT->segment_num)));
     }
 
     stm_thread_local_t *tl = stm_all_thread_locals;
@@ -380,7 +398,7 @@
         /* look at all objs on the shadow stack (they are old but may
            be uncommitted so far, so only exist in the associated_segment_num).
 
-           IF they are uncommitted new objs, trace in the actual segment,
+           IF they are uncommitted overflow objs, trace in the actual segment,
            otherwise, since we just executed a minor collection, they were
            all synced to the sharing seg0. Thus we can trace them there.
 
@@ -392,17 +410,17 @@
            If 'tl' is currently running, its 'last_associated_segment_num'
            field is the segment number that contains the correct
            version of its overflowed objects. */
-        char *segment_base = get_segment_base(tl->last_associated_segment_num);
+        struct stm_priv_segment_info_s *pseg = get_priv_segment(tl->last_associated_segment_num);
 
         struct stm_shadowentry_s *current = tl->shadowstack;
         struct stm_shadowentry_s *base = tl->shadowstack_base;
         while (current-- != base) {
             if ((((uintptr_t)current->ss) & 3) == 0) {
-                mark_visit_possibly_new_object(segment_base, current->ss);
+                mark_visit_possibly_new_object(current->ss, pseg);
             }
         }
 
-        mark_visit_possibly_new_object(segment_base, tl->thread_local_obj);
+        mark_visit_possibly_new_object(tl->thread_local_obj, pseg);
 
         tl = tl->next;
     } while (tl != stm_all_thread_locals);
@@ -413,8 +431,8 @@
     for (i = 1; i < NB_SEGMENTS; i++) {
         if (get_priv_segment(i)->transaction_state != TS_NONE) {
             mark_visit_possibly_new_object(
-                get_segment_base(i),
-                get_priv_segment(i)->threadlocal_at_start_of_transaction);
+                get_priv_segment(i)->threadlocal_at_start_of_transaction,
+                get_priv_segment(i));
 
             stm_rewind_jmp_enum_shadowstack(
                 get_segment(i)->running_thread,
@@ -423,49 +441,6 @@
     }
 }
 
-static void ready_new_objects(void)
-{
-#pragma push_macro("STM_PSEGMENT")
-#pragma push_macro("STM_SEGMENT")
-#undef STM_PSEGMENT
-#undef STM_SEGMENT
-    /* objs in new_objects only have garbage in the sharing seg0,
-       since it is used to mark objs as visited, we must make
-       sure the flag is cleared at the start of a major collection.
-       (XXX: ^^^ may be optional if we have the part below)
-
-       Also, we need to be able to recognize these objects in order
-       to only trace them in the segment they are valid in. So we
-       also make sure to set WB_EXECUTED in the sharing seg0. No
-       other objs than new_objects have WB_EXECUTED in seg0 (since
-       there can only be committed versions there).
-    */
-
-    long i;
-    for (i = 1; i < NB_SEGMENTS; i++) {
-        struct stm_priv_segment_info_s *pseg = get_priv_segment(i);
-        struct list_s *lst = pseg->new_objects;
-
-        LIST_FOREACH_R(lst, object_t* /*item*/,
-            ({
-                struct object_s *realobj;
-                /* WB_EXECUTED always set in this segment */
-                assert(realobj = (struct object_s*)REAL_ADDRESS(pseg->pub.segment_base, item));
-                assert(realobj->stm_flags & GCFLAG_WB_EXECUTED);
-
-                /* clear VISITED (garbage) and ensure WB_EXECUTED in seg0 */
-                mark_visited_test_and_clear(item);
-                realobj = (struct object_s*)REAL_ADDRESS(stm_object_pages, item);
-                realobj->stm_flags |= GCFLAG_WB_EXECUTED;
-
-                /* make sure this flag is cleared as well */
-                realobj->stm_flags &= ~GCFLAG_FINALIZATION_ORDERING;
-            }));
-    }
-#pragma pop_macro("STM_SEGMENT")
-#pragma pop_macro("STM_PSEGMENT")
-}
-
 
 static void clean_up_segment_lists(void)
 {
@@ -494,11 +469,10 @@
                 ({
                     struct object_s *realobj = (struct object_s *)
                         REAL_ADDRESS(pseg->pub.segment_base, (uintptr_t)item);
+                    assert(!(realobj->stm_flags & GCFLAG_WRITE_BARRIER));
+                    realobj->stm_flags |= GCFLAG_WRITE_BARRIER;
 
-                    assert(realobj->stm_flags & GCFLAG_WB_EXECUTED);
-                    assert(!(realobj->stm_flags & GCFLAG_WRITE_BARRIER));
-
-                    realobj->stm_flags |= GCFLAG_WRITE_BARRIER;
+                    OPT_ASSERT(!(realobj->stm_flags & GCFLAG_CARDS_SET));
                 }));
             list_clear(lst);
         } else {
@@ -507,12 +481,31 @@
                modified_old_objs. */
         }
 
-        /* remove from new_objects all objects that die */
-        lst = pseg->new_objects;
+        lst = pseg->old_objects_with_cards_set;
+        LIST_FOREACH_R(lst, object_t* /*item*/,
+            ({
+                struct object_s *realobj = (struct object_s *)
+                    REAL_ADDRESS(pseg->pub.segment_base, item);
+                OPT_ASSERT(realobj->stm_flags & GCFLAG_WRITE_BARRIER);
+
+                /* mark marked cards as old if it survives, otherwise
+                   CLEAR, as their spot could get reused */
+                uint8_t mark_value = mark_visited_test(item) ?
+                    pseg->pub.transaction_read_version : CARD_CLEAR;
+                _reset_object_cards(pseg, item, mark_value, false,
+                                    mark_value == CARD_CLEAR);
+            }));
+        list_clear(lst);
+
+
+        /* remove from large_overflow_objects all objects that die */
+        lst = pseg->large_overflow_objects;
         uintptr_t n = list_count(lst);
         while (n-- > 0) {
             object_t *obj = (object_t *)list_item(lst, n);
             if (!mark_visited_test(obj)) {
+                if (obj_should_use_cards(pseg->pub.segment_base, obj))
+                    _reset_object_cards(pseg, obj, CARD_CLEAR, false, true);
                 list_set_item(lst, n, list_pop_item(lst));
             }
         }
@@ -524,6 +517,7 @@
 
 static inline bool largemalloc_keep_object_at(char *data)
 {
+    /* XXX: identical to smallmalloc_keep_object_at()? */
     /* this is called by _stm_largemalloc_sweep() */
     object_t *obj = (object_t *)(data - stm_object_pages);
     //dprintf(("keep obj %p ? -> %d\n", obj, mark_visited_test(obj)));
@@ -646,17 +640,42 @@
     dprintf((" | commit log entries before: %ld\n",
              _stm_count_cl_entries()));
 
+
     /* free all commit log entries. all segments are on the most recent
        revision now. */
+    uint64_t allocd_before = pages_ctl.total_allocated;
     clean_up_commit_log_entries();
+    /* check if freeing the log entries actually freed a considerable
+       amount itself. Then we don't want to also trace the whole heap
+       and just leave major gc right here.
+       The problem is apparent from raytrace.py, but may disappear if
+       we have card marking that also reduces the size of commit log
+       entries */
+    if ((pages_ctl.total_allocated < pages_ctl.total_allocated_bound)
+        && (allocd_before - pages_ctl.total_allocated > 0.3 * allocd_before)) {
+        /* 0.3 should mean that we are at about 50% of the way to the
+           allocated_bound again */
+#ifndef STM_TESTS
+        /* we freed a considerable amount just by freeing commit log entries */
+        pages_ctl.major_collection_requested = false; // reset_m_gc_requested
+
+        dprintf(("STOP AFTER FREEING CL ENTRIES: -%ld\n",
+                 (long)(allocd_before - pages_ctl.total_allocated)));
+        dprintf((" | used after collection:  %ld\n",
+                (long)pages_ctl.total_allocated));
+        dprintf((" `----------------------------------------------\n"));
+        if (must_abort())
+            abort_with_mutex();
+
+        return;
+#endif
+    }
 
     /* only necessary because of assert that fails otherwise (XXX) */
     acquire_all_privatization_locks();
 
     DEBUG_EXPECT_SEGFAULT(false);
 
-    ready_new_objects();
-
     /* marking */
     LIST_CREATE(marked_objects_to_trace);
     mark_visit_from_modified_objects();
diff --git a/rpython/translator/stm/src_stm/stm/gcpage.h b/rpython/translator/stm/src_stm/stm/gcpage.h
--- a/rpython/translator/stm/src_stm/stm/gcpage.h
+++ b/rpython/translator/stm/src_stm/stm/gcpage.h
@@ -7,7 +7,6 @@
 #define GC_MIN                 (NB_NURSERY_PAGES * 4096 * 8)
 #define GC_MAJOR_COLLECT       1.82
 
-
 static struct list_s *testing_prebuilt_objs;
 static char *uninitialized_page_start;   /* within segment 0 */
 static char *uninitialized_page_stop;
diff --git a/rpython/translator/stm/src_stm/stm/misc.c b/rpython/translator/stm/src_stm/stm/misc.c
--- a/rpython/translator/stm/src_stm/stm/misc.c
+++ b/rpython/translator/stm/src_stm/stm/misc.c
@@ -42,6 +42,11 @@
     return (obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) == 0;
 }
 
+bool _stm_was_written_card(object_t *obj)
+{
+    return obj->stm_flags & _STM_GCFLAG_CARDS_SET;
+}
+
 long _stm_count_cl_entries()
 {
     struct stm_commit_log_entry_s *cl = &commit_log_root;
@@ -80,6 +85,13 @@
     return list_count(STM_PSEGMENT->objects_pointing_to_nursery);
 }
 
+long _stm_count_old_objects_with_cards_set(void)
+{
+    if (STM_PSEGMENT->old_objects_with_cards_set == NULL)
+        return -1;
+    return list_count(STM_PSEGMENT->old_objects_with_cards_set);
+}
+
 object_t *_stm_enum_modified_old_objects(long index)
 {
     return (object_t *)list_item(
@@ -92,6 +104,27 @@
         STM_PSEGMENT->objects_pointing_to_nursery, index);
 }
 
+object_t *_stm_enum_old_objects_with_cards_set(long index)
+{
+    return (object_t *)list_item(
+        STM_PSEGMENT->old_objects_with_cards_set, index);
+}
+
+
+uint8_t _stm_get_card_value(object_t *obj, long idx)
+{
+    struct stm_read_marker_s *cards = get_read_marker(STM_SEGMENT->segment_base,
+                                                      (uintptr_t)obj);
+    return cards[get_index_to_card_index(idx)].rm;
+}
+
+uint8_t _stm_get_transaction_read_version()
+{
+    return STM_SEGMENT->transaction_read_version;
+}
+
+
+
 static struct stm_commit_log_entry_s *_last_cl_entry;
 static long _last_cl_entry_index;
 void _stm_start_enum_last_cl_entry()
diff --git a/rpython/translator/stm/src_stm/stm/nursery.c b/rpython/translator/stm/src_stm/stm/nursery.c
--- a/rpython/translator/stm/src_stm/stm/nursery.c
+++ b/rpython/translator/stm/src_stm/stm/nursery.c
@@ -38,7 +38,7 @@
 }
 
 static inline bool _is_from_same_transaction(object_t *obj) {
-    return _is_young(obj) || (obj->stm_flags & GCFLAG_WB_EXECUTED);
+    return _is_young(obj) || IS_OVERFLOW_OBJ(STM_PSEGMENT, obj);
 }
 
 long stm_can_move(object_t *obj)
@@ -132,18 +132,159 @@
         nobj_sync_now = ((uintptr_t)nobj) | FLAG_SYNC_LARGE;
     }
 
-    /* if this is not during commit, we will add them to the new_objects
-       list and push them to other segments on commit. Thus we can add
-       the WB_EXECUTED flag so that they don't end up in modified_old_objects */
+    /* if this is not during commit, we make them overflow objects
+       and push them to other segments on commit. */
     assert(!(nobj->stm_flags & GCFLAG_WB_EXECUTED));
+    assert((nobj->stm_flags & -GCFLAG_OVERFLOW_NUMBER_bit0) == 0);
     if (!STM_PSEGMENT->minor_collect_will_commit_now) {
-        nobj->stm_flags |= GCFLAG_WB_EXECUTED;
+        nobj->stm_flags |= STM_PSEGMENT->overflow_number;
     }
 
     /* Must trace the object later */
     LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, nobj_sync_now);
+    _cards_cleared_in_object(get_priv_segment(STM_SEGMENT->segment_num), nobj, true);
 }
 
+static void _cards_cleared_in_object(struct stm_priv_segment_info_s *pseg, object_t *obj,
+                                     bool strict) /* strict = MARKED_OLD not allowed */
+{
+#ifndef NDEBUG
+    struct object_s *realobj = (struct object_s *)REAL_ADDRESS(pseg->pub.segment_base, obj);
+    size_t size = stmcb_size_rounded_up(realobj);
+
+    if (size < _STM_MIN_CARD_OBJ_SIZE)
+        return;                 /* too small for cards */
+
+    assert(!(realobj->stm_flags & GCFLAG_CARDS_SET));
+
+    if (!stmcb_obj_supports_cards(realobj))
+        return;
+
+    uintptr_t offset_itemsize[2] = {0, 0};
+    stmcb_get_card_base_itemsize(realobj, offset_itemsize);
+    struct stm_read_marker_s *cards = get_read_marker(pseg->pub.segment_base, (uintptr_t)obj);
+    uintptr_t card_index = 1;
+    size_t real_idx_count = (size - offset_itemsize[0]) / offset_itemsize[1];
+    uintptr_t last_card_index = get_index_to_card_index(real_idx_count - 1); /* max valid index */
+
+    while (card_index <= last_card_index) {
+        assert(cards[card_index].rm == CARD_CLEAR
+               || (cards[card_index].rm != CARD_MARKED
+                   && cards[card_index].rm < pseg->pub.transaction_read_version)
+               || (!strict && cards[card_index].rm != CARD_MARKED));
+        card_index++;
+    }
+#endif
+}
+
+static void _verify_cards_cleared_in_all_lists(struct stm_priv_segment_info_s *pseg)
+{
+#ifndef NDEBUG
+    struct list_s *list = pseg->modified_old_objects;
+    struct stm_undo_s *undo = (struct stm_undo_s *)list->items;
+    struct stm_undo_s *end = (struct stm_undo_s *)(list->items + list->count);
+
+    for (; undo < end; undo++) {
+        _cards_cleared_in_object(pseg, undo->object, false);
+    }
+    LIST_FOREACH_R(
+        pseg->large_overflow_objects, object_t * /*item*/,
+        _cards_cleared_in_object(pseg, item, false));
+    LIST_FOREACH_R(
+        pseg->objects_pointing_to_nursery, object_t * /*item*/,
+        _cards_cleared_in_object(pseg, item, false));
+    LIST_FOREACH_R(
+        pseg->old_objects_with_cards_set, object_t * /*item*/,
+        _cards_cleared_in_object(pseg, item, false));
+#endif
+}
+
+static void _reset_object_cards(struct stm_priv_segment_info_s *pseg,
+                                object_t *obj, uint8_t mark_value,
+                                bool mark_all, bool really_clear)
+{
+#pragma push_macro("STM_PSEGMENT")
+#pragma push_macro("STM_SEGMENT")
+#undef STM_PSEGMENT
+#undef STM_SEGMENT
+    dprintf(("_reset_object_cards(%p, mark=%d, mark_all=%d, really_clear=%d)\n",
+             obj, mark_value, mark_all, really_clear));
+    struct object_s *realobj = (struct object_s *)REAL_ADDRESS(pseg->pub.segment_base, obj);
+    size_t size = stmcb_size_rounded_up(realobj);
+    OPT_ASSERT(size >= _STM_MIN_CARD_OBJ_SIZE);
+
+    uintptr_t offset_itemsize[2];
+    stmcb_get_card_base_itemsize(realobj, offset_itemsize);
+    size = (size - offset_itemsize[0]) / offset_itemsize[1];
+
+    /* really_clear only used for freed new objs in minor collections, as
+       they need to clear cards even if they are set to transaction_read_version */
+    assert(IMPLY(really_clear, mark_value == CARD_CLEAR && !mark_all));
+    assert(IMPLY(mark_value == CARD_CLEAR, !mark_all)); /* not necessary */
+    assert(IMPLY(mark_all,
+                 mark_value == pseg->pub.transaction_read_version)); /* set *all* to OLD */
+
+    struct stm_read_marker_s *cards = get_read_marker(pseg->pub.segment_base, (uintptr_t)obj);
+    uintptr_t card_index = 1;
+    uintptr_t last_card_index = get_index_to_card_index(size - 1); /* max valid index */
+
+    /* dprintf(("mark cards of %p, size %lu with %d, all: %d\n",
+                obj, size, mark_value, mark_all));
+       dprintf(("obj has %lu cards\n", last_card_index));*/
+    while (card_index <= last_card_index) {
+        if (mark_all || cards[card_index].rm == CARD_MARKED
+            || (really_clear && cards[card_index].rm != CARD_CLEAR)) {
+            /* dprintf(("mark card %lu,wl:%lu of %p with %d\n", */
+            /*          card_index, card_lock_idx, obj, mark_value)); */
+            cards[card_index].rm = mark_value;
+        }
+        card_index++;
+    }
+
+    realobj->stm_flags &= ~GCFLAG_CARDS_SET;
+
+#pragma pop_macro("STM_SEGMENT")
+#pragma pop_macro("STM_PSEGMENT")
+}
+
+
+static void _trace_card_object(object_t *obj)
+{
+    assert(!_is_in_nursery(obj));
+    assert(obj->stm_flags & GCFLAG_CARDS_SET);
+    assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
+
+    dprintf(("_trace_card_object(%p)\n", obj));
+
+    struct object_s *realobj = (struct object_s *)REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
+    size_t size = stmcb_size_rounded_up(realobj);
+    uintptr_t offset_itemsize[2];
+    stmcb_get_card_base_itemsize(realobj, offset_itemsize);
+    size = (size - offset_itemsize[0]) / offset_itemsize[1];
+
+    struct stm_read_marker_s *cards = get_read_marker(STM_SEGMENT->segment_base, (uintptr_t)obj);
+    uintptr_t card_index = 1;
+    uintptr_t last_card_index = get_index_to_card_index(size - 1); /* max valid index */
+
+    /* XXX: merge ranges */
+    while (card_index <= last_card_index) {
+        if (cards[card_index].rm == CARD_MARKED) {
+            /* clear or set to old: */
+            cards[card_index].rm = STM_SEGMENT->transaction_read_version;
+
+            uintptr_t start = get_card_index_to_index(card_index);
+            uintptr_t stop = get_card_index_to_index(card_index + 1);
+
+            dprintf(("trace_cards on %p with start:%lu stop:%lu\n",
+                     obj, start, stop));
+            stmcb_trace_cards(realobj, &minor_trace_if_young,
+                              start, stop);
+        }
+
+        card_index++;
+    }
+    obj->stm_flags &= ~GCFLAG_CARDS_SET;
+}
 
 static void collect_roots_in_nursery(void)
 {
@@ -177,15 +318,20 @@
 static inline void _collect_now(object_t *obj)
 {
     assert(!_is_young(obj));
+    assert(!(obj->stm_flags & GCFLAG_CARDS_SET));
 
     //dprintf(("_collect_now: %p\n", obj));
 
-    assert(!(obj->stm_flags & GCFLAG_WRITE_BARRIER));
+    if (!(obj->stm_flags & GCFLAG_WRITE_BARRIER)) {
+        /* Trace the 'obj' to replace pointers to nursery with pointers
+           outside the nursery, possibly forcing nursery objects out and
+           adding them to 'objects_pointing_to_nursery' as well. */
+        char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
+        stmcb_trace((struct object_s *)realobj, &minor_trace_if_young);
 
-    char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
-    stmcb_trace((struct object_s *)realobj, &minor_trace_if_young);
-
-    obj->stm_flags |= GCFLAG_WRITE_BARRIER;
+        obj->stm_flags |= GCFLAG_WRITE_BARRIER;
+    }
+    /* else traced in collect_cardrefs_to_nursery if necessary */
 }
 
 
@@ -201,25 +347,29 @@
         assert(!_is_in_nursery(obj));
 
         _collect_now(obj);
+        assert(!(obj->stm_flags & GCFLAG_CARDS_SET));
 
         if (obj_sync_now & FLAG_SYNC_LARGE) {
+            /* XXX: SYNC_LARGE even set for small objs right now */
             /* this is a newly allocated obj in this transaction. We must
                either synchronize the object to other segments now, or
-               add the object to new_objects list */
-            if (STM_PSEGMENT->minor_collect_will_commit_now) {
-                acquire_privatization_lock(STM_SEGMENT->segment_num);
+               add the object to large_overflow_objects list */
+            struct stm_priv_segment_info_s *pseg = get_priv_segment(STM_SEGMENT->segment_num);
+            if (pseg->minor_collect_will_commit_now) {
+                acquire_privatization_lock(pseg->pub.segment_num);
                 synchronize_object_enqueue(obj);
-                release_privatization_lock(STM_SEGMENT->segment_num);
+                release_privatization_lock(pseg->pub.segment_num);
             } else {
-                LIST_APPEND(STM_PSEGMENT->new_objects, obj);
+                LIST_APPEND(STM_PSEGMENT->large_overflow_objects, obj);
             }
+            _cards_cleared_in_object(pseg, obj, false);
         }
 
         /* the list could have moved while appending */
         lst = STM_PSEGMENT->objects_pointing_to_nursery;
     }
 
-    /* flush all new objects to other segments now */
+    /* flush all overflow objects to other segments now */
     if (STM_PSEGMENT->minor_collect_will_commit_now) {
         acquire_privatization_lock(STM_SEGMENT->segment_num);
         synchronize_objects_flush();
@@ -230,6 +380,30 @@
     }
 }
 
+
+static void collect_cardrefs_to_nursery(void)
+{
+    dprintf(("collect_cardrefs_to_nursery\n"));
+    struct list_s *lst = STM_PSEGMENT->old_objects_with_cards_set;
+
+    while (!list_is_empty(lst)) {
+        object_t *obj = (object_t*)list_pop_item(lst);
+
+        assert(!_is_young(obj));
+
+        if (!(obj->stm_flags & GCFLAG_CARDS_SET)) {
+            /* sometimes we remove the CARDS_SET in the WB slowpath, see core.c */
+            continue;
+        }
+
+        /* traces cards, clears marked cards or marks them old if necessary */
+        _trace_card_object(obj);
+
+        assert(!(obj->stm_flags & GCFLAG_CARDS_SET));
+    }
+}
+
+
 static void collect_objs_still_young_but_with_finalizers(void)
 {
     struct list_s *lst = STM_PSEGMENT->finalizers->objects_with_finalizers;
@@ -313,6 +487,13 @@
     dprintf(("minor_collection commit=%d\n", (int)commit));
 
     STM_PSEGMENT->minor_collect_will_commit_now = commit;
+    if (!commit) {
+        /* 'STM_PSEGMENT->overflow_number' is used now by this collection,
+           in the sense that it's copied to the overflow objects */
+        STM_PSEGMENT->overflow_number_has_been_used = true;
+    }
+
+    collect_cardrefs_to_nursery();
 
     collect_roots_in_nursery();
 
@@ -320,6 +501,7 @@
         collect_objs_still_young_but_with_finalizers();
 
     collect_oldrefs_to_nursery();
+    assert(list_is_empty(STM_PSEGMENT->old_objects_with_cards_set));
 
     /* now all surviving nursery objects have been moved out */
     acquire_privatization_lock(STM_SEGMENT->segment_num);
diff --git a/rpython/translator/stm/src_stm/stm/nursery.h b/rpython/translator/stm/src_stm/stm/nursery.h
--- a/rpython/translator/stm/src_stm/stm/nursery.h
+++ b/rpython/translator/stm/src_stm/stm/nursery.h
@@ -2,6 +2,14 @@
 #define NSE_SIGPAUSE   _STM_NSE_SIGNAL_MAX
 #define NSE_SIGABORT   _STM_NSE_SIGNAL_ABORT
 
+static uint32_t highest_overflow_number;
+
+static void _cards_cleared_in_object(struct stm_priv_segment_info_s *pseg, object_t *obj,
+                                     bool strict);
+static void _reset_object_cards(struct stm_priv_segment_info_s *pseg,
+                                object_t *obj, uint8_t mark_value,
+                                bool mark_all, bool really_clear);
+
 static void minor_collection(bool commit);
 static void check_nursery_at_transaction_start(void);
 static size_t throw_away_nursery(struct stm_priv_segment_info_s *pseg);
diff --git a/rpython/translator/stm/src_stm/stm/setup.c b/rpython/translator/stm/src_stm/stm/setup.c
--- a/rpython/translator/stm/src_stm/stm/setup.c
+++ b/rpython/translator/stm/src_stm/stm/setup.c
@@ -100,10 +100,11 @@
         pr->pub.segment_num = i;
         pr->pub.segment_base = segment_base;
         pr->modified_old_objects = list_create();
-        pr->new_objects = list_create();
+        pr->large_overflow_objects = list_create();
         pr->young_weakrefs = list_create();
         pr->old_weakrefs = list_create();
         pr->objects_pointing_to_nursery = list_create();
+        pr->old_objects_with_cards_set = list_create();
         pr->young_outside_nursery = tree_create();
         pr->nursery_objects_shadows = tree_create();
         pr->callbacks_on_commit_and_abort[0] = tree_create();
@@ -112,6 +113,8 @@
         pr->old_objects_with_light_finalizers = list_create();
 
         pr->last_commit_log_entry = &commit_log_root;
+        pr->overflow_number = GCFLAG_OVERFLOW_NUMBER_bit0 * i;
+        highest_overflow_number = pr->overflow_number;
         pr->pub.transaction_read_version = 0xff;
     }
 
@@ -147,9 +150,10 @@
         struct stm_priv_segment_info_s *pr = get_priv_segment(i);
         assert(list_is_empty(pr->objects_pointing_to_nursery));
         list_free(pr->objects_pointing_to_nursery);
+        list_free(pr->old_objects_with_cards_set);
         list_free(pr->modified_old_objects);
-        assert(list_is_empty(pr->new_objects));
-        list_free(pr->new_objects);
+        assert(list_is_empty(pr->large_overflow_objects));
+        list_free(pr->large_overflow_objects);
         list_free(pr->young_weakrefs);
         list_free(pr->old_weakrefs);
         tree_free(pr->young_outside_nursery);
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -77,7 +77,15 @@
 #define _STM_NSE_SIGNAL_ABORT             1
 #define _STM_NSE_SIGNAL_MAX               2
 
+#define _STM_CARD_MARKED 1      /* should always be 1... */
+#define _STM_GCFLAG_CARDS_SET          0x8
+#define _STM_CARD_SIZE                 32     /* must be >= 32 */
+#define _STM_MIN_CARD_COUNT            17
+#define _STM_MIN_CARD_OBJ_SIZE         (_STM_CARD_SIZE * _STM_MIN_CARD_COUNT)
+
 void _stm_write_slowpath(object_t *);
+void _stm_write_slowpath_card(object_t *, uintptr_t);
+char _stm_write_slowpath_card_extra(object_t *);
 object_t *_stm_allocate_slowpath(ssize_t);
 object_t *_stm_allocate_external(ssize_t);
 void _stm_become_inevitable(const char*);
@@ -87,9 +95,11 @@
 char *_stm_real_address(object_t *o);
 #ifdef STM_TESTS
 #include 
+uint8_t _stm_get_transaction_read_version();
+uint8_t _stm_get_card_value(object_t *obj, long idx);
 bool _stm_was_read(object_t *obj);
 bool _stm_was_written(object_t *obj);
-
+bool _stm_was_written_card(object_t *obj);
 bool _stm_is_accessible_page(uintptr_t pagenum);
 
 void _stm_test_switch(stm_thread_local_t *tl);
@@ -125,7 +135,8 @@
 object_t *_stm_next_last_cl_entry();
 void _stm_start_enum_last_cl_entry();
 long _stm_count_cl_entries();
-
+long _stm_count_old_objects_with_cards_set(void);
+object_t *_stm_enum_old_objects_with_cards_set(long index);
 uint64_t _stm_total_allocated(void);
 #endif
 
@@ -156,6 +167,22 @@
 
 extern ssize_t stmcb_size_rounded_up(struct object_s *);
 void stmcb_trace(struct object_s *obj, void visit(object_t **));
+/* a special trace-callback that is only called for the marked
+   ranges of indices (using stm_write_card(o, index)) */
+extern void stmcb_trace_cards(struct object_s *, void (object_t **),
+                              uintptr_t start, uintptr_t stop);
+/* this function will be called on objects that support cards.
+   It returns the base_offset (in bytes) inside the object from
+   where the indices start, and item_size (in bytes) for the size of
+   one item */
+extern void stmcb_get_card_base_itemsize(struct object_s *,
+                                         uintptr_t offset_itemsize[2]);
+/* returns whether this object supports cards. we will only call
+   stmcb_get_card_base_itemsize on objs that do so. */
+extern long stmcb_obj_supports_cards(struct object_s *);
+
+
+
 
 __attribute__((always_inline))
 static inline void stm_read(object_t *obj)
@@ -173,6 +200,14 @@
 
 
 __attribute__((always_inline))
+static inline void stm_write_card(object_t *obj, uintptr_t index)
+{
+    if (UNLIKELY((obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER) != 0))
+        _stm_write_slowpath_card(obj, index);
+}
+
+
+__attribute__((always_inline))
 static inline object_t *stm_allocate(ssize_t size_rounded_up)
 {
     OPT_ASSERT(size_rounded_up >= 16);
@@ -327,14 +362,8 @@
 
 
 /* dummies for now: */
-__attribute__((always_inline))
-static inline void stm_write_card(object_t *obj, uintptr_t index)
-{
-    stm_write(obj);
-}
+static inline void stm_flush_timing(stm_thread_local_t *tl, int verbose) {}
 
-
-static inline void stm_flush_timing(stm_thread_local_t *tl, int verbose) {}
 /* ==================== END ==================== */
 
 static void (*stmcb_expand_marker)(char *segment_base, uintptr_t odd_number,

From noreply at buildbot.pypy.org  Sat Feb 28 12:19:48 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Sat, 28 Feb 2015 12:19:48 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c8: remove dummy implementations
Message-ID: <20150228111948.C65001C009F@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: stmgc-c8
Changeset: r76194:19f6c9fde7a6
Date: 2015-02-28 09:32 +0100
http://bitbucket.org/pypy/pypy/changeset/19f6c9fde7a6/

Log:	remove dummy implementations

diff --git a/rpython/translator/stm/src_stm/stmgcintf.c b/rpython/translator/stm/src_stm/stmgcintf.c
--- a/rpython/translator/stm/src_stm/stmgcintf.c
+++ b/rpython/translator/stm/src_stm/stmgcintf.c
@@ -10,27 +10,6 @@
 __thread uintptr_t pypy_stm_nursery_low_fill_mark_saved;
 
 
-/* C8: not implemented properly yet: */
-char _stm_write_slowpath_card_extra(object_t *obj)
-{
-    /* the PyPy JIT calls this function directly if it finds that an
-       array doesn't have the GCFLAG_CARDS_SET */
-    stm_write(obj);
-    return 0;
-}
-
-long _stm_write_slowpath_card_extra_base(void)
-{
-    /* for the PyPy JIT: _stm_write_slowpath_card_extra_base[obj >> 4]
-       is the byte that must be set to CARD_MARKED.  The logic below
-       does the same, but more explicitly. */
-    /* result should only appear in JIT-paths that are never executed
-       because the above func returns false */
-    return 42;
-}
-/* C8: not implemented properly yet ^^^^^^^^^^^^^^^^^^ */
-
-
 
 extern Signed pypy_stmcb_size_rounded_up(void*);
 extern void pypy_stmcb_get_card_base_itemsize(void*, uintptr_t[]);
diff --git a/rpython/translator/stm/src_stm/stmgcintf.h b/rpython/translator/stm/src_stm/stmgcintf.h
--- a/rpython/translator/stm/src_stm/stmgcintf.h
+++ b/rpython/translator/stm/src_stm/stmgcintf.h
@@ -42,10 +42,6 @@
 
 /* C8: not implemented properly yet: */
 extern void stmcb_commit_soon(void);
-#define _STM_CARD_SIZE                 32     /* must be >= 32 */
-#define _STM_CARD_MARKED 100
-char _stm_write_slowpath_card_extra(object_t *obj);
-long _stm_write_slowpath_card_extra_base(void);
 typedef struct {
     stm_thread_local_t *tl;
     char *segment_base;    /* base to interpret the 'object' below */

From noreply at buildbot.pypy.org  Sat Feb 28 12:19:49 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Sat, 28 Feb 2015 12:19:49 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c8: import stmgc 8ca689687bfa
Message-ID: <20150228111949.E20621C009F@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: stmgc-c8
Changeset: r76195:3d008eb21392
Date: 2015-02-28 12:18 +0100
http://bitbucket.org/pypy/pypy/changeset/3d008eb21392/

Log:	import stmgc 8ca689687bfa

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-17b08c18f7b4
+8ca689687bfa
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -978,6 +978,9 @@
     struct stm_read_marker_s *cards = get_read_marker(STM_SEGMENT->segment_base,
                                                       (uintptr_t)obj);
     uintptr_t card_index = get_index_to_card_index(index);
+    if (cards[card_index].rm == CARD_MARKED)
+        return;
+
     if (!IS_OVERFLOW_OBJ(STM_PSEGMENT, obj)
         && !(cards[card_index].rm == CARD_MARKED
              || cards[card_index].rm == STM_SEGMENT->transaction_read_version)) {

From noreply at buildbot.pypy.org  Sat Feb 28 12:19:51 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Sat, 28 Feb 2015 12:19:51 +0100 (CET)
Subject: [pypy-commit] pypy stmgc-c8: (arigo,
	remi) fix jit support for card marking in c8
Message-ID: <20150228111951.25ED01C009F@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: stmgc-c8
Changeset: r76196:1257140b7eef
Date: 2015-02-28 12:19 +0100
http://bitbucket.org/pypy/pypy/changeset/1257140b7eef/

Log:	(arigo, remi) fix jit support for card marking in c8

diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -88,6 +88,8 @@
         self._build_wb_slowpath(False)
         self._build_wb_slowpath(True)
         self._build_wb_slowpath(False, for_frame=True)
+        if gc_ll_descr.stm:
+            self._build_stm_wb_card_slowpath(False)
         # only one of those
         self.build_frame_realloc_slowpath()
         if self.cpu.supports_floats:
@@ -95,6 +97,8 @@
             self._build_failure_recovery(True, withfloats=True)
             self._build_wb_slowpath(False, withfloats=True)
             self._build_wb_slowpath(True, withfloats=True)
+            if gc_ll_descr.stm:
+                self._build_stm_wb_card_slowpath(True)
         self._build_propagate_exception_path()
 
         if gc_ll_descr.get_malloc_slowpath_addr() is not None:
@@ -390,4 +394,3 @@
                 (r_uint(descr_number), r_uint(rawstart),
                     r_uint(rawstart + codeendpos)))
     debug_stop("jit-backend-addr")
-
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -54,6 +54,7 @@
         self.malloc_slowpath = 0
         self.malloc_slowpath_varsize = 0
         self.wb_slowpath = [0, 0, 0, 0, 0]
+        self.wb_card_slowpath = [0, 0]
         self.setup_failure_recovery()
         self.datablockwrapper = None
         self.stack_check_slowpath = 0
@@ -363,6 +364,26 @@
         rawstart = mc.materialize(self.cpu.asmmemmgr, [])
         self.stack_check_slowpath = rawstart
 
+    def _build_stm_wb_card_slowpath(self, withfloats):
+        mc = codebuf.MachineCodeBlockWrapper()
+
+        self._push_all_regs_to_frame(mc, [], withfloats, callee_only=True)
+
+        mc.MOV_rs(esi.value, WORD) #index
+        mc.MOV_rs(edi.value, 2*WORD) #obj
+
+        mc.PUSH(r11) # for alignment
+        func = rstm.adr_write_slowpath_card
+        mc.CALL(imm(func))
+        mc.POP(r11)
+
+        self._pop_all_regs_from_frame(mc, [], withfloats, callee_only=True)
+        mc.RET16_i(2 * WORD)
+
+        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
+        self.wb_card_slowpath[withfloats] = rawstart
+
+
     def _build_wb_slowpath(self, withcards, withfloats=False, for_frame=False):
         descr = self.cpu.gc_ll_descr.write_barrier_descr
         exc0, exc1 = None, None
@@ -2366,10 +2387,11 @@
         # Write only a CALL to the helper prepared in advance, passing it as
         # argument the address of the structure we are writing into
         # (the first argument to COND_CALL_GC_WB).
+        withfloats = self._regalloc is not None and bool(self._regalloc.xrm.reg_bindings)
         helper_num = card_marking
         if is_frame:
             helper_num = 4
-        elif self._regalloc is not None and self._regalloc.xrm.reg_bindings:
+        elif withfloats:
             helper_num += 2
         if self.wb_slowpath[helper_num] == 0:    # tests only
             assert not we_are_translated()
@@ -2400,6 +2422,7 @@
             # So here, we can simply write again a 'JNS', which will be
             # taken if GCFLAG_CARDS_SET is still not set.
             if stm:
+                # here it's actually the result of _stm_write_slowpath_card_extra
                 mc.J_il8(rx86.Conditions['Z'], 0) # patched later
             else:
                 mc.J_il8(rx86.Conditions['NS'], 0) # patched later
@@ -2415,10 +2438,9 @@
             loc_index = arglocs[1]
 
             if stm:
-                # must write the value CARD_MARKED into the byte at:
-                #     write_locks_base + (object >> 4) + (index / CARD_SIZE)
+                # if CARD_MARKED, we are done
+                #     (object >> 4) + (index / CARD_SIZE) + 1
                 #
-                write_locks_base = rstm.adr__stm_write_slowpath_card_extra_base
                 if rstm.CARD_SIZE == 32:
                     card_bits = 5
                 elif rstm.CARD_SIZE == 64:
@@ -2428,12 +2450,12 @@
                 else:
                     raise AssertionError("CARD_SIZE should be 32/64/128")
                 #
-                # idea:  mov r11, write_locks_base<<4
-                #        add r11, loc_base    # the object
+                # idea:
+                #        mov r11, loc_base    # the object
                 #        and r11, ~15         # align
                 #        lea r11, [loc_index + r11<<(card_bits-4)]
                 #        shr r11, card_bits
-                #        mov [r11], card_marked
+                #        cmp [r11+1], card_marked
                 #
                 # this assumes that the value computed up to the
                 # "shr r11, card_bits" instruction does not overflow
@@ -2444,15 +2466,13 @@
                 # and 2**X, for X <= 56).
                 #
                 r11 = X86_64_SCRATCH_REG
-                initial_value = write_locks_base << 4
                 if isinstance(loc_index, RegLoc):
                     if isinstance(loc_base, RegLoc):
-                        mc.MOV_ri(r11.value, initial_value)
-                        mc.ADD_rr(r11.value, loc_base.value)
+                        mc.MOV_ri(r11.value, loc_base.value)
                         mc.AND_ri(r11.value, ~15)
                     else:
                         assert isinstance(loc_base, ImmedLoc)
-                        initial_value += loc_base.value & ~15
+                        initial_value = loc_base.value & ~15
                         mc.MOV_ri(r11.value, initial_value)
                     mc.LEA_ra(r11.value, (self.SEGMENT_NO,
                                           loc_index.value,
@@ -2462,7 +2482,7 @@
                     mc.SHR_ri(r11.value, card_bits)
                 else:
                     assert isinstance(loc_index, ImmedLoc)
-                    initial_value += (loc_index.value >> card_bits) << 4
+                    initial_value = (loc_index.value >> card_bits) << 4
                     if isinstance(loc_base, RegLoc):
                         mc.MOV_ri(r11.value, initial_value)
                         mc.ADD_rr(r11.value, loc_base.value)
@@ -2473,8 +2493,18 @@
                         initial_value >>= 4
                         mc.MOV_ri(r11.value, initial_value)
                 #
-                mc.MOV8_mi((self.SEGMENT_NO, r11.value, 0),
+                mc.CMP8_mi((self.SEGMENT_GC, r11.value, 1),
                            rstm.CARD_MARKED)
+                mc.J_il8(rx86.Conditions['E'], 0) # patched later
+                before_loc = mc.get_relative_pos()
+                # slowpath: call _stm_write_slowpath_card
+                mc.PUSH(loc_base)
+                mc.PUSH(loc_index)
+                mc.CALL(imm(self.wb_card_slowpath[withfloats]))
+
+                offset = mc.get_relative_pos() - before_loc
+                assert 0 < offset <= 127
+                mc.overwrite(before_loc-1, chr(offset))
 
             elif isinstance(loc_index, RegLoc):
                 if IS_X86_64 and isinstance(loc_base, RegLoc):
@@ -2811,7 +2841,7 @@
         # It is only supported if 'translate_support_code' is
         # true; otherwise, the execute_token() was done with a
         # dummy value for the stack location THREADLOCAL_OFS
-        # 
+        #
         assert self.cpu.translate_support_code
         assert isinstance(resloc, RegLoc)
         self.mc.MOV_rs(resloc.value, THREADLOCAL_OFS)
diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py
--- a/rpython/jit/backend/x86/rx86.py
+++ b/rpython/jit/backend/x86/rx86.py
@@ -516,6 +516,7 @@
     CMP32_mi = insn(rex_nw, '\x81', orbyte(7<<3), mem_reg_plus_const(1), immediate(2))
     CMP16_mi = insn('\x66', rex_nw, '\x81', orbyte(7<<3), mem_reg_plus_const(1), immediate(2, 'h'))
     CMP8_ri = insn(rex_fw, '\x80', byte_register(1), '\xF8', immediate(2, 'b'))
+    CMP8_mi = insn(rex_nw, '\x80', orbyte(7<<3), mem_reg_plus_const(1), immediate(2, 'b'))
 
     AND8_rr = insn(rex_fw, '\x20', byte_register(1), byte_register(2,8), '\xC0')
 
diff --git a/rpython/memory/gc/stmgc.py b/rpython/memory/gc/stmgc.py
--- a/rpython/memory/gc/stmgc.py
+++ b/rpython/memory/gc/stmgc.py
@@ -11,6 +11,7 @@
 from rpython.rlib.rarithmetic import LONG_BIT, r_uint
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.translator.stm import stmgcintf
+from rpython.rlib import rstm
 
 WORD = LONG_BIT // 8
 NULL = llmemory.NULL
@@ -33,9 +34,9 @@
 
     VISIT_FPTR = lltype.Ptr(lltype.FuncType([llmemory.Address], lltype.Void))
 
-    JIT_WB_IF_FLAG = 0x01            # value of _STM_GCFLAG_WRITE_BARRIER
-    JIT_WB_CARDS_SET = 0x08          # value of _STM_GCFLAG_CARDS_SET
-    stm_fast_alloc = 66*1024         # value of _STM_FAST_ALLOC in stmgc.h
+    JIT_WB_IF_FLAG = 0x1   # from stmgc.h
+    JIT_WB_CARDS_SET = 0x8 # from stmgc.h
+    stm_fast_alloc = rstm.FAST_ALLOC
     minimal_size_in_nursery = 16     # hard-coded lower limit
 
     TRANSLATION_PARAMS = {
diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -30,11 +30,16 @@
 adr_write_slowpath = CFlexSymbolic('((long)&_stm_write_slowpath)')
 adr_write_slowpath_card_extra = (
     CFlexSymbolic('((long)&_stm_write_slowpath_card_extra)'))
-adr__stm_write_slowpath_card_extra_base = (
-   CFlexSymbolic('(_stm_write_slowpath_card_extra_base()-0x4000000000000000L)'))
+adr_write_slowpath_card = (
+    CFlexSymbolic('((long)&_stm_write_slowpath_card)'))
+
 CARD_MARKED = CFlexSymbolic('_STM_CARD_MARKED')
 CARD_SIZE   = CFlexSymbolic('_STM_CARD_SIZE')
 
+GCFLAG_CARDS_SET = CFlexSymbolic('_STM_GCFLAG_CARDS_SET')
+GCFLAG_WRITE_BARRIER = CFlexSymbolic('_STM_GCFLAG_WRITE_BARRIER')
+FAST_ALLOC = CFlexSymbolic('_STM_FAST_ALLOC')
+
 adr_pypy__rewind_jmp_copy_stack_slice = (
     CFlexSymbolic('((long)&pypy__rewind_jmp_copy_stack_slice)'))
 adr_pypy_stm_commit_if_not_atomic = (

From noreply at buildbot.pypy.org  Sat Feb 28 16:54:22 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Sat, 28 Feb 2015 16:54:22 +0100 (CET)
Subject: [pypy-commit] pypy llvm-translation-backend: Skip tests that test
 ordered dicts with refcounting.
Message-ID: <20150228155422.ACA251C0113@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: llvm-translation-backend
Changeset: r76197:e580a050b66b
Date: 2015-02-07 12:35 +0100
http://bitbucket.org/pypy/pypy/changeset/e580a050b66b/

Log:	Skip tests that test ordered dicts with refcounting.

diff --git a/rpython/translator/llvm/test/test_genllvm.py b/rpython/translator/llvm/test/test_genllvm.py
--- a/rpython/translator/llvm/test/test_genllvm.py
+++ b/rpython/translator/llvm/test/test_genllvm.py
@@ -671,10 +671,14 @@
     def getcompiled(self, func, argtypes):
         return _LLVMMixin.getcompiled(self, func, argtypes, backendopt=False)
 
+    def test_r_dict_exceptions(self):
+        py.test.skip("ordered dicts don't seem to work with refcounting")
+
 
 class TestTypedOptimizedTestCaseLLVM(_LLVMMixin, test_backendoptimized
                                                  .TestTypedOptimizedTestCase):
-    pass
+    def test_r_dict_exceptions(self):
+        py.test.skip("ordered dicts don't seem to work with refcounting")
 
 class TestTypedOptimizedSwitchTestCaseLLVM(_LLVMMixin,
                                            test_backendoptimized

From noreply at buildbot.pypy.org  Sat Feb 28 17:00:17 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Sat, 28 Feb 2015 17:00:17 +0100 (CET)
Subject: [pypy-commit] pypy llvm-translation-backend: Pass more immutability
	hints to LLVM.
Message-ID: <20150228160017.2427A1C03F2@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: llvm-translation-backend
Changeset: r76198:1b6be80b1ea6
Date: 2015-02-28 16:51 +0100
http://bitbucket.org/pypy/pypy/changeset/1b6be80b1ea6/

Log:	Pass more immutability hints to LLVM.

diff --git a/rpython/translator/llvm/genllvm.py b/rpython/translator/llvm/genllvm.py
--- a/rpython/translator/llvm/genllvm.py
+++ b/rpython/translator/llvm/genllvm.py
@@ -659,7 +659,7 @@
                     .format(self.typestr, groupname, i))
         struct_type = StructType()
         struct_type.setup('group_' + obj.name, fields, False)
-        database.f.write('{} = global {}\n'.format(
+        database.f.write('{} = constant {}\n'.format(
                 groupname, struct_type.repr_type_and_value(group)))
 
 
@@ -1216,9 +1216,13 @@
 
     def _get_element(self, result, var, *fields):
         if result.type is not LLVMVoid:
+            if var.type.lltype.TO._hints.get('immutable'):
+                metadata= ', !invariant.load !0'
+            else:
+                metadata=''
             t = self._tmp(PtrType.tmp(result.type))
             self._get_element_ptr(var, fields, t)
-            self.w('{result.V} = load {t.TV}'.format(**locals()))
+            self.w('{result.V} = load {t.TV}{metadata}'.format(**locals()))
     op_getfield = op_bare_getfield = _get_element
     op_getinteriorfield = op_bare_getinteriorfield = _get_element
     op_getarrayitem = op_bare_getarrayitem = _get_element
@@ -1835,6 +1839,7 @@
                 '}}\n'.format(raise_=get_repr(exctrans.rpyexc_raise_ptr),
                               type=get_repr(self.ovf_err[0]),
                               inst=get_repr(self.ovf_err[1])))
+        f.write('!0 = metadata !{ }\n')
 
     def gen_source(self):
         global database

From noreply at buildbot.pypy.org  Sat Feb 28 20:53:20 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Sat, 28 Feb 2015 20:53:20 +0100 (CET)
Subject: [pypy-commit] pypy default: Remove unused import and variable
 assignments in flowcontext.py.
Message-ID: <20150228195320.7BD6C1C0498@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: 
Changeset: r76199:374151605651
Date: 2015-02-28 20:52 +0100
http://bitbucket.org/pypy/pypy/changeset/374151605651/

Log:	Remove unused import and variable assignments in flowcontext.py.

diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py
--- a/rpython/flowspace/flowcontext.py
+++ b/rpython/flowspace/flowcontext.py
@@ -7,7 +7,6 @@
 import __builtin__
 
 from rpython.tool.error import source_lines
-from rpython.translator.simplify import eliminate_empty_blocks
 from rpython.rlib import rstackovf
 from rpython.flowspace.argument import CallSpec
 from rpython.flowspace.model import (Constant, Variable, Block, Link,
@@ -124,7 +123,6 @@
 
     def guessexception(self, ctx, *cases):
         block = self.crnt_block
-        bvars = vars = vars2 = block.getvariables()
         links = []
         for case in [None] + list(cases):
             if case is not None:

From noreply at buildbot.pypy.org  Sat Feb 28 23:26:46 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 28 Feb 2015 23:26:46 +0100 (CET)
Subject: [pypy-commit] pypy object-dtype2: translation fix,
	there must be a better way
Message-ID: <20150228222646.23BDE1C009F@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: object-dtype2
Changeset: r76208:bcee1fdd5f52
Date: 2015-03-01 00:23 +0200
http://bitbucket.org/pypy/pypy/changeset/bcee1fdd5f52/

Log:	translation fix, there must be a better way

diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -1648,6 +1648,7 @@
     def read(self, arr, i, offset, dtype=None):
         return self.box(self._read(arr.storage, i, offset))
 
+    @jit.dont_look_inside
     def _write(self, storage, i, offset, w_obj):
         if we_are_translated():
             value = rffi.cast(lltype.Signed, cast_instance_to_gcref(w_obj))
@@ -1656,6 +1657,7 @@
             _all_objs_for_tests.append(w_obj)
         raw_storage_setitem_unaligned(storage, i + offset, value)
 
+    @jit.dont_look_inside
     def _read(self, storage, i, offset):
         res = raw_storage_getitem_unaligned(self.T, storage, i + offset)
         if we_are_translated():

From noreply at buildbot.pypy.org  Sat Feb 28 23:26:44 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 28 Feb 2015 23:26:44 +0100 (CET)
Subject: [pypy-commit] pypy object-dtype2: put space as an attribute on
	type.Types
Message-ID: <20150228222644.D55EF1C009F@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: object-dtype2
Changeset: r76207:320bc55d7975
Date: 2015-02-28 20:07 +0200
http://bitbucket.org/pypy/pypy/changeset/320bc55d7975/

Log:	put space as an attribute on type.Types

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -3,7 +3,7 @@
 """
 import re
 from pypy.interpreter import special
-from pypy.interpreter.baseobjspace import InternalSpaceCache, W_Root
+from pypy.interpreter.baseobjspace import InternalSpaceCache, W_Root, ObjSpace
 from pypy.interpreter.error import OperationError
 from rpython.rlib.objectmodel import specialize, instantiate
 from rpython.rlib.nonconst import NonConstant
@@ -47,7 +47,7 @@
     def lookup(self, name):
         return self.getdictvalue(self, name)
 
-class FakeSpace(object):
+class FakeSpace(ObjSpace):
     w_ValueError = W_TypeObject("ValueError")
     w_TypeError = W_TypeObject("TypeError")
     w_IndexError = W_TypeObject("IndexError")
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -431,7 +431,7 @@
 
                 self.names.append(name)
                 self.fields[name] = offset, dtype
-            self.itemtype = types.RecordType()
+            self.itemtype = types.RecordType(space)
 
         if self.is_flexible():
             self.elsize = size
@@ -446,7 +446,7 @@
                 endian = NPY.OPPBYTE if self.is_native() else NPY.NATBYTE
             elif newendian != NPY.IGNORE:
                 endian = newendian
-        itemtype = self.itemtype.__class__(endian in (NPY.NATIVE, NPY.NATBYTE))
+        itemtype = self.itemtype.__class__(space, endian in (NPY.NATIVE, NPY.NATBYTE))
         fields = self.fields
         if fields is None:
             fields = {}
@@ -485,7 +485,7 @@
         fields[fldname] = (offset, subdtype)
         offset += subdtype.elsize
         names.append(fldname)
-    return W_Dtype(types.RecordType(), NPY.VOID, NPY.VOIDLTR, NPY.VOIDLTR,
+    return W_Dtype(types.RecordType(space), NPY.VOID, NPY.VOIDLTR, NPY.VOIDLTR,
                    space.gettypefor(boxes.W_VoidBox),
                    names=names, fields=fields, elsize=offset)
 
@@ -544,7 +544,7 @@
         if size == 1:
             return subdtype
         size *= subdtype.elsize
-        return W_Dtype(types.VoidType(), NPY.VOID, NPY.VOIDLTR, NPY.VOIDLTR,
+        return W_Dtype(types.VoidType(space), NPY.VOID, NPY.VOIDLTR, NPY.VOIDLTR,
                        space.gettypefor(boxes.W_VoidBox),
                        shape=shape, subdtype=subdtype, elsize=size)
 
@@ -655,7 +655,7 @@
 
 def new_string_dtype(space, size, char=NPY.STRINGLTR):
     return W_Dtype(
-        types.StringType(),
+        types.StringType(space),
         elsize=size,
         num=NPY.STRING,
         kind=NPY.STRINGLTR,
@@ -665,7 +665,7 @@
 
 
 def new_unicode_dtype(space, size):
-    itemtype = types.UnicodeType()
+    itemtype = types.UnicodeType(space)
     return W_Dtype(
         itemtype,
         elsize=size * itemtype.get_element_size(),
@@ -678,7 +678,7 @@
 
 def new_void_dtype(space, size):
     return W_Dtype(
-        types.VoidType(),
+        types.VoidType(space),
         elsize=size,
         num=NPY.VOID,
         kind=NPY.VOIDLTR,
@@ -690,126 +690,126 @@
 class DtypeCache(object):
     def __init__(self, space):
         self.w_booldtype = W_Dtype(
-            types.Bool(),
+            types.Bool(space),
             num=NPY.BOOL,
             kind=NPY.GENBOOLLTR,
             char=NPY.BOOLLTR,
             w_box_type=space.gettypefor(boxes.W_BoolBox),
         )
         self.w_int8dtype = W_Dtype(
-            types.Int8(),
+            types.Int8(space),
             num=NPY.BYTE,
             kind=NPY.SIGNEDLTR,
             char=NPY.BYTELTR,
             w_box_type=space.gettypefor(boxes.W_Int8Box),
         )
         self.w_uint8dtype = W_Dtype(
-            types.UInt8(),
+            types.UInt8(space),
             num=NPY.UBYTE,
             kind=NPY.UNSIGNEDLTR,
             char=NPY.UBYTELTR,
             w_box_type=space.gettypefor(boxes.W_UInt8Box),
         )
         self.w_int16dtype = W_Dtype(
-            types.Int16(),
+            types.Int16(space),
             num=NPY.SHORT,
             kind=NPY.SIGNEDLTR,
             char=NPY.SHORTLTR,
             w_box_type=space.gettypefor(boxes.W_Int16Box),
         )
         self.w_uint16dtype = W_Dtype(
-            types.UInt16(),
+            types.UInt16(space),
             num=NPY.USHORT,
             kind=NPY.UNSIGNEDLTR,
             char=NPY.USHORTLTR,
             w_box_type=space.gettypefor(boxes.W_UInt16Box),
         )
         self.w_int32dtype = W_Dtype(
-            types.Int32(),
+            types.Int32(space),
             num=NPY.INT,
             kind=NPY.SIGNEDLTR,
             char=NPY.INTLTR,
             w_box_type=space.gettypefor(boxes.W_Int32Box),
         )
         self.w_uint32dtype = W_Dtype(
-            types.UInt32(),
+            types.UInt32(space),
             num=NPY.UINT,
             kind=NPY.UNSIGNEDLTR,
             char=NPY.UINTLTR,
             w_box_type=space.gettypefor(boxes.W_UInt32Box),
         )
         self.w_longdtype = W_Dtype(
-            types.Long(),
+            types.Long(space),
             num=NPY.LONG,
             kind=NPY.SIGNEDLTR,
             char=NPY.LONGLTR,
             w_box_type=space.gettypefor(boxes.W_LongBox),
         )
         self.w_ulongdtype = W_Dtype(
-            types.ULong(),
+            types.ULong(space),
             num=NPY.ULONG,
             kind=NPY.UNSIGNEDLTR,
             char=NPY.ULONGLTR,
             w_box_type=space.gettypefor(boxes.W_ULongBox),
         )
         self.w_int64dtype = W_Dtype(
-            types.Int64(),
+            types.Int64(space),
             num=NPY.LONGLONG,
             kind=NPY.SIGNEDLTR,
             char=NPY.LONGLONGLTR,
             w_box_type=space.gettypefor(boxes.W_Int64Box),
         )
         self.w_uint64dtype = W_Dtype(
-            types.UInt64(),
+            types.UInt64(space),
             num=NPY.ULONGLONG,
             kind=NPY.UNSIGNEDLTR,
             char=NPY.ULONGLONGLTR,
             w_box_type=space.gettypefor(boxes.W_UInt64Box),
         )
         self.w_float32dtype = W_Dtype(
-            types.Float32(),
+            types.Float32(space),
             num=NPY.FLOAT,
             kind=NPY.FLOATINGLTR,
             char=NPY.FLOATLTR,
             w_box_type=space.gettypefor(boxes.W_Float32Box),
         )
         self.w_float64dtype = W_Dtype(
-            types.Float64(),
+            types.Float64(space),
             num=NPY.DOUBLE,
             kind=NPY.FLOATINGLTR,
             char=NPY.DOUBLELTR,
             w_box_type=space.gettypefor(boxes.W_Float64Box),
         )
         self.w_floatlongdtype = W_Dtype(
-            types.FloatLong(),
+            types.FloatLong(space),
             num=NPY.LONGDOUBLE,
             kind=NPY.FLOATINGLTR,
             char=NPY.LONGDOUBLELTR,
             w_box_type=space.gettypefor(boxes.W_FloatLongBox),
         )
         self.w_complex64dtype = W_Dtype(
-            types.Complex64(),
+            types.Complex64(space),
             num=NPY.CFLOAT,
             kind=NPY.COMPLEXLTR,
             char=NPY.CFLOATLTR,
             w_box_type=space.gettypefor(boxes.W_Complex64Box),
         )
         self.w_complex128dtype = W_Dtype(
-            types.Complex128(),
+            types.Complex128(space),
             num=NPY.CDOUBLE,
             kind=NPY.COMPLEXLTR,
             char=NPY.CDOUBLELTR,
             w_box_type=space.gettypefor(boxes.W_Complex128Box),
         )
         self.w_complexlongdtype = W_Dtype(
-            types.ComplexLong(),
+            types.ComplexLong(space),
             num=NPY.CLONGDOUBLE,
             kind=NPY.COMPLEXLTR,
             char=NPY.CLONGDOUBLELTR,
             w_box_type=space.gettypefor(boxes.W_ComplexLongBox),
         )
         self.w_stringdtype = W_Dtype(
-            types.StringType(),
+            types.StringType(space),
             elsize=0,
             num=NPY.STRING,
             kind=NPY.STRINGLTR,
@@ -817,7 +817,7 @@
             w_box_type=space.gettypefor(boxes.W_StringBox),
         )
         self.w_unicodedtype = W_Dtype(
-            types.UnicodeType(),
+            types.UnicodeType(space),
             elsize=0,
             num=NPY.UNICODE,
             kind=NPY.UNICODELTR,
@@ -825,7 +825,7 @@
             w_box_type=space.gettypefor(boxes.W_UnicodeBox),
         )
         self.w_voiddtype = W_Dtype(
-            types.VoidType(),
+            types.VoidType(space),
             elsize=0,
             num=NPY.VOID,
             kind=NPY.VOIDLTR,
@@ -833,28 +833,28 @@
             w_box_type=space.gettypefor(boxes.W_VoidBox),
         )
         self.w_float16dtype = W_Dtype(
-            types.Float16(),
+            types.Float16(space),
             num=NPY.HALF,
             kind=NPY.FLOATINGLTR,
             char=NPY.HALFLTR,
             w_box_type=space.gettypefor(boxes.W_Float16Box),
         )
         self.w_intpdtype = W_Dtype(
-            types.Long(),
+            types.Long(space),
             num=NPY.LONG,
             kind=NPY.SIGNEDLTR,
             char=NPY.INTPLTR,
             w_box_type=space.gettypefor(boxes.W_LongBox),
         )
         self.w_uintpdtype = W_Dtype(
-            types.ULong(),
+            types.ULong(space),
             num=NPY.ULONG,
             kind=NPY.UNSIGNEDLTR,
             char=NPY.UINTPLTR,
             w_box_type=space.gettypefor(boxes.W_ULongBox),
         )
         self.w_objectdtype = W_Dtype(
-            types.ObjectType(),
+            types.ObjectType(space),
             num=NPY.OBJECT,
             kind=NPY.OBJECTLTR,
             char=NPY.OBJECTLTR,
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -3,7 +3,7 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.objspace.std.floatobject import float2string
 from pypy.objspace.std.complexobject import str_format
-from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.baseobjspace import W_Root, ObjSpace
 from rpython.rlib import clibffi, jit, rfloat, rcomplex
 from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rlib.rarithmetic import widen, byteswap, r_ulonglong, \
@@ -112,11 +112,12 @@
     return dispatcher
 
 class BaseType(object):
-    _immutable_fields_ = ['native']
+    _immutable_fields_ = ['native', 'space']
 
-    def __init__(self, native=True):
-        assert native is True or native is False 
+    def __init__(self, space, native=True):
+        assert isinstance(space, ObjSpace)
         self.native = native
+        self.space = space
 
     def __repr__(self):
         return self.__class__.__name__
@@ -307,7 +308,7 @@
 
     @raw_unary_op
     def rint(self, v):
-        float64 = Float64()
+        float64 = Float64(self.space)
         return float64.rint(float64.box(v))
 
 class Bool(BaseType, Primitive):
@@ -400,7 +401,7 @@
     def round(self, v, decimals=0):
         if decimals != 0:
             return v
-        return Float64().box(self.unbox(v))
+        return Float64(self.space).box(self.unbox(v))
 
 class Integer(Primitive):
     _mixin_ = True
@@ -1675,7 +1676,15 @@
 
     @specialize.argtype(1)
     def box(self, w_obj):
-        assert isinstance(w_obj, W_Root)
+        if isinstance(w_obj, W_Root):
+            pass
+        elif isinstance(w_obj, int):
+            w_obj = self.space.newint(w_obj)
+        elif isinstance(w_obj, lltype.Number):
+            w_obj = self.space.newint(w_obj)
+        else:
+            raise oefmt(self.space.w_NotImplementedError,
+                "cannot create object array/scalar from lltype")
         return self.BoxType(w_obj)
 
     def str_format(self, box):

From noreply at buildbot.pypy.org  Sat Feb 28 23:26:43 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 28 Feb 2015 23:26:43 +0100 (CET)
Subject: [pypy-commit] pypy object-dtype2: start again,
 reuse the basic ideas from object-dtype
Message-ID: <20150228222643.7AD071C009F@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: object-dtype2
Changeset: r76206:a41e39f68799
Date: 2015-02-27 17:00 +0200
http://bitbucket.org/pypy/pypy/changeset/a41e39f68799/

Log:	start again, reuse the basic ideas from object-dtype

diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py
--- a/pypy/module/micronumpy/base.py
+++ b/pypy/module/micronumpy/base.py
@@ -34,11 +34,13 @@
 
     @staticmethod
     def from_shape(space, shape, dtype, order='C', w_instance=None, zero=True):
-        from pypy.module.micronumpy import concrete
+        from pypy.module.micronumpy import concrete, descriptor, boxes
         from pypy.module.micronumpy.strides import calc_strides
         strides, backstrides = calc_strides(shape, dtype.base, order)
         impl = concrete.ConcreteArray(shape, dtype.base, order, strides,
                                       backstrides, zero=zero)
+        if dtype == descriptor.get_dtype_cache(space).w_objectdtype:
+            impl.fill(space, boxes.W_ObjectBox(space.w_None))
         if w_instance:
             return wrap_impl(space, space.type(w_instance), w_instance, impl)
         return W_NDimArray(impl)
diff --git a/pypy/module/micronumpy/boxes.py b/pypy/module/micronumpy/boxes.py
--- a/pypy/module/micronumpy/boxes.py
+++ b/pypy/module/micronumpy/boxes.py
@@ -607,6 +607,15 @@
         #    arr.storage[i] = arg[i]
         return W_UnicodeBox(arr, 0, arr.dtype)
 
+class W_ObjectBox(W_GenericBox):
+    descr__new__, _get_dtype, descr_reduce = new_dtype_getter(NPY.OBJECT)
+
+    def __init__(self, w_obj):
+        self.w_obj = w_obj
+
+    def convert_to(self, space, dtype):
+        return self # XXX
+
 
 W_GenericBox.typedef = TypeDef("numpy.generic",
     __new__ = interp2app(W_GenericBox.descr__new__.im_func),
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -67,6 +67,7 @@
     w_unicode = W_TypeObject("unicode")
     w_complex = W_TypeObject("complex")
     w_dict = W_TypeObject("dict")
+    w_object = W_TypeObject("object")
 
     def __init__(self):
         """NOT_RPYTHON"""
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -56,7 +56,7 @@
         self.char = char
         self.w_box_type = w_box_type
         if byteorder is None:
-            if itemtype.get_element_size() == 1:
+            if itemtype.get_element_size() == 1 or isinstance(itemtype, types.ObjectType):
                 byteorder = NPY.IGNORE
             else:
                 byteorder = NPY.NATIVE
@@ -112,6 +112,9 @@
     def is_str(self):
         return self.num == NPY.STRING
 
+    def is_object(self):
+        return self.num == NPY.OBJECT
+
     def is_str_or_unicode(self):
         return self.num == NPY.STRING or self.num == NPY.UNICODE
 
@@ -585,8 +588,7 @@
         if w_dtype is dtype.w_box_type:
             return dtype
     if space.isinstance_w(w_dtype, space.w_type):
-        raise oefmt(space.w_NotImplementedError,
-            "cannot create dtype with type '%N'", w_dtype)
+        return cache.w_objectdtype
     raise oefmt(space.w_TypeError, "data type not understood")
 
 
@@ -851,6 +853,13 @@
             char=NPY.UINTPLTR,
             w_box_type=space.gettypefor(boxes.W_ULongBox),
         )
+        self.w_objectdtype = W_Dtype(
+            types.ObjectType(),
+            num=NPY.OBJECT,
+            kind=NPY.OBJECTLTR,
+            char=NPY.OBJECTLTR,
+            w_box_type=space.gettypefor(boxes.W_ObjectBox),
+        )
         aliases = {
             NPY.BOOL:        ['bool_', 'bool8'],
             NPY.BYTE:        ['byte'],
@@ -869,6 +878,7 @@
             NPY.CLONGDOUBLE: ['clongdouble', 'clongfloat'],
             NPY.STRING:      ['string_', 'str'],
             NPY.UNICODE:     ['unicode_'],
+            NPY.OBJECT:      ['object_'],
         }
         self.alternate_constructors = {
             NPY.BOOL:     [space.w_bool],
@@ -887,6 +897,8 @@
             NPY.UNICODE:  [space.w_unicode],
             NPY.VOID:     [space.gettypefor(boxes.W_GenericBox)],
                            #space.w_buffer,  # XXX no buffer in space
+            NPY.OBJECT:   [space.gettypefor(boxes.W_ObjectBox),
+                           space.w_object],
         }
         float_dtypes = [self.w_float16dtype, self.w_float32dtype,
                         self.w_float64dtype, self.w_floatlongdtype]
@@ -906,7 +918,7 @@
             self.w_int64dtype, self.w_uint64dtype,
             ] + float_dtypes + complex_dtypes + [
             self.w_stringdtype, self.w_unicodedtype, self.w_voiddtype,
-            self.w_intpdtype, self.w_uintpdtype,
+            self.w_intpdtype, self.w_uintpdtype, self.w_objectdtype,
         ]
         self.float_dtypes_by_num_bytes = sorted(
             (dtype.elsize, dtype)
@@ -958,6 +970,7 @@
             'USHORT': self.w_uint16dtype,
             'FLOAT': self.w_float32dtype,
             'BOOL': self.w_booldtype,
+            'OBJECT': self.w_objectdtype,
         }
 
         typeinfo_partial = {
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -472,11 +472,8 @@
         class O(object):
             pass
         for o in [object, O]:
-            if '__pypy__' not in sys.builtin_module_names:
-                assert np.dtype(o).str == '|O8'
-            else:
-                exc = raises(NotImplementedError, "np.dtype(o)")
-                assert exc.value[0] == "cannot create dtype with type '%s'" % o.__name__
+            print np.dtype(o).byteorder
+            assert np.dtype(o).str == '|O8'
 
 class AppTestTypes(BaseAppTestDtypes):
     def test_abstract_types(self):
@@ -1354,9 +1351,21 @@
         import sys
         class Polynomial(object):
             pass
-        if '__pypy__' in sys.builtin_module_names:
-            exc = raises(NotImplementedError, array, Polynomial())
-            assert exc.value.message.find('unable to create dtype from objects') >= 0
-        else:
-            a = array(Polynomial())
-            assert a.shape == ()
+        a = array(Polynomial())
+        assert a.shape == ()
+
+    def test_uninitialized_object_array_is_filled_by_None(self):
+        import numpy as np
+
+        a = np.ndarray([5], dtype="O")
+
+        assert a[0] == None
+
+    def test_object_arrays_add(self):
+        import numpy as np
+
+        a = np.array(["foo"], dtype=object)
+        b = np.array(["bar"], dtype=object)
+
+        res = a + b
+        assert res[0] == "foobar"
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -3150,11 +3150,7 @@
         assert b[35] == 200
         b[[slice(25, 30)]] = range(5)
         assert all(a[:5] == range(5))
-        import sys
-        if '__pypy__' not in sys.builtin_module_names:
-            raises(TypeError, 'b[[[slice(25, 125)]]]')
-        else:
-            raises(NotImplementedError, 'b[[[slice(25, 125)]]]')
+        raises(IndexError, 'b[[[slice(25, 125)]]]')
 
     def test_cumsum(self):
         from numpy import arange
diff --git a/pypy/module/micronumpy/test/test_selection.py b/pypy/module/micronumpy/test/test_selection.py
--- a/pypy/module/micronumpy/test/test_selection.py
+++ b/pypy/module/micronumpy/test/test_selection.py
@@ -12,14 +12,11 @@
             exp = sorted(range(len(exp)), key=exp.__getitem__)
             c = a.copy()
             res = a.argsort()
-            assert (res == exp).all(), '%r\n%r\n%r' % (a,res,exp)
+            assert (res == exp).all(), 'Failed sortng %r\na=%r\nres=%r\nexp=%r' % (dtype,a,res,exp)
             assert (a == c).all() # not modified
 
             a = arange(100, dtype=dtype)
             assert (a.argsort() == a).all()
-        import sys
-        if '__pypy__' in sys.builtin_module_names:
-            raises(NotImplementedError, 'arange(10,dtype="float16").argsort()')
 
     def test_argsort_ndim(self):
         from numpy import array
@@ -63,14 +60,13 @@
                       'i2', complex]:
             a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
             exp = sorted(list(a))
-            res = a.copy()
-            res.sort()
-            assert (res == exp).all(), '%r\n%r\n%r' % (a,res,exp)
+            a.sort()
+            assert (a == exp).all(), 'Failed sorting %r\n%r\n%r' % (dtype, a, exp)
 
             a = arange(100, dtype=dtype)
             c = a.copy()
             a.sort()
-            assert (a == c).all()
+            assert (a == c).all(), 'Failed sortng %r\na=%r\nc=%r' % (dtype,a,c)
 
     def test_sort_nonnative(self):
         from numpy import array
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -3,8 +3,9 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.objspace.std.floatobject import float2string
 from pypy.objspace.std.complexobject import str_format
+from pypy.interpreter.baseobjspace import W_Root
 from rpython.rlib import clibffi, jit, rfloat, rcomplex
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rlib.rarithmetic import widen, byteswap, r_ulonglong, \
     most_neg_value_of, LONG_BIT
 from rpython.rlib.rawstorage import (alloc_raw_storage,
@@ -14,7 +15,9 @@
                                        pack_float80, unpack_float80)
 from rpython.rlib.rstruct.nativefmttable import native_is_bigendian
 from rpython.rlib.rstruct.runpack import runpack
-from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.annlowlevel import cast_instance_to_gcref,\
+     cast_gcref_to_instance
+from rpython.rtyper.lltypesystem import lltype, rffi, llmemory
 from rpython.tool.sourcetools import func_with_new_name
 from pypy.module.micronumpy import boxes
 from pypy.module.micronumpy.concrete import SliceArray, VoidBoxStorage
@@ -112,6 +115,7 @@
     _immutable_fields_ = ['native']
 
     def __init__(self, native=True):
+        assert native is True or native is False 
         self.native = native
 
     def __repr__(self):
@@ -1623,6 +1627,80 @@
         BoxType = boxes.W_ComplexLongBox
         ComponentBoxType = boxes.W_FloatLongBox
 
+_all_objs_for_tests = [] # for tests
+
+class ObjectType(BaseType):
+    T = lltype.Signed
+    BoxType = boxes.W_ObjectBox
+
+    def get_element_size(self):
+        return rffi.sizeof(lltype.Signed)
+
+    def coerce(self, space, dtype, w_item):
+        if isinstance(w_item, boxes.W_ObjectBox):
+            return w_item
+        return boxes.W_ObjectBox(w_item)
+
+    def store(self, arr, i, offset, box):
+        self._write(arr.storage, i, offset, self.unbox(box))
+
+    def read(self, arr, i, offset, dtype=None):
+        return self.box(self._read(arr.storage, i, offset))
+
+    def _write(self, storage, i, offset, w_obj):
+        if we_are_translated():
+            value = rffi.cast(lltype.Signed, cast_instance_to_gcref(w_obj))
+        else:
+            value = len(_all_objs_for_tests)
+            _all_objs_for_tests.append(w_obj)
+        raw_storage_setitem_unaligned(storage, i + offset, value)
+
+    def _read(self, storage, i, offset):
+        res = raw_storage_getitem_unaligned(self.T, storage, i + offset)
+        if we_are_translated():
+            gcref = rffi.cast(llmemory.GCREF, res)
+            w_obj = cast_gcref_to_instance(W_Root, gcref)
+        else:
+            w_obj = _all_objs_for_tests[res]
+        return w_obj
+
+    def fill(self, storage, width, box, start, stop, offset):
+        value = self.unbox(box)
+        for i in xrange(start, stop, width):
+            self._write(storage, i, offset, value)
+
+    def unbox(self, box):
+        assert isinstance(box, self.BoxType)
+        return box.w_obj
+
+    @specialize.argtype(1)
+    def box(self, w_obj):
+        assert isinstance(w_obj, W_Root)
+        return self.BoxType(w_obj)
+
+    def str_format(self, box):
+        return 'Object as string'
+        #return space.str_w(space.repr(self.unbox(box)))
+
+    def to_builtin_type(self, space, box):
+        assert isinstance(box, self.BoxType)
+        return box.w_obj
+
+    @staticmethod
+    def for_computation(v):
+        return v
+
+    @simple_binary_op
+    def add(self, v1, v2):
+        return v1
+        #return self.space.add(v1, v2)
+
+    @raw_binary_op
+    def eq(self, v1, v2):
+        return True
+        #return self.space.eq_w(v1, v2)
+
+
 class FlexibleType(BaseType):
     def get_element_size(self):
         return rffi.sizeof(self.T)
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -431,7 +431,9 @@
         w_rhs = numpify(space, w_rhs)
         w_ldtype = _get_dtype(space, w_lhs)
         w_rdtype = _get_dtype(space, w_rhs)
-        if w_ldtype.is_str() and w_rdtype.is_str() and \
+        if w_ldtype.is_object() or w_rdtype.is_object():
+            pass
+        elif w_ldtype.is_str() and w_rdtype.is_str() and \
                 self.comparison_func:
             pass
         elif (w_ldtype.is_str() or w_rdtype.is_str()) and \
@@ -945,6 +947,10 @@
         return dt1
     if dt1 is None:
         return dt2
+
+    if dt1.num == NPY.OBJECT or dt2.num == NPY.OBJECT:
+        return get_dtype_cache(space).w_objectdtype
+
     # dt1.num should be <= dt2.num
     if dt1.num > dt2.num:
         dt1, dt2 = dt2, dt1
@@ -1059,6 +1065,7 @@
     uint64_dtype = get_dtype_cache(space).w_uint64dtype
     complex_dtype = get_dtype_cache(space).w_complex128dtype
     float_dtype = get_dtype_cache(space).w_float64dtype
+    object_dtype = get_dtype_cache(space).w_objectdtype
     if isinstance(w_obj, boxes.W_GenericBox):
         dtype = w_obj.get_dtype(space)
         return find_binop_result_dtype(space, dtype, current_guess)
@@ -1092,9 +1099,10 @@
                 return variable_dtype(space,
                                                    'S%d' % space.len_w(w_obj))
         return current_guess
-    raise oefmt(space.w_NotImplementedError,
-                'unable to create dtype from objects, "%T" instance not '
-                'supported', w_obj)
+    return object_dtype
+    #raise oefmt(space.w_NotImplementedError,
+    #            'unable to create dtype from objects, "%T" instance not '
+    #            'supported', w_obj)
 
 
 def ufunc_dtype_caller(space, ufunc_name, op_name, nin, comparison_func,