[pypy-commit] stmgc default: hg merge hashtable
arigo
noreply at buildbot.pypy.org
Mon Jan 19 18:35:45 CET 2015
Author: Armin Rigo <arigo at tunes.org>
Branch:
Changeset: r1552:957947bc7ad9
Date: 2015-01-19 18:36 +0100
http://bitbucket.org/pypy/stmgc/changeset/957947bc7ad9/
Log: hg merge hashtable
diff --git a/c7/demo/demo_hashtable1.c b/c7/demo/demo_hashtable1.c
new file mode 100644
--- /dev/null
+++ b/c7/demo/demo_hashtable1.c
@@ -0,0 +1,217 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+#include <pthread.h>
+#include <semaphore.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+
+#include "stmgc.h"
+
+#define NUMTHREADS 4
+
+
+typedef TLPREFIX struct node_s node_t;
+typedef TLPREFIX struct dict_s dict_t;
+
+
+struct node_s {
+ struct object_s header;
+ int typeid;
+ intptr_t freevalue;
+};
+
+struct dict_s {
+ struct node_s hdr;
+ stm_hashtable_t *hashtable;
+};
+
+#define TID_NODE 0x01234567
+#define TID_DICT 0x56789ABC
+#define TID_DICTENTRY 0x6789ABCD
+
+
+static sem_t done;
+__thread stm_thread_local_t stm_thread_local;
+
+// global and per-thread-data
+time_t default_seed;
+dict_t *global_dict;
+
+struct thread_data {
+ unsigned int thread_seed;
+};
+__thread struct thread_data td;
+
+
+ssize_t stmcb_size_rounded_up(struct object_s *ob)
+{
+ if (((struct node_s*)ob)->typeid == TID_NODE)
+ return sizeof(struct node_s);
+ if (((struct node_s*)ob)->typeid == TID_DICT)
+ return sizeof(struct dict_s);
+ if (((struct node_s*)ob)->typeid == TID_DICTENTRY)
+ return sizeof(struct stm_hashtable_entry_s);
+ abort();
+}
+
+void stmcb_trace(struct object_s *obj, void visit(object_t **))
+{
+ struct node_s *n;
+ n = (struct node_s*)obj;
+ if (n->typeid == TID_NODE) {
+ return;
+ }
+ if (n->typeid == TID_DICT) {
+ stm_hashtable_tracefn(((struct dict_s *)n)->hashtable, visit);
+ return;
+ }
+ if (n->typeid == TID_DICTENTRY) {
+ object_t **ref = &((struct stm_hashtable_entry_s *)obj)->object;
+ visit(ref);
+ return;
+ }
+ abort();
+}
+
+void stmcb_commit_soon() {}
+long stmcb_obj_supports_cards(struct object_s *obj)
+{
+ return 0;
+}
+void stmcb_trace_cards(struct object_s *obj, void cb(object_t **),
+ uintptr_t start, uintptr_t stop) {
+ abort();
+}
+void stmcb_get_card_base_itemsize(struct object_s *obj,
+ uintptr_t offset_itemsize[2]) {
+ abort();
+}
+
+int get_rand(int max)
+{
+ if (max == 0)
+ return 0;
+ return (int)(rand_r(&td.thread_seed) % (unsigned int)max);
+}
+
+
+void populate_hashtable(int keymin, int keymax)
+{
+ int i;
+ int diff = get_rand(keymax - keymin);
+ for (i = 0; i < keymax - keymin; i++) {
+ int key = keymin + i + diff;
+ if (key >= keymax)
+ key -= (keymax - keymin);
+ object_t *o = stm_allocate(sizeof(struct node_s));
+ ((node_t *)o)->typeid = TID_NODE;
+ ((node_t *)o)->freevalue = key;
+ assert(global_dict->hdr.freevalue == 42);
+ stm_hashtable_write((object_t *)global_dict, global_dict->hashtable,
+ key, o, &stm_thread_local);
+ }
+}
+
+void setup_thread(void)
+{
+ memset(&td, 0, sizeof(struct thread_data));
+ td.thread_seed = default_seed++;
+}
+
+void *demo_random(void *arg)
+{
+ int threadnum = (uintptr_t)arg;
+ int status;
+ rewind_jmp_buf rjbuf;
+ stm_register_thread_local(&stm_thread_local);
+ stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf);
+
+ setup_thread();
+
+ volatile int start_count = 0;
+
+ stm_start_transaction(&stm_thread_local);
+ ++start_count;
+ assert(start_count == 1); // all the writes that follow must not conflict
+ populate_hashtable(1291 * threadnum, 1291 * (threadnum + 1));
+ stm_commit_transaction();
+
+ stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
+ stm_unregister_thread_local(&stm_thread_local);
+
+ status = sem_post(&done); assert(status == 0);
+ return NULL;
+}
+
+void newthread(void*(*func)(void*), void *arg)
+{
+ pthread_t th;
+ int status = pthread_create(&th, NULL, func, arg);
+ if (status != 0)
+ abort();
+ pthread_detach(th);
+ printf("started new thread\n");
+}
+
+void setup_globals(void)
+{
+ stm_hashtable_t *my_hashtable = stm_hashtable_create();
+ struct dict_s new_templ = {
+ .hdr = {
+ .typeid = TID_DICT,
+ .freevalue = 42,
+ },
+ .hashtable = my_hashtable,
+ };
+
+ stm_start_inevitable_transaction(&stm_thread_local);
+ global_dict = (dict_t *)stm_setup_prebuilt(
+ (object_t* )(uintptr_t)&new_templ);
+ assert(global_dict->hashtable);
+ stm_commit_transaction();
+}
+
+
+int main(void)
+{
+ int i, status;
+ rewind_jmp_buf rjbuf;
+
+ stm_hashtable_entry_userdata = TID_DICTENTRY;
+
+ /* pick a random seed from the time in seconds.
+ A bit pointless for now... because the interleaving of the
+ threads is really random. */
+ default_seed = time(NULL);
+ printf("running with seed=%lld\n", (long long)default_seed);
+
+ status = sem_init(&done, 0, 0);
+ assert(status == 0);
+
+
+ stm_setup();
+ stm_register_thread_local(&stm_thread_local);
+ stm_rewind_jmp_enterframe(&stm_thread_local, &rjbuf);
+
+ setup_globals();
+
+ for (i = 0; i < NUMTHREADS; i++) {
+ newthread(demo_random, (void *)(uintptr_t)i);
+ }
+
+ for (i=0; i < NUMTHREADS; i++) {
+ status = sem_wait(&done);
+ assert(status == 0);
+ printf("thread finished\n");
+ }
+
+ printf("Test OK!\n");
+
+ stm_rewind_jmp_leaveframe(&stm_thread_local, &rjbuf);
+ stm_unregister_thread_local(&stm_thread_local);
+ stm_teardown();
+
+ return 0;
+}
diff --git a/c7/stm/core.c b/c7/stm/core.c
--- a/c7/stm/core.c
+++ b/c7/stm/core.c
@@ -372,6 +372,7 @@
assert(tree_is_cleared(STM_PSEGMENT->nursery_objects_shadows));
assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[0]));
assert(tree_is_cleared(STM_PSEGMENT->callbacks_on_commit_and_abort[1]));
+ assert(list_is_empty(STM_PSEGMENT->young_objects_with_light_finalizers));
assert(STM_PSEGMENT->objects_pointing_to_nursery == NULL);
assert(STM_PSEGMENT->large_overflow_objects == NULL);
assert(STM_PSEGMENT->finalizers == NULL);
@@ -972,6 +973,8 @@
(int)pseg->transaction_state);
}
+ abort_finalizers(pseg);
+
/* throw away the content of the nursery */
long bytes_in_nursery = throw_away_nursery(pseg);
@@ -1060,8 +1063,6 @@
/* invoke the callbacks */
invoke_and_clear_user_callbacks(1); /* for abort */
- abort_finalizers();
-
if (is_abort(STM_SEGMENT->nursery_end)) {
/* done aborting */
STM_SEGMENT->nursery_end = pause_signalled ? NSE_SIGPAUSE
diff --git a/c7/stm/finalizer.c b/c7/stm/finalizer.c
--- a/c7/stm/finalizer.c
+++ b/c7/stm/finalizer.c
@@ -58,28 +58,73 @@
STM_PSEGMENT->finalizers = NULL;
}
-static void _abort_finalizers(void)
+static void abort_finalizers(struct stm_priv_segment_info_s *pseg)
{
/* like _commit_finalizers(), but forget everything from the
current transaction */
- if (STM_PSEGMENT->finalizers->run_finalizers != NULL) {
- if (STM_PSEGMENT->finalizers->running_next != NULL) {
- *STM_PSEGMENT->finalizers->running_next = (uintptr_t)-1;
+ if (pseg->finalizers != NULL) {
+ if (pseg->finalizers->run_finalizers != NULL) {
+ if (pseg->finalizers->running_next != NULL) {
+ *pseg->finalizers->running_next = (uintptr_t)-1;
+ }
+ list_free(pseg->finalizers->run_finalizers);
}
- list_free(STM_PSEGMENT->finalizers->run_finalizers);
+ list_free(pseg->finalizers->objects_with_finalizers);
+ free(pseg->finalizers);
+ pseg->finalizers = NULL;
}
- list_free(STM_PSEGMENT->finalizers->objects_with_finalizers);
- free(STM_PSEGMENT->finalizers);
- STM_PSEGMENT->finalizers = NULL;
+
+ /* call the light finalizers for objects that are about to
+ be forgotten from the current transaction */
+ char *old_gs_register = STM_SEGMENT->segment_base;
+ bool must_fix_gs = old_gs_register != pseg->pub.segment_base;
+
+ struct list_s *lst = pseg->young_objects_with_light_finalizers;
+ long i, count = list_count(lst);
+ if (lst > 0) {
+ for (i = 0; i < count; i++) {
+ object_t *obj = (object_t *)list_item(lst, i);
+ assert(_is_young(obj));
+ if (must_fix_gs) {
+ set_gs_register(pseg->pub.segment_base);
+ must_fix_gs = false;
+ }
+ stmcb_light_finalizer(obj);
+ }
+ list_clear(lst);
+ }
+
+ /* also deals with overflow objects: they are at the tail of
+ old_objects_with_light_finalizers (this list is kept in order
+ and we cannot add any already-committed object) */
+ lst = pseg->old_objects_with_light_finalizers;
+ count = list_count(lst);
+ while (count > 0) {
+ object_t *obj = (object_t *)list_item(lst, --count);
+ if (!IS_OVERFLOW_OBJ(pseg, obj))
+ break;
+ lst->count = count;
+ if (must_fix_gs) {
+ set_gs_register(pseg->pub.segment_base);
+ must_fix_gs = false;
+ }
+ stmcb_light_finalizer(obj);
+ }
+
+ if (STM_SEGMENT->segment_base != old_gs_register)
+ set_gs_register(old_gs_register);
}
void stm_enable_light_finalizer(object_t *obj)
{
- if (_is_young(obj))
+ if (_is_young(obj)) {
LIST_APPEND(STM_PSEGMENT->young_objects_with_light_finalizers, obj);
- else
+ }
+ else {
+ assert(_is_from_same_transaction(obj));
LIST_APPEND(STM_PSEGMENT->old_objects_with_light_finalizers, obj);
+ }
}
object_t *stm_allocate_with_finalizer(ssize_t size_rounded_up)
@@ -108,7 +153,7 @@
struct list_s *lst = STM_PSEGMENT->young_objects_with_light_finalizers;
long i, count = list_count(lst);
for (i = 0; i < count; i++) {
- object_t* obj = (object_t *)list_item(lst, i);
+ object_t *obj = (object_t *)list_item(lst, i);
assert(_is_young(obj));
object_t *TLPREFIX *pforwarded_array = (object_t *TLPREFIX *)obj;
@@ -138,7 +183,7 @@
long i, count = list_count(lst);
lst->count = 0;
for (i = 0; i < count; i++) {
- object_t* obj = (object_t *)list_item(lst, i);
+ object_t *obj = (object_t *)list_item(lst, i);
if (!mark_visited_test(obj)) {
/* not marked: object dies */
/* we're calling the light finalizer in the same
@@ -345,6 +390,24 @@
LIST_FREE(_finalizer_emptystack);
}
+static void mark_visit_from_finalizer1(char *base, struct finalizers_s *f)
+{
+ if (f != NULL && f->run_finalizers != NULL) {
+ LIST_FOREACH_R(f->run_finalizers, object_t * /*item*/,
+ mark_visit_object(item, base));
+ }
+}
+
+static void mark_visit_from_finalizer_pending(void)
+{
+ long j;
+ for (j = 1; j <= NB_SEGMENTS; j++) {
+ struct stm_priv_segment_info_s *pseg = get_priv_segment(j);
+ mark_visit_from_finalizer1(pseg->pub.segment_base, pseg->finalizers);
+ }
+ mark_visit_from_finalizer1(stm_object_pages, &g_finalizers);
+}
+
static void _execute_finalizers(struct finalizers_s *f)
{
if (f->run_finalizers == NULL)
diff --git a/c7/stm/finalizer.h b/c7/stm/finalizer.h
--- a/c7/stm/finalizer.h
+++ b/c7/stm/finalizer.h
@@ -6,6 +6,7 @@
uintptr_t *running_next;
};
+static void mark_visit_from_finalizer_pending(void);
static void deal_with_young_objects_with_finalizers(void);
static void deal_with_old_objects_with_finalizers(void);
static void deal_with_objects_with_finalizers(void);
@@ -14,18 +15,13 @@
static void teardown_finalizer(void);
static void _commit_finalizers(void);
-static void _abort_finalizers(void);
+static void abort_finalizers(struct stm_priv_segment_info_s *);
#define commit_finalizers() do { \
if (STM_PSEGMENT->finalizers != NULL) \
_commit_finalizers(); \
} while (0)
-#define abort_finalizers() do { \
- if (STM_PSEGMENT->finalizers != NULL) \
- _abort_finalizers(); \
-} while (0)
-
/* regular finalizers (objs from already-committed transactions) */
static struct finalizers_s g_finalizers;
diff --git a/c7/stm/forksupport.c b/c7/stm/forksupport.c
--- a/c7/stm/forksupport.c
+++ b/c7/stm/forksupport.c
@@ -201,9 +201,6 @@
just release these locks early */
s_mutex_unlock();
- /* Open a new profiling file, if any */
- forksupport_open_new_profiling_file();
-
/* Move the copy of the mmap over the old one, overwriting it
and thus freeing the old mapping in this process
*/
diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c
--- a/c7/stm/gcpage.c
+++ b/c7/stm/gcpage.c
@@ -344,6 +344,8 @@
LIST_APPEND(mark_objects_to_trace, obj);
}
+#define TRACE_FOR_MAJOR_COLLECTION (&mark_record_trace)
+
static void mark_trace(object_t *obj, char *segment_base)
{
assert(list_is_empty(mark_objects_to_trace));
@@ -352,7 +354,7 @@
/* trace into the object (the version from 'segment_base') */
struct object_s *realobj =
(struct object_s *)REAL_ADDRESS(segment_base, obj);
- stmcb_trace(realobj, &mark_record_trace);
+ stmcb_trace(realobj, TRACE_FOR_MAJOR_COLLECTION);
if (list_is_empty(mark_objects_to_trace))
break;
@@ -629,6 +631,7 @@
mark_visit_from_modified_objects();
mark_visit_from_markers();
mark_visit_from_roots();
+ mark_visit_from_finalizer_pending();
LIST_FREE(mark_objects_to_trace);
/* finalizer support: will mark as WL_VISITED all objects with a
diff --git a/c7/stm/hashtable.c b/c7/stm/hashtable.c
new file mode 100644
--- /dev/null
+++ b/c7/stm/hashtable.c
@@ -0,0 +1,390 @@
+/*
+Design of stmgc's "hashtable" objects
+=====================================
+
+A "hashtable" is theoretically a lazily-filled array of objects of
+length 2**64. Initially it is full of NULLs. It's obviously
+implemented as a dictionary in which NULL objects are not needed.
+
+A real dictionary can be implemented on top of it, by using the index
+`hash(key)` in the hashtable, and storing a list of `(key, value)`
+pairs at that index (usually only one, unless there is a hash
+collision).
+
+The main operations on a hashtable are reading or writing an object at a
+given index. It might support in the future enumerating the indexes of
+non-NULL objects.
+
+There are two markers for every index (a read and a write marker).
+This is unlike regular arrays, which have only two markers in total.
+
+
+Implementation
+--------------
+
+First idea: have the hashtable in raw memory, pointing to "entry"
+objects. The entry objects themselves point to the user-specified
+objects. The entry objects have the read/write markers. Every entry
+object, once created, stays around. It is only removed by the next
+major GC if it points to NULL and its read/write markers are not set
+in any currently-running transaction.
+
+References
+----------
+
+Inspired by: http://ppl.stanford.edu/papers/podc011-bronson.pdf
+*/
+
+
+uint32_t stm_hashtable_entry_userdata;
+
+
+#define INITIAL_HASHTABLE_SIZE 8
+#define PERTURB_SHIFT 5
+#define RESIZING_LOCK 0
+
+typedef struct {
+ uintptr_t mask;
+
+ /* 'resize_counter' start at an odd value, and is decremented (by
+ 6) for every new item put in 'items'. When it crosses 0, we
+ instead allocate a bigger table and change 'resize_counter' to
+ be a regular pointer to it (which is then even). The whole
+ structure is immutable then.
+
+ The field 'resize_counter' also works as a write lock: changes
+ go via the intermediate value RESIZING_LOCK (0).
+ */
+ uintptr_t resize_counter;
+
+ stm_hashtable_entry_t *items[INITIAL_HASHTABLE_SIZE];
+} stm_hashtable_table_t;
+
+#define IS_EVEN(p) (((p) & 1) == 0)
+
+struct stm_hashtable_s {
+ stm_hashtable_table_t *table;
+ stm_hashtable_table_t initial_table;
+ uint64_t additions;
+};
+
+
+static inline void init_table(stm_hashtable_table_t *table, uintptr_t itemcount)
+{
+ table->mask = itemcount - 1;
+ table->resize_counter = itemcount * 4 + 1;
+ memset(table->items, 0, itemcount * sizeof(stm_hashtable_entry_t *));
+}
+
+stm_hashtable_t *stm_hashtable_create(void)
+{
+ stm_hashtable_t *hashtable = malloc(sizeof(stm_hashtable_t));
+ assert(hashtable);
+ hashtable->table = &hashtable->initial_table;
+ hashtable->additions = 0;
+ init_table(&hashtable->initial_table, INITIAL_HASHTABLE_SIZE);
+ return hashtable;
+}
+
+void stm_hashtable_free(stm_hashtable_t *hashtable)
+{
+ uintptr_t rc = hashtable->initial_table.resize_counter;
+ free(hashtable);
+ while (IS_EVEN(rc)) {
+ assert(rc != RESIZING_LOCK);
+
+ stm_hashtable_table_t *table = (stm_hashtable_table_t *)rc;
+ rc = table->resize_counter;
+ free(table);
+ }
+}
+
+static bool _stm_was_read_by_anybody(object_t *obj)
+{
+ long i;
+ for (i = 1; i <= NB_SEGMENTS; i++) {
+ char *remote_base = get_segment_base(i);
+ uint8_t remote_version = get_segment(i)->transaction_read_version;
+ if (was_read_remote(remote_base, obj, remote_version))
+ return true;
+ }
+ return false;
+}
+
+#define VOLATILE_HASHTABLE(p) ((volatile stm_hashtable_t *)(p))
+#define VOLATILE_TABLE(p) ((volatile stm_hashtable_table_t *)(p))
+
+static void _insert_clean(stm_hashtable_table_t *table,
+ stm_hashtable_entry_t *entry)
+{
+ uintptr_t mask = table->mask;
+ uintptr_t i = entry->index & mask;
+ if (table->items[i] == NULL) {
+ table->items[i] = entry;
+ return;
+ }
+
+ uintptr_t perturb = entry->index;
+ while (1) {
+ i = (i << 2) + i + perturb + 1;
+ i &= mask;
+ if (table->items[i] == NULL) {
+ table->items[i] = entry;
+ return;
+ }
+
+ perturb >>= PERTURB_SHIFT;
+ }
+}
+
+static void _stm_rehash_hashtable(stm_hashtable_t *hashtable,
+ uintptr_t biggercount,
+ bool remove_unread)
+{
+ dprintf(("rehash %p to %ld, remove_unread=%d\n",
+ hashtable, biggercount, (int)remove_unread));
+
+ size_t size = (offsetof(stm_hashtable_table_t, items)
+ + biggercount * sizeof(stm_hashtable_entry_t *));
+ stm_hashtable_table_t *biggertable = malloc(size);
+ assert(biggertable); // XXX
+
+ stm_hashtable_table_t *table = hashtable->table;
+ table->resize_counter = (uintptr_t)biggertable;
+ /* ^^^ this unlocks the table by writing a non-zero value to
+ table->resize_counter, but the new value is a pointer to the
+ new bigger table, so IS_EVEN() is still true */
+
+ init_table(biggertable, biggercount);
+
+ uintptr_t j, mask = table->mask;
+ uintptr_t rc = biggertable->resize_counter;
+ for (j = 0; j <= mask; j++) {
+ stm_hashtable_entry_t *entry = table->items[j];
+ if (entry == NULL)
+ continue;
+ if (remove_unread) {
+ if (entry->object == NULL &&
+ !_stm_was_read_by_anybody((object_t *)entry))
+ continue;
+ }
+ _insert_clean(biggertable, entry);
+ rc -= 6;
+ }
+ biggertable->resize_counter = rc;
+
+ write_fence(); /* make sure that 'biggertable' is valid here,
+ and make sure 'table->resize_counter' is updated
+ ('table' must be immutable from now on). */
+ VOLATILE_HASHTABLE(hashtable)->table = biggertable;
+}
+
+stm_hashtable_entry_t *stm_hashtable_lookup(object_t *hashtableobj,
+ stm_hashtable_t *hashtable,
+ uintptr_t index)
+{
+ stm_hashtable_table_t *table;
+ uintptr_t mask;
+ uintptr_t i;
+ stm_hashtable_entry_t *entry;
+
+ restart:
+ /* classical dict lookup logic */
+ table = VOLATILE_HASHTABLE(hashtable)->table;
+ mask = table->mask; /* read-only field */
+ i = index & mask;
+ entry = VOLATILE_TABLE(table)->items[i];
+ if (entry != NULL) {
+ if (entry->index == index)
+ return entry; /* found at the first try */
+
+ uintptr_t perturb = index;
+ while (1) {
+ i = (i << 2) + i + perturb + 1;
+ i &= mask;
+ entry = VOLATILE_TABLE(table)->items[i];
+ if (entry != NULL) {
+ if (entry->index == index)
+ return entry; /* found */
+ }
+ else
+ break;
+ perturb >>= PERTURB_SHIFT;
+ }
+ }
+ /* here, we didn't find the 'entry' with the correct index. */
+
+ uintptr_t rc = VOLATILE_TABLE(table)->resize_counter;
+
+ /* if rc is RESIZING_LOCK (which is 0, so even), a concurrent thread
+ is writing to the hashtable. Or, if rc is another even number, it is
+ actually a pointer to the next version of the table, installed
+ just now. In both cases, this thread must simply spin loop.
+ */
+ if (IS_EVEN(rc)) {
+ spin_loop();
+ goto restart;
+ }
+ /* in the other cases, we need to grab the RESIZING_LOCK.
+ */
+ if (!__sync_bool_compare_and_swap(&table->resize_counter,
+ rc, RESIZING_LOCK)) {
+ goto restart;
+ }
+ /* we now have the lock. The only table with a non-even value of
+ 'resize_counter' should be the last one in the chain, so if we
+ succeeded in locking it, check this. */
+ assert(table == hashtable->table);
+
+ /* Check that 'table->items[i]' is still NULL,
+ i.e. hasn't been populated under our feet.
+ */
+ if (table->items[i] != NULL) {
+ table->resize_counter = rc; /* unlock */
+ goto restart;
+ }
+ /* if rc is greater than 6, there is enough room for a new
+ item in the current table.
+ */
+ if (rc > 6) {
+ /* we can only enter here once! If we allocate stuff, we may
+ run the GC, and so 'hashtableobj' might move afterwards. */
+ if (_is_in_nursery(hashtableobj)) {
+ entry = (stm_hashtable_entry_t *)
+ stm_allocate(sizeof(stm_hashtable_entry_t));
+ entry->userdata = stm_hashtable_entry_userdata;
+ entry->index = index;
+ entry->object = NULL;
+ }
+ else {
+ /* for a non-nursery 'hashtableobj', we pretend that the
+ 'entry' object we're about to return was already
+ existing all along, with NULL in all segments. If the
+ caller of this function is going to modify the 'object'
+ field, it will call stm_write(entry) first, which will
+ correctly schedule 'entry' for write propagation. We
+ do that even if 'hashtableobj' was created by the
+ running transaction: the new 'entry' object is created
+ as if it was older than the transaction.
+
+ Note the following difference: if 'hashtableobj' is
+ still in the nursery (case above), the 'entry' object
+ is also allocated from the nursery, and after a minor
+ collection it ages as an old-but-created-by-the-
+ current-transaction object. We could try to emulate
+ this here, or to create young 'entry' objects, but
+ doing either of these would require careful
+ synchronization with other pieces of the code that may
+ change.
+ */
+ acquire_privatization_lock();
+ char *p = allocate_outside_nursery_large(
+ sizeof(stm_hashtable_entry_t));
+ entry = (stm_hashtable_entry_t *)(p - stm_object_pages);
+
+ long j;
+ for (j = 0; j <= NB_SEGMENTS; j++) {
+ struct stm_hashtable_entry_s *e;
+ e = (struct stm_hashtable_entry_s *)
+ REAL_ADDRESS(get_segment_base(j), entry);
+ e->header.stm_flags = GCFLAG_WRITE_BARRIER;
+ e->userdata = stm_hashtable_entry_userdata;
+ e->index = index;
+ e->object = NULL;
+ }
+ release_privatization_lock();
+ }
+ write_fence(); /* make sure 'entry' is fully initialized here */
+ table->items[i] = entry;
+ hashtable->additions += 1;
+ write_fence(); /* make sure 'table->items' is written here */
+ VOLATILE_TABLE(table)->resize_counter = rc - 6; /* unlock */
+ return entry;
+ }
+ else {
+ /* if rc is smaller than 6, we must allocate a new bigger table.
+ */
+ uintptr_t biggercount = table->mask + 1;
+ if (biggercount < 50000)
+ biggercount *= 4;
+ else
+ biggercount *= 2;
+ _stm_rehash_hashtable(hashtable, biggercount, /*remove_unread=*/false);
+ goto restart;
+ }
+}
+
+object_t *stm_hashtable_read(object_t *hobj, stm_hashtable_t *hashtable,
+ uintptr_t key)
+{
+ stm_hashtable_entry_t *e = stm_hashtable_lookup(hobj, hashtable, key);
+ stm_read((object_t *)e);
+ return e->object;
+}
+
+void stm_hashtable_write(object_t *hobj, stm_hashtable_t *hashtable,
+ uintptr_t key, object_t *nvalue,
+ stm_thread_local_t *tl)
+{
+ STM_PUSH_ROOT(*tl, nvalue);
+ stm_hashtable_entry_t *e = stm_hashtable_lookup(hobj, hashtable, key);
+ stm_write((object_t *)e);
+ STM_POP_ROOT(*tl, nvalue);
+ e->object = nvalue;
+}
+
+static void _stm_compact_hashtable(stm_hashtable_t *hashtable)
+{
+ stm_hashtable_table_t *table = hashtable->table;
+ assert(!IS_EVEN(table->resize_counter));
+
+ if (hashtable->additions * 4 > table->mask) {
+ hashtable->additions = 0;
+ uintptr_t initial_rc = (table->mask + 1) * 4 + 1;
+ uintptr_t num_entries_times_6 = initial_rc - table->resize_counter;
+ uintptr_t count = INITIAL_HASHTABLE_SIZE;
+ while (count * 4 < num_entries_times_6)
+ count *= 2;
+ /* sanity-check: 'num_entries_times_6 < initial_rc', and so 'count'
+ can never grow larger than the current table size. */
+ assert(count <= table->mask + 1);
+
+ _stm_rehash_hashtable(hashtable, count, /*remove_unread=*/true);
+ }
+
+ table = hashtable->table;
+ assert(!IS_EVEN(table->resize_counter));
+
+ if (table != &hashtable->initial_table) {
+ uintptr_t rc = hashtable->initial_table.resize_counter;
+ while (1) {
+ assert(IS_EVEN(rc));
+ assert(rc != RESIZING_LOCK);
+
+ stm_hashtable_table_t *old_table = (stm_hashtable_table_t *)rc;
+ if (old_table == table)
+ break;
+ rc = old_table->resize_counter;
+ free(old_table);
+ }
+ hashtable->initial_table.resize_counter = (uintptr_t)table;
+ }
+}
+
+void stm_hashtable_tracefn(stm_hashtable_t *hashtable, void trace(object_t **))
+{
+ if (trace == TRACE_FOR_MAJOR_COLLECTION)
+ _stm_compact_hashtable(hashtable);
+
+ stm_hashtable_table_t *table;
+ table = VOLATILE_HASHTABLE(hashtable)->table;
+
+ uintptr_t j, mask = table->mask;
+ for (j = 0; j <= mask; j++) {
+ stm_hashtable_entry_t *volatile *pentry;
+ pentry = &VOLATILE_TABLE(table)->items[j];
+ if (*pentry != NULL) {
+ trace((object_t **)pentry);
+ }
+ }
+}
diff --git a/c7/stm/nursery.c b/c7/stm/nursery.c
--- a/c7/stm/nursery.c
+++ b/c7/stm/nursery.c
@@ -44,6 +44,10 @@
tree_contains(STM_PSEGMENT->young_outside_nursery, (uintptr_t)obj));
}
+static inline bool _is_from_same_transaction(object_t *obj) {
+ return _is_young(obj) || IS_OVERFLOW_OBJ(STM_PSEGMENT, obj);
+}
+
long stm_can_move(object_t *obj)
{
/* 'long' return value to avoid using 'bool' in the public interface */
@@ -329,6 +333,7 @@
}
+#define TRACE_FOR_MINOR_COLLECTION (&minor_trace_if_young)
static inline void _collect_now(object_t *obj)
{
@@ -342,7 +347,7 @@
outside the nursery, possibly forcing nursery objects out and
adding them to 'objects_pointing_to_nursery' as well. */
char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
- stmcb_trace((struct object_s *)realobj, &minor_trace_if_young);
+ stmcb_trace((struct object_s *)realobj, TRACE_FOR_MINOR_COLLECTION);
obj->stm_flags |= GCFLAG_WRITE_BARRIER;
}
diff --git a/c7/stm/prof.c b/c7/stm/prof.c
--- a/c7/stm/prof.c
+++ b/c7/stm/prof.c
@@ -74,7 +74,13 @@
return false;
}
-static void forksupport_open_new_profiling_file(void)
+static void prof_forksupport_prepare(void)
+{
+ if (profiling_file != NULL)
+ fflush(profiling_file);
+}
+
+static void prof_forksupport_child(void)
{
if (close_timing_log() && profiling_basefn != NULL) {
char filename[1024];
@@ -98,6 +104,15 @@
expand_marker = default_expand_marker;
profiling_expand_marker = expand_marker;
+ static bool fork_support_ready = false;
+ if (!fork_support_ready) {
+ int res = pthread_atfork(prof_forksupport_prepare,
+ NULL, prof_forksupport_child);
+ if (res != 0)
+ stm_fatalerror("pthread_atfork() failed: %m");
+ fork_support_ready = true;
+ }
+
if (!open_timing_log(profiling_file_name))
return -1;
diff --git a/c7/stm/prof.h b/c7/stm/prof.h
deleted file mode 100644
--- a/c7/stm/prof.h
+++ /dev/null
@@ -1,2 +0,0 @@
-
-static void forksupport_open_new_profiling_file(void);
diff --git a/c7/stmgc.c b/c7/stmgc.c
--- a/c7/stmgc.c
+++ b/c7/stmgc.c
@@ -15,7 +15,6 @@
#include "stm/fprintcolor.h"
#include "stm/weakref.h"
#include "stm/marker.h"
-#include "stm/prof.h"
#include "stm/finalizer.h"
#include "stm/misc.c"
@@ -39,3 +38,4 @@
#include "stm/prof.c"
#include "stm/rewind_setjmp.c"
#include "stm/finalizer.c"
+#include "stm/hashtable.c"
diff --git a/c7/stmgc.h b/c7/stmgc.h
--- a/c7/stmgc.h
+++ b/c7/stmgc.h
@@ -508,7 +508,7 @@
/* Support for light finalizers. This is a simple version of
finalizers that guarantees not to do anything fancy, like not
resurrecting objects. */
-void (*stmcb_light_finalizer)(object_t *);
+extern void (*stmcb_light_finalizer)(object_t *);
void stm_enable_light_finalizer(object_t *);
/* Support for regular finalizers. Unreachable objects with
@@ -525,9 +525,34 @@
transaction. For older objects, the finalizer is called from a
random thread between regular transactions, in a new custom
transaction. */
-void (*stmcb_finalizer)(object_t *);
+extern void (*stmcb_finalizer)(object_t *);
object_t *stm_allocate_with_finalizer(ssize_t size_rounded_up);
+/* Hashtables. Keys are 64-bit unsigned integers, values are
+ 'object_t *'. Note that the type 'stm_hashtable_t' is not an
+ object type at all; you need to allocate and free it explicitly.
+ If you want to embed the hashtable inside an 'object_t' you
+ probably need a light finalizer to do the freeing. */
+typedef struct stm_hashtable_s stm_hashtable_t;
+typedef TLPREFIX struct stm_hashtable_entry_s stm_hashtable_entry_t;
+
+stm_hashtable_t *stm_hashtable_create(void);
+void stm_hashtable_free(stm_hashtable_t *);
+stm_hashtable_entry_t *stm_hashtable_lookup(object_t *, stm_hashtable_t *,
+ uintptr_t key);
+object_t *stm_hashtable_read(object_t *, stm_hashtable_t *, uintptr_t key);
+void stm_hashtable_write(object_t *, stm_hashtable_t *, uintptr_t key,
+ object_t *nvalue, stm_thread_local_t *);
+extern uint32_t stm_hashtable_entry_userdata;
+void stm_hashtable_tracefn(stm_hashtable_t *, void (object_t **));
+
+struct stm_hashtable_entry_s {
+ struct object_s header;
+ uint32_t userdata;
+ uintptr_t index;
+ object_t *object;
+};
+
/* ==================== END ==================== */
#endif
diff --git a/c7/test/support.py b/c7/test/support.py
--- a/c7/test/support.py
+++ b/c7/test/support.py
@@ -165,6 +165,19 @@
void stm_enable_light_finalizer(object_t *);
void (*stmcb_finalizer)(object_t *);
+
+typedef struct stm_hashtable_s stm_hashtable_t;
+stm_hashtable_t *stm_hashtable_create(void);
+void stm_hashtable_free(stm_hashtable_t *);
+bool _check_hashtable_read(object_t *, stm_hashtable_t *, uintptr_t key);
+object_t *hashtable_read_result;
+bool _check_hashtable_write(object_t *, stm_hashtable_t *, uintptr_t key,
+ object_t *nvalue, stm_thread_local_t *tl);
+uint32_t stm_hashtable_entry_userdata;
+void stm_hashtable_tracefn(stm_hashtable_t *, void (object_t **));
+
+void _set_hashtable(object_t *obj, stm_hashtable_t *h);
+stm_hashtable_t *_get_hashtable(object_t *obj);
""")
@@ -240,6 +253,19 @@
CHECKED(stm_become_globally_unique_transaction(tl, "TESTGUT"));
}
+object_t *hashtable_read_result;
+
+bool _check_hashtable_read(object_t *hobj, stm_hashtable_t *h, uintptr_t key)
+{
+ CHECKED(hashtable_read_result = stm_hashtable_read(hobj, h, key));
+}
+
+bool _check_hashtable_write(object_t *hobj, stm_hashtable_t *h, uintptr_t key,
+ object_t *nvalue, stm_thread_local_t *tl)
+{
+ CHECKED(stm_hashtable_write(hobj, h, key, nvalue, tl));
+}
+
#undef CHECKED
@@ -268,6 +294,20 @@
return *WEAKREF_PTR(obj, size);
}
+void _set_hashtable(object_t *obj, stm_hashtable_t *h)
+{
+ stm_char *field_addr = ((stm_char*)obj);
+ field_addr += SIZEOF_MYOBJ; /* header */
+ *(stm_hashtable_t *TLPREFIX *)field_addr = h;
+}
+
+stm_hashtable_t *_get_hashtable(object_t *obj)
+{
+ stm_char *field_addr = ((stm_char*)obj);
+ field_addr += SIZEOF_MYOBJ; /* header */
+ return *(stm_hashtable_t *TLPREFIX *)field_addr;
+}
+
void _set_ptr(object_t *obj, int n, object_t *v)
{
long nrefs = (long)((myobj_t*)obj)->type_id - 421420;
@@ -296,7 +336,14 @@
ssize_t stmcb_size_rounded_up(struct object_s *obj)
{
struct myobj_s *myobj = (struct myobj_s*)obj;
+ assert(myobj->type_id != 0);
if (myobj->type_id < 421420) {
+ if (myobj->type_id == 421419) { /* hashtable */
+ return sizeof(struct myobj_s) + 1 * sizeof(void*);
+ }
+ if (myobj->type_id == 421418) { /* hashtable entry */
+ return sizeof(struct stm_hashtable_entry_s);
+ }
/* basic case: tid equals 42 plus the size of the object */
assert(myobj->type_id >= 42 + sizeof(struct myobj_s));
assert((myobj->type_id - 42) >= 16);
@@ -316,6 +363,17 @@
{
int i;
struct myobj_s *myobj = (struct myobj_s*)obj;
+ if (myobj->type_id == 421419) {
+ /* hashtable */
+ stm_hashtable_t *h = *((stm_hashtable_t **)(myobj + 1));
+ stm_hashtable_tracefn(h, visit);
+ return;
+ }
+ if (myobj->type_id == 421418) {
+ /* hashtable entry */
+ object_t **ref = &((struct stm_hashtable_entry_s *)myobj)->object;
+ visit(ref);
+ }
if (myobj->type_id < 421420) {
/* basic case: no references */
return;
@@ -334,6 +392,8 @@
{
int i;
struct myobj_s *myobj = (struct myobj_s*)obj;
+ assert(myobj->type_id != 421419);
+ assert(myobj->type_id != 421418);
if (myobj->type_id < 421420) {
/* basic case: no references */
return;
@@ -404,6 +464,7 @@
CARD_SIZE = lib._STM_CARD_SIZE # 16b at least
NB_SEGMENTS = lib.STM_NB_SEGMENTS
FAST_ALLOC = lib._STM_FAST_ALLOC
+lib.stm_hashtable_entry_userdata = 421418
class Conflict(Exception):
pass
@@ -441,6 +502,18 @@
lib._set_weakref(o, point_to_obj)
return o
+def stm_allocate_hashtable():
+ o = lib.stm_allocate(16)
+ tid = 421419
+ lib._set_type_id(o, tid)
+ h = lib.stm_hashtable_create()
+ lib._set_hashtable(o, h)
+ return o
+
+def get_hashtable(o):
+ assert lib._get_type_id(o) == 421419
+ return lib._get_hashtable(o)
+
def stm_get_weakref(o):
return lib._get_weakref(o)
@@ -558,7 +631,6 @@
-
SHADOWSTACK_LENGTH = 1000
_keepalive = weakref.WeakKeyDictionary()
diff --git a/c7/test/test_finalizer.py b/c7/test/test_finalizer.py
--- a/c7/test/test_finalizer.py
+++ b/c7/test/test_finalizer.py
@@ -9,6 +9,7 @@
#
@ffi.callback("void(object_t *)")
def light_finalizer(obj):
+ assert stm_get_obj_size(obj) == 48
segnum = lib.current_segment_num()
tlnum = '?'
for n, tl in enumerate(self.tls):
@@ -20,6 +21,10 @@
lib.stmcb_light_finalizer = light_finalizer
self._light_finalizer_keepalive = light_finalizer
+ def teardown_method(self, meth):
+ lib.stmcb_light_finalizer = ffi.NULL
+ BaseTest.teardown_method(self, meth)
+
def expect_finalized(self, objs, from_tlnum=None):
assert [obj for (obj, tlnum) in self.light_finalizers_called] == objs
if from_tlnum is not None:
@@ -49,6 +54,15 @@
self.commit_transaction()
self.expect_finalized([])
+ def test_young_light_finalizer_aborts(self):
+ self.start_transaction()
+ lp1 = stm_allocate(48)
+ lib.stm_enable_light_finalizer(lp1)
+ self.expect_finalized([])
+ self.abort_transaction()
+ self.start_transaction()
+ self.expect_finalized([lp1], from_tlnum=0)
+
def test_old_light_finalizer(self):
self.start_transaction()
lp1 = stm_allocate(48)
@@ -99,15 +113,47 @@
stm_major_collect()
self.expect_finalized([lp1], from_tlnum=1)
+ def test_old_light_finalizer_aborts(self):
+ self.start_transaction()
+ lp1 = stm_allocate(48)
+ lib.stm_enable_light_finalizer(lp1)
+ self.push_root(lp1)
+ self.commit_transaction()
+ #
+ self.start_transaction()
+ self.expect_finalized([])
+ self.abort_transaction()
+ self.expect_finalized([])
+
+ def test_overflow_light_finalizer_aborts(self):
+ self.start_transaction()
+ lp1 = stm_allocate(48)
+ lib.stm_enable_light_finalizer(lp1)
+ self.push_root(lp1)
+ stm_minor_collect()
+ lp1 = self.pop_root()
+ self.push_root(lp1)
+ self.expect_finalized([])
+ self.abort_transaction()
+ self.expect_finalized([lp1], from_tlnum=0)
+
class TestRegularFinalizer(BaseTest):
+ expect_content_character = None
+ run_major_collect_in_finalizer = False
def setup_method(self, meth):
BaseTest.setup_method(self, meth)
#
@ffi.callback("void(object_t *)")
def finalizer(obj):
+ print "finalizing!", obj
+ assert stm_get_obj_size(obj) in [16, 32, 48, 56]
+ if self.expect_content_character is not None:
+ assert stm_get_char(obj) == self.expect_content_character
self.finalizers_called.append(obj)
+ if self.run_major_collect_in_finalizer:
+ stm_major_collect()
self.finalizers_called = []
lib.stmcb_finalizer = finalizer
self._finalizer_keepalive = finalizer
@@ -137,6 +183,21 @@
stm_major_collect()
self.expect_finalized([lp1, lp2, lp3])
+ def test_finalizer_from_other_thread(self):
+ self.start_transaction()
+ lp1 = stm_allocate_with_finalizer(48)
+ stm_set_char(lp1, 'H')
+ self.expect_content_character = 'H'
+ print lp1
+ #
+ self.switch(1)
+ self.start_transaction()
+ stm_major_collect()
+ self.expect_finalized([]) # marked as dead, but wrong thread
+ #
+ self.switch(0)
+ self.expect_finalized([lp1]) # now it has been finalized
+
def test_finalizer_ordering(self):
self.start_transaction()
lp1 = stm_allocate_with_finalizer_refs(1)
@@ -148,7 +209,7 @@
stm_major_collect()
self.expect_finalized([lp3])
- def test_finalizer_extra_transation(self):
+ def test_finalizer_extra_transaction(self):
self.start_transaction()
lp1 = stm_allocate_with_finalizer(32)
print lp1
@@ -182,3 +243,12 @@
stm_major_collect()
self.switch(0)
self.expect_finalized([lp2, lp1])
+
+ def test_run_major_collect_in_finalizer(self):
+ self.run_major_collect_in_finalizer = True
+ self.start_transaction()
+ lp1 = stm_allocate_with_finalizer(32)
+ lp2 = stm_allocate_with_finalizer(32)
+ lp3 = stm_allocate_with_finalizer(32)
+ print lp1, lp2, lp3
+ stm_major_collect()
diff --git a/c7/test/test_hashtable.py b/c7/test/test_hashtable.py
new file mode 100644
--- /dev/null
+++ b/c7/test/test_hashtable.py
@@ -0,0 +1,414 @@
+from support import *
+import random
+import py, sys
+
+
+def htget(o, key):
+ h = get_hashtable(o)
+ res = lib._check_hashtable_read(o, h, key)
+ if res:
+ raise Conflict
+ return lib.hashtable_read_result
+
+def htset(o, key, nvalue, tl):
+ h = get_hashtable(o)
+ res = lib._check_hashtable_write(o, h, key, nvalue, tl)
+ if res:
+ raise Conflict
+
+
+class BaseTestHashtable(BaseTest):
+
+ def setup_method(self, meth):
+ BaseTest.setup_method(self, meth)
+ #
+ @ffi.callback("void(object_t *)")
+ def light_finalizer(obj):
+ print 'light_finalizer:', obj
+ try:
+ assert lib._get_type_id(obj) == 421419
+ self.seen_hashtables -= 1
+ except:
+ self.errors.append(sys.exc_info()[2])
+ raise
+
+ lib.stmcb_light_finalizer = light_finalizer
+ self._light_finalizer_keepalive = light_finalizer
+ self.seen_hashtables = 0
+ self.errors = []
+
+ def teardown_method(self, meth):
+ BaseTest.teardown_method(self, meth)
+ lib.stmcb_light_finalizer = ffi.NULL
+ assert self.errors == []
+ assert self.seen_hashtables == 0
+
+ def allocate_hashtable(self):
+ h = stm_allocate_hashtable()
+ lib.stm_enable_light_finalizer(h)
+ self.seen_hashtables += 1
+ return h
+
+
+class TestHashtable(BaseTestHashtable):
+
+ def test_empty(self):
+ self.start_transaction()
+ h = self.allocate_hashtable()
+ for i in range(100):
+ index = random.randrange(0, 1<<64)
+ got = htget(h, index)
+ assert got == ffi.NULL
+
+ def test_set_value(self):
+ self.start_transaction()
+ tl0 = self.tls[self.current_thread]
+ h = self.allocate_hashtable()
+ lp1 = stm_allocate(16)
+ htset(h, 12345678901, lp1, tl0)
+ assert htget(h, 12345678901) == lp1
+ for i in range(64):
+ index = 12345678901 ^ (1 << i)
+ assert htget(h, index) == ffi.NULL
+ assert htget(h, 12345678901) == lp1
+
+ def test_no_conflict(self):
+ lp1 = stm_allocate_old(16)
+ lp2 = stm_allocate_old(16)
+ #
+ self.start_transaction()
+ tl0 = self.tls[self.current_thread]
+ h = self.allocate_hashtable()
+ self.push_root(h)
+ stm_set_char(lp1, 'A')
+ htset(h, 1234, lp1, tl0)
+ self.commit_transaction()
+ #
+ self.start_transaction()
+ h = self.pop_root()
+ stm_set_char(lp2, 'B')
+ htset(h, 9991234, lp2, tl0)
+ #
+ self.switch(1)
+ self.start_transaction()
+ lp1b = htget(h, 1234)
+ assert lp1b != ffi.NULL
+ assert stm_get_char(lp1b) == 'A'
+ assert lp1b == lp1
+ self.commit_transaction()
+ #
+ self.switch(0)
+ assert htget(h, 9991234) == lp2
+ assert stm_get_char(lp2) == 'B'
+ assert htget(h, 1234) == lp1
+ htset(h, 1234, ffi.NULL, tl0)
+ self.commit_transaction()
+ #
+ self.start_transaction()
+ stm_major_collect() # to get rid of the hashtable object
+
+ def test_conflict(self):
+ lp1 = stm_allocate_old(16)
+ lp2 = stm_allocate_old(16)
+ #
+ self.start_transaction()
+ h = self.allocate_hashtable()
+ self.push_root(h)
+ self.commit_transaction()
+ #
+ self.start_transaction()
+ h = self.pop_root()
+ self.push_root(h)
+ tl0 = self.tls[self.current_thread]
+ htset(h, 1234, lp1, tl0)
+ #
+ self.switch(1)
+ self.start_transaction()
+ tl1 = self.tls[self.current_thread]
+ py.test.raises(Conflict, "htset(h, 1234, lp2, tl1)")
+ #
+ self.switch(0)
+ self.pop_root()
+ stm_major_collect() # to get rid of the hashtable object
+ self.commit_transaction()
+
+ def test_keepalive_minor(self):
+ self.start_transaction()
+ h = self.allocate_hashtable()
+ self.push_root(h)
+ lp1 = stm_allocate(16)
+ stm_set_char(lp1, 'N')
+ tl0 = self.tls[self.current_thread]
+ htset(h, 1234, lp1, tl0)
+ stm_minor_collect()
+ h = self.pop_root()
+ lp1b = htget(h, 1234)
+ assert lp1b != ffi.NULL
+ assert stm_get_char(lp1b) == 'N'
+ assert lp1b != lp1
+
+ def test_keepalive_major(self):
+ lp1 = stm_allocate_old(16)
+ #
+ self.start_transaction()
+ h = self.allocate_hashtable()
+ self.push_root(h)
+ stm_set_char(lp1, 'N')
+ tl0 = self.tls[self.current_thread]
+ htset(h, 1234, lp1, tl0)
+ self.commit_transaction()
+ #
+ self.start_transaction()
+ stm_major_collect()
+ h = self.pop_root()
+ lp1b = htget(h, 1234)
+ assert lp1b == lp1
+ assert stm_get_char(lp1b) == 'N'
+ #
+ stm_major_collect() # to get rid of the hashtable object
+ self.commit_transaction()
+
+ def test_minor_collect_bug1(self):
+ self.start_transaction()
+ lp1 = stm_allocate(32)
+ self.push_root(lp1)
+ h = self.allocate_hashtable()
+ self.push_root(h)
+ stm_minor_collect()
+ h = self.pop_root()
+ lp1 = self.pop_root()
+ print 'h', h # 0xa040010
+ print 'lp1', lp1 # 0xa040040
+ tl0 = self.tls[self.current_thread]
+ htset(h, 1, lp1, tl0)
+ self.commit_transaction()
+ #
+ self.start_transaction()
+ assert htget(h, 1) == lp1
+ stm_major_collect() # to get rid of the hashtable object
+
+ def test_minor_collect_bug1_different_thread(self):
+ self.start_transaction()
+ lp1 = stm_allocate(32)
+ self.push_root(lp1)
+ h = self.allocate_hashtable()
+ self.push_root(h)
+ stm_minor_collect()
+ h = self.pop_root()
+ lp1 = self.pop_root()
+ print 'h', h # 0xa040010
+ print 'lp1', lp1 # 0xa040040
+ tl0 = self.tls[self.current_thread]
+ htset(h, 1, lp1, tl0)
+ self.commit_transaction()
+ #
+ self.switch(1) # in a different thread
+ self.start_transaction()
+ assert htget(h, 1) == lp1
+ stm_major_collect() # to get rid of the hashtable object
+
+
+class TestRandomHashtable(BaseTestHashtable):
+
+ def setup_method(self, meth):
+ BaseTestHashtable.setup_method(self, meth)
+ self.values = []
+ self.mirror = None
+ self.roots = []
+ self.other_thread = ([], [])
+
+ def push_roots(self):
+ assert self.roots is None
+ self.roots = []
+ for k, hitems in self.mirror.items():
+ assert lib._get_type_id(k) == 421419
+ for key, value in hitems.items():
+ assert lib._get_type_id(value) < 1000
+ self.push_root(value)
+ self.roots.append(key)
+ self.push_root(k)
+ self.roots.append(None)
+ for v in self.values:
+ self.push_root(v)
+ self.mirror = None
+
+ def pop_roots(self):
+ assert self.mirror is None
+ for i in reversed(range(len(self.values))):
+ self.values[i] = self.pop_root()
+ assert stm_get_char(self.values[i]) == chr((i + 1) & 255)
+ self.mirror = {}
+ for r in reversed(self.roots):
+ obj = self.pop_root()
+ if r is None:
+ assert lib._get_type_id(obj) == 421419
+ self.mirror[obj] = curhitems = {}
+ else:
+ assert lib._get_type_id(obj) < 1000
+ curhitems[r] = obj
+ self.roots = None
+
+ def exchange_threads(self):
+ old_thread = (self.values, self.roots)
+ self.switch(1 - self.current_thread)
+ (self.values, self.roots) = self.other_thread
+ self.mirror = None
+ self.other_thread = old_thread
+
+ def test_random_single_thread(self):
+ import random
+ #
+ for i in range(100):
+ print "start_transaction"
+ self.start_transaction()
+ self.pop_roots()
+ for j in range(10):
+ r = random.random()
+ if r < 0.05:
+ h = self.allocate_hashtable()
+ print "allocate_hashtable ->", h
+ self.mirror[h] = {}
+ elif r < 0.10:
+ print "stm_minor_collect"
+ self.push_roots()
+ stm_minor_collect()
+ self.pop_roots()
+ elif r < 0.11:
+ print "stm_major_collect"
+ self.push_roots()
+ stm_major_collect()
+ self.pop_roots()
+ elif r < 0.5:
+ if not self.mirror: continue
+ h = random.choice(self.mirror.keys())
+ if not self.mirror[h]: continue
+ key = random.choice(self.mirror[h].keys())
+ value = self.mirror[h][key]
+ print "htget(%r, %r) == %r" % (h, key, value)
+ self.push_roots()
+ self.push_root(value)
+ result = htget(h, key)
+ value = self.pop_root()
+ assert result == value
+ self.pop_roots()
+ elif r < 0.6:
+ if not self.mirror: continue
+ h = random.choice(self.mirror.keys())
+ key = random.randrange(0, 40)
+ if key in self.mirror[h]: continue
+ print "htget(%r, %r) == NULL" % (h, key)
+ self.push_roots()
+ assert htget(h, key) == ffi.NULL
+ self.pop_roots()
+ elif r < 0.63:
+ if not self.mirror: continue
+ h, _ = self.mirror.popitem()
+ print "popped", h
+ elif r < 0.75:
+ obj = stm_allocate(32)
+ self.values.append(obj)
+ stm_set_char(obj, chr(len(self.values) & 255))
+ else:
+ if not self.mirror or not self.values: continue
+ h = random.choice(self.mirror.keys())
+ key = random.randrange(0, 32)
+ value = random.choice(self.values)
+ print "htset(%r, %r, %r)" % (h, key, value)
+ self.push_roots()
+ tl = self.tls[self.current_thread]
+ htset(h, key, value, tl)
+ self.pop_roots()
+ self.mirror[h][key] = value
+ self.push_roots()
+ print "commit_transaction"
+ self.commit_transaction()
+ #
+ self.start_transaction()
+ self.become_inevitable()
+ self.pop_roots()
+ stm_major_collect() # to get rid of the hashtable objects
+
+ def test_random_multiple_threads(self):
+ import random
+ self.start_transaction()
+ self.exchange_threads()
+ self.start_transaction()
+ self.pop_roots()
+ #
+ for j in range(1000):
+ r = random.random()
+ if r > 0.9:
+ if r > 0.95:
+ self.push_roots()
+ self.commit_transaction()
+ self.start_transaction()
+ self.pop_roots()
+ else:
+ self.push_roots()
+ self.exchange_threads()
+ self.pop_roots()
+ continue
+
+ if r < 0.05:
+ h = self.allocate_hashtable()
+ print "allocate_hashtable ->", h
+ self.mirror[h] = {}
+ elif r < 0.10:
+ print "stm_minor_collect"
+ self.push_roots()
+ stm_minor_collect()
+ self.pop_roots()
+ elif r < 0.11:
+ print "stm_major_collect"
+ self.push_roots()
+ stm_major_collect()
+ self.pop_roots()
+ elif r < 0.5:
+ if not self.mirror: continue
+ h = random.choice(self.mirror.keys())
+ if not self.mirror[h]: continue
+ key = random.choice(self.mirror[h].keys())
+ value = self.mirror[h][key]
+ print "htget(%r, %r) == %r" % (h, key, value)
+ self.push_roots()
+ self.push_root(value)
+ result = htget(h, key)
+ value = self.pop_root()
+ assert result == value
+ self.pop_roots()
+ elif r < 0.6:
+ if not self.mirror: continue
+ h = random.choice(self.mirror.keys())
+ key = random.randrange(0, 40)
+ if key in self.mirror[h]: continue
+ print "htget(%r, %r) == NULL" % (h, key)
+ self.push_roots()
+ assert htget(h, key) == ffi.NULL
+ self.pop_roots()
+ elif r < 0.63:
+ if not self.mirror: continue
+ h, _ = self.mirror.popitem()
+ print "popped", h
+ elif r < 0.75:
+ obj = stm_allocate(32)
+ self.values.append(obj)
+ stm_set_char(obj, chr(len(self.values) & 255))
+ else:
+ if not self.mirror or not self.values: continue
+ h = random.choice(self.mirror.keys())
+ key = random.randrange(0, 32)
+ value = random.choice(self.values)
+ print "htset(%r, %r, %r)" % (h, key, value)
+ self.push_roots()
+ tl = self.tls[self.current_thread]
+ htset(h, key, value, tl)
+ self.pop_roots()
+ self.mirror[h][key] = value
+ #
+ print 'closing down...'
+ self.become_inevitable()
+ self.commit_transaction()
+ self.exchange_threads()
+ self.pop_roots()
+ self.become_inevitable()
+ stm_major_collect() # to get rid of the hashtable objects
More information about the pypy-commit
mailing list