[pypy-commit] stmgc c7-refactor: in-progress
arigo
noreply at buildbot.pypy.org
Sun Feb 9 18:31:30 CET 2014
Author: Armin Rigo <arigo at tunes.org>
Branch: c7-refactor
Changeset: r715:3650a3e0e16c
Date: 2014-02-09 17:50 +0100
http://bitbucket.org/pypy/stmgc/changeset/3650a3e0e16c/
Log: in-progress
diff --git a/c7/core.c b/c7/core.c
deleted file mode 100644
--- a/c7/core.c
+++ /dev/null
@@ -1,511 +0,0 @@
-#define _GNU_SOURCE
-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-#include <string.h>
-#include <unistd.h>
-#include <sys/mman.h>
-
-#include <pthread.h>
-
-#include "core.h"
-#include "list.h"
-#include "reader_writer_lock.h"
-#include "nursery.h"
-#include "pages.h"
-#include "stmsync.h"
-#include "largemalloc.h"
-
-
-char *object_pages;
-static int num_threads_started;
-uint8_t write_locks[READMARKER_END - READMARKER_START];
-volatile uint8_t inevitable_lock __attribute__((aligned(64))); /* cache-line alignment */
-long global_age = 0;
-
-struct _thread_local1_s* _stm_dbg_get_tl(int thread)
-{
- if (thread == -1)
- return (struct _thread_local1_s*)real_address((object_t*)_STM_TL);
- return (struct _thread_local1_s*)REAL_ADDRESS(get_thread_base(thread), _STM_TL);
-}
-
-bool _stm_was_read_remote(char *base, object_t *obj)
-{
- struct read_marker_s *marker = (struct read_marker_s *)
- (base + (((uintptr_t)obj) >> 4));
- struct _thread_local1_s *other_TL1 = (struct _thread_local1_s*)
- (base + (uintptr_t)_STM_TL);
- return (marker->rm == other_TL1->transaction_read_version);
-}
-
-bool _stm_was_read(object_t *obj)
-{
- read_marker_t *marker = (read_marker_t *)(((uintptr_t)obj) >> 4);
- return (marker->rm == _STM_TL->transaction_read_version);
-}
-
-bool _stm_was_written(object_t *obj)
-{
- /* if the obj was written to in the current transaction
- and doesn't trigger the write-barrier slowpath */
- return !(obj->stm_flags & GCFLAG_WRITE_BARRIER);
-}
-
-
-
-static void push_modified_to_other_threads()
-{
- /* WE HAVE THE EXCLUSIVE LOCK HERE */
-
- struct stm_list_s *modified = _STM_TL->modified_objects;
- char *local_base = _STM_TL->thread_base;
- char *remote_base = get_thread_base(1 - _STM_TL->thread_num);
- bool conflicted = 0;
-
- STM_LIST_FOREACH(
- modified,
- ({
- if (!conflicted)
- conflicted = _stm_was_read_remote(remote_base, item);
-
- /* clear the write-lock */
- uintptr_t lock_idx = (((uintptr_t)item) >> 4) - READMARKER_START;
- assert(write_locks[lock_idx] == _STM_TL->thread_num + 1);
- write_locks[lock_idx] = 0;
-
- _stm_move_object(item,
- REAL_ADDRESS(local_base, item),
- REAL_ADDRESS(remote_base, item));
- }));
-
- if (conflicted) {
- struct _thread_local1_s *remote_TL = (struct _thread_local1_s *)
- REAL_ADDRESS(remote_base, _STM_TL);
- remote_TL->need_abort = 1;
- }
-}
-
-
-
-void _stm_write_slowpath(object_t *obj)
-{
- uintptr_t pagenum = ((uintptr_t)obj) / 4096;
- assert(pagenum < NB_PAGES);
- assert(!_stm_is_young(obj));
-
- LIST_APPEND(_STM_TL->old_objects_to_trace, obj);
-
- /* for old objects from the same transaction we don't need
- to privatize the pages */
- if (obj->stm_flags & GCFLAG_NOT_COMMITTED) {
- obj->stm_flags &= ~GCFLAG_WRITE_BARRIER;
- return;
- }
-
- /* privatize if SHARED_PAGE */
- uintptr_t pagenum2, pages;
- if (obj->stm_flags & GCFLAG_SMALL) {
- pagenum2 = pagenum;
- pages = 1;
- } else {
- _stm_chunk_pages((struct object_s*)REAL_ADDRESS(get_thread_base(0), obj),
- &pagenum2, &pages);
- assert(pagenum == pagenum2);
- /* assert(pages == (stmcb_size(real_address(obj)) + 4095) / 4096);
- not true if obj spans two pages, but is itself smaller than 1 */
- }
-
- for (pagenum2 += pages - 1; pagenum2 >= pagenum; pagenum2--)
- stm_pages_privatize(pagenum2);
-
-
- /* claim the write-lock for this object (XXX: maybe a fastpath
- for prev_owner == lock_num?) */
- uintptr_t lock_idx = (((uintptr_t)obj) >> 4) - READMARKER_START;
- uint8_t lock_num = _STM_TL->thread_num + 1;
- uint8_t prev_owner;
- uint8_t retries = 0;
- retry:
- do {
- prev_owner = __sync_val_compare_and_swap(&write_locks[lock_idx],
- 0, lock_num);
-
- /* if there was no lock-holder or we already have the lock */
- if ((!prev_owner) || (prev_owner == lock_num))
- break;
-
- struct _thread_local1_s* other_tl = _stm_dbg_get_tl(prev_owner - 1);
- if ((_STM_TL->age < other_tl->age) || (_STM_TL->active == 2)) {
- /* we must succeed! */
- other_tl->need_abort = 1;
- _stm_start_safe_point(0);
- /* XXX: not good, maybe should be signalled by other thread */
- usleep(1);
- _stm_stop_safe_point(0);
- goto retry;
- } else if (retries < 1) {
- _stm_start_safe_point(0);
- usleep(1);
- _stm_stop_safe_point(0);
- retries++;
- goto retry;
- }
-
- stm_abort_transaction();
- /* XXX: only abort if we are younger */
- } while (1);
-
- /* remove the write-barrier ONLY if we have the write-lock */
- obj->stm_flags &= ~GCFLAG_WRITE_BARRIER;
-
- if (prev_owner == 0) {
- /* otherwise, we have the lock and already added it to
- modified_objects / read-marker */
- stm_read(obj);
- LIST_APPEND(_STM_TL->modified_objects, obj);
- }
-}
-
-void _stm_setup_static_thread(void)
-{
- int thread_num = __sync_fetch_and_add(&num_threads_started, 1);
- assert(thread_num < 2); /* only 2 threads for now */
-
- _stm_restore_local_state(thread_num);
-
- _STM_TL->nursery_current = (localchar_t*)(FIRST_NURSERY_PAGE * 4096);
- memset((void*)real_address((object_t*)NURSERY_CURRENT(_STM_TL)),
- 0x0, (FIRST_AFTER_NURSERY_PAGE - FIRST_NURSERY_PAGE) * 4096); /* clear nursery */
-
- _STM_TL->shadow_stack = NULL;
- _STM_TL->shadow_stack_base = NULL;
-
- _STM_TL->old_objects_to_trace = stm_list_create();
-
- _STM_TL->modified_objects = stm_list_create();
- _STM_TL->uncommitted_objects = stm_list_create();
- assert(!_STM_TL->active);
- _stm_assert_clean_tl();
-}
-
-void stm_setup(void)
-{
- _stm_reset_shared_lock();
- _stm_reset_pages();
-
- inevitable_lock = 0;
-
- /* Check that some values are acceptable */
- assert(4096 <= ((uintptr_t)_STM_TL));
- assert(((uintptr_t)_STM_TL) == ((uintptr_t)_STM_TL));
- assert(((uintptr_t)_STM_TL) + sizeof(*_STM_TL) <= 8192);
- assert(2 <= FIRST_READMARKER_PAGE);
- assert(FIRST_READMARKER_PAGE * 4096UL <= READMARKER_START);
- assert(READMARKER_START < READMARKER_END);
- assert(READMARKER_END <= 4096UL * FIRST_OBJECT_PAGE);
- assert(FIRST_OBJECT_PAGE < NB_PAGES);
- assert((NB_NURSERY_PAGES * 4096) % NURSERY_SECTION == 0);
-
- object_pages = mmap(NULL, TOTAL_MEMORY,
- PROT_READ | PROT_WRITE,
- MAP_PAGES_FLAGS, -1, 0);
- if (object_pages == MAP_FAILED) {
- perror("object_pages mmap");
- abort();
- }
-
- long i;
- for (i = 0; i < NB_THREADS; i++) {
- char *thread_base = get_thread_base(i);
-
- /* In each thread's section, the first page is where TLPREFIX'ed
- NULL accesses land. We mprotect it so that accesses fail. */
- mprotect(thread_base, 4096, PROT_NONE);
-
- /* Fill the TLS page (page 1) with 0xDD */
- memset(REAL_ADDRESS(thread_base, 4096), 0xDD, 4096);
- /* Make a "hole" at _STM_TL / _STM_TL */
- memset(REAL_ADDRESS(thread_base, _STM_TL), 0, sizeof(*_STM_TL));
-
- /* Pages in range(2, FIRST_READMARKER_PAGE) are never used */
- if (FIRST_READMARKER_PAGE > 2)
- mprotect(thread_base + 8192, (FIRST_READMARKER_PAGE - 2) * 4096UL,
- PROT_NONE);
-
- struct _thread_local1_s *th =
- (struct _thread_local1_s *)REAL_ADDRESS(thread_base, _STM_TL);
-
- th->thread_num = i;
- th->thread_base = thread_base;
-
- if (i > 0) {
- int res;
- res = remap_file_pages(
- thread_base + FIRST_AFTER_NURSERY_PAGE * 4096UL,
- (NB_PAGES - FIRST_AFTER_NURSERY_PAGE) * 4096UL,
- 0, FIRST_AFTER_NURSERY_PAGE, 0);
-
- if (res != 0) {
- perror("remap_file_pages");
- abort();
- }
- }
- }
-
- for (i = FIRST_NURSERY_PAGE; i < FIRST_AFTER_NURSERY_PAGE; i++)
- stm_set_page_flag(i, PRIVATE_PAGE); /* nursery is private.
- or should it be UNCOMMITTED??? */
-
- num_threads_started = 0;
-
- assert(HEAP_PAGES < NB_PAGES - FIRST_AFTER_NURSERY_PAGE);
- assert(HEAP_PAGES > 10);
-
- uintptr_t first_heap = stm_pages_reserve(HEAP_PAGES);
- char *heap = REAL_ADDRESS(get_thread_base(0), first_heap * 4096UL);
- assert(memset(heap, 0xcd, HEAP_PAGES * 4096)); // testing
- stm_largemalloc_init(heap, HEAP_PAGES * 4096UL);
-
- for (i = 0; i < NB_THREADS; i++) {
- _stm_setup_static_thread();
- }
-}
-
-
-
-void _stm_teardown_static_thread(int thread_num)
-{
- _stm_restore_local_state(thread_num);
-
- _stm_assert_clean_tl();
- _stm_reset_shared_lock();
-
- stm_list_free(_STM_TL->modified_objects);
- _STM_TL->modified_objects = NULL;
-
- assert(stm_list_is_empty(_STM_TL->uncommitted_objects));
- stm_list_free(_STM_TL->uncommitted_objects);
-
- assert(_STM_TL->old_objects_to_trace->count == 0);
- stm_list_free(_STM_TL->old_objects_to_trace);
-
- _stm_restore_local_state(-1); // invalid
-}
-
-void stm_teardown(void)
-{
- for (; num_threads_started > 0; num_threads_started--) {
- _stm_teardown_static_thread(num_threads_started - 1);
- }
-
- assert(inevitable_lock == 0);
- munmap(object_pages, TOTAL_MEMORY);
- _stm_reset_pages();
- memset(write_locks, 0, sizeof(write_locks));
- object_pages = NULL;
-}
-
-
-
-static void reset_transaction_read_version(void)
-{
- /* force-reset all read markers to 0 */
-
- /* XXX measure the time taken by this madvise() and the following
- zeroing of pages done lazily by the kernel; compare it with using
- 16-bit read_versions.
- */
- /* XXX try to use madvise() on smaller ranges of memory. In my
- measures, we could gain a factor 2 --- not really more, even if
- the range of virtual addresses below is very large, as long as it
- is already mostly non-reserved pages. (The following call keeps
- them non-reserved; apparently the kernel just skips them very
- quickly.)
- */
- int res = madvise((void*)real_address
- ((object_t*) (FIRST_READMARKER_PAGE * 4096UL)),
- (FIRST_OBJECT_PAGE - FIRST_READMARKER_PAGE) * 4096UL,
- MADV_DONTNEED);
- if (res < 0) {
- perror("madvise");
- abort();
- }
- _STM_TL->transaction_read_version = 1;
-}
-
-
-void stm_become_inevitable(char* msg)
-{
- if (_STM_TL->active == 2)
- return;
- assert(_STM_TL->active == 1);
- fprintf(stderr, "%c", 'I'+_STM_TL->thread_num*32);
-
- uint8_t our_lock = _STM_TL->thread_num + 1;
- do {
- _stm_start_safe_point(LOCK_COLLECT);
- _stm_stop_safe_point(LOCK_COLLECT|LOCK_EXCLUSIVE);
-
- if (!inevitable_lock)
- break;
-
- _stm_start_safe_point(LOCK_EXCLUSIVE|LOCK_COLLECT);
- _stm_stop_safe_point(LOCK_COLLECT);
- } while (1);
-
- inevitable_lock = our_lock;
- _STM_TL->active = 2;
-
- _stm_start_safe_point(LOCK_EXCLUSIVE|LOCK_COLLECT);
- _stm_stop_safe_point(LOCK_COLLECT);
-}
-
-void stm_start_inevitable_transaction()
-{
- stm_start_transaction(NULL);
- stm_become_inevitable("stm_start_inevitable_transaction");
-}
-
-void stm_start_transaction(jmpbufptr_t *jmpbufptr)
-{
- /* GS invalid before this point! */
- _stm_stop_safe_point(LOCK_COLLECT|THREAD_YIELD);
-
- assert(!_STM_TL->active);
-
- uint8_t old_rv = _STM_TL->transaction_read_version;
- _STM_TL->transaction_read_version = old_rv + 1;
- if (UNLIKELY(old_rv == 0xff))
- reset_transaction_read_version();
-
- assert(stm_list_is_empty(_STM_TL->modified_objects));
-
- nursery_on_start();
-
- _STM_TL->jmpbufptr = jmpbufptr;
- _STM_TL->active = 1;
- _STM_TL->need_abort = 0;
- /* global_age is approximate -> no synchronization required */
- _STM_TL->age = global_age++;
- /* XXX: only increment our age on commit, not abort? that way we
- are more likely to succeed next time, thus prevent starvation
- (may be fairer, but should probably be done per pthread??) */
-
- fprintf(stderr, "%c", 'S'+_STM_TL->thread_num*32);
-}
-
-
-void stm_stop_transaction(void)
-{
- assert(_STM_TL->active);
-
- /* do the minor_collection here and not in nursery_on_commit,
- since here we can still run concurrently with other threads
- as we don't hold the exclusive lock yet. */
- _stm_minor_collect();
-
- /* Some operations require us to have the EXCLUSIVE lock */
- if (_STM_TL->active == 1) {
- while (1) {
- _stm_start_safe_point(LOCK_COLLECT);
- usleep(1); /* XXX: better algorithm that allows
- for waiting on a mutex */
- _stm_stop_safe_point(LOCK_COLLECT|LOCK_EXCLUSIVE);
-
- if (!inevitable_lock)
- break;
-
- _stm_start_safe_point(LOCK_COLLECT|LOCK_EXCLUSIVE);
- _stm_stop_safe_point(LOCK_COLLECT);
- }
- /* we have the exclusive lock */
- } else {
- /* inevitable! no other transaction could have committed
- or aborted us */
- _stm_start_safe_point(LOCK_COLLECT);
- _stm_stop_safe_point(LOCK_EXCLUSIVE|LOCK_COLLECT);
- inevitable_lock = 0;
- }
-
- _STM_TL->jmpbufptr = NULL; /* cannot abort any more */
-
- /* push uncommitted objects to other threads */
- nursery_on_commit();
-
- /* copy modified object versions to other threads */
- push_modified_to_other_threads();
- stm_list_clear(_STM_TL->modified_objects);
-
-
- _STM_TL->active = 0;
-
- fprintf(stderr, "%c", 'C'+_STM_TL->thread_num*32);
-
- _stm_start_safe_point(LOCK_EXCLUSIVE|LOCK_COLLECT|THREAD_YIELD);
- /* GS invalid after this point! */
-}
-
-
-static void reset_modified_from_other_threads()
-{
- /* pull the right versions from other threads in order
- to reset our pages as part of an abort */
-
- struct stm_list_s *modified = _STM_TL->modified_objects;
- char *local_base = _STM_TL->thread_base;
- char *remote_base = get_thread_base(1 - _STM_TL->thread_num);
-
- STM_LIST_FOREACH(
- modified,
- ({
- /* note: same as push_modified_to... but src/dst swapped
- TODO: unify both... */
-
- /* check at least the first page (required by move_obj() */
- assert(stm_get_page_flag((uintptr_t)item / 4096) == PRIVATE_PAGE);
-
- _stm_move_object(item,
- REAL_ADDRESS(remote_base, item),
- REAL_ADDRESS(local_base, item));
-
- /* copying from the other thread re-added the
- WRITE_BARRIER flag */
- assert(item->stm_flags & GCFLAG_WRITE_BARRIER);
-
- /* write all changes to the object before we release the
- write lock below */
- write_fence();
-
- /* clear the write-lock */
- uintptr_t lock_idx = (((uintptr_t)item) >> 4) - READMARKER_START;
- assert(write_locks[lock_idx]);
- write_locks[lock_idx] = 0;
- }));
-}
-
-
-void stm_abort_transaction(void)
-{
- /* here we hold the shared lock as a reader or writer */
- assert(_STM_TL->active == 1);
-
- nursery_on_abort();
-
- assert(_STM_TL->jmpbufptr != NULL);
- assert(_STM_TL->jmpbufptr != (jmpbufptr_t *)-1); /* for tests only */
- _STM_TL->active = 0;
- _STM_TL->need_abort = 0;
-
- /* reset all the modified objects (incl. re-adding GCFLAG_WRITE_BARRIER) */
- reset_modified_from_other_threads();
- stm_list_clear(_STM_TL->modified_objects);
-
- jmpbufptr_t *buf = _STM_TL->jmpbufptr; /* _STM_TL not valid during safe-point */
- fprintf(stderr, "%c", 'A'+_STM_TL->thread_num*32);
-
- _stm_start_safe_point(LOCK_COLLECT|THREAD_YIELD);
- /* GS invalid after this point! */
-
- __builtin_longjmp(*buf, 1);
-}
diff --git a/c7/core.h b/c7/core.h
deleted file mode 100644
--- a/c7/core.h
+++ /dev/null
@@ -1,277 +0,0 @@
-#ifndef _STM_CORE_H
-#define _STM_CORE_H
-
-#include <stddef.h>
-#include <stdint.h>
-#include <stdbool.h>
-#include <assert.h>
-#include <limits.h>
-
-#if LONG_MAX == 2147483647
-# error "Requires a 64-bit environment"
-#endif
-
-#if BYTE_ORDER == 1234
-# define LENDIAN 1 // little endian
-#elif BYTE_ORDER == 4321
-# define LENDIAN 0 // big endian
-#else
-# error "Unsupported endianness"
-#endif
-
-
-#define NB_PAGES (6*256*256) // 6*256MB
-#define NB_THREADS 2
-#define MAP_PAGES_FLAGS (MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE)
-#define LARGE_OBJECT_WORDS 36
-#define NB_NURSERY_PAGES 1024 // 4MB
-#define LENGTH_SHADOW_STACK 163840
-
-#define NURSERY_SECTION (32*4096)
-/* (NB_NURSERY_PAGE * 4096) % NURSERY_SECTION == 0 */
-
-
-#define TOTAL_MEMORY (NB_PAGES * 4096UL * NB_THREADS)
-#define READMARKER_END ((NB_PAGES * 4096UL) >> 4)
-#define FIRST_OBJECT_PAGE ((READMARKER_END + 4095) / 4096UL)
-#define FIRST_NURSERY_PAGE FIRST_OBJECT_PAGE
-#define READMARKER_START ((FIRST_OBJECT_PAGE * 4096UL) >> 4)
-#define FIRST_READMARKER_PAGE (READMARKER_START / 4096UL)
-#define FIRST_AFTER_NURSERY_PAGE (FIRST_OBJECT_PAGE + NB_NURSERY_PAGES)
-#define HEAP_PAGES (((NB_PAGES - FIRST_AFTER_NURSERY_PAGE) * 3) / 4)
-
-
-
-enum {
- /* set if the write-barrier slowpath needs to trigger. set on all
- old objects if there was no write-barrier on it in the same
- transaction and no collection inbetween. */
- GCFLAG_WRITE_BARRIER = (1 << 0),
- /* set on objects which are in pages visible to others (SHARED
- or PRIVATE), but not committed yet. So only visible from
- this transaction. */
- GCFLAG_NOT_COMMITTED = (1 << 1),
- /* only used during collections to mark an obj as moved out of the
- generation it was in */
- GCFLAG_MOVED = (1 << 2),
- /* objects smaller than one page and even smaller than
- LARGE_OBJECT_WORDS * 8 bytes */
- GCFLAG_SMALL = (1 << 3),
-};
-
-
-
-#define TLPREFIX __attribute__((address_space(256)))
-
-typedef TLPREFIX struct _thread_local1_s _thread_local1_t;
-typedef TLPREFIX struct object_s object_t;
-typedef TLPREFIX struct alloc_for_size_s alloc_for_size_t;
-typedef TLPREFIX struct read_marker_s read_marker_t;
-typedef TLPREFIX char localchar_t;
-typedef void* jmpbufptr_t[5]; /* for use with __builtin_setjmp() */
-
-/* Structure of objects
- --------------------
-
- Objects manipulated by the user program, and managed by this library,
- must start with a "struct object_s" field. Pointers to any user object
- must use the "TLPREFIX struct foo *" type --- don't forget TLPREFIX.
- The best is to use typedefs like above.
-
- The object_s part contains some fields reserved for the STM library,
- as well as a 32-bit integer field that can be freely used by the user
- program. However, right now this field must be read-only --- i.e. it
- must never be modified on any object that may already belong to a
- past transaction; you can only set it on just-allocated objects. The
- best is to consider it as a field that is written to only once on
- newly allocated objects.
-*/
-
-
-struct object_s {
- uint8_t stm_flags; /* reserved for the STM library */
- /* make sure it doesn't get bigger than 4 bytes for performance
- reasons */
-};
-
-struct read_marker_s {
- uint8_t rm;
-};
-
-struct alloc_for_size_s {
- localchar_t *next;
- uint16_t start, stop;
- bool flag_partial_page;
-};
-
-struct _thread_local1_s {
- jmpbufptr_t *jmpbufptr;
- uint8_t transaction_read_version;
-
- /* unsynchronized/inaccurate start age of transaction
- XXX: may be replaced with size_of(read/write-set) */
- long age;
-
- /* static threads, not pthreads */
- int thread_num;
- char *thread_base;
-
- uint8_t active; /* 1 normal, 2 inevitable, 0 no trans. */
- bool need_abort;
-
- object_t **old_shadow_stack;
- object_t **shadow_stack;
- object_t **shadow_stack_base;
-
- union {
- localchar_t *nursery_current;
- uint32_t nursery_current_halfwords[2];
- };
-
- struct stm_list_s *modified_objects;
-
- struct alloc_for_size_s alloc[LARGE_OBJECT_WORDS];
- struct stm_list_s *uncommitted_objects;
-
- struct stm_list_s *old_objects_to_trace;
-};
-#define _STM_TL ((_thread_local1_t *)4352)
-
-
-
-extern char *object_pages; /* start of MMAP region */
-extern uint8_t write_locks[READMARKER_END - READMARKER_START];
-
-/* this should use llvm's coldcc calling convention,
- but it's not exposed to C code so far */
-void _stm_write_slowpath(object_t *);
-
-
-/* ==================== HELPERS ==================== */
-#ifdef NDEBUG
-#define OPT_ASSERT(cond) do { if (!(cond)) __builtin_unreachable(); } while (0)
-#else
-#define OPT_ASSERT(cond) assert(cond)
-#endif
-#define LIKELY(x) __builtin_expect(x, true)
-#define UNLIKELY(x) __builtin_expect(x, false)
-#define IMPLY(a, b) (!(a) || (b))
-
-#define REAL_ADDRESS(object_pages, src) ((object_pages) + (uintptr_t)(src))
-
-
-static inline struct object_s *real_address(object_t *src)
-{
- return (struct object_s*)REAL_ADDRESS(_STM_TL->thread_base, src);
-}
-
-static inline char *_stm_real_address(object_t *o)
-{
- if (o == NULL)
- return NULL;
- assert(FIRST_OBJECT_PAGE * 4096 <= (uintptr_t)o
- && (uintptr_t)o < NB_PAGES * 4096);
- return (char*)real_address(o);
-}
-
-static inline object_t *_stm_tl_address(char *ptr)
-{
- if (ptr == NULL)
- return NULL;
-
- uintptr_t res = ptr - _STM_TL->thread_base;
- assert(FIRST_OBJECT_PAGE * 4096 <= res
- && res < NB_PAGES * 4096);
- return (object_t*)res;
-}
-
-static inline char *get_thread_base(long thread_num)
-{
- return object_pages + thread_num * (NB_PAGES * 4096UL);
-}
-
-
-static inline void spin_loop(void)
-{
- asm("pause" : : : "memory");
-}
-
-
-static inline void write_fence(void)
-{
- /* This function inserts a "write fence". The goal is to make
- sure that past writes are really pushed to memory before
- the future writes. We assume that the corresponding "read
- fence" effect is done automatically by a corresponding
- __sync_bool_compare_and_swap(). */
-#if defined(__amd64__) || defined(__i386__)
- /* this is only a compiler barrier, which is enough on x86 */
- asm("" : : : "memory");
-#else
- /* general fall-back, but we might have more efficient
- alternative on some other platforms too */
- __sync_synchronize();
-#endif
-}
-
-
-
-/* ==================== API ==================== */
-
-static inline void stm_read(object_t *obj)
-{
- ((read_marker_t *)(((uintptr_t)obj) >> 4))->rm =
- _STM_TL->transaction_read_version;
-}
-
-static inline void stm_write(object_t *obj)
-{
- if (UNLIKELY(obj->stm_flags & GCFLAG_WRITE_BARRIER))
- _stm_write_slowpath(obj);
-}
-
-static inline void stm_push_root(object_t *obj)
-{
- *(_STM_TL->shadow_stack++) = obj;
-}
-
-static inline object_t *stm_pop_root(void)
-{
- return *(--_STM_TL->shadow_stack);
-}
-
-/* must be provided by the user of this library */
-extern size_t stmcb_size(struct object_s *);
-extern void stmcb_trace(struct object_s *, void (object_t **));
-
-char* _stm_restore_local_state(int thread_num);
-void stm_teardown(void);
-void stm_teardown_pthread(void);
-bool _stm_is_in_transaction(void);
-void _stm_assert_clean_tl(void);
-
-bool _stm_was_read(object_t *obj);
-bool _stm_was_written(object_t *obj);
-
-object_t *stm_allocate(size_t size);
-void stm_setup(void);
-void stm_setup_pthread(void);
-
-void stm_start_transaction(jmpbufptr_t *jmpbufptr);
-void stm_stop_transaction(void);
-
-
-object_t *_stm_allocate_old(size_t size);
-
-object_t *stm_allocate_prebuilt(size_t size);
-
-void stm_abort_transaction(void);
-
-void _stm_minor_collect();
-void stm_become_inevitable(char* msg);
-void stm_start_inevitable_transaction();
-
-struct _thread_local1_s* _stm_dbg_get_tl(int thread); /* -1 is current thread */
-
-
-#endif
diff --git a/c7/stm/core.c b/c7/stm/core.c
new file mode 100644
--- /dev/null
+++ b/c7/stm/core.c
@@ -0,0 +1,6 @@
+
+
+void _stm_write_slowpath(object_t *obj)
+{
+ abort();
+}
diff --git a/c7/stm/core.h b/c7/stm/core.h
new file mode 100644
--- /dev/null
+++ b/c7/stm/core.h
@@ -0,0 +1,47 @@
+
+#define NB_PAGES (1500*256) // 1500MB
+#define NB_THREADS 2
+#define MAP_PAGES_FLAGS (MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE)
+#define LARGE_OBJECT_WORDS 36
+#define NB_NURSERY_PAGES 1024 // 4MB
+
+#define NURSERY_SECTION_SIZE (24*4096)
+
+
+#define TOTAL_MEMORY (NB_PAGES * 4096UL * NB_THREADS)
+#define READMARKER_END ((NB_PAGES * 4096UL) >> 4)
+#define START_OBJECT_PAGE ((READMARKER_END + 4095) / 4096UL)
+#define START_NURSERY_PAGE START_OBJECT_PAGE
+#define READMARKER_START ((START_OBJECT_PAGE * 4096UL) >> 4)
+#define START_READMARKER_PAGE (READMARKER_START / 4096UL)
+#define STOP_NURSERY_PAGE (START_NURSERY_PAGE + NB_NURSERY_PAGES)
+
+
+enum {
+ /* set if the write-barrier slowpath needs to trigger. set on all
+ old objects if there was no write-barrier on it in the same
+ transaction and no collection inbetween. */
+ GCFLAG_WRITE_BARRIER = _STM_GCFLAG_WRITE_BARRIER,
+ /* set on objects which are in pages visible to others (SHARED
+ or PRIVATE), but not committed yet. So only visible from
+ this transaction. */
+ //GCFLAG_NOT_COMMITTED = _STM_GCFLAG_WRITE_BARRIER << 1,
+ /* only used during collections to mark an obj as moved out of the
+ generation it was in */
+ //GCFLAG_MOVED = _STM_GCFLAG_WRITE_BARRIER << 2,
+ /* objects smaller than one page and even smaller than
+ LARGE_OBJECT_WORDS * 8 bytes */
+ //GCFLAG_SMALL = _STM_GCFLAG_WRITE_BARRIER << 3,
+};
+
+
+#define STM_PREGION ((stm_priv_region_info_t *)STM_REGION)
+
+typedef TLPREFIX struct stm_priv_region_info_s stm_priv_region_info_t;
+
+struct stm_priv_region_info_s {
+ struct stm_region_info_s pub;
+};
+
+
+#define REAL_ADDRESS(thread_base, src) ((thread_base) + (uintptr_t)(src))
diff --git a/c7/stm/gcpage.c b/c7/stm/gcpage.c
new file mode 100644
--- /dev/null
+++ b/c7/stm/gcpage.c
@@ -0,0 +1,11 @@
+
+stm_char *_stm_allocate_slowpath(ssize_t size_rounded_up)
+{
+ abort();
+}
+
+
+object_t *stm_allocate_prebuilt(ssize_t size_rounded_up)
+{
+ abort();
+}
diff --git a/c7/stm/misc.c b/c7/stm/misc.c
new file mode 100644
--- /dev/null
+++ b/c7/stm/misc.c
@@ -0,0 +1,34 @@
+#include <stdlib.h>
+
+
+char *_stm_real_address(object_t *o)
+{
+ if (o == NULL)
+ return NULL;
+
+ assert(START_OBJECT_PAGE * 4096UL <= (uintptr_t)o
+ && (uintptr_t)o < NB_PAGES * 4096UL);
+ return REAL_ADDRESS(STM_REGION->region_base, o);
+}
+
+object_t *_stm_region_address(char *ptr)
+{
+ if (ptr == NULL)
+ return NULL;
+
+ uintptr_t res = ptr - STM_REGION->region_base;
+ assert(START_OBJECT_PAGE * 4096UL <= res
+ && res < NB_PAGES * 4096UL);
+ return (object_t*)res;
+}
+
+bool _stm_was_read(object_t *obj)
+{
+ return ((stm_read_marker_t *)(((uintptr_t)obj) >> 4))->rm ==
+ STM_REGION->transaction_read_version;
+}
+
+bool _stm_was_written(object_t *obj)
+{
+ return !(obj->stm_flags & GCFLAG_WRITE_BARRIER);
+}
diff --git a/c7/stm/setup.c b/c7/stm/setup.c
new file mode 100644
--- /dev/null
+++ b/c7/stm/setup.c
@@ -0,0 +1,86 @@
+
+
+void stm_setup(void)
+{
+#if 0
+ _stm_reset_shared_lock();
+ _stm_reset_pages();
+
+ inevitable_lock = 0;
+
+ /* Check that some values are acceptable */
+ assert(4096 <= ((uintptr_t)_STM_TL));
+ assert(((uintptr_t)_STM_TL) == ((uintptr_t)_STM_TL));
+ assert(((uintptr_t)_STM_TL) + sizeof(*_STM_TL) <= 8192);
+ assert(2 <= FIRST_READMARKER_PAGE);
+ assert(FIRST_READMARKER_PAGE * 4096UL <= READMARKER_START);
+ assert(READMARKER_START < READMARKER_END);
+ assert(READMARKER_END <= 4096UL * FIRST_OBJECT_PAGE);
+ assert(FIRST_OBJECT_PAGE < NB_PAGES);
+ assert((NB_NURSERY_PAGES * 4096) % NURSERY_SECTION == 0);
+
+ object_pages = mmap(NULL, TOTAL_MEMORY,
+ PROT_READ | PROT_WRITE,
+ MAP_PAGES_FLAGS, -1, 0);
+ if (object_pages == MAP_FAILED) {
+ perror("object_pages mmap");
+ abort();
+ }
+
+ long i;
+ for (i = 0; i < NB_THREADS; i++) {
+ char *thread_base = get_thread_base(i);
+
+ /* In each thread's section, the first page is where TLPREFIX'ed
+ NULL accesses land. We mprotect it so that accesses fail. */
+ mprotect(thread_base, 4096, PROT_NONE);
+
+ /* Fill the TLS page (page 1) with 0xDD */
+ memset(REAL_ADDRESS(thread_base, 4096), 0xDD, 4096);
+ /* Make a "hole" at _STM_TL / _STM_TL */
+ memset(REAL_ADDRESS(thread_base, _STM_TL), 0, sizeof(*_STM_TL));
+
+ /* Pages in range(2, FIRST_READMARKER_PAGE) are never used */
+ if (FIRST_READMARKER_PAGE > 2)
+ mprotect(thread_base + 8192, (FIRST_READMARKER_PAGE - 2) * 4096UL,
+ PROT_NONE);
+
+ struct _thread_local1_s *th =
+ (struct _thread_local1_s *)REAL_ADDRESS(thread_base, _STM_TL);
+
+ th->thread_num = i;
+ th->thread_base = thread_base;
+
+ if (i > 0) {
+ int res;
+ res = remap_file_pages(
+ thread_base + FIRST_AFTER_NURSERY_PAGE * 4096UL,
+ (NB_PAGES - FIRST_AFTER_NURSERY_PAGE) * 4096UL,
+ 0, FIRST_AFTER_NURSERY_PAGE, 0);
+
+ if (res != 0) {
+ perror("remap_file_pages");
+ abort();
+ }
+ }
+ }
+
+ for (i = FIRST_NURSERY_PAGE; i < FIRST_AFTER_NURSERY_PAGE; i++)
+ stm_set_page_flag(i, PRIVATE_PAGE); /* nursery is private.
+ or should it be UNCOMMITTED??? */
+
+ num_threads_started = 0;
+
+ assert(HEAP_PAGES < NB_PAGES - FIRST_AFTER_NURSERY_PAGE);
+ assert(HEAP_PAGES > 10);
+
+ uintptr_t first_heap = stm_pages_reserve(HEAP_PAGES);
+ char *heap = REAL_ADDRESS(get_thread_base(0), first_heap * 4096UL);
+ assert(memset(heap, 0xcd, HEAP_PAGES * 4096)); // testing
+ stm_largemalloc_init(heap, HEAP_PAGES * 4096UL);
+
+ for (i = 0; i < NB_THREADS; i++) {
+ _stm_setup_static_thread();
+ }
+#endif
+}
diff --git a/c7/stmgc.c b/c7/stmgc.c
new file mode 100644
--- /dev/null
+++ b/c7/stmgc.c
@@ -0,0 +1,7 @@
+#include "stmgc.h"
+#include "stm/core.h"
+
+#include "stm/misc.c"
+#include "stm/core.c"
+#include "stm/gcpage.c"
+#include "stm/setup.c"
diff --git a/c7/stmgc.h b/c7/stmgc.h
--- a/c7/stmgc.h
+++ b/c7/stmgc.h
@@ -9,8 +9,11 @@
#include <stddef.h>
#include <stdint.h>
+#include <stdbool.h>
#include <assert.h>
#include <limits.h>
+#include <endian.h>
+#include <unistd.h>
#if LONG_MAX == 2147483647
# error "Requires a 64-bit environment"
@@ -25,21 +28,13 @@
#endif
-enum {
- /* set if the write-barrier slowpath needs to trigger. set on all
- old objects if there was no write-barrier on it in the same
- transaction and no collection inbetween. */
- GCFLAG_WRITE_BARRIER = (1 << 0),
-};
-
-
#define TLPREFIX __attribute__((address_space(256)))
typedef TLPREFIX struct object_s object_t;
typedef TLPREFIX struct stm_region_info_s stm_region_info_t;
typedef TLPREFIX struct stm_read_marker_s stm_read_marker_t;
typedef TLPREFIX char stm_char;
-typedef void* stm_jmpbufptr_t[5]; /* for use with __builtin_setjmp() */
+typedef void* stm_jmpbuf_t[5]; /* for use with __builtin_setjmp() */
struct stm_read_marker_s {
uint8_t rm;
@@ -47,18 +42,19 @@
struct stm_region_info_s {
uint8_t transaction_read_version;
- uint8_t active; /* 0 = no, 1 = active, 2 = inevitable */
stm_char *nursery_current;
- uint64_t nursery_block_end;
- char *thread_base;
+ uintptr_t nursery_section_end;
+ char *region_base;
+ struct stm_thread_local_s *running_thread;
+ stm_jmpbuf_t *jmpbuf_ptr;
};
#define STM_REGION ((stm_region_info_t *)4352)
typedef struct stm_thread_local_s {
+ /* every thread should handle the shadow stack itself */
object_t **shadowstack, **shadowstack_base;
- stm_jmpbufptr_t jmpbuf;
- /* the following fields are handled automatically by the library */
- int region_number;
+ /* the next fields are handled automatically by the library */
+ stm_region_info_t *running_in_region;
struct stm_thread_local_s *prev, *next;
} stm_thread_local_t;
@@ -68,9 +64,15 @@
stm_char *_stm_allocate_slowpath(ssize_t);
void _stm_become_inevitable(char*);
-bool _stm_was_read(object_t *object);
-bool _stm_was_written(object_t *object);
-stm_thread_local_t *_stm_test_switch(stm_thread_local_t *);
+#ifdef STM_TESTS
+bool _stm_was_read(object_t *obj);
+bool _stm_was_written(object_t *obj);
+bool _stm_in_nursery(object_t *obj);
+char *_stm_real_address(object_t *o);
+object_t *_stm_region_address(char *ptr);
+#endif
+
+#define _STM_GCFLAG_WRITE_BARRIER 0x01
/* ==================== HELPERS ==================== */
@@ -110,7 +112,7 @@
static inline void stm_write(object_t *obj)
{
- if (UNLIKELY(obj->stm_flags & GCFLAG_WRITE_BARRIER))
+ if (UNLIKELY(obj->stm_flags & _STM_GCFLAG_WRITE_BARRIER))
_stm_write_slowpath(obj);
}
@@ -128,7 +130,7 @@
stm_char *p = STM_REGION->nursery_current;
stm_char *end = p + size_rounded_up;
STM_REGION->nursery_current = end;
- if (UNLIKELY((uint64_t)end > STM_REGION->nursery_block_end))
+ if (UNLIKELY((uintptr_t)end > STM_REGION->nursery_section_end))
p = _stm_allocate_slowpath(size_rounded_up);
return (object_t *)p;
}
@@ -140,18 +142,20 @@
void stm_register_thread_local(stm_thread_local_t *tl);
void stm_unregister_thread_local(stm_thread_local_t *tl);
-void stm_start_transaction(stm_thread_local_t *tl);
+void stm_start_transaction(stm_thread_local_t *tl, stm_jmpbuf_t *jmpbuf);
void stm_start_inevitable_transaction(stm_thread_local_t *tl);
void stm_commit_transaction(void);
void stm_abort_transaction(void);
-#define STM_START_TRANSACTION(tl) ({ \
- int _restart = __builtin_setjmp((tl)->jmpbuf); \
- stm_start_transaction(tl); \
- _restart; })
+#define STM_START_TRANSACTION(tl) ({ \
+ stm_jmpbuf_t _buf; \
+ int _restart = __builtin_setjmp(_buf); \
+ stm_start_transaction(tl, _buf); \
+ _restart; \
+})
static inline void stm_become_inevitable(char* msg) {
- if (STM_REGION->active == 1)
+ if (STM_REGION->jmpbuf_ptr != NULL)
_stm_become_inevitable(msg);
}
diff --git a/c7/test/support.py b/c7/test/support.py
--- a/c7/test/support.py
+++ b/c7/test/support.py
@@ -27,43 +27,40 @@
ffi = cffi.FFI()
ffi.cdef("""
typedef ... object_t;
-typedef ... jmpbufptr_t;
#define SIZEOF_MYOBJ ...
typedef struct {
object_t **shadowstack, **shadowstack_base;
- stm_jmpbufptr_t jmpbuf;
...;
} stm_thread_local_t;
void stm_read(object_t *obj);
-void stm_write(object_t *obj);
+/*void stm_write(object_t *obj); use _checked_stm_write() instead */
object_t *stm_allocate(ssize_t size_rounded_up);
object_t *stm_allocate_prebuilt(ssize_t size_rounded_up);
void stm_setup(void);
+
+bool _checked_stm_write(object_t *obj);
+bool _stm_was_read(object_t *obj);
+bool _stm_was_written(object_t *obj);
+""")
+
+
+TEMPORARILY_DISABLED = """
void stm_teardown(void);
void stm_register_thread_local(stm_thread_local_t *tl);
void stm_unregister_thread_local(stm_thread_local_t *tl);
-void stm_start_transaction(stm_thread_local_t *tl);
+void stm_start_transaction(stm_thread_local_t *tl, stm_jmpbuf_t *jmpbuf);
void stm_start_inevitable_transaction(stm_thread_local_t *tl);
void stm_commit_transaction(void);
void stm_abort_transaction(void);
void stm_become_inevitable(char* msg);
+bool _stm_in_nursery(object_t *obj);
+char *_stm_real_address(object_t *obj);
+object_t *_stm_region_address(char *ptr);
-bool _checked_stm_write(object_t *object);
-bool _stm_was_read(object_t *object);
-bool _stm_was_written(object_t *object);
-stm_thread_local_t *_stm_test_switch(stm_thread_local_t *);
-
-char *_stm_real_address(object_t *o);
-object_t *_stm_region_address(char *ptr);
-bool _stm_is_young(object_t *o);
-""")
-
-
-TEMPORARILY_DISABLED = """
void _stm_start_safe_point(uint8_t);
void _stm_stop_safe_point(uint8_t);
bool _stm_check_stop_safe_point(void);
@@ -130,7 +127,7 @@
#include <string.h>
#include <assert.h>
-#include "../stmgc.h"
+#include "stmgc.h"
struct myobj_s {
struct object_s hdr;
@@ -144,7 +141,7 @@
return obj->stm_flags;
}
-
+#if 0
bool _checked_stm_become_inevitable() {
jmpbufptr_t here;
int tn = _STM_TL->thread_num;
@@ -158,21 +155,23 @@
_stm_dbg_get_tl(tn)->jmpbufptr = (jmpbufptr_t*)-1;
return 1;
}
+#endif
bool _checked_stm_write(object_t *object) {
- jmpbufptr_t here;
- int tn = _STM_TL->thread_num;
+ stm_jmpbuf_t here;
+ stm_region_info_t *region = STM_REGION;
if (__builtin_setjmp(here) == 0) { // returned directly
- assert(_STM_TL->jmpbufptr == (jmpbufptr_t*)-1);
- _STM_TL->jmpbufptr = &here;
- stm_write(object);
- _STM_TL->jmpbufptr = (jmpbufptr_t*)-1;
- return 0;
+ assert(region->jmpbuf_ptr == (stm_jmpbuf_t *)-1);
+ region->jmpbuf_ptr = &here;
+ stm_write(object);
+ region->jmpbuf_ptr = (stm_jmpbuf_t *)-1;
+ return 0;
}
- _stm_dbg_get_tl(tn)->jmpbufptr = (jmpbufptr_t*)-1;
+ region->jmpbuf_ptr = (stm_jmpbuf_t *)-1;
return 1;
}
+#if 0
bool _stm_stop_transaction(void) {
jmpbufptr_t here;
int tn = _STM_TL->thread_num;
@@ -214,6 +213,7 @@
_stm_dbg_get_tl(tn)->jmpbufptr = (jmpbufptr_t*)-1;
return 1;
}
+#endif
void _set_type_id(object_t *obj, uint32_t h)
@@ -228,7 +228,7 @@
void _set_ptr(object_t *obj, int n, object_t *v)
{
- localchar_t *field_addr = ((localchar_t*)obj);
+ stm_char *field_addr = ((stm_char*)obj);
field_addr += SIZEOF_MYOBJ; /* header */
field_addr += n * sizeof(void*); /* field */
object_t * TLPREFIX * field = (object_t * TLPREFIX *)field_addr;
@@ -237,7 +237,7 @@
object_t * _get_ptr(object_t *obj, int n)
{
- localchar_t *field_addr = ((localchar_t*)obj);
+ stm_char *field_addr = ((stm_char*)obj);
field_addr += SIZEOF_MYOBJ; /* header */
field_addr += n * sizeof(void*); /* field */
object_t * TLPREFIX * field = (object_t * TLPREFIX *)field_addr;
@@ -294,7 +294,7 @@
pass
def is_in_nursery(o):
- return lib._stm_is_young(o)
+ return lib._stm_in_nursery(o)
def stm_allocate_old(size):
o = lib._stm_allocate_old(size)
@@ -333,8 +333,8 @@
def stm_get_real_address(obj):
return lib._stm_real_address(ffi.cast('object_t*', obj))
-def stm_get_tl_address(ptr):
- return int(ffi.cast('uintptr_t', lib._stm_tl_address(ptr)))
+def stm_get_region_address(ptr):
+ return int(ffi.cast('uintptr_t', lib._stm_region_address(ptr)))
def stm_read(o):
lib.stm_read(o)
More information about the pypy-commit
mailing list