[pypy-commit] stmgc default: remove special segment 0 and introduce simple stm_allocate_old for tests
Raemi
noreply at buildbot.pypy.org
Wed Sep 3 11:29:06 CEST 2014
Author: Remi Meier <remi.meier at inf.ethz.ch>
Branch:
Changeset: r1337:0d99d2f929cd
Date: 2014-09-03 11:30 +0200
http://bitbucket.org/pypy/stmgc/changeset/0d99d2f929cd/
Log: remove special segment 0 and introduce simple stm_allocate_old for
tests
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -33,8 +33,6 @@
#define NB_READMARKER_PAGES (FIRST_OBJECT_PAGE - FIRST_READMARKER_PAGE)
-#define USE_REMAP_FILE_PAGES 1
-
enum /* stm_flags */ {
GCFLAG_WRITE_BARRIER = _STM_GCFLAG_WRITE_BARRIER,
};
@@ -52,6 +50,7 @@
struct list_s *modified_old_objects;
struct list_s *objects_pointing_to_nursery;
+ uint8_t privatization_lock;
/* For debugging */
#ifndef NDEBUG
@@ -96,3 +95,17 @@
llvmfix/no-memset-creation-with-addrspace.diff. */
asm("/* workaround for llvm bug */");
}
+
+static inline void acquire_privatization_lock(void)
+{
+ uint8_t *lock = (uint8_t *)REAL_ADDRESS(STM_SEGMENT->segment_base,
+ &STM_PSEGMENT->privatization_lock);
+ spinlock_acquire(*lock);
+}
+
+static inline void release_privatization_lock(void)
+{
+ uint8_t *lock = (uint8_t *)REAL_ADDRESS(STM_SEGMENT->segment_base,
+ &STM_PSEGMENT->privatization_lock);
+ spinlock_release(*lock);
+}
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
new file mode 100644
--- /dev/null
+++ b/c8/stm/gcpage.c
@@ -0,0 +1,55 @@
+#ifndef _STM_CORE_H_
+# error "must be compiled via stmgc.c"
+#endif
+
+
+static void setup_gcpage(void)
+{
+ uninitialized_page_start = stm_object_pages + END_NURSERY_PAGE * 4096UL;
+ uninitialized_page_stop = stm_object_pages + NB_PAGES * 4096UL;
+}
+
+static void teardown_gcpage(void)
+{
+}
+
+static void setup_N_pages(char *pages_addr, uint64_t num)
+{
+ pages_initialize_private((pages_addr - stm_object_pages) / 4096UL, num);
+}
+
+
+static char *allocate_outside_nursery_large(uint64_t size)
+{
+ /* XXX: real allocation */
+ char *addr = uninitialized_page_start;
+
+ char *start = uninitialized_page_start;
+ if (addr + size > start) { /* XXX: always for now */
+ uintptr_t npages;
+ npages = (addr + size - start) / 4096UL + 1;
+ if (uninitialized_page_stop - start < npages * 4096UL) {
+ stm_fatalerror("out of memory!"); /* XXX */
+ }
+ setup_N_pages(start, npages);
+ if (!__sync_bool_compare_and_swap(&uninitialized_page_start,
+ start,
+ start + npages * 4096UL)) {
+ stm_fatalerror("uninitialized_page_start changed?");
+ }
+ }
+
+ return addr;
+}
+
+object_t *_stm_allocate_old(ssize_t size_rounded_up)
+{
+ /* only for tests xxx but stm_setup_prebuilt() uses this now too */
+ char *p = allocate_outside_nursery_large(size_rounded_up);
+ memset(p, 0, size_rounded_up);
+
+ object_t *o = (object_t *)(p - stm_object_pages);
+ o->stm_flags = GCFLAG_WRITE_BARRIER;
+
+ return o;
+}
diff --git a/c8/stm/gcpage.h b/c8/stm/gcpage.h
new file mode 100644
--- /dev/null
+++ b/c8/stm/gcpage.h
@@ -0,0 +1,7 @@
+
+
+static char *uninitialized_page_start; /* within segment 0 */
+static char *uninitialized_page_stop;
+
+static void setup_gcpage(void);
+static void teardown_gcpage(void);
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -18,7 +18,7 @@
_stm_nursery_start = NURSERY_START;
long i;
- for (i = 1; i <= NB_SEGMENTS; i++) {
+ for (i = 0; i < NB_SEGMENTS; i++) {
get_segment(i)->nursery_current = (stm_char *)NURSERY_START;
get_segment(i)->nursery_end = NURSERY_END;
}
@@ -122,7 +122,7 @@
_stm_nursery_start = NURSERY_END - free_count;
long i;
- for (i = 1; i <= NB_SEGMENTS; i++) {
+ for (i = 0; i < NB_SEGMENTS; i++) {
if ((uintptr_t)get_segment(i)->nursery_current < _stm_nursery_start)
get_segment(i)->nursery_current = (stm_char *)_stm_nursery_start;
}
diff --git a/c8/stm/pagecopy.c b/c8/stm/pagecopy.c
new file mode 100644
--- /dev/null
+++ b/c8/stm/pagecopy.c
@@ -0,0 +1,68 @@
+
+#define PAGECOPY_128(dest, src) \
+ asm volatile("movdqa (%0), %%xmm0\n" \
+ "movdqa 16(%0), %%xmm1\n" \
+ "movdqa 32(%0), %%xmm2\n" \
+ "movdqa 48(%0), %%xmm3\n" \
+ "movdqa %%xmm0, (%1)\n" \
+ "movdqa %%xmm1, 16(%1)\n" \
+ "movdqa %%xmm2, 32(%1)\n" \
+ "movdqa %%xmm3, 48(%1)\n" \
+ "movdqa 64(%0), %%xmm0\n" \
+ "movdqa 80(%0), %%xmm1\n" \
+ "movdqa 96(%0), %%xmm2\n" \
+ "movdqa 112(%0), %%xmm3\n" \
+ "movdqa %%xmm0, 64(%1)\n" \
+ "movdqa %%xmm1, 80(%1)\n" \
+ "movdqa %%xmm2, 96(%1)\n" \
+ "movdqa %%xmm3, 112(%1)\n" \
+ : \
+ : "r"(src), "r"(dest) \
+ : "xmm0", "xmm1", "xmm2", "xmm3", "memory")
+
+static void pagecopy(void *dest, const void *src)
+{
+ unsigned long i;
+ for (i = 0; i < 4096 / 128; i++) {
+ PAGECOPY_128(dest + 128*i, src + 128*i);
+ }
+}
+
+#if 0
+static void pagecopy_256(void *dest, const void *src)
+{
+ PAGECOPY_128(dest, src );
+ PAGECOPY_128(dest + 128, src + 128);
+}
+#endif
+
+#if 0 /* XXX enable if detected on the cpu */
+static void pagecopy_ymm8(void *dest, const void *src)
+{
+ asm volatile("0:\n"
+ "vmovdqa (%0), %%ymm0\n"
+ "vmovdqa 32(%0), %%ymm1\n"
+ "vmovdqa 64(%0), %%ymm2\n"
+ "vmovdqa 96(%0), %%ymm3\n"
+ "vmovdqa 128(%0), %%ymm4\n"
+ "vmovdqa 160(%0), %%ymm5\n"
+ "vmovdqa 192(%0), %%ymm6\n"
+ "vmovdqa 224(%0), %%ymm7\n"
+ "addq $256, %0\n"
+ "vmovdqa %%ymm0, (%1)\n"
+ "vmovdqa %%ymm1, 32(%1)\n"
+ "vmovdqa %%ymm2, 64(%1)\n"
+ "vmovdqa %%ymm3, 96(%1)\n"
+ "vmovdqa %%ymm4, 128(%1)\n"
+ "vmovdqa %%ymm5, 160(%1)\n"
+ "vmovdqa %%ymm6, 192(%1)\n"
+ "vmovdqa %%ymm7, 224(%1)\n"
+ "addq $256, %1\n"
+ "cmpq %2, %0\n"
+ "jne 0b"
+ : "=r"(src), "=r"(dest)
+ : "r"((char *)src + 4096), "0"(src), "1"(dest)
+ : "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5", "xmm6", "xmm7");
+}
+#endif
diff --git a/c8/stm/pagecopy.h b/c8/stm/pagecopy.h
new file mode 100644
--- /dev/null
+++ b/c8/stm/pagecopy.h
@@ -0,0 +1,2 @@
+
+static void pagecopy(void *dest, const void *src); // 4096 bytes
diff --git a/c8/stm/pages.c b/c8/stm/pages.c
new file mode 100644
--- /dev/null
+++ b/c8/stm/pages.c
@@ -0,0 +1,46 @@
+#ifndef _STM_CORE_H_
+# error "must be compiled via stmgc.c"
+#endif
+
+
+/************************************************************/
+
+static void setup_pages(void)
+{
+}
+
+static void teardown_pages(void)
+{
+ memset(pages_privatized, 0, sizeof(pages_privatized));
+}
+
+/************************************************************/
+
+
+static void pages_initialize_private(uintptr_t pagenum, uintptr_t count)
+{
+ dprintf(("pages_initialize_private: 0x%ld - 0x%ld\n", pagenum,
+ pagenum + count));
+ assert(pagenum < NB_PAGES);
+ if (count == 0)
+ return;
+
+ long i;
+ for (i = 0; i < NB_SEGMENTS; i++) {
+ spinlock_acquire(get_priv_segment(i)->privatization_lock);
+ }
+
+ while (count-->0) {
+ for (i = 0; i < NB_SEGMENTS; i++) {
+ uint64_t bitmask = 1UL << i;
+ volatile struct page_shared_s *ps = (volatile struct page_shared_s *)
+ &pages_privatized[pagenum + count - PAGE_FLAG_START];
+
+ ps->by_segment |= bitmask;
+ }
+ }
+
+ for (i = NB_SEGMENTS-1; i >= 0; i--) {
+ spinlock_release(get_priv_segment(i)->privatization_lock);
+ }
+}
diff --git a/c8/stm/pages.h b/c8/stm/pages.h
new file mode 100644
--- /dev/null
+++ b/c8/stm/pages.h
@@ -0,0 +1,31 @@
+
+#define PAGE_FLAG_START END_NURSERY_PAGE
+#define PAGE_FLAG_END NB_PAGES
+
+#define USE_REMAP_FILE_PAGES
+
+struct page_shared_s {
+#if NB_SEGMENTS <= 8
+ uint8_t by_segment;
+#elif NB_SEGMENTS <= 16
+ uint16_t by_segment;
+#elif NB_SEGMENTS <= 32
+ uint32_t by_segment;
+#elif NB_SEGMENTS <= 64
+ uint64_t by_segment;
+#else
+# error "NB_SEGMENTS > 64 not supported right now"
+#endif
+};
+
+static struct page_shared_s pages_privatized[PAGE_FLAG_END - PAGE_FLAG_START];
+
+static void pages_initialize_private(uintptr_t pagenum, uintptr_t count);
+
+
+static inline bool is_private_page(long segnum, uintptr_t pagenum)
+{
+ assert(pagenum >= PAGE_FLAG_START);
+ uint64_t bitmask = 1UL << segnum;
+ return (pages_privatized[pagenum - PAGE_FLAG_START].by_segment & bitmask);
+}
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -54,14 +54,8 @@
static void setup_protection_settings(void)
{
- /* The segment 0 is not used to run transactions, but contains the
- shared copy of the pages. We mprotect all pages before so that
- accesses fail, up to and including the pages corresponding to the
- nurseries of the other segments. */
- mprotect(stm_object_pages, END_NURSERY_PAGE * 4096UL, PROT_NONE);
-
long i;
- for (i = 1; i <= NB_SEGMENTS; i++) {
+ for (i = 0; i < NB_SEGMENTS; i++) {
char *segment_base = get_segment_base(i);
/* In each segment, the first page is where TLPREFIX'ed
@@ -97,7 +91,7 @@
setup_protection_settings();
long i;
- for (i = 1; i <= NB_SEGMENTS; i++) {
+ for (i = 0; i < NB_SEGMENTS; i++) {
char *segment_base = get_segment_base(i);
/* Fill the TLS page (page 1) with 0xDC, for debugging */
@@ -108,7 +102,7 @@
/* Initialize STM_PSEGMENT */
struct stm_priv_segment_info_s *pr = get_priv_segment(i);
- assert(1 <= i && i < 255); /* 255 is WL_VISITED in gcpage.c */
+ assert(0 <= i && i < 255); /* 255 is WL_VISITED in gcpage.c */
pr->pub.segment_num = i;
pr->pub.segment_base = segment_base;
pr->modified_old_objects = list_create();
@@ -127,6 +121,8 @@
setup_sync();
setup_nursery();
+ setup_gcpage();
+ setup_pages();
}
void stm_teardown(void)
@@ -136,7 +132,7 @@
assert(!_has_mutex());
long i;
- for (i = 1; i <= NB_SEGMENTS; i++) {
+ for (i = 0; i < NB_SEGMENTS; i++) {
struct stm_priv_segment_info_s *pr = get_priv_segment(i);
assert(list_is_empty(pr->objects_pointing_to_nursery));
list_free(pr->objects_pointing_to_nursery);
@@ -148,6 +144,8 @@
close_fd_mmap(stm_object_pages_fd);
teardown_sync();
+ teardown_gcpage();
+ teardown_pages();
}
static pthread_t *_get_cpth(stm_thread_local_t *tl)
@@ -175,7 +173,7 @@
/* assign numbers consecutively, but that's for tests; we could also
assign the same number to all of them and they would get their own
numbers automatically. */
- num = (num % NB_SEGMENTS) + 1;
+ num = (num + 1) % NB_SEGMENTS;
tl->associated_segment_num = num;
*_get_cpth(tl) = pthread_self();
set_gs_register(get_segment_base(num));
diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -73,7 +73,7 @@
assert(_is_tl_registered(tl));
int num = tl->associated_segment_num;
- if (sync_ctl.in_use1[num - 1] == 0) {
+ if (sync_ctl.in_use1[num] == 0) {
/* fast-path: we can get the same segment number than the one
we had before. The value stored in GS is still valid. */
#ifdef STM_TESTS
@@ -88,8 +88,8 @@
the condition variable. */
int retries;
for (retries = 0; retries < NB_SEGMENTS; retries++) {
- num = (num % NB_SEGMENTS) + 1;
- if (sync_ctl.in_use1[num - 1] == 0) {
+ num = num % NB_SEGMENTS;
+ if (sync_ctl.in_use1[num] == 0) {
/* we're getting 'num', a different number. */
dprintf(("acquired different segment: %d->%d\n", tl->associated_segment_num, num));
tl->associated_segment_num = num;
@@ -105,7 +105,7 @@
return false;
got_num:
- sync_ctl.in_use1[num - 1] = 1;
+ sync_ctl.in_use1[num] = 1;
assert(STM_SEGMENT->segment_num == num);
assert(STM_SEGMENT->running_thread == NULL);
STM_SEGMENT->running_thread = tl;
@@ -119,8 +119,8 @@
assert(STM_SEGMENT->running_thread == tl);
STM_SEGMENT->running_thread = NULL;
- assert(sync_ctl.in_use1[tl->associated_segment_num - 1] == 1);
- sync_ctl.in_use1[tl->associated_segment_num - 1] = 0;
+ assert(sync_ctl.in_use1[tl->associated_segment_num] == 1);
+ sync_ctl.in_use1[tl->associated_segment_num] = 0;
}
__attribute__((unused))
@@ -132,7 +132,7 @@
bool _stm_in_transaction(stm_thread_local_t *tl)
{
int num = tl->associated_segment_num;
- assert(1 <= num && num <= NB_SEGMENTS);
+ assert(0 <= num && num < NB_SEGMENTS);
return get_segment(num)->running_thread == tl;
}
diff --git a/c8/stmgc.c b/c8/stmgc.c
--- a/c8/stmgc.c
+++ b/c8/stmgc.c
@@ -2,7 +2,10 @@
#include "stmgc.h"
#include "stm/atomic.h"
#include "stm/list.h"
+#include "stm/pagecopy.h"
#include "stm/core.h"
+#include "stm/pages.h"
+#include "stm/gcpage.h"
#include "stm/nursery.h"
#include "stm/sync.h"
#include "stm/setup.h"
@@ -10,8 +13,11 @@
#include "stm/rewind_setjmp.h"
#include "stm/list.c"
+#include "stm/pagecopy.c"
#include "stm/nursery.c"
#include "stm/core.c"
+#include "stm/pages.c"
+#include "stm/gcpage.c"
#include "stm/sync.c"
#include "stm/setup.c"
#include "stm/fprintcolor.c"
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -62,13 +62,15 @@
void _stm_write_slowpath(object_t *);
object_t *_stm_allocate_slowpath(ssize_t);
+
+object_t *_stm_allocate_old(ssize_t size_rounded_up);
+char *_stm_real_address(object_t *o);
#ifdef STM_TESTS
#include <stdbool.h>
bool _stm_was_read(object_t *obj);
bool _stm_was_written(object_t *obj);
long stm_can_move(object_t *obj);
-char *_stm_real_address(object_t *o);
void _stm_test_switch(stm_thread_local_t *tl);
char *_stm_get_segment_base(long index);
diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -39,6 +39,7 @@
bool _stm_in_transaction(stm_thread_local_t *tl);
int _stm_get_flags(object_t *obj);
+object_t *_stm_allocate_old(ssize_t size_rounded_up);
long stm_can_move(object_t *obj);
char *_stm_real_address(object_t *o);
void _stm_test_switch(stm_thread_local_t *tl);
More information about the pypy-commit
mailing list