[pypy-commit] stmgc gc-small-uniform: in-progress
arigo
noreply at buildbot.pypy.org
Sun Apr 13 18:25:16 CEST 2014
Author: Armin Rigo <arigo at tunes.org>
Branch: gc-small-uniform
Changeset: r1156:96ab628450d6
Date: 2014-04-12 14:57 +0200
http://bitbucket.org/pypy/stmgc/changeset/96ab628450d6/
Log: in-progress
diff --git a/c7/stm/core.c b/c7/stm/core.c
--- a/c7/stm/core.c
+++ b/c7/stm/core.c
@@ -332,129 +332,107 @@
}
}
-static void synchronize_object_now(object_t *obj)
+static inline void _synchronize_fragment(stm_char *frag, ssize_t frag_size)
+{
+ /* First copy the object into the shared page, if needed */
+ uintptr_t page = ((uintptr_t)obj) / 4096UL;
+
+ char *src = REAL_ADDRESS(STM_SEGMENT->segment_base, frag);
+ char *dst = REAL_ADDRESS(stm_object_pages, frag);
+ if (is_private_page(STM_SEGMENT->segment_num, page))
+ memcpy(dst, src, frag_size);
+ else
+ EVENTUALLY(memcmp(dst, src, frag_size) == 0); /* same page */
+
+ /* Then enqueue this object (or fragemnt of object) */
+ if (STM_PSEGMENT->sq_len == SYNC_QUEUE_SIZE)
+ synchronize_objects_flush();
+ STM_PSEGMENT->sq_fragments[STM_PSEGMENT->sq_len] = frag;
+ STM_PSEGMENT->sq_fragsizes[STM_PSEGMENT->sq_len] = frag_size;
+ ++STM_PSEGMENT->sq_len;
+}
+
+static void synchronize_object_enqueue(object_t *obj)
{
/* Copy around the version of 'obj' that lives in our own segment.
It is first copied into the shared pages, and then into other
- segments' own private pages.
+ segments' own private pages. (The second part might be done
+ later; call synchronize_objects_flush() to flush this queue.)
*/
assert(!_is_young(obj));
assert(obj->stm_flags & GCFLAG_WRITE_BARRIER);
+ ssize_t obj_size = stmcb_size_rounded_up((struct object_s *)dst);
+ OPT_ASSERT(obj_size >= 16);
+ if (LIKELY(is_small_uniform(obj))) {
+ _synchronize_fragment((stm_char *)obj, obj_size);
+ return;
+ }
+
+ /* else, a more complicated case for large objects, to copy
+ around data only within the needed pages
+ */
uintptr_t start = (uintptr_t)obj;
- uintptr_t first_page = start / 4096UL;
- char *realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
+ uintptr_t end = start + obj_size;
+
+ do {
+ uintptr_t copy_up_to = (start + 4096) & ~4095; /* end of page */
+ if (copy_up_to >= end) {
+ copy_up_to = end; /* this is the last fragment */
+ }
+ uintptr_t copy_size = copy_up_to - start;
+
+ /* double-check that the result fits in one page */
+ assert(copy_size > 0);
+ assert(copy_size + (start & 4095) <= 4096);
+
+ _synchronize_fragment((stm_char *)start, copy_size);
+
+ start = copy_up_to;
+
+ } while (start != end);
+}
+
+static void synchronize_objects_flush(void)
+{
+
+ /* Do a full memory barrier. We must make sure that other
+ CPUs see the changes we did to the shared page ("S", in
+ synchronize_object_enqueue()) before we check the other segments
+ with is_private_page() (below). Otherwise, we risk the
+ following: this CPU writes "S" but the writes are not visible yet;
+ then it checks is_private_page() and gets false, and does nothing
+ more; just afterwards another CPU sets its own private_page bit
+ and copies the page; but it risks doing so before seeing the "S"
+ writes.
+ */
+ long j = STM_PSEGMENT->sq_len;
+ if (j == 0)
+ return;
+ STM_PSEGMENT->sq_len = 0;
+
+ __sync_synchronize();
+
long i, myself = STM_SEGMENT->segment_num;
+ do {
+ --j;
+ stm_char *frag = STM_PSEGMENT->sq_fragments[j];
+ uintptr_t page = ((uintptr_t)frag) / 4096UL;
+ if (!any_other_private_page(myself, page))
+ continue;
- if (is_small_uniform(obj)) {
- /* First copy the object into the shared page, if needed */
- char *src = REAL_ADDRESS(STM_SEGMENT->segment_base, start);
- char *dst = REAL_ADDRESS(stm_object_pages, start);
- ssize_t obj_size = 0; /* computed lazily, only if needed */
-
- if (is_private_page(myself, first_page)) {
- obj_size = stmcb_size_rounded_up((struct object_s *)realobj);
- memcpy(dst, src, obj_size);
- }
- else {
- assert(memcmp(dst, src, /* already identical */
- stmcb_size_rounded_up((struct object_s *)realobj)) == 0);
- }
+ ssize_t frag_size = STM_PSEGMENT->sq_fragsizes[j];
for (i = 1; i <= NB_SEGMENTS; i++) {
if (i == myself)
continue;
- src = REAL_ADDRESS(stm_object_pages, start);
- dst = REAL_ADDRESS(get_segment_base(i), start);
- if (is_private_page(i, first_page)) {
- /* The page is a private page. We need to diffuse this
- object from the shared page to this private page. */
- if (obj_size == 0) {
- obj_size =
- stmcb_size_rounded_up((struct object_s *)src);
- }
- memcpy(dst, src, obj_size);
- }
- else {
- assert(memcmp(dst, src, /* already identical */
- stmcb_size_rounded_up((struct object_s *)src)) == 0);
- }
- }
- }
- else {
- ssize_t obj_size = stmcb_size_rounded_up((struct object_s *)realobj);
- assert(obj_size >= 16);
- uintptr_t end = start + obj_size;
- uintptr_t last_page = (end - 1) / 4096UL;
-
- for (; first_page <= last_page; first_page++) {
-
- uintptr_t copy_size;
- if (first_page == last_page) {
- /* this is the final fragment */
- copy_size = end - start;
- }
- else {
- /* this is a non-final fragment, going up to the
- page's end */
- copy_size = 4096 - (start & 4095);
- }
- /* double-check that the result fits in one page */
- assert(copy_size > 0);
- assert(copy_size + (start & 4095) <= 4096);
-
- /* First copy the object into the shared page, if needed */
- char *src = REAL_ADDRESS(STM_SEGMENT->segment_base, start);
- char *dst = REAL_ADDRESS(stm_object_pages, start);
- if (is_private_page(myself, first_page)) {
- if (copy_size == 4096)
- pagecopy(dst, src);
- else
- memcpy(dst, src, copy_size);
- }
- else {
- EVENTUALLY(memcmp(dst, src, copy_size) == 0); /* same page */
- }
-
- /* Do a full memory barrier. We must make sure that other
- CPUs see the changes we did to the shared page ("S",
- above) before we check the other segments below with
- is_private_page(). Otherwise, we risk the following:
- this CPU writes "S" but the writes are not visible yet;
- then it checks is_private_page() and gets false, and does
- nothing more; just afterwards another CPU sets its own
- private_page bit and copies the page; but it risks doing
- so before seeing the "S" writes.
-
- XXX what is the cost of this? If it's high, then we
- should reorganize the code so that we buffer the second
- parts and do them by bunch of N, after just one call to
- __sync_synchronize()...
- */
- __sync_synchronize();
-
- for (i = 1; i <= NB_SEGMENTS; i++) {
- if (i == myself)
- continue;
-
- src = REAL_ADDRESS(stm_object_pages, start);
- dst = REAL_ADDRESS(get_segment_base(i), start);
- if (is_private_page(i, first_page)) {
- /* The page is a private page. We need to diffuse this
- fragment of object from the shared page to this private
- page. */
- if (copy_size == 4096)
- pagecopy(dst, src);
- else
- memcpy(dst, src, copy_size);
- }
- else {
- EVENTUALLY(!memcmp(dst, src, copy_size)); /* same page */
- }
- }
-
- start = (start + 4096) & ~4095;
+ char *src = REAL_ADDRESS(stm_object_pages, frag);
+ char *dst = REAL_ADDRESS(get_segment_base(i), frag);
+ if (is_private_page(i, page))
+ memcpy(dst, src, frag_size);
+ else
+ EVENTUALLY(memcmp(dst, src, frag_size) == 0); /* same page */
}
}
}
diff --git a/c7/stm/core.h b/c7/stm/core.h
--- a/c7/stm/core.h
+++ b/c7/stm/core.h
@@ -59,6 +59,8 @@
GCFLAG_OVERFLOW_NUMBER_bit0 = 0x4 /* must be last */
};
+#define SYNC_QUEUE_SIZE 31
+
/************************************************************/
@@ -155,6 +157,12 @@
/* This is for smallmalloc.c */
struct small_malloc_data_s small_malloc_data;
+
+ /* The sync queue is used to minimize the number of __sync_synchronize
+ calls needed. */
+ stm_char *sq_fragments[SYNC_QUEUE_SIZE];
+ int sq_fragsizes[SYNC_QUEUE_SIZE];
+ int sq_len;
};
enum /* safe_point */ {
@@ -226,4 +234,5 @@
}
static void copy_object_to_shared(object_t *obj, int source_segment_num);
-static void synchronize_object_now(object_t *obj);
+static void synchronize_object_enqueue(object_t *obj);
+static void synchronize_objects_flush(void);
diff --git a/c7/stm/nursery.c b/c7/stm/nursery.c
--- a/c7/stm/nursery.c
+++ b/c7/stm/nursery.c
@@ -56,7 +56,6 @@
/************************************************************/
#define GCWORD_MOVED ((object_t *) -1)
-#define FLAG_SYNC_LARGE 0x01
static void minor_trace_if_young(object_t **pobj)
@@ -145,8 +144,7 @@
}
/* Must trace the object later */
- uintptr_t nobj_sync_now = (uintptr_t)nobj | !is_small_uniform(nobj);
- LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, nobj_sync_now);
+ LIST_APPEND(STM_PSEGMENT->objects_pointing_to_nursery, nobj);
}
static void collect_roots_in_nursery(void)
@@ -179,29 +177,20 @@
static void collect_oldrefs_to_nursery(void)
{
struct list_s *lst = STM_PSEGMENT->objects_pointing_to_nursery;
+ assert(STM_PSEGMENT->minor_collect_will_commit_now);
while (!list_is_empty(lst)) {
- uintptr_t obj_sync_now = list_pop_item(lst);
- object_t *obj = (object_t *)(obj_sync_now & ~FLAG_SYNC_LARGE);
+ object_t *obj = (object_t *)list_pop_item(lst);
_collect_now(obj);
- if (obj_sync_now & FLAG_SYNC_LARGE) {
- /* this was a large object. We must either synchronize the
- object to other segments now (after we added the
- WRITE_BARRIER flag and traced into it to fix its
- content); or add the object to 'large_overflow_objects'.
- */
- if (STM_PSEGMENT->minor_collect_will_commit_now) {
- synchronize_object_now(obj);
- }
- else
- LIST_APPEND(STM_PSEGMENT->large_overflow_objects, obj);
- }
+ synchronize_object_enqueue(obj);
/* the list could have moved while appending */
lst = STM_PSEGMENT->objects_pointing_to_nursery;
}
+
+ synchronize_objects_flush();
}
static void collect_modified_old_objects(void)
@@ -284,6 +273,9 @@
following shortcut */
collect_modified_old_objects();
}
+ else {
+ abort(); // handle specially the objects_pointing_to_nursery already there
+ }
collect_roots_in_nursery();
diff --git a/c7/stm/pages.h b/c7/stm/pages.h
--- a/c7/stm/pages.h
+++ b/c7/stm/pages.h
@@ -67,6 +67,20 @@
return (pages_privatized[pagenum - PAGE_FLAG_START].by_segment & bitmask);
}
+static inline bool any_private_page(uintptr_t pagenum)
+{
+ assert(pagenum >= PAGE_FLAG_START);
+ return pages_privatized[pagenum - PAGE_FLAG_START].by_segment != 0;
+}
+
+static inline bool any_other_private_page(long exclsegnum, uintptr_t pagenum)
+{
+ assert(pagenum >= PAGE_FLAG_START);
+ uint64_t bitmask = 1UL << (exclsegnum - 1);
+ return ((pages_privatized[pagenum - PAGE_FLAG_START].by_segment & ~bitmask)
+ != 0);
+}
+
static inline void page_check_and_reshare(uintptr_t pagenum)
{
if (pages_privatized[pagenum - PAGE_FLAG_START].by_segment != 0)
More information about the pypy-commit
mailing list