[pypy-svn] r79496 - in pypy/branch/jit-free-asm/pypy/jit/backend/llsupport: . test
arigo at codespeak.net
arigo at codespeak.net
Thu Nov 25 10:21:38 CET 2010
Author: arigo
Date: Thu Nov 25 10:21:36 2010
New Revision: 79496
Added:
pypy/branch/jit-free-asm/pypy/jit/backend/llsupport/codebuf.py
pypy/branch/jit-free-asm/pypy/jit/backend/llsupport/test/test_codebuf.py
Log:
Initial work.
Added: pypy/branch/jit-free-asm/pypy/jit/backend/llsupport/codebuf.py
==============================================================================
--- (empty file)
+++ pypy/branch/jit-free-asm/pypy/jit/backend/llsupport/codebuf.py Thu Nov 25 10:21:36 2010
@@ -0,0 +1,132 @@
+import sys
+from pypy.rlib.rarithmetic import intmask, r_uint
+from pypy.rlib.objectmodel import we_are_translated
+from pypy.rlib import rmmap
+from pypy.rpython.lltypesystem import lltype, llmemory, rffi
+
+
+class AsmMemoryManager(object):
+ LARGE_ALLOC_SIZE = 1024 * 1024 # 1MB
+ MIN_FRAGMENT = 64
+ NUM_INDICES = 32 # good for all sizes between 64 bytes and ~490 KB
+
+ def __init__(self, large_alloc_size = LARGE_ALLOC_SIZE,
+ min_fragment = MIN_FRAGMENT,
+ num_indices = NUM_INDICES):
+ self.total_memory_allocated = r_uint(0)
+ self.large_alloc_size = large_alloc_size
+ self.min_fragment = min_fragment
+ self.num_indices = num_indices
+ self.free_blocks = {} # map {start: stop}
+ self.free_blocks_end = {} # map {stop: start}
+ self.blocks_by_size = [[] for i in range(self.num_indices)]
+ self._allocated = []
+
+ def malloc(self, minsize, maxsize):
+ """Allocate executable memory, between minsize and maxsize bytes,
+ and return a pair (start, stop). Does not perform any rounding
+ of minsize and maxsize.
+ """
+ result = self._allocate_block(minsize)
+ (start, stop) = result
+ smaller_stop = start + maxsize
+ if smaller_stop + self.min_fragment <= stop:
+ result = (start, smaller_stop)
+ self._add_free_block(smaller_stop, stop)
+ return result # pair (start, stop)
+
+ def free(self, start, stop):
+ """Free a block (start, stop) returned by a previous malloc()."""
+ self._add_free_block(start, stop)
+
+ def _allocate_large_block(self, minsize):
+ # Compute 'size' from 'minsize': it must be rounded up to
+ # 'large_alloc_size'. Additionally, we use the following line
+ # to limit how many mmap() requests the OS will see in total:
+ minsize = max(minsize, intmask(self.total_memory_allocated >> 4))
+ size = minsize + self.large_alloc_size - 1
+ size = (size // self.large_alloc_size) * self.large_alloc_size
+ data = rmmap.alloc(size)
+ if not we_are_translated():
+ self._allocated.append((data, size))
+ if sys.maxint > 2147483647:
+ # Hack to make sure that mcs are not within 32-bits of one
+ # another for testing purposes
+ rmmap.hint.pos += 0x80000000 - size
+ self.total_memory_allocated += size
+ data = rffi.cast(lltype.Signed, data)
+ return self._add_free_block(data, data + size)
+
+ def _get_index(self, length):
+ i = 0
+ while length > self.min_fragment:
+ length = (length * 3) >> 2
+ i += 1
+ if i == self.num_indices - 1:
+ break
+ return i
+
+ def _add_free_block(self, start, stop):
+ # Merge with the block on the left
+ if start in self.free_blocks_end:
+ left_start = self.free_blocks_end[start]
+ self._del_free_block(left_start, start)
+ start = left_start
+ # Merge with the block on the right
+ if stop in self.free_blocks:
+ right_stop = self.free_blocks[stop]
+ self._del_free_block(stop, right_stop)
+ stop = right_stop
+ # Add it to the dicts
+ self.free_blocks[start] = stop
+ self.free_blocks_end[stop] = start
+ i = self._get_index(stop - start)
+ self.blocks_by_size[i].append(start)
+ return start
+
+ def _del_free_block(self, start, stop):
+ del self.free_blocks[start]
+ del self.free_blocks_end[stop]
+ i = self._get_index(stop - start)
+ self.blocks_by_size[i].remove(start)
+
+ def _allocate_block(self, length):
+ # First look in the group of index i0 if there is a block that is
+ # big enough. Following an idea found in the Linux malloc.c, we
+ # prefer the oldest entries rather than the newest one, to let
+ # them have enough time to coalesce into bigger blocks. It makes
+ # a big difference on the purely random test (30% of total usage).
+ i0 = self._get_index(length)
+ bbs = self.blocks_by_size[i0]
+ for j in range(len(bbs)):
+ start = bbs[j]
+ stop = self.free_blocks[start]
+ if start + length <= stop:
+ del bbs[j]
+ break # found a block big enough
+ else:
+ # Then look in the larger groups
+ i = i0 + 1
+ while i < self.num_indices:
+ if len(self.blocks_by_size[i]) > 0:
+ # any block found in a larger group is big enough
+ start = self.blocks_by_size[i].pop(0)
+ stop = self.free_blocks[start]
+ break
+ i += 1
+ else:
+ # Exhausted the memory. Allocate the resulting block.
+ start = self._allocate_large_block(length)
+ stop = self.free_blocks[start]
+ i = self._get_index(stop - start)
+ assert self.blocks_by_size[i][-1] == start
+ self.blocks_by_size[i].pop()
+ #
+ del self.free_blocks[start]
+ del self.free_blocks_end[stop]
+ return (start, stop)
+
+ def _delete(self):
+ "NOT_RPYTHON"
+ for data, size in self._allocated:
+ rmmap.free(data, size)
Added: pypy/branch/jit-free-asm/pypy/jit/backend/llsupport/test/test_codebuf.py
==============================================================================
--- (empty file)
+++ pypy/branch/jit-free-asm/pypy/jit/backend/llsupport/test/test_codebuf.py Thu Nov 25 10:21:36 2010
@@ -0,0 +1,149 @@
+import random
+from pypy.jit.backend.llsupport.codebuf import AsmMemoryManager
+
+
+def test_get_index():
+ memmgr = AsmMemoryManager(min_fragment=8,
+ num_indices=5)
+ index = 0
+ for length in range(100):
+ assert memmgr._get_index(length) == index
+ if length in [8, 11, 15, 21]:
+ index += 1
+
+def test_get_index_default_values():
+ memmgr = AsmMemoryManager()
+ jumps = [64, 86, 115, 154, 206, 275, 367, 490, 654, 873, 1165,
+ 1554, 2073, 2765, 3687, 4917, 6557, 8743, 11658, 15545,
+ 20727, 27637, 36850, 49134, 65513, 87351, 116469, 155293,
+ 207058, 276078, 368105]
+ for i, jump in enumerate(jumps):
+ assert memmgr._get_index(jump) == i
+ assert memmgr._get_index(jump + 1) == i + 1
+
+def test_add_free_block():
+ memmgr = AsmMemoryManager(min_fragment=8,
+ num_indices=5)
+ memmgr._add_free_block(10, 18)
+ assert memmgr.free_blocks == {10: 18}
+ assert memmgr.free_blocks_end == {18: 10}
+ assert memmgr.blocks_by_size == [[10], [], [], [], []]
+ memmgr._add_free_block(20, 30)
+ assert memmgr.free_blocks == {10: 18, 20: 30}
+ assert memmgr.free_blocks_end == {18: 10, 30: 20}
+ assert memmgr.blocks_by_size == [[10], [20], [], [], []]
+ memmgr._add_free_block(18, 20) # merge both left and right
+ assert memmgr.free_blocks == {10: 30}
+ assert memmgr.free_blocks_end == {30: 10}
+ assert memmgr.blocks_by_size == [[], [], [], [10], []]
+
+def test_allocate_block():
+ memmgr = AsmMemoryManager(min_fragment=8,
+ num_indices=5)
+ memmgr._add_free_block(10, 18)
+ memmgr._add_free_block(20, 30)
+ (start, stop) = memmgr._allocate_block(4)
+ assert (start, stop) == (10, 18)
+ assert memmgr.free_blocks == {20: 30}
+ assert memmgr.free_blocks_end == {30: 20}
+ assert memmgr.blocks_by_size == [[], [20], [], [], []]
+ (start, stop) = memmgr._allocate_block(4)
+ assert (start, stop) == (20, 30)
+ assert memmgr.free_blocks == {}
+ assert memmgr.free_blocks_end == {}
+ assert memmgr.blocks_by_size == [[], [], [], [], []]
+
+def test_malloc_without_fragment():
+ memmgr = AsmMemoryManager(min_fragment=8,
+ num_indices=5)
+ memmgr._add_free_block(10, 18)
+ memmgr._add_free_block(20, 30)
+ for minsize in range(1, 11):
+ for maxsize in range(minsize, 14):
+ (start, stop) = memmgr.malloc(minsize, maxsize)
+ if minsize <= 8:
+ assert (start, stop) == (10, 18)
+ else:
+ assert (start, stop) == (20, 30)
+ memmgr._add_free_block(start, stop)
+ memmgr._add_free_block(40, 49)
+ (start, stop) = memmgr.malloc(10, 10)
+ assert (start, stop) == (20, 30)
+
+def test_malloc_with_fragment():
+ for reqsize in range(1, 33):
+ memmgr = AsmMemoryManager(min_fragment=8,
+ num_indices=5)
+ memmgr._add_free_block(12, 44)
+ (start, stop) = memmgr.malloc(reqsize, reqsize)
+ if reqsize + 8 <= 32:
+ assert (start, stop) == (12, 12 + reqsize)
+ assert memmgr.free_blocks == {stop: 44}
+ assert memmgr.free_blocks_end == {44: stop}
+ assert [stop] in memmgr.blocks_by_size
+ else:
+ assert (start, stop) == (12, 44)
+ assert memmgr.free_blocks == {}
+ assert memmgr.free_blocks_end == {}
+ assert memmgr.blocks_by_size == [[], [], [], [], []]
+
+
+class TestAsmMemoryManager:
+
+ def setup_method(self, _):
+ self.memmgr = AsmMemoryManager(min_fragment=8,
+ num_indices=10,
+ large_alloc_size=8192)
+
+ def teardown_method(self, _):
+ self.memmgr._delete()
+
+ def test_malloc_simple(self):
+ for i in range(100):
+ while self.memmgr.total_memory_allocated < 16384:
+ reqsize = random.randrange(1, 200)
+ (start, stop) = self.memmgr.malloc(reqsize, reqsize)
+ assert reqsize <= stop - start < reqsize + 8
+ assert self.memmgr.total_memory_allocated in [8192, 16384]
+ self.teardown_method(None)
+ self.setup_method(None)
+
+ def test_random(self):
+ got = []
+ real_use = 0
+ prev_total = 0
+ iterations_without_allocating_more = 0
+ while True:
+ #
+ if got and (random.random() < 0.4 or len(got) == 1000):
+ # free
+ start, stop = got.pop(random.randrange(0, len(got)))
+ self.memmgr.free(start, stop)
+ real_use -= (stop - start)
+ assert real_use >= 0
+ #
+ else:
+ # allocate
+ reqsize = random.randrange(1, 200)
+ if random.random() < 0.5:
+ reqmaxsize = reqsize
+ else:
+ reqmaxsize = reqsize + random.randrange(0, 200)
+ (start, stop) = self.memmgr.malloc(reqsize, reqmaxsize)
+ assert reqsize <= stop - start < reqmaxsize + 8
+ for otherstart, otherstop in got: # no overlap
+ assert otherstop <= start or stop <= otherstart
+ got.append((start, stop))
+ real_use += (stop - start)
+ if self.memmgr.total_memory_allocated == prev_total:
+ iterations_without_allocating_more += 1
+ if iterations_without_allocating_more == 40000:
+ break # ok
+ else:
+ new_total = self.memmgr.total_memory_allocated
+ iterations_without_allocating_more = 0
+ print real_use, new_total
+ # We seem to never see a printed value greater
+ # than 131072. Be reasonable and allow up to 147456.
+ assert new_total <= 147456
+ prev_total = new_total
More information about the Pypy-commit
mailing list