[pypy-commit] pypy numpy-record-dtypes: progress on storing record boxes (and reading). Not quite working , pdb left
fijal
noreply at buildbot.pypy.org
Mon Feb 13 22:40:36 CET 2012
Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-record-dtypes
Changeset: r52433:bd3a6f174333
Date: 2012-02-13 23:40 +0200
http://bitbucket.org/pypy/pypy/changeset/bd3a6f174333/
Log: progress on storing record boxes (and reading). Not quite working ,
pdb left where it's left
diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -170,24 +170,35 @@
pass
class W_VoidBox(W_FlexibleBox):
- def __init__(self, dtype, arr):
- self.arr = arr
- self.dtype = dtype
+ def __init__(self, arr, i):
+ self.arr = arr # we have to keep array alive
+ self.i = i
def get_dtype(self, space):
- return self.dtype
+ return self.arr.dtype
@unwrap_spec(item=str)
def descr_getitem(self, space, item):
try:
- ofs, dtype = self.dtype.fields[item]
+ ofs, dtype = self.arr.dtype.fields[item]
except KeyError:
- raise OperationError(space.w_KeyError, space.wrap("Field %s does not exist" % item))
- return dtype.itemtype.read(dtype, self.arr,
- dtype.itemtype.get_element_size(), 0, ofs)
+ raise OperationError(space.w_IndexError,
+ space.wrap("Field %s does not exist" % item))
+ self.arr.dtype.itemtype.get_element_size()
+ return dtype.itemtype.read(self.arr,
+ dtype.itemtype.get_element_size(), self.i,
+ ofs)
- def __del__(self):
- lltype.free(self.arr, flavor='raw', track_allocation=False)
+ @unwrap_spec(item=str)
+ def descr_setitem(self, space, item, w_value):
+ try:
+ ofs, dtype = self.arr.dtype.fields[item]
+ except KeyError:
+ raise OperationError(space.w_IndexError,
+ space.wrap("Field %s does not exist" % item))
+ dtype.itemtype.store(self.arr,
+ dtype.itemtype.get_element_size(), 0, ofs,
+ dtype.coerce(space, w_value))
class W_CharacterBox(W_FlexibleBox):
pass
@@ -328,6 +339,7 @@
W_VoidBox.typedef = TypeDef("void", W_FlexibleBox.typedef,
__module__ = "numpypy",
__getitem__ = interp2app(W_VoidBox.descr_getitem),
+ __setitem__ = interp2app(W_VoidBox.descr_setitem),
)
W_CharacterBox.typedef = TypeDef("character", W_FlexibleBox.typedef,
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -8,7 +8,6 @@
from pypy.module.micronumpy import types, interp_boxes
from pypy.rlib.objectmodel import specialize
from pypy.rlib.rarithmetic import LONG_BIT, r_longlong, r_ulonglong
-from pypy.rpython.lltypesystem import lltype
UNSIGNEDLTR = "u"
@@ -19,8 +18,6 @@
STRINGLTR = 'S'
UNICODELTR = 'U'
-VOID_STORAGE = lltype.Array(lltype.Char, hints={'nolength': True, 'render_as_void': True})
-
class W_Dtype(Wrappable):
_immutable_fields_ = ["itemtype", "num", "kind"]
@@ -37,29 +34,22 @@
self.fields = fields
self.fieldnames = fieldnames
- def malloc(self, length):
- # XXX find out why test_zjit explodes with tracking of allocations
- return lltype.malloc(VOID_STORAGE,
- self.itemtype.get_element_size() * length,
- zero=True, flavor="raw",
- track_allocation=False, add_memory_pressure=True)
-
@specialize.argtype(1)
def box(self, value):
return self.itemtype.box(value)
def coerce(self, space, w_item):
- return self.itemtype.coerce(space, w_item)
+ return self.itemtype.coerce(space, self, w_item)
- def getitem(self, storage, i):
- return self.itemtype.read(self, storage, self.itemtype.get_element_size(), i, 0)
+ def getitem(self, arr, i):
+ return self.itemtype.read(arr, self.itemtype.get_element_size(), i, 0)
- def getitem_bool(self, storage, i):
+ def getitem_bool(self, arr, i):
isize = self.itemtype.get_element_size()
- return self.itemtype.read_bool(storage, isize, i, 0)
+ return self.itemtype.read_bool(arr.storage, isize, i, 0)
- def setitem(self, storage, i, box):
- self.itemtype.store(storage, self.itemtype.get_element_size(), i, 0, box)
+ def setitem(self, arr, i, box):
+ self.itemtype.store(arr, self.itemtype.get_element_size(), i, 0, box)
def fill(self, storage, box, start, stop):
self.itemtype.fill(storage, self.itemtype.get_element_size(), box, start, stop, 0)
@@ -138,7 +128,7 @@
ofs_and_items.append((offset, subdtype.itemtype))
offset += subdtype.itemtype.get_element_size()
fieldnames.append(fldname)
- itemtype = types.RecordType(ofs_and_items)
+ itemtype = types.RecordType(ofs_and_items, offset)
return W_Dtype(itemtype, 20, VOIDLTR, "void" + str(8 * itemtype.get_element_size()),
"V", space.gettypefor(interp_boxes.W_VoidBox), fields=fields,
fieldnames=fieldnames)
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -340,7 +340,7 @@
count_driver.jit_merge_point(arr=arr, frame=frame, iter=iter, s=s,
shapelen=shapelen)
iter = frame.get_final_iter()
- s += arr.dtype.getitem_bool(arr.storage, iter.offset)
+ s += arr.dtype.getitem_bool(arr, iter.offset)
frame.next(shapelen)
return s
@@ -361,7 +361,7 @@
filter_driver.jit_merge_point(concr=concr, argi=argi, ri=ri,
frame=frame, v=v, res=res, sig=sig,
shapelen=shapelen, self=self)
- if concr.dtype.getitem_bool(concr.storage, argi.offset):
+ if concr.dtype.getitem_bool(concr, argi.offset):
v = sig.eval(frame, self)
res.setitem(ri.offset, v)
ri = ri.next(1)
@@ -382,7 +382,7 @@
filter_set_driver.jit_merge_point(idx=idx, idxi=idxi, sig=sig,
frame=frame, arr=arr,
shapelen=shapelen)
- if idx.dtype.getitem_bool(idx.storage, idxi.offset):
+ if idx.dtype.getitem_bool(idx, idxi.offset):
sig.eval(frame, arr)
frame.next_from_second(1)
frame.next_first(shapelen)
@@ -904,7 +904,7 @@
if parent is not None:
self.storage = parent.storage
else:
- self.storage = dtype.malloc(size)
+ self.storage = dtype.itemtype.malloc(size)
self.order = order
self.dtype = dtype
if self.strides is None:
@@ -923,7 +923,7 @@
return self.dtype
def getitem(self, item):
- return self.dtype.getitem(self.storage, item)
+ return self.dtype.getitem(self, item)
def setitem(self, item, value):
self.invalidated()
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -11,8 +11,10 @@
class MockDtype(object):
- def malloc(self, size):
- return None
+ class itemtype(object):
+ @classmethod
+ def malloc(size):
+ return None
class TestNumArrayDirect(object):
@@ -1796,6 +1798,11 @@
def test_zeros(self):
from _numpypy import zeros
a = zeros(2, dtype=[('x', int), ('y', float)])
- raises(KeyError, 'a[0]["xyz"]')
+ raises(IndexError, 'a[0]["xyz"]')
assert a[0]['x'] == 0
assert a[0]['y'] == 0
+ raises(ValueError, "a[0] = (1, 2, 3)")
+ a[0]['x'] = 13
+ assert a[0]['x'] == 13
+ a[1] = (1, 2)
+ assert a[1]['y'] == 2
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -11,6 +11,10 @@
from pypy.rpython.lltypesystem import lltype, rffi
from pypy.rlib.rstruct.runpack import runpack
from pypy.tool.sourcetools import func_with_new_name
+from pypy.rlib import jit
+
+VOID_STORAGE = lltype.Array(lltype.Char, hints={'nolength': True,
+ 'render_as_void': True})
def simple_unary_op(func):
specialize.argtype(1)(func)
@@ -65,6 +69,13 @@
# exp = sin = cos = tan = arcsin = arccos = arctan = arcsinh = \
# arctanh = _unimplemented_ufunc
+ def malloc(self, length):
+ # XXX find out why test_zjit explodes with tracking of allocations
+ return lltype.malloc(VOID_STORAGE,
+ self.get_element_size() * length,
+ zero=True, flavor="raw",
+ track_allocation=False, add_memory_pressure=True)
+
class Primitive(object):
_mixin_ = True
@@ -79,7 +90,7 @@
assert isinstance(box, self.BoxType)
return box.value
- def coerce(self, space, w_item):
+ def coerce(self, space, dtype, w_item):
if isinstance(w_item, self.BoxType):
return w_item
return self.coerce_subtype(space, space.gettypefor(self.BoxType), w_item)
@@ -104,19 +115,19 @@
return libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T),
width, storage, i, offset)
- def read(self, dtype, storage, width, i, offset):
- return self.box(self._read(storage, width, i, offset))
+ def read(self, arr, width, i, offset):
+ return self.box(self._read(arr.storage, width, i, offset))
- def read_bool(self, storage, width, i, offset):
- return bool(self.for_computation(self._read(storage, width, i, offset)))
+ def read_bool(self, arr, width, i, offset):
+ return bool(self.for_computation(self._read(arr.storage, width, i, offset)))
def _write(self, storage, width, i, offset, value):
libffi.array_setitem(clibffi.cast_type_to_ffitype(self.T),
width, storage, i, offset, value)
- def store(self, storage, width, i, offset, box):
- self._write(storage, width, i, offset, self.unbox(box))
+ def store(self, arr, width, i, offset, box):
+ self._write(arr.storage, width, i, offset, self.unbox(box))
def fill(self, storage, width, box, start, stop, offset):
value = self.unbox(box)
@@ -595,10 +606,9 @@
format_code = "d"
class CompositeType(BaseType):
- def __init__(self, offsets_and_types):
- self.offsets_and_types = offsets_and_types
- last_item = offsets_and_types[-1]
- self.size = last_item[0] + last_item[1].get_element_size()
+ def __init__(self, offsets_and_fields, size):
+ self.offsets_and_fields = offsets_and_fields
+ self.size = size
def get_element_size(self):
return self.size
@@ -614,7 +624,10 @@
class StringType(BaseType, BaseStringType):
T = lltype.Char
-VoidType = StringType # why not?
+
+class VoidType(BaseType, BaseStringType):
+ T = lltype.Char
+
NonNativeVoidType = VoidType
NonNativeStringType = StringType
@@ -624,11 +637,41 @@
NonNativeUnicodeType = UnicodeType
class RecordType(CompositeType):
- def read(self, dtype, storage, width, i, offset):
- arr = dtype.malloc(1)
- for j in range(width):
- arr[j] = storage[i + j]
- return interp_boxes.W_VoidBox(dtype, arr)
+ T = lltype.Char
+
+ def read(self, arr, width, i, offset):
+ return interp_boxes.W_VoidBox(arr, i)
+
+ @jit.unroll_safe
+ def coerce(self, space, dtype, w_item):
+ from pypy.module.micronumpy.interp_numarray import W_NDimArray
+ # we treat every sequence as sequence, no special support
+ # for arrays
+ if not space.issequence_w(w_item):
+ raise OperationError(space.w_TypeError, space.wrap(
+ "expected sequence"))
+ if len(self.offsets_and_fields) != space.int_w(space.len(w_item)):
+ raise OperationError(space.w_ValueError, space.wrap(
+ "wrong length"))
+ items_w = space.fixedview(w_item)
+ # XXX optimize it out one day, but for now we just allocate an
+ # array
+ arr = W_NDimArray(1, [1], dtype)
+ for i in range(len(items_w)):
+ subdtype = dtype.fields[dtype.fieldnames[i]][1]
+ ofs, itemtype = self.offsets_and_fields[i]
+ w_item = items_w[i]
+ w_box = itemtype.coerce(space, subdtype, w_item)
+ width = itemtype.get_element_size()
+ import pdb
+ pdb.set_trace()
+ itemtype.store(arr, width, 0, ofs, w_box)
+ return interp_boxes.W_VoidBox(arr, 0)
+
+ @jit.unroll_safe
+ def store(self, arr, width, i, ofs, box):
+ for k in range(width):
+ arr[k + i] = box.arr.storage[k + box.i]
for tp in [Int32, Int64]:
if tp.T == lltype.Signed:
More information about the pypy-commit
mailing list