[pypy-commit] pypy numpy-full-fromstring: Wrote some really ugly code to deal with all the corner cases of numpy fromstring error handling

[jterrace]

Author: Jeff Terrace <jterrace at gmail.com>
Branch: numpy-full-fromstring
Changeset: r50475:8c97ceda4c34
Date: 2011-12-13 15:51 -0500
http://bitbucket.org/pypy/pypy/changeset/8c97ceda4c34/

Log:	Wrote some really ugly code to deal with all the corner cases of
	numpy fromstring error handling

diff --git a/pypy/module/micronumpy/interp_support.py b/pypy/module/micronumpy/interp_support.py
--- a/pypy/module/micronumpy/interp_support.py
+++ b/pypy/module/micronumpy/interp_support.py
@@ -2,14 +2,17 @@
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.module.micronumpy import interp_dtype
+from pypy.objspace.std.strutil import strip_spaces
 
 
 FLOAT_SIZE = rffi.sizeof(lltype.Float)
 
 def _fromstring_text(space, s, count, sep, length, dtype):
-    import string
     from pypy.module.micronumpy.interp_numarray import W_NDimArray
 
+    sep_stripped = strip_spaces(sep)
+    skip_bad_vals = True if len(sep_stripped) == 0 else False
+
     A = []
     num_items = 0
     ptr = 0
@@ -18,10 +21,28 @@
         nextptr = s.find(sep, ptr)
         if nextptr < 0:
             nextptr = length
-        piece = s[ptr:nextptr]
-        #FIXME: need to check piece.isspace() also, but does not translate
-        if len(piece) > 0:
-            val = dtype.coerce(space, space.wrap(piece))
+        piece = strip_spaces(s[ptr:nextptr])
+        if len(piece) > 0 or not skip_bad_vals:
+            if len(piece) == 0 and not skip_bad_vals:
+                val = dtype.itemtype.default_fromstring(space)
+            else:
+                try:
+                    val = dtype.coerce(space, space.wrap(piece))
+                except OperationError, e:
+                    if not e.match(space, space.w_ValueError):
+                        raise
+                    gotit = False
+                    while not gotit and len(piece) > 0:
+                        piece = piece[:-1]
+                        try:
+                            val = dtype.coerce(space, space.wrap(piece))
+                            gotit = True
+                        except OperationError, e:
+                            if not e.match(space, space.w_ValueError):
+                                raise
+                    if not gotit:
+                        val = dtype.itemtype.default_fromstring(space)
+                    nextptr = length
             A.append(val)
             num_items += 1
         ptr = nextptr + 1
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1173,7 +1173,7 @@
         cls.w_float64val = cls.space.wrap(struct.pack('d', 300.4))
 
     def test_fromstring(self):
-        from numpypy import fromstring, uint8, float32, int32
+        from numpypy import fromstring, array, uint8, float32, int32
         a = fromstring(self.data)
         for i in range(4):
             assert a[i] == i + 1
@@ -1201,12 +1201,28 @@
         assert g[0] == 1
         assert g[1] == 2
         assert g[2] == 3
-        #FIXME: below should work
-        #h = fromstring("1, , 2, 3", dtype=uint8, sep=",")
-        #assert len(h) == 3
-        #assert h[0] == 1
-        #assert h[1] == 2
-        #assert h[2] == 3
+        h = fromstring("1, , 2, 3", dtype=uint8, sep=",")
+        assert h.tolist() == [1,0,2,3]
+        i = fromstring("1    2 3", dtype=uint8, sep=" ")
+        assert i.tolist() == [1,2,3]
+        j = fromstring("1\t\t\t\t2\t3", dtype=uint8, sep="\t")
+        assert j.tolist() == [1,2,3]
+        k = fromstring("1,x,2,3", dtype=uint8, sep=",")
+        assert k.tolist() == [1,0]
+        l = fromstring("1,x,2,3", dtype='float32', sep=",")
+        assert l.tolist() == [1.0,-1.0]
+        m = fromstring("1,,2,3", sep=",")
+        assert m.tolist() == [1.0,-1.0,2.0,3.0]
+        n = fromstring("3.4 2.0 3.8 2.2", dtype=int32, sep=" ")
+        assert n.tolist() == [3]
+        o = fromstring("1.0 2f.0f 3.8 2.2", dtype=float32, sep=" ")
+        assert len(o) == 2
+        assert o[0] == 1.0
+        assert o[1] == 2.0
+        p = fromstring("1.0,,2.0,3.0", sep=",")
+        assert p.tolist() == [1.0, -1.0, 2.0, 3.0]
+        q = fromstring("1.0,,2.0,3.0", sep=" ")
+        assert q.tolist() == [1.0]
         
     def test_fromstring_types(self):
         from numpypy import fromstring
@@ -1241,8 +1257,6 @@
         raises(ValueError, fromstring, "\x01\x03\x03", dtype=uint16)
         #5 bytes is larger than 3 bytes
         raises(ValueError, fromstring, "\x01\x02\x03", count=5, dtype=uint8)
-        #can't cast floats to ints with fromstring
-        raises(ValueError, fromstring, "3.4 2.0 3.8 2.2", dtype=int32, sep=" ")
 
 
 class AppTestRepr(BaseNumpyAppTest):
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -87,6 +87,9 @@
     def _coerce(self, space, w_item):
         raise NotImplementedError
 
+    def default_fromstring(self, space):
+        raise NotImplementedError
+
     def read(self, storage, width, i, offset):
         return self.box(libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T),
             width, storage, i, offset
@@ -213,6 +216,9 @@
 
     def for_computation(self, v):
         return widen(v)
+    
+    def default_fromstring(self, space):
+        return self._coerce(space, space.wrap(0))
 
     @simple_binary_op
     def div(self, v1, v2):
@@ -320,6 +326,9 @@
     def for_computation(self, v):
         return float(v)
 
+    def default_fromstring(self, space):
+        return self._coerce(space, space.wrap(-1.0))
+
     @simple_binary_op
     def div(self, v1, v2):
         try: