[pypy-commit] pypy propogate-nans: fix off-by-one in edge case

[mattip]

Author: mattip <matti.picus at gmail.com>
Branch: propogate-nans
Changeset: r80588:9ea713544f54
Date: 2015-11-08 08:34 +0200
http://bitbucket.org/pypy/pypy/changeset/9ea713544f54/

Log:	fix off-by-one in edge case

diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -1853,10 +1853,14 @@
         assert a.view('S16')[0] == '\x01' + '\x00' * 7 + '\x02'
 
     def test_half_conversions(self):
+        from numpy import array, arange
+        from math import isnan, isinf
+        e = array([0, -1, -float('inf'), float('nan'), 6], dtype='float16')
+        assert map(isnan, e) == [False, False, False, True, False]
+        assert map(isinf, e) == [False, False, True, False, False]
         # numpy preserves value for uint16 -> cast_as_float16 -> 
         #     convert_to_float64 -> convert_to_float16 -> uint16
         #  even for float16 various float16 nans
-        from numpy import array, arange
         all_f16 = arange(0xfe00, 0xffff, dtype='uint16')
         all_f16.dtype = 'float16'
         all_f32 = array(all_f16, dtype='float32')
diff --git a/rpython/rlib/rstruct/ieee.py b/rpython/rlib/rstruct/ieee.py
--- a/rpython/rlib/rstruct/ieee.py
+++ b/rpython/rlib/rstruct/ieee.py
@@ -149,13 +149,13 @@
         exp = MAX_EXP - MIN_EXP + 2
     elif rfloat.isnan(x):
         asint = cast(ULONGLONG, float2longlong(x))
-        mant = asint & ((r_ulonglong(1) << 52) - 1)
+        mant = asint & ((r_ulonglong(1) << 51) - 1)
         sign = asint >> 63
         # shift off lower bits, perhaps losing data
         if MANT_DIG <= 52:
             mant = mant >> (52 - MANT_DIG)
         if mant == 0:
-            mant = r_ulonglong(1) << (MANT_DIG-2)
+            mant = r_ulonglong(1) << (MANT_DIG - 2) - 1
         exp = MAX_EXP - MIN_EXP + 2
     elif x == 0.0:
         mant = r_ulonglong(0)
@@ -214,7 +214,9 @@
         exp = MAX_EXP - MIN_EXP + 2
     elif rfloat.isnan(x):  # rfloat.isnan(x):
         asint = cast(ULONGLONG, float2longlong(x))
-        mant = asint & ((r_ulonglong(1) << 52) - 1)
+        mant = asint & ((r_ulonglong(1) << 51) - 1)
+        if mant == 0:
+            mant = r_ulonglong(1) << (MANT_DIG-2) - 1
         sign = asint < 0
         exp = MAX_EXP - MIN_EXP + 2
     elif x == 0.0: