[pypy-commit] pypy set-strategies: merge default
l.diekmann
noreply at buildbot.pypy.org
Wed Feb 1 13:48:44 CET 2012
Author: l.diekmann
Branch: set-strategies
Changeset: r52011:1317685b0c35
Date: 2012-01-30 14:16 +0000
http://bitbucket.org/pypy/pypy/changeset/1317685b0c35/
Log: merge default
diff too long, truncating to 10000 out of 13220 lines
diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -2,6 +2,9 @@
*.py[co]
*~
.*.swp
+.idea
+.project
+.pydevproject
syntax: regexp
^testresult$
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -73,8 +73,12 @@
class Field(object):
def __init__(self, name, offset, size, ctype, num, is_bitfield):
- for k in ('name', 'offset', 'size', 'ctype', 'num', 'is_bitfield'):
- self.__dict__[k] = locals()[k]
+ self.__dict__['name'] = name
+ self.__dict__['offset'] = offset
+ self.__dict__['size'] = size
+ self.__dict__['ctype'] = ctype
+ self.__dict__['num'] = num
+ self.__dict__['is_bitfield'] = is_bitfield
def __setattr__(self, name, value):
raise AttributeError(name)
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -20,6 +20,8 @@
# 2. Altered source versions must be plainly marked as such, and must not be
# misrepresented as being the original software.
# 3. This notice may not be removed or altered from any source distribution.
+#
+# Note: This software has been modified for use in PyPy.
from ctypes import c_void_p, c_int, c_double, c_int64, c_char_p, cdll
from ctypes import POINTER, byref, string_at, CFUNCTYPE, cast
@@ -27,7 +29,6 @@
from collections import OrderedDict
import datetime
import sys
-import time
import weakref
from threading import _get_ident as thread_get_ident
@@ -606,7 +607,7 @@
def authorizer(userdata, action, arg1, arg2, dbname, source):
try:
return int(callback(action, arg1, arg2, dbname, source))
- except Exception, e:
+ except Exception:
return SQLITE_DENY
c_authorizer = AUTHORIZER(authorizer)
@@ -653,7 +654,7 @@
if not aggregate_ptr[0]:
try:
aggregate = cls()
- except Exception, e:
+ except Exception:
msg = ("user-defined aggregate's '__init__' "
"method raised error")
sqlite.sqlite3_result_error(context, msg, len(msg))
@@ -667,7 +668,7 @@
params = _convert_params(context, argc, c_params)
try:
aggregate.step(*params)
- except Exception, e:
+ except Exception:
msg = ("user-defined aggregate's 'step' "
"method raised error")
sqlite.sqlite3_result_error(context, msg, len(msg))
@@ -683,7 +684,7 @@
aggregate = self.aggregate_instances[aggregate_ptr[0]]
try:
val = aggregate.finalize()
- except Exception, e:
+ except Exception:
msg = ("user-defined aggregate's 'finalize' "
"method raised error")
sqlite.sqlite3_result_error(context, msg, len(msg))
@@ -771,7 +772,7 @@
self.statement.item = None
self.statement.exhausted = True
- if self.statement.kind == DML or self.statement.kind == DDL:
+ if self.statement.kind == DML:
self.statement.reset()
self.rowcount = -1
@@ -791,7 +792,7 @@
if self.statement.kind == DML:
self.connection._begin()
else:
- raise ProgrammingError, "executemany is only for DML statements"
+ raise ProgrammingError("executemany is only for DML statements")
self.rowcount = 0
for params in many_params:
@@ -861,8 +862,6 @@
except StopIteration:
return None
- return nextrow
-
def fetchmany(self, size=None):
self._check_closed()
self._check_reset()
@@ -915,7 +914,7 @@
def __init__(self, connection, sql):
self.statement = None
if not isinstance(sql, str):
- raise ValueError, "sql must be a string"
+ raise ValueError("sql must be a string")
self.con = connection
self.sql = sql # DEBUG ONLY
first_word = self._statement_kind = sql.lstrip().split(" ")[0].upper()
@@ -944,8 +943,8 @@
raise self.con._get_exception(ret)
self.con._remember_statement(self)
if _check_remaining_sql(next_char.value):
- raise Warning, "One and only one statement required: %r" % (
- next_char.value,)
+ raise Warning("One and only one statement required: %r" % (
+ next_char.value,))
# sql_char should remain alive until here
self._build_row_cast_map()
@@ -1016,7 +1015,7 @@
elif type(param) is buffer:
sqlite.sqlite3_bind_blob(self.statement, idx, str(param), len(param), SQLITE_TRANSIENT)
else:
- raise InterfaceError, "parameter type %s is not supported" % str(type(param))
+ raise InterfaceError("parameter type %s is not supported" % str(type(param)))
def set_params(self, params):
ret = sqlite.sqlite3_reset(self.statement)
@@ -1045,11 +1044,11 @@
for idx in range(1, sqlite.sqlite3_bind_parameter_count(self.statement) + 1):
param_name = sqlite.sqlite3_bind_parameter_name(self.statement, idx)
if param_name is None:
- raise ProgrammingError, "need named parameters"
+ raise ProgrammingError("need named parameters")
param_name = param_name[1:]
try:
param = params[param_name]
- except KeyError, e:
+ except KeyError:
raise ProgrammingError("missing parameter '%s'" %param)
self.set_param(idx, param)
@@ -1260,7 +1259,7 @@
params = _convert_params(context, nargs, c_params)
try:
val = real_cb(*params)
- except Exception, e:
+ except Exception:
msg = "user-defined function raised exception"
sqlite.sqlite3_result_error(context, msg, len(msg))
else:
diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -13,7 +13,7 @@
Sources for time zone and DST data: http://www.twinsun.com/tz/tz-link.htm
This was originally copied from the sandbox of the CPython CVS repository.
-Thanks to Tim Peters for suggesting using it.
+Thanks to Tim Peters for suggesting using it.
"""
import time as _time
@@ -271,6 +271,8 @@
raise ValueError("%s()=%d, must be in -1439..1439" % (name, offset))
def _check_date_fields(year, month, day):
+ if not isinstance(year, (int, long)):
+ raise TypeError('int expected')
if not MINYEAR <= year <= MAXYEAR:
raise ValueError('year must be in %d..%d' % (MINYEAR, MAXYEAR), year)
if not 1 <= month <= 12:
@@ -280,6 +282,8 @@
raise ValueError('day must be in 1..%d' % dim, day)
def _check_time_fields(hour, minute, second, microsecond):
+ if not isinstance(hour, (int, long)):
+ raise TypeError('int expected')
if not 0 <= hour <= 23:
raise ValueError('hour must be in 0..23', hour)
if not 0 <= minute <= 59:
@@ -543,61 +547,76 @@
self = object.__new__(cls)
- self.__days = d
- self.__seconds = s
- self.__microseconds = us
+ self._days = d
+ self._seconds = s
+ self._microseconds = us
if abs(d) > 999999999:
raise OverflowError("timedelta # of days is too large: %d" % d)
return self
def __repr__(self):
- if self.__microseconds:
+ if self._microseconds:
return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__,
- self.__days,
- self.__seconds,
- self.__microseconds)
- if self.__seconds:
+ self._days,
+ self._seconds,
+ self._microseconds)
+ if self._seconds:
return "%s(%d, %d)" % ('datetime.' + self.__class__.__name__,
- self.__days,
- self.__seconds)
- return "%s(%d)" % ('datetime.' + self.__class__.__name__, self.__days)
+ self._days,
+ self._seconds)
+ return "%s(%d)" % ('datetime.' + self.__class__.__name__, self._days)
def __str__(self):
- mm, ss = divmod(self.__seconds, 60)
+ mm, ss = divmod(self._seconds, 60)
hh, mm = divmod(mm, 60)
s = "%d:%02d:%02d" % (hh, mm, ss)
- if self.__days:
+ if self._days:
def plural(n):
return n, abs(n) != 1 and "s" or ""
- s = ("%d day%s, " % plural(self.__days)) + s
- if self.__microseconds:
- s = s + ".%06d" % self.__microseconds
+ s = ("%d day%s, " % plural(self._days)) + s
+ if self._microseconds:
+ s = s + ".%06d" % self._microseconds
return s
- days = property(lambda self: self.__days, doc="days")
- seconds = property(lambda self: self.__seconds, doc="seconds")
- microseconds = property(lambda self: self.__microseconds,
- doc="microseconds")
-
def total_seconds(self):
return ((self.days * 86400 + self.seconds) * 10**6
+ self.microseconds) / 1e6
+ # Read-only field accessors
+ @property
+ def days(self):
+ """days"""
+ return self._days
+
+ @property
+ def seconds(self):
+ """seconds"""
+ return self._seconds
+
+ @property
+ def microseconds(self):
+ """microseconds"""
+ return self._microseconds
+
def __add__(self, other):
if isinstance(other, timedelta):
# for CPython compatibility, we cannot use
# our __class__ here, but need a real timedelta
- return timedelta(self.__days + other.__days,
- self.__seconds + other.__seconds,
- self.__microseconds + other.__microseconds)
+ return timedelta(self._days + other._days,
+ self._seconds + other._seconds,
+ self._microseconds + other._microseconds)
return NotImplemented
__radd__ = __add__
def __sub__(self, other):
if isinstance(other, timedelta):
- return self + -other
+ # for CPython compatibility, we cannot use
+ # our __class__ here, but need a real timedelta
+ return timedelta(self._days - other._days,
+ self._seconds - other._seconds,
+ self._microseconds - other._microseconds)
return NotImplemented
def __rsub__(self, other):
@@ -606,17 +625,17 @@
return NotImplemented
def __neg__(self):
- # for CPython compatibility, we cannot use
- # our __class__ here, but need a real timedelta
- return timedelta(-self.__days,
- -self.__seconds,
- -self.__microseconds)
+ # for CPython compatibility, we cannot use
+ # our __class__ here, but need a real timedelta
+ return timedelta(-self._days,
+ -self._seconds,
+ -self._microseconds)
def __pos__(self):
return self
def __abs__(self):
- if self.__days < 0:
+ if self._days < 0:
return -self
else:
return self
@@ -625,81 +644,81 @@
if isinstance(other, (int, long)):
# for CPython compatibility, we cannot use
# our __class__ here, but need a real timedelta
- return timedelta(self.__days * other,
- self.__seconds * other,
- self.__microseconds * other)
+ return timedelta(self._days * other,
+ self._seconds * other,
+ self._microseconds * other)
return NotImplemented
__rmul__ = __mul__
def __div__(self, other):
if isinstance(other, (int, long)):
- usec = ((self.__days * (24*3600L) + self.__seconds) * 1000000 +
- self.__microseconds)
+ usec = ((self._days * (24*3600L) + self._seconds) * 1000000 +
+ self._microseconds)
return timedelta(0, 0, usec // other)
return NotImplemented
__floordiv__ = __div__
- # Comparisons.
+ # Comparisons of timedelta objects with other.
def __eq__(self, other):
if isinstance(other, timedelta):
- return self.__cmp(other) == 0
+ return self._cmp(other) == 0
else:
return False
def __ne__(self, other):
if isinstance(other, timedelta):
- return self.__cmp(other) != 0
+ return self._cmp(other) != 0
else:
return True
def __le__(self, other):
if isinstance(other, timedelta):
- return self.__cmp(other) <= 0
+ return self._cmp(other) <= 0
else:
_cmperror(self, other)
def __lt__(self, other):
if isinstance(other, timedelta):
- return self.__cmp(other) < 0
+ return self._cmp(other) < 0
else:
_cmperror(self, other)
def __ge__(self, other):
if isinstance(other, timedelta):
- return self.__cmp(other) >= 0
+ return self._cmp(other) >= 0
else:
_cmperror(self, other)
def __gt__(self, other):
if isinstance(other, timedelta):
- return self.__cmp(other) > 0
+ return self._cmp(other) > 0
else:
_cmperror(self, other)
- def __cmp(self, other):
+ def _cmp(self, other):
assert isinstance(other, timedelta)
- return cmp(self.__getstate(), other.__getstate())
+ return cmp(self._getstate(), other._getstate())
def __hash__(self):
- return hash(self.__getstate())
+ return hash(self._getstate())
def __nonzero__(self):
- return (self.__days != 0 or
- self.__seconds != 0 or
- self.__microseconds != 0)
+ return (self._days != 0 or
+ self._seconds != 0 or
+ self._microseconds != 0)
# Pickle support.
__safe_for_unpickling__ = True # For Python 2.2
- def __getstate(self):
- return (self.__days, self.__seconds, self.__microseconds)
+ def _getstate(self):
+ return (self._days, self._seconds, self._microseconds)
def __reduce__(self):
- return (self.__class__, self.__getstate())
+ return (self.__class__, self._getstate())
timedelta.min = timedelta(-999999999)
timedelta.max = timedelta(days=999999999, hours=23, minutes=59, seconds=59,
@@ -749,25 +768,26 @@
return self
_check_date_fields(year, month, day)
self = object.__new__(cls)
- self.__year = year
- self.__month = month
- self.__day = day
+ self._year = year
+ self._month = month
+ self._day = day
return self
# Additional constructors
+ @classmethod
def fromtimestamp(cls, t):
"Construct a date from a POSIX timestamp (like time.time())."
y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t)
return cls(y, m, d)
- fromtimestamp = classmethod(fromtimestamp)
+ @classmethod
def today(cls):
"Construct a date from time.time()."
t = _time.time()
return cls.fromtimestamp(t)
- today = classmethod(today)
+ @classmethod
def fromordinal(cls, n):
"""Contruct a date from a proleptic Gregorian ordinal.
@@ -776,16 +796,24 @@
"""
y, m, d = _ord2ymd(n)
return cls(y, m, d)
- fromordinal = classmethod(fromordinal)
# Conversions to string
def __repr__(self):
- "Convert to formal string, for repr()."
+ """Convert to formal string, for repr().
+
+ >>> dt = datetime(2010, 1, 1)
+ >>> repr(dt)
+ 'datetime.datetime(2010, 1, 1, 0, 0)'
+
+ >>> dt = datetime(2010, 1, 1, tzinfo=timezone.utc)
+ >>> repr(dt)
+ 'datetime.datetime(2010, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)'
+ """
return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__,
- self.__year,
- self.__month,
- self.__day)
+ self._year,
+ self._month,
+ self._day)
# XXX These shouldn't depend on time.localtime(), because that
# clips the usable dates to [1970 .. 2038). At least ctime() is
# easily done without using strftime() -- that's better too because
@@ -793,12 +821,20 @@
def ctime(self):
"Format a la ctime()."
- return tmxxx(self.__year, self.__month, self.__day).ctime()
+ return tmxxx(self._year, self._month, self._day).ctime()
def strftime(self, fmt):
"Format using strftime()."
return _wrap_strftime(self, fmt, self.timetuple())
+ def __format__(self, fmt):
+ if not isinstance(fmt, (str, unicode)):
+ raise ValueError("__format__ excepts str or unicode, not %s" %
+ fmt.__class__.__name__)
+ if len(fmt) != 0:
+ return self.strftime(fmt)
+ return str(self)
+
def isoformat(self):
"""Return the date formatted according to ISO.
@@ -808,29 +844,31 @@
- http://www.w3.org/TR/NOTE-datetime
- http://www.cl.cam.ac.uk/~mgk25/iso-time.html
"""
- return "%04d-%02d-%02d" % (self.__year, self.__month, self.__day)
+ return "%04d-%02d-%02d" % (self._year, self._month, self._day)
__str__ = isoformat
- def __format__(self, format):
- if not isinstance(format, (str, unicode)):
- raise ValueError("__format__ excepts str or unicode, not %s" %
- format.__class__.__name__)
- if not format:
- return str(self)
- return self.strftime(format)
+ # Read-only field accessors
+ @property
+ def year(self):
+ """year (1-9999)"""
+ return self._year
- # Read-only field accessors
- year = property(lambda self: self.__year,
- doc="year (%d-%d)" % (MINYEAR, MAXYEAR))
- month = property(lambda self: self.__month, doc="month (1-12)")
- day = property(lambda self: self.__day, doc="day (1-31)")
+ @property
+ def month(self):
+ """month (1-12)"""
+ return self._month
+
+ @property
+ def day(self):
+ """day (1-31)"""
+ return self._day
# Standard conversions, __cmp__, __hash__ (and helpers)
def timetuple(self):
"Return local time tuple compatible with time.localtime()."
- return _build_struct_time(self.__year, self.__month, self.__day,
+ return _build_struct_time(self._year, self._month, self._day,
0, 0, 0, -1)
def toordinal(self):
@@ -839,24 +877,24 @@
January 1 of year 1 is day 1. Only the year, month and day values
contribute to the result.
"""
- return _ymd2ord(self.__year, self.__month, self.__day)
+ return _ymd2ord(self._year, self._month, self._day)
def replace(self, year=None, month=None, day=None):
"""Return a new date with new values for the specified fields."""
if year is None:
- year = self.__year
+ year = self._year
if month is None:
- month = self.__month
+ month = self._month
if day is None:
- day = self.__day
+ day = self._day
_check_date_fields(year, month, day)
return date(year, month, day)
- # Comparisons.
+ # Comparisons of date objects with other.
def __eq__(self, other):
if isinstance(other, date):
- return self.__cmp(other) == 0
+ return self._cmp(other) == 0
elif hasattr(other, "timetuple"):
return NotImplemented
else:
@@ -864,7 +902,7 @@
def __ne__(self, other):
if isinstance(other, date):
- return self.__cmp(other) != 0
+ return self._cmp(other) != 0
elif hasattr(other, "timetuple"):
return NotImplemented
else:
@@ -872,7 +910,7 @@
def __le__(self, other):
if isinstance(other, date):
- return self.__cmp(other) <= 0
+ return self._cmp(other) <= 0
elif hasattr(other, "timetuple"):
return NotImplemented
else:
@@ -880,7 +918,7 @@
def __lt__(self, other):
if isinstance(other, date):
- return self.__cmp(other) < 0
+ return self._cmp(other) < 0
elif hasattr(other, "timetuple"):
return NotImplemented
else:
@@ -888,7 +926,7 @@
def __ge__(self, other):
if isinstance(other, date):
- return self.__cmp(other) >= 0
+ return self._cmp(other) >= 0
elif hasattr(other, "timetuple"):
return NotImplemented
else:
@@ -896,21 +934,21 @@
def __gt__(self, other):
if isinstance(other, date):
- return self.__cmp(other) > 0
+ return self._cmp(other) > 0
elif hasattr(other, "timetuple"):
return NotImplemented
else:
_cmperror(self, other)
- def __cmp(self, other):
+ def _cmp(self, other):
assert isinstance(other, date)
- y, m, d = self.__year, self.__month, self.__day
- y2, m2, d2 = other.__year, other.__month, other.__day
+ y, m, d = self._year, self._month, self._day
+ y2, m2, d2 = other._year, other._month, other._day
return cmp((y, m, d), (y2, m2, d2))
def __hash__(self):
"Hash."
- return hash(self.__getstate())
+ return hash(self._getstate())
# Computations
@@ -922,9 +960,9 @@
def __add__(self, other):
"Add a date to a timedelta."
if isinstance(other, timedelta):
- t = tmxxx(self.__year,
- self.__month,
- self.__day + other.days)
+ t = tmxxx(self._year,
+ self._month,
+ self._day + other.days)
self._checkOverflow(t.year)
result = date(t.year, t.month, t.day)
return result
@@ -966,9 +1004,9 @@
ISO calendar algorithm taken from
http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm
"""
- year = self.__year
+ year = self._year
week1monday = _isoweek1monday(year)
- today = _ymd2ord(self.__year, self.__month, self.__day)
+ today = _ymd2ord(self._year, self._month, self._day)
# Internally, week and day have origin 0
week, day = divmod(today - week1monday, 7)
if week < 0:
@@ -985,18 +1023,18 @@
__safe_for_unpickling__ = True # For Python 2.2
- def __getstate(self):
- yhi, ylo = divmod(self.__year, 256)
- return ("%c%c%c%c" % (yhi, ylo, self.__month, self.__day), )
+ def _getstate(self):
+ yhi, ylo = divmod(self._year, 256)
+ return ("%c%c%c%c" % (yhi, ylo, self._month, self._day), )
def __setstate(self, string):
if len(string) != 4 or not (1 <= ord(string[2]) <= 12):
raise TypeError("not enough arguments")
- yhi, ylo, self.__month, self.__day = map(ord, string)
- self.__year = yhi * 256 + ylo
+ yhi, ylo, self._month, self._day = map(ord, string)
+ self._year = yhi * 256 + ylo
def __reduce__(self):
- return (self.__class__, self.__getstate())
+ return (self.__class__, self._getstate())
_date_class = date # so functions w/ args named "date" can get at the class
@@ -1118,62 +1156,80 @@
return self
_check_tzinfo_arg(tzinfo)
_check_time_fields(hour, minute, second, microsecond)
- self.__hour = hour
- self.__minute = minute
- self.__second = second
- self.__microsecond = microsecond
+ self._hour = hour
+ self._minute = minute
+ self._second = second
+ self._microsecond = microsecond
self._tzinfo = tzinfo
return self
# Read-only field accessors
- hour = property(lambda self: self.__hour, doc="hour (0-23)")
- minute = property(lambda self: self.__minute, doc="minute (0-59)")
- second = property(lambda self: self.__second, doc="second (0-59)")
- microsecond = property(lambda self: self.__microsecond,
- doc="microsecond (0-999999)")
- tzinfo = property(lambda self: self._tzinfo, doc="timezone info object")
+ @property
+ def hour(self):
+ """hour (0-23)"""
+ return self._hour
+
+ @property
+ def minute(self):
+ """minute (0-59)"""
+ return self._minute
+
+ @property
+ def second(self):
+ """second (0-59)"""
+ return self._second
+
+ @property
+ def microsecond(self):
+ """microsecond (0-999999)"""
+ return self._microsecond
+
+ @property
+ def tzinfo(self):
+ """timezone info object"""
+ return self._tzinfo
# Standard conversions, __hash__ (and helpers)
- # Comparisons.
+ # Comparisons of time objects with other.
def __eq__(self, other):
if isinstance(other, time):
- return self.__cmp(other) == 0
+ return self._cmp(other) == 0
else:
return False
def __ne__(self, other):
if isinstance(other, time):
- return self.__cmp(other) != 0
+ return self._cmp(other) != 0
else:
return True
def __le__(self, other):
if isinstance(other, time):
- return self.__cmp(other) <= 0
+ return self._cmp(other) <= 0
else:
_cmperror(self, other)
def __lt__(self, other):
if isinstance(other, time):
- return self.__cmp(other) < 0
+ return self._cmp(other) < 0
else:
_cmperror(self, other)
def __ge__(self, other):
if isinstance(other, time):
- return self.__cmp(other) >= 0
+ return self._cmp(other) >= 0
else:
_cmperror(self, other)
def __gt__(self, other):
if isinstance(other, time):
- return self.__cmp(other) > 0
+ return self._cmp(other) > 0
else:
_cmperror(self, other)
- def __cmp(self, other):
+ def _cmp(self, other):
assert isinstance(other, time)
mytz = self._tzinfo
ottz = other._tzinfo
@@ -1187,23 +1243,23 @@
base_compare = myoff == otoff
if base_compare:
- return cmp((self.__hour, self.__minute, self.__second,
- self.__microsecond),
- (other.__hour, other.__minute, other.__second,
- other.__microsecond))
+ return cmp((self._hour, self._minute, self._second,
+ self._microsecond),
+ (other._hour, other._minute, other._second,
+ other._microsecond))
if myoff is None or otoff is None:
# XXX Buggy in 2.2.2.
raise TypeError("cannot compare naive and aware times")
- myhhmm = self.__hour * 60 + self.__minute - myoff
- othhmm = other.__hour * 60 + other.__minute - otoff
- return cmp((myhhmm, self.__second, self.__microsecond),
- (othhmm, other.__second, other.__microsecond))
+ myhhmm = self._hour * 60 + self._minute - myoff
+ othhmm = other._hour * 60 + other._minute - otoff
+ return cmp((myhhmm, self._second, self._microsecond),
+ (othhmm, other._second, other._microsecond))
def __hash__(self):
"""Hash."""
tzoff = self._utcoffset()
if not tzoff: # zero or None
- return hash(self.__getstate()[0])
+ return hash(self._getstate()[0])
h, m = divmod(self.hour * 60 + self.minute - tzoff, 60)
if 0 <= h < 24:
return hash(time(h, m, self.second, self.microsecond))
@@ -1227,14 +1283,14 @@
def __repr__(self):
"""Convert to formal string, for repr()."""
- if self.__microsecond != 0:
- s = ", %d, %d" % (self.__second, self.__microsecond)
- elif self.__second != 0:
- s = ", %d" % self.__second
+ if self._microsecond != 0:
+ s = ", %d, %d" % (self._second, self._microsecond)
+ elif self._second != 0:
+ s = ", %d" % self._second
else:
s = ""
s= "%s(%d, %d%s)" % ('datetime.' + self.__class__.__name__,
- self.__hour, self.__minute, s)
+ self._hour, self._minute, s)
if self._tzinfo is not None:
assert s[-1:] == ")"
s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")"
@@ -1246,8 +1302,8 @@
This is 'HH:MM:SS.mmmmmm+zz:zz', or 'HH:MM:SS+zz:zz' if
self.microsecond == 0.
"""
- s = _format_time(self.__hour, self.__minute, self.__second,
- self.__microsecond)
+ s = _format_time(self._hour, self._minute, self._second,
+ self._microsecond)
tz = self._tzstr()
if tz:
s += tz
@@ -1255,14 +1311,6 @@
__str__ = isoformat
- def __format__(self, format):
- if not isinstance(format, (str, unicode)):
- raise ValueError("__format__ excepts str or unicode, not %s" %
- format.__class__.__name__)
- if not format:
- return str(self)
- return self.strftime(format)
-
def strftime(self, fmt):
"""Format using strftime(). The date part of the timestamp passed
to underlying strftime should not be used.
@@ -1270,10 +1318,18 @@
# The year must be >= 1900 else Python's strftime implementation
# can raise a bogus exception.
timetuple = (1900, 1, 1,
- self.__hour, self.__minute, self.__second,
+ self._hour, self._minute, self._second,
0, 1, -1)
return _wrap_strftime(self, fmt, timetuple)
+ def __format__(self, fmt):
+ if not isinstance(fmt, (str, unicode)):
+ raise ValueError("__format__ excepts str or unicode, not %s" %
+ fmt.__class__.__name__)
+ if len(fmt) != 0:
+ return self.strftime(fmt)
+ return str(self)
+
# Timezone functions
def utcoffset(self):
@@ -1350,10 +1406,10 @@
__safe_for_unpickling__ = True # For Python 2.2
- def __getstate(self):
- us2, us3 = divmod(self.__microsecond, 256)
+ def _getstate(self):
+ us2, us3 = divmod(self._microsecond, 256)
us1, us2 = divmod(us2, 256)
- basestate = ("%c" * 6) % (self.__hour, self.__minute, self.__second,
+ basestate = ("%c" * 6) % (self._hour, self._minute, self._second,
us1, us2, us3)
if self._tzinfo is None:
return (basestate,)
@@ -1363,13 +1419,13 @@
def __setstate(self, string, tzinfo):
if len(string) != 6 or ord(string[0]) >= 24:
raise TypeError("an integer is required")
- self.__hour, self.__minute, self.__second, us1, us2, us3 = \
+ self._hour, self._minute, self._second, us1, us2, us3 = \
map(ord, string)
- self.__microsecond = (((us1 << 8) | us2) << 8) | us3
+ self._microsecond = (((us1 << 8) | us2) << 8) | us3
self._tzinfo = tzinfo
def __reduce__(self):
- return (time, self.__getstate())
+ return (time, self._getstate())
_time_class = time # so functions w/ args named "time" can get at the class
@@ -1378,9 +1434,11 @@
time.resolution = timedelta(microseconds=1)
class datetime(date):
+ """datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]])
- # XXX needs docstrings
- # See http://www.zope.org/Members/fdrake/DateTimeWiki/TimeZoneInfo
+ The year, month and day arguments are required. tzinfo may be None, or an
+ instance of a tzinfo subclass. The remaining arguments may be ints or longs.
+ """
def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0,
microsecond=0, tzinfo=None):
@@ -1393,24 +1451,43 @@
_check_time_fields(hour, minute, second, microsecond)
self = date.__new__(cls, year, month, day)
# XXX This duplicates __year, __month, __day for convenience :-(
- self.__year = year
- self.__month = month
- self.__day = day
- self.__hour = hour
- self.__minute = minute
- self.__second = second
- self.__microsecond = microsecond
+ self._year = year
+ self._month = month
+ self._day = day
+ self._hour = hour
+ self._minute = minute
+ self._second = second
+ self._microsecond = microsecond
self._tzinfo = tzinfo
return self
# Read-only field accessors
- hour = property(lambda self: self.__hour, doc="hour (0-23)")
- minute = property(lambda self: self.__minute, doc="minute (0-59)")
- second = property(lambda self: self.__second, doc="second (0-59)")
- microsecond = property(lambda self: self.__microsecond,
- doc="microsecond (0-999999)")
- tzinfo = property(lambda self: self._tzinfo, doc="timezone info object")
+ @property
+ def hour(self):
+ """hour (0-23)"""
+ return self._hour
+ @property
+ def minute(self):
+ """minute (0-59)"""
+ return self._minute
+
+ @property
+ def second(self):
+ """second (0-59)"""
+ return self._second
+
+ @property
+ def microsecond(self):
+ """microsecond (0-999999)"""
+ return self._microsecond
+
+ @property
+ def tzinfo(self):
+ """timezone info object"""
+ return self._tzinfo
+
+ @classmethod
def fromtimestamp(cls, t, tz=None):
"""Construct a datetime from a POSIX timestamp (like time.time()).
@@ -1438,37 +1515,42 @@
if tz is not None:
result = tz.fromutc(result)
return result
- fromtimestamp = classmethod(fromtimestamp)
+ @classmethod
def utcfromtimestamp(cls, t):
"Construct a UTC datetime from a POSIX timestamp (like time.time())."
- if 1 - (t % 1.0) < 0.0000005:
- t = float(int(t)) + 1
- if t < 0:
- t -= 1
+ t, frac = divmod(t, 1.0)
+ us = round(frac * 1e6)
+
+ # If timestamp is less than one microsecond smaller than a
+ # full second, us can be rounded up to 1000000. In this case,
+ # roll over to seconds, otherwise, ValueError is raised
+ # by the constructor.
+ if us == 1000000:
+ t += 1
+ us = 0
y, m, d, hh, mm, ss, weekday, jday, dst = _time.gmtime(t)
- us = int((t % 1.0) * 1000000)
ss = min(ss, 59) # clamp out leap seconds if the platform has them
return cls(y, m, d, hh, mm, ss, us)
- utcfromtimestamp = classmethod(utcfromtimestamp)
# XXX This is supposed to do better than we *can* do by using time.time(),
# XXX if the platform supports a more accurate way. The C implementation
# XXX uses gettimeofday on platforms that have it, but that isn't
# XXX available from Python. So now() may return different results
# XXX across the implementations.
+ @classmethod
def now(cls, tz=None):
"Construct a datetime from time.time() and optional time zone info."
t = _time.time()
return cls.fromtimestamp(t, tz)
- now = classmethod(now)
+ @classmethod
def utcnow(cls):
"Construct a UTC datetime from time.time()."
t = _time.time()
return cls.utcfromtimestamp(t)
- utcnow = classmethod(utcnow)
+ @classmethod
def combine(cls, date, time):
"Construct a datetime from a given date and a given time."
if not isinstance(date, _date_class):
@@ -1478,7 +1560,6 @@
return cls(date.year, date.month, date.day,
time.hour, time.minute, time.second, time.microsecond,
time.tzinfo)
- combine = classmethod(combine)
def timetuple(self):
"Return local time tuple compatible with time.localtime()."
@@ -1504,7 +1585,7 @@
def date(self):
"Return the date part."
- return date(self.__year, self.__month, self.__day)
+ return date(self._year, self._month, self._day)
def time(self):
"Return the time part, with tzinfo None."
@@ -1564,8 +1645,8 @@
def ctime(self):
"Format a la ctime()."
- t = tmxxx(self.__year, self.__month, self.__day, self.__hour,
- self.__minute, self.__second)
+ t = tmxxx(self._year, self._month, self._day, self._hour,
+ self._minute, self._second)
return t.ctime()
def isoformat(self, sep='T'):
@@ -1580,10 +1661,10 @@
Optional argument sep specifies the separator between date and
time, default 'T'.
"""
- s = ("%04d-%02d-%02d%c" % (self.__year, self.__month, self.__day,
+ s = ("%04d-%02d-%02d%c" % (self._year, self._month, self._day,
sep) +
- _format_time(self.__hour, self.__minute, self.__second,
- self.__microsecond))
+ _format_time(self._hour, self._minute, self._second,
+ self._microsecond))
off = self._utcoffset()
if off is not None:
if off < 0:
@@ -1596,13 +1677,13 @@
return s
def __repr__(self):
- "Convert to formal string, for repr()."
- L = [self.__year, self.__month, self.__day, # These are never zero
- self.__hour, self.__minute, self.__second, self.__microsecond]
+ """Convert to formal string, for repr()."""
+ L = [self._year, self._month, self._day, # These are never zero
+ self._hour, self._minute, self._second, self._microsecond]
if L[-1] == 0:
del L[-1]
if L[-1] == 0:
- del L[-1]
+ del L[-1]
s = ", ".join(map(str, L))
s = "%s(%s)" % ('datetime.' + self.__class__.__name__, s)
if self._tzinfo is not None:
@@ -1675,7 +1756,7 @@
def __eq__(self, other):
if isinstance(other, datetime):
- return self.__cmp(other) == 0
+ return self._cmp(other) == 0
elif hasattr(other, "timetuple") and not isinstance(other, date):
return NotImplemented
else:
@@ -1683,7 +1764,7 @@
def __ne__(self, other):
if isinstance(other, datetime):
- return self.__cmp(other) != 0
+ return self._cmp(other) != 0
elif hasattr(other, "timetuple") and not isinstance(other, date):
return NotImplemented
else:
@@ -1691,7 +1772,7 @@
def __le__(self, other):
if isinstance(other, datetime):
- return self.__cmp(other) <= 0
+ return self._cmp(other) <= 0
elif hasattr(other, "timetuple") and not isinstance(other, date):
return NotImplemented
else:
@@ -1699,7 +1780,7 @@
def __lt__(self, other):
if isinstance(other, datetime):
- return self.__cmp(other) < 0
+ return self._cmp(other) < 0
elif hasattr(other, "timetuple") and not isinstance(other, date):
return NotImplemented
else:
@@ -1707,7 +1788,7 @@
def __ge__(self, other):
if isinstance(other, datetime):
- return self.__cmp(other) >= 0
+ return self._cmp(other) >= 0
elif hasattr(other, "timetuple") and not isinstance(other, date):
return NotImplemented
else:
@@ -1715,13 +1796,13 @@
def __gt__(self, other):
if isinstance(other, datetime):
- return self.__cmp(other) > 0
+ return self._cmp(other) > 0
elif hasattr(other, "timetuple") and not isinstance(other, date):
return NotImplemented
else:
_cmperror(self, other)
- def __cmp(self, other):
+ def _cmp(self, other):
assert isinstance(other, datetime)
mytz = self._tzinfo
ottz = other._tzinfo
@@ -1737,12 +1818,12 @@
base_compare = myoff == otoff
if base_compare:
- return cmp((self.__year, self.__month, self.__day,
- self.__hour, self.__minute, self.__second,
- self.__microsecond),
- (other.__year, other.__month, other.__day,
- other.__hour, other.__minute, other.__second,
- other.__microsecond))
+ return cmp((self._year, self._month, self._day,
+ self._hour, self._minute, self._second,
+ self._microsecond),
+ (other._year, other._month, other._day,
+ other._hour, other._minute, other._second,
+ other._microsecond))
if myoff is None or otoff is None:
# XXX Buggy in 2.2.2.
raise TypeError("cannot compare naive and aware datetimes")
@@ -1756,13 +1837,13 @@
"Add a datetime and a timedelta."
if not isinstance(other, timedelta):
return NotImplemented
- t = tmxxx(self.__year,
- self.__month,
- self.__day + other.days,
- self.__hour,
- self.__minute,
- self.__second + other.seconds,
- self.__microsecond + other.microseconds)
+ t = tmxxx(self._year,
+ self._month,
+ self._day + other.days,
+ self._hour,
+ self._minute,
+ self._second + other.seconds,
+ self._microsecond + other.microseconds)
self._checkOverflow(t.year)
result = datetime(t.year, t.month, t.day,
t.hour, t.minute, t.second,
@@ -1780,11 +1861,11 @@
days1 = self.toordinal()
days2 = other.toordinal()
- secs1 = self.__second + self.__minute * 60 + self.__hour * 3600
- secs2 = other.__second + other.__minute * 60 + other.__hour * 3600
+ secs1 = self._second + self._minute * 60 + self._hour * 3600
+ secs2 = other._second + other._minute * 60 + other._hour * 3600
base = timedelta(days1 - days2,
secs1 - secs2,
- self.__microsecond - other.__microsecond)
+ self._microsecond - other._microsecond)
if self._tzinfo is other._tzinfo:
return base
myoff = self._utcoffset()
@@ -1792,13 +1873,13 @@
if myoff == otoff:
return base
if myoff is None or otoff is None:
- raise TypeError, "cannot mix naive and timezone-aware time"
+ raise TypeError("cannot mix naive and timezone-aware time")
return base + timedelta(minutes = otoff-myoff)
def __hash__(self):
tzoff = self._utcoffset()
if tzoff is None:
- return hash(self.__getstate()[0])
+ return hash(self._getstate()[0])
days = _ymd2ord(self.year, self.month, self.day)
seconds = self.hour * 3600 + (self.minute - tzoff) * 60 + self.second
return hash(timedelta(days, seconds, self.microsecond))
@@ -1807,12 +1888,12 @@
__safe_for_unpickling__ = True # For Python 2.2
- def __getstate(self):
- yhi, ylo = divmod(self.__year, 256)
- us2, us3 = divmod(self.__microsecond, 256)
+ def _getstate(self):
+ yhi, ylo = divmod(self._year, 256)
+ us2, us3 = divmod(self._microsecond, 256)
us1, us2 = divmod(us2, 256)
- basestate = ("%c" * 10) % (yhi, ylo, self.__month, self.__day,
- self.__hour, self.__minute, self.__second,
+ basestate = ("%c" * 10) % (yhi, ylo, self._month, self._day,
+ self._hour, self._minute, self._second,
us1, us2, us3)
if self._tzinfo is None:
return (basestate,)
@@ -1820,14 +1901,14 @@
return (basestate, self._tzinfo)
def __setstate(self, string, tzinfo):
- (yhi, ylo, self.__month, self.__day, self.__hour,
- self.__minute, self.__second, us1, us2, us3) = map(ord, string)
- self.__year = yhi * 256 + ylo
- self.__microsecond = (((us1 << 8) | us2) << 8) | us3
+ (yhi, ylo, self._month, self._day, self._hour,
+ self._minute, self._second, us1, us2, us3) = map(ord, string)
+ self._year = yhi * 256 + ylo
+ self._microsecond = (((us1 << 8) | us2) << 8) | us3
self._tzinfo = tzinfo
def __reduce__(self):
- return (self.__class__, self.__getstate())
+ return (self.__class__, self._getstate())
datetime.min = datetime(1, 1, 1)
@@ -2009,7 +2090,7 @@
Because we know z.d said z was in daylight time (else [5] would have held and
we would have stopped then), and we know z.d != z'.d (else [8] would have held
-and we we have stopped then), and there are only 2 possible values dst() can
+and we have stopped then), and there are only 2 possible values dst() can
return in Eastern, it follows that z'.d must be 0 (which it is in the example,
but the reasoning doesn't depend on the example -- it depends on there being
two possible dst() outcomes, one zero and the other non-zero). Therefore
diff --git a/lib_pypy/numpypy/__init__.py b/lib_pypy/numpypy/__init__.py
--- a/lib_pypy/numpypy/__init__.py
+++ b/lib_pypy/numpypy/__init__.py
@@ -1,2 +1,2 @@
from _numpypy import *
-from .fromnumeric import *
+from .core import *
diff --git a/lib_pypy/numpypy/core/__init__.py b/lib_pypy/numpypy/core/__init__.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/numpypy/core/__init__.py
@@ -0,0 +1,2 @@
+from .fromnumeric import *
+from .numeric import *
diff --git a/lib_pypy/numpypy/core/_methods.py b/lib_pypy/numpypy/core/_methods.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/numpypy/core/_methods.py
@@ -0,0 +1,98 @@
+# Array methods which are called by the both the C-code for the method
+# and the Python code for the NumPy-namespace function
+
+import _numpypy as mu
+um = mu
+#from numpypy.core import umath as um
+from numpypy.core.numeric import asanyarray
+
+def _amax(a, axis=None, out=None, skipna=False, keepdims=False):
+ return um.maximum.reduce(a, axis=axis,
+ out=out, skipna=skipna, keepdims=keepdims)
+
+def _amin(a, axis=None, out=None, skipna=False, keepdims=False):
+ return um.minimum.reduce(a, axis=axis,
+ out=out, skipna=skipna, keepdims=keepdims)
+
+def _sum(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False):
+ return um.add.reduce(a, axis=axis, dtype=dtype,
+ out=out, skipna=skipna, keepdims=keepdims)
+
+def _prod(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False):
+ return um.multiply.reduce(a, axis=axis, dtype=dtype,
+ out=out, skipna=skipna, keepdims=keepdims)
+
+def _mean(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False):
+ arr = asanyarray(a)
+
+ # Upgrade bool, unsigned int, and int to float64
+ if dtype is None and arr.dtype.kind in ['b','u','i']:
+ ret = um.add.reduce(arr, axis=axis, dtype='f8',
+ out=out, skipna=skipna, keepdims=keepdims)
+ else:
+ ret = um.add.reduce(arr, axis=axis, dtype=dtype,
+ out=out, skipna=skipna, keepdims=keepdims)
+ rcount = mu.count_reduce_items(arr, axis=axis,
+ skipna=skipna, keepdims=keepdims)
+ if isinstance(ret, mu.ndarray):
+ ret = um.true_divide(ret, rcount,
+ casting='unsafe', subok=False)
+ else:
+ ret = ret / float(rcount)
+ return ret
+
+def _var(a, axis=None, dtype=None, out=None, ddof=0,
+ skipna=False, keepdims=False):
+ arr = asanyarray(a)
+
+ # First compute the mean, saving 'rcount' for reuse later
+ if dtype is None and arr.dtype.kind in ['b','u','i']:
+ arrmean = um.add.reduce(arr, axis=axis, dtype='f8',
+ skipna=skipna, keepdims=True)
+ else:
+ arrmean = um.add.reduce(arr, axis=axis, dtype=dtype,
+ skipna=skipna, keepdims=True)
+ rcount = mu.count_reduce_items(arr, axis=axis,
+ skipna=skipna, keepdims=True)
+ if isinstance(arrmean, mu.ndarray):
+ arrmean = um.true_divide(arrmean, rcount,
+ casting='unsafe', subok=False)
+ else:
+ arrmean = arrmean / float(rcount)
+
+ # arr - arrmean
+ x = arr - arrmean
+
+ # (arr - arrmean) ** 2
+ if arr.dtype.kind == 'c':
+ x = um.multiply(x, um.conjugate(x)).real
+ else:
+ x = um.multiply(x, x)
+
+ # add.reduce((arr - arrmean) ** 2, axis)
+ ret = um.add.reduce(x, axis=axis, dtype=dtype, out=out,
+ skipna=skipna, keepdims=keepdims)
+
+ # add.reduce((arr - arrmean) ** 2, axis) / (n - ddof)
+ if not keepdims and isinstance(rcount, mu.ndarray):
+ rcount = rcount.squeeze(axis=axis)
+ rcount -= ddof
+ if isinstance(ret, mu.ndarray):
+ ret = um.true_divide(ret, rcount,
+ casting='unsafe', subok=False)
+ else:
+ ret = ret / float(rcount)
+
+ return ret
+
+def _std(a, axis=None, dtype=None, out=None, ddof=0,
+ skipna=False, keepdims=False):
+ ret = _var(a, axis=axis, dtype=dtype, out=out, ddof=ddof,
+ skipna=skipna, keepdims=keepdims)
+
+ if isinstance(ret, mu.ndarray):
+ ret = um.sqrt(ret)
+ else:
+ ret = um.sqrt(ret)
+
+ return ret
diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/numpypy/core/arrayprint.py
@@ -0,0 +1,789 @@
+"""Array printing function
+
+$Id: arrayprint.py,v 1.9 2005/09/13 13:58:44 teoliphant Exp $
+"""
+__all__ = ["array2string", "set_printoptions", "get_printoptions"]
+__docformat__ = 'restructuredtext'
+
+#
+# Written by Konrad Hinsen <hinsenk at ere.umontreal.ca>
+# last revision: 1996-3-13
+# modified by Jim Hugunin 1997-3-3 for repr's and str's (and other details)
+# and by Perry Greenfield 2000-4-1 for numarray
+# and by Travis Oliphant 2005-8-22 for numpy
+
+import sys
+import _numpypy as _nt
+from _numpypy import maximum, minimum, absolute, not_equal, isinf, isnan, isna
+#from _numpypy import format_longfloat, datetime_as_string, datetime_data
+from .fromnumeric import ravel
+
+
+def product(x, y): return x*y
+
+_summaryEdgeItems = 3 # repr N leading and trailing items of each dimension
+_summaryThreshold = 1000 # total items > triggers array summarization
+
+_float_output_precision = 8
+_float_output_suppress_small = False
+_line_width = 75
+_nan_str = 'nan'
+_inf_str = 'inf'
+_na_str = 'NA'
+_formatter = None # formatting function for array elements
+
+if sys.version_info[0] >= 3:
+ from functools import reduce
+
+def set_printoptions(precision=None, threshold=None, edgeitems=None,
+ linewidth=None, suppress=None,
+ nanstr=None, infstr=None, nastr=None,
+ formatter=None):
+ """
+ Set printing options.
+
+ These options determine the way floating point numbers, arrays and
+ other NumPy objects are displayed.
+
+ Parameters
+ ----------
+ precision : int, optional
+ Number of digits of precision for floating point output (default 8).
+ threshold : int, optional
+ Total number of array elements which trigger summarization
+ rather than full repr (default 1000).
+ edgeitems : int, optional
+ Number of array items in summary at beginning and end of
+ each dimension (default 3).
+ linewidth : int, optional
+ The number of characters per line for the purpose of inserting
+ line breaks (default 75).
+ suppress : bool, optional
+ Whether or not suppress printing of small floating point values
+ using scientific notation (default False).
+ nanstr : str, optional
+ String representation of floating point not-a-number (default nan).
+ infstr : str, optional
+ String representation of floating point infinity (default inf).
+ nastr : str, optional
+ String representation of NA missing value (default NA).
+ formatter : dict of callables, optional
+ If not None, the keys should indicate the type(s) that the respective
+ formatting function applies to. Callables should return a string.
+ Types that are not specified (by their corresponding keys) are handled
+ by the default formatters. Individual types for which a formatter
+ can be set are::
+
+ - 'bool'
+ - 'int'
+ - 'timedelta' : a `numpy.timedelta64`
+ - 'datetime' : a `numpy.datetime64`
+ - 'float'
+ - 'longfloat' : 128-bit floats
+ - 'complexfloat'
+ - 'longcomplexfloat' : composed of two 128-bit floats
+ - 'numpy_str' : types `numpy.string_` and `numpy.unicode_`
+ - 'str' : all other strings
+
+ Other keys that can be used to set a group of types at once are::
+
+ - 'all' : sets all types
+ - 'int_kind' : sets 'int'
+ - 'float_kind' : sets 'float' and 'longfloat'
+ - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat'
+ - 'str_kind' : sets 'str' and 'numpystr'
+
+ See Also
+ --------
+ get_printoptions, set_string_function, array2string
+
+ Notes
+ -----
+ `formatter` is always reset with a call to `set_printoptions`.
+
+ Examples
+ --------
+ Floating point precision can be set:
+
+ >>> np.set_printoptions(precision=4)
+ >>> print np.array([1.123456789])
+ [ 1.1235]
+
+ Long arrays can be summarised:
+
+ >>> np.set_printoptions(threshold=5)
+ >>> print np.arange(10)
+ [0 1 2 ..., 7 8 9]
+
+ Small results can be suppressed:
+
+ >>> eps = np.finfo(float).eps
+ >>> x = np.arange(4.)
+ >>> x**2 - (x + eps)**2
+ array([ -4.9304e-32, -4.4409e-16, 0.0000e+00, 0.0000e+00])
+ >>> np.set_printoptions(suppress=True)
+ >>> x**2 - (x + eps)**2
+ array([-0., -0., 0., 0.])
+
+ A custom formatter can be used to display array elements as desired:
+
+ >>> np.set_printoptions(formatter={'all':lambda x: 'int: '+str(-x)})
+ >>> x = np.arange(3)
+ >>> x
+ array([int: 0, int: -1, int: -2])
+ >>> np.set_printoptions() # formatter gets reset
+ >>> x
+ array([0, 1, 2])
+
+ To put back the default options, you can use:
+
+ >>> np.set_printoptions(edgeitems=3,infstr='inf',
+ ... linewidth=75, nanstr='nan', precision=8,
+ ... suppress=False, threshold=1000, formatter=None)
+ """
+
+ global _summaryThreshold, _summaryEdgeItems, _float_output_precision, \
+ _line_width, _float_output_suppress_small, _nan_str, _inf_str, \
+ _na_str, _formatter
+ if linewidth is not None:
+ _line_width = linewidth
+ if threshold is not None:
+ _summaryThreshold = threshold
+ if edgeitems is not None:
+ _summaryEdgeItems = edgeitems
+ if precision is not None:
+ _float_output_precision = precision
+ if suppress is not None:
+ _float_output_suppress_small = not not suppress
+ if nanstr is not None:
+ _nan_str = nanstr
+ if infstr is not None:
+ _inf_str = infstr
+ if nastr is not None:
+ _na_str = nastr
+ _formatter = formatter
+
+def get_printoptions():
+ """
+ Return the current print options.
+
+ Returns
+ -------
+ print_opts : dict
+ Dictionary of current print options with keys
+
+ - precision : int
+ - threshold : int
+ - edgeitems : int
+ - linewidth : int
+ - suppress : bool
+ - nanstr : str
+ - infstr : str
+ - formatter : dict of callables
+
+ For a full description of these options, see `set_printoptions`.
+
+ See Also
+ --------
+ set_printoptions, set_string_function
+
+ """
+ d = dict(precision=_float_output_precision,
+ threshold=_summaryThreshold,
+ edgeitems=_summaryEdgeItems,
+ linewidth=_line_width,
+ suppress=_float_output_suppress_small,
+ nanstr=_nan_str,
+ infstr=_inf_str,
+ nastr=_na_str,
+ formatter=_formatter)
+ return d
+
+def _leading_trailing(a):
+ import numeric as _nc
+ if a.ndim == 1:
+ if len(a) > 2*_summaryEdgeItems:
+ b = _nc.concatenate((a[:_summaryEdgeItems],
+ a[-_summaryEdgeItems:]))
+ else:
+ b = a
+ else:
+ if len(a) > 2*_summaryEdgeItems:
+ l = [_leading_trailing(a[i]) for i in range(
+ min(len(a), _summaryEdgeItems))]
+ l.extend([_leading_trailing(a[-i]) for i in range(
+ min(len(a), _summaryEdgeItems),0,-1)])
+ else:
+ l = [_leading_trailing(a[i]) for i in range(0, len(a))]
+ b = _nc.concatenate(tuple(l))
+ return b
+
+def _boolFormatter(x):
+ if isna(x):
+ return str(x).replace('NA', _na_str, 1)
+ elif x:
+ return ' True'
+ else:
+ return 'False'
+
+
+def repr_format(x):
+ if isna(x):
+ return str(x).replace('NA', _na_str, 1)
+ else:
+ return repr(x)
+
+def _array2string(a, max_line_width, precision, suppress_small, separator=' ',
+ prefix="", formatter=None):
+
+ if max_line_width is None:
+ max_line_width = _line_width
+
+ if precision is None:
+ precision = _float_output_precision
+
+ if suppress_small is None:
+ suppress_small = _float_output_suppress_small
+
+ if formatter is None:
+ formatter = _formatter
+
+ if a.size > _summaryThreshold:
+ summary_insert = "..., "
+ data = _leading_trailing(a)
+ else:
+ summary_insert = ""
+ data = ravel(a)
+
+ formatdict = {'bool' : _boolFormatter,
+ 'int' : IntegerFormat(data),
+ 'float' : FloatFormat(data, precision, suppress_small),
+ 'longfloat' : LongFloatFormat(precision),
+ #'complexfloat' : ComplexFormat(data, precision,
+ # suppress_small),
+ #'longcomplexfloat' : LongComplexFormat(precision),
+ #'datetime' : DatetimeFormat(data),
+ #'timedelta' : TimedeltaFormat(data),
+ 'numpystr' : repr_format,
+ 'str' : str}
+
+ if formatter is not None:
+ fkeys = [k for k in formatter.keys() if formatter[k] is not None]
+ if 'all' in fkeys:
+ for key in formatdict.keys():
+ formatdict[key] = formatter['all']
+ if 'int_kind' in fkeys:
+ for key in ['int']:
+ formatdict[key] = formatter['int_kind']
+ if 'float_kind' in fkeys:
+ for key in ['float', 'longfloat']:
+ formatdict[key] = formatter['float_kind']
+ if 'complex_kind' in fkeys:
+ for key in ['complexfloat', 'longcomplexfloat']:
+ formatdict[key] = formatter['complex_kind']
+ if 'str_kind' in fkeys:
+ for key in ['numpystr', 'str']:
+ formatdict[key] = formatter['str_kind']
+ for key in formatdict.keys():
+ if key in fkeys:
+ formatdict[key] = formatter[key]
+
+ try:
+ format_function = a._format
+ msg = "The `_format` attribute is deprecated in Numpy 2.0 and " \
+ "will be removed in 2.1. Use the `formatter` kw instead."
+ import warnings
+ warnings.warn(msg, DeprecationWarning)
+ except AttributeError:
+ # find the right formatting function for the array
+ dtypeobj = a.dtype.type
+ if issubclass(dtypeobj, _nt.bool_):
+ format_function = formatdict['bool']
+ elif issubclass(dtypeobj, _nt.integer):
+ #if issubclass(dtypeobj, _nt.timedelta64):
+ # format_function = formatdict['timedelta']
+ #else:
+ format_function = formatdict['int']
+ elif issubclass(dtypeobj, _nt.floating):
+ #if issubclass(dtypeobj, _nt.longfloat):
+ # format_function = formatdict['longfloat']
+ #else:
+ format_function = formatdict['float']
+ elif issubclass(dtypeobj, _nt.complexfloating):
+ if issubclass(dtypeobj, _nt.clongfloat):
+ format_function = formatdict['longcomplexfloat']
+ else:
+ format_function = formatdict['complexfloat']
+ elif issubclass(dtypeobj, (_nt.unicode_, _nt.string_)):
+ format_function = formatdict['numpystr']
+ elif issubclass(dtypeobj, _nt.datetime64):
+ format_function = formatdict['datetime']
+ else:
+ format_function = formatdict['str']
+
+ # skip over "["
+ next_line_prefix = " "
+ # skip over array(
+ next_line_prefix += " "*len(prefix)
+
+ lst = _formatArray(a, format_function, len(a.shape), max_line_width,
+ next_line_prefix, separator,
+ _summaryEdgeItems, summary_insert)[:-1]
+ return lst
+
+def _convert_arrays(obj):
+ import numeric as _nc
+ newtup = []
+ for k in obj:
+ if isinstance(k, _nc.ndarray):
+ k = k.tolist()
+ elif isinstance(k, tuple):
+ k = _convert_arrays(k)
+ newtup.append(k)
+ return tuple(newtup)
+
+
+def array2string(a, max_line_width=None, precision=None,
+ suppress_small=None, separator=' ', prefix="",
+ style=repr, formatter=None):
+ """
+ Return a string representation of an array.
+
+ Parameters
+ ----------
+ a : ndarray
+ Input array.
+ max_line_width : int, optional
+ The maximum number of columns the string should span. Newline
+ characters splits the string appropriately after array elements.
+ precision : int, optional
+ Floating point precision. Default is the current printing
+ precision (usually 8), which can be altered using `set_printoptions`.
+ suppress_small : bool, optional
+ Represent very small numbers as zero. A number is "very small" if it
+ is smaller than the current printing precision.
+ separator : str, optional
+ Inserted between elements.
+ prefix : str, optional
+ An array is typically printed as::
+
+ 'prefix(' + array2string(a) + ')'
+
+ The length of the prefix string is used to align the
+ output correctly.
+ style : function, optional
+ A function that accepts an ndarray and returns a string. Used only
+ when the shape of `a` is equal to ``()``, i.e. for 0-D arrays.
+ formatter : dict of callables, optional
+ If not None, the keys should indicate the type(s) that the respective
+ formatting function applies to. Callables should return a string.
+ Types that are not specified (by their corresponding keys) are handled
+ by the default formatters. Individual types for which a formatter
+ can be set are::
+
+ - 'bool'
+ - 'int'
+ - 'timedelta' : a `numpy.timedelta64`
+ - 'datetime' : a `numpy.datetime64`
+ - 'float'
+ - 'longfloat' : 128-bit floats
+ - 'complexfloat'
+ - 'longcomplexfloat' : composed of two 128-bit floats
+ - 'numpy_str' : types `numpy.string_` and `numpy.unicode_`
+ - 'str' : all other strings
+
+ Other keys that can be used to set a group of types at once are::
+
+ - 'all' : sets all types
+ - 'int_kind' : sets 'int'
+ - 'float_kind' : sets 'float' and 'longfloat'
+ - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat'
+ - 'str_kind' : sets 'str' and 'numpystr'
+
+ Returns
+ -------
+ array_str : str
+ String representation of the array.
+
+ Raises
+ ------
+ TypeError : if a callable in `formatter` does not return a string.
+
+ See Also
+ --------
+ array_str, array_repr, set_printoptions, get_printoptions
+
+ Notes
+ -----
+ If a formatter is specified for a certain type, the `precision` keyword is
+ ignored for that type.
+
+ Examples
+ --------
+ >>> x = np.array([1e-16,1,2,3])
+ >>> print np.array2string(x, precision=2, separator=',',
+ ... suppress_small=True)
+ [ 0., 1., 2., 3.]
+
+ >>> x = np.arange(3.)
+ >>> np.array2string(x, formatter={'float_kind':lambda x: "%.2f" % x})
+ '[0.00 1.00 2.00]'
+
+ >>> x = np.arange(3)
+ >>> np.array2string(x, formatter={'int':lambda x: hex(x)})
+ '[0x0L 0x1L 0x2L]'
+
+ """
+
+ if a.shape == ():
+ x = a.item()
+ if isna(x):
+ lst = str(x).replace('NA', _na_str, 1)
+ else:
+ try:
+ lst = a._format(x)
+ msg = "The `_format` attribute is deprecated in Numpy " \
+ "2.0 and will be removed in 2.1. Use the " \
+ "`formatter` kw instead."
+ import warnings
+ warnings.warn(msg, DeprecationWarning)
+ except AttributeError:
+ if isinstance(x, tuple):
+ x = _convert_arrays(x)
+ lst = style(x)
+ elif reduce(product, a.shape) == 0:
+ # treat as a null array if any of shape elements == 0
+ lst = "[]"
+ else:
+ lst = _array2string(a, max_line_width, precision, suppress_small,
+ separator, prefix, formatter=formatter)
+ return lst
+
+def _extendLine(s, line, word, max_line_len, next_line_prefix):
+ if len(line.rstrip()) + len(word.rstrip()) >= max_line_len:
+ s += line.rstrip() + "\n"
+ line = next_line_prefix
+ line += word
+ return s, line
+
+
+def _formatArray(a, format_function, rank, max_line_len,
+ next_line_prefix, separator, edge_items, summary_insert):
+ """formatArray is designed for two modes of operation:
+
+ 1. Full output
+
+ 2. Summarized output
+
+ """
+ if rank == 0:
+ obj = a.item()
+ if isinstance(obj, tuple):
+ obj = _convert_arrays(obj)
+ return str(obj)
+
+ if summary_insert and 2*edge_items < len(a):
+ leading_items, trailing_items, summary_insert1 = \
+ edge_items, edge_items, summary_insert
+ else:
+ leading_items, trailing_items, summary_insert1 = 0, len(a), ""
+
+ if rank == 1:
+ s = ""
+ line = next_line_prefix
+ for i in xrange(leading_items):
+ word = format_function(a[i]) + separator
+ s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)
+
+ if summary_insert1:
+ s, line = _extendLine(s, line, summary_insert1, max_line_len, next_line_prefix)
+
+ for i in xrange(trailing_items, 1, -1):
+ word = format_function(a[-i]) + separator
+ s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)
+
+ word = format_function(a[-1])
+ s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)
+ s += line + "]\n"
+ s = '[' + s[len(next_line_prefix):]
+ else:
+ s = '['
+ sep = separator.rstrip()
+ for i in xrange(leading_items):
+ if i > 0:
+ s += next_line_prefix
+ s += _formatArray(a[i], format_function, rank-1, max_line_len,
+ " " + next_line_prefix, separator, edge_items,
+ summary_insert)
+ s = s.rstrip() + sep.rstrip() + '\n'*max(rank-1,1)
+
+ if summary_insert1:
+ s += next_line_prefix + summary_insert1 + "\n"
+
+ for i in xrange(trailing_items, 1, -1):
+ if leading_items or i != trailing_items:
+ s += next_line_prefix
+ s += _formatArray(a[-i], format_function, rank-1, max_line_len,
+ " " + next_line_prefix, separator, edge_items,
+ summary_insert)
+ s = s.rstrip() + sep.rstrip() + '\n'*max(rank-1,1)
+ if leading_items or trailing_items > 1:
+ s += next_line_prefix
+ s += _formatArray(a[-1], format_function, rank-1, max_line_len,
+ " " + next_line_prefix, separator, edge_items,
+ summary_insert).rstrip()+']\n'
+ return s
+
+class FloatFormat(object):
+ def __init__(self, data, precision, suppress_small, sign=False):
+ self.precision = precision
+ self.suppress_small = suppress_small
+ self.sign = sign
+ self.exp_format = False
+ self.large_exponent = False
+ self.max_str_len = 0
+ #try:
+ self.fillFormat(data)
+ #except (TypeError, NotImplementedError):
+ # if reduce(data) fails, this instance will not be called, just
+ # instantiated in formatdict.
+ #pass
+
+ def fillFormat(self, data):
+ import numeric as _nc
+ # XXX pypy unimplemented
+ #errstate = _nc.seterr(all='ignore')
+ try:
+ special = isnan(data) | isinf(data) | isna(data)
+ special[isna(data)] = False
+ valid = not_equal(data, 0) & ~special
+ valid[isna(data)] = False
+ non_zero = absolute(data.compress(valid))
+ if len(non_zero) == 0:
+ max_val = 0.
+ min_val = 0.
+ else:
+ max_val = maximum.reduce(non_zero, skipna=True)
+ min_val = minimum.reduce(non_zero, skipna=True)
+ if max_val >= 1.e8:
+ self.exp_format = True
+ if not self.suppress_small and (min_val < 0.0001
+ or max_val/min_val > 1000.):
+ self.exp_format = True
+ finally:
+ pass
+ # XXX pypy unimplemented
+ #_nc.seterr(**errstate)
+
+ if self.exp_format:
+ self.large_exponent = 0 < min_val < 1e-99 or max_val >= 1e100
+ self.max_str_len = 8 + self.precision
+ if self.large_exponent:
+ self.max_str_len += 1
+ if self.sign:
+ format = '%+'
+ else:
+ format = '%'
+ format = format + '%d.%de' % (self.max_str_len, self.precision)
+ else:
+ format = '%%.%df' % (self.precision,)
+ if len(non_zero):
+ precision = max([_digits(x, self.precision, format)
+ for x in non_zero])
+ else:
+ precision = 0
+ precision = min(self.precision, precision)
+ self.max_str_len = len(str(int(max_val))) + precision + 2
+ if special.any():
+ self.max_str_len = max(self.max_str_len,
+ len(_nan_str),
+ len(_inf_str)+1,
+ len(_na_str))
+ if self.sign:
+ format = '%#+'
+ else:
+ format = '%#'
+ format = format + '%d.%df' % (self.max_str_len, precision)
+
+ self.special_fmt = '%%%ds' % (self.max_str_len,)
+ self.format = format
+
+ def __call__(self, x, strip_zeros=True):
+ import numeric as _nc
+ #err = _nc.seterr(invalid='ignore')
+ try:
+ if isna(x):
+ return self.special_fmt % (str(x).replace('NA', _na_str, 1),)
+ elif isnan(x):
+ if self.sign:
+ return self.special_fmt % ('+' + _nan_str,)
+ else:
+ return self.special_fmt % (_nan_str,)
+ elif isinf(x):
+ if x > 0:
+ if self.sign:
+ return self.special_fmt % ('+' + _inf_str,)
+ else:
+ return self.special_fmt % (_inf_str,)
+ else:
+ return self.special_fmt % ('-' + _inf_str,)
+ finally:
+ pass
+ #_nc.seterr(**err)
+
+ s = self.format % x
+ if self.large_exponent:
+ # 3-digit exponent
+ expsign = s[-3]
+ if expsign == '+' or expsign == '-':
+ s = s[1:-2] + '0' + s[-2:]
+ elif self.exp_format:
+ # 2-digit exponent
+ if s[-3] == '0':
+ s = ' ' + s[:-3] + s[-2:]
+ elif strip_zeros:
+ z = s.rstrip('0')
+ s = z + ' '*(len(s)-len(z))
+ return s
+
+
+def _digits(x, precision, format):
+ s = format % x
+ z = s.rstrip('0')
+ return precision - len(s) + len(z)
+
+
+_MAXINT = sys.maxint
+_MININT = -sys.maxint-1
+class IntegerFormat(object):
+ def __init__(self, data):
+ try:
+ max_str_len = max(len(str(maximum.reduce(data, skipna=True))),
+ len(str(minimum.reduce(data, skipna=True))))
+ self.format = '%' + str(max_str_len) + 'd'
+ except TypeError, NotImplementedError:
+ # if reduce(data) fails, this instance will not be called, just
+ # instantiated in formatdict.
+ pass
+ except ValueError:
+ # this occurs when everything is NA
+ pass
+
+ def __call__(self, x):
+ if isna(x):
+ return str(x).replace('NA', _na_str, 1)
+ elif _MININT < x < _MAXINT:
+ return self.format % x
+ else:
+ return "%s" % x
+
+class LongFloatFormat(object):
+ # XXX Have to add something to determine the width to use a la FloatFormat
+ # Right now, things won't line up properly
+ def __init__(self, precision, sign=False):
+ self.precision = precision
+ self.sign = sign
+
+ def __call__(self, x):
+ if isna(x):
+ return str(x).replace('NA', _na_str, 1)
+ elif isnan(x):
+ if self.sign:
+ return '+' + _nan_str
+ else:
+ return ' ' + _nan_str
+ elif isinf(x):
+ if x > 0:
+ if self.sign:
+ return '+' + _inf_str
+ else:
+ return ' ' + _inf_str
+ else:
+ return '-' + _inf_str
+ elif x >= 0:
+ if self.sign:
+ return '+' + format_longfloat(x, self.precision)
+ else:
+ return ' ' + format_longfloat(x, self.precision)
+ else:
+ return format_longfloat(x, self.precision)
+
+
+class LongComplexFormat(object):
+ def __init__(self, precision):
+ self.real_format = LongFloatFormat(precision)
+ self.imag_format = LongFloatFormat(precision, sign=True)
+
+ def __call__(self, x):
+ if isna(x):
+ return str(x).replace('NA', _na_str, 1)
+ else:
+ r = self.real_format(x.real)
+ i = self.imag_format(x.imag)
+ return r + i + 'j'
+
+
+class ComplexFormat(object):
+ def __init__(self, x, precision, suppress_small):
+ self.real_format = FloatFormat(x.real, precision, suppress_small)
+ self.imag_format = FloatFormat(x.imag, precision, suppress_small,
+ sign=True)
+
+ def __call__(self, x):
+ if isna(x):
+ return str(x).replace('NA', _na_str, 1)
+ else:
+ r = self.real_format(x.real, strip_zeros=False)
+ i = self.imag_format(x.imag, strip_zeros=False)
+ if not self.imag_format.exp_format:
+ z = i.rstrip('0')
+ i = z + 'j' + ' '*(len(i)-len(z))
+ else:
+ i = i + 'j'
+ return r + i
+
+class DatetimeFormat(object):
+ def __init__(self, x, unit=None,
+ timezone=None, casting='same_kind'):
+ # Get the unit from the dtype
+ if unit is None:
+ if x.dtype.kind == 'M':
+ unit = datetime_data(x.dtype)[0]
+ else:
+ unit = 's'
+
+ # If timezone is default, make it 'local' or 'UTC' based on the unit
+ if timezone is None:
+ # Date units -> UTC, time units -> local
+ if unit in ('Y', 'M', 'W', 'D'):
+ self.timezone = 'UTC'
+ else:
+ self.timezone = 'local'
+ else:
+ self.timezone = timezone
+ self.unit = unit
+ self.casting = casting
+
+ def __call__(self, x):
+ if isna(x):
+ return str(x).replace('NA', _na_str, 1)
+ else:
+ return "'%s'" % datetime_as_string(x,
+ unit=self.unit,
+ timezone=self.timezone,
+ casting=self.casting)
+
+class TimedeltaFormat(object):
+ def __init__(self, data):
+ if data.dtype.kind == 'm':
+ v = data.view('i8')
+ max_str_len = max(len(str(maximum.reduce(v))),
+ len(str(minimum.reduce(v))))
+ self.format = '%' + str(max_str_len) + 'd'
+
+ def __call__(self, x):
+ if isna(x):
+ return str(x).replace('NA', _na_str, 1)
+ else:
+ return self.format % x.astype('i8')
+
diff --git a/lib_pypy/numpypy/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
rename from lib_pypy/numpypy/fromnumeric.py
rename to lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -30,7 +30,7 @@
'rank', 'size', 'around', 'round_', 'mean', 'std', 'var', 'squeeze',
'amax', 'amin',
]
-
+
def take(a, indices, axis=None, out=None, mode='raise'):
"""
Take elements from an array along an axis.
@@ -85,7 +85,7 @@
array([4, 3, 6])
"""
- raise NotImplemented('Waiting on interp level method')
+ raise NotImplementedError('Waiting on interp level method')
# not deprecated --- copy if necessary, view otherwise
@@ -149,6 +149,7 @@
[5, 6]])
"""
+ assert order == 'C'
if not hasattr(a, 'reshape'):
a = numpypy.array(a)
return a.reshape(newshape)
@@ -273,7 +274,7 @@
[-1, -2, -3, -4, -5]]])
"""
- raise NotImplemented('Waiting on interp level method')
+ raise NotImplementedError('Waiting on interp level method')
def repeat(a, repeats, axis=None):
@@ -315,7 +316,7 @@
[3, 4]])
"""
- raise NotImplemented('Waiting on interp level method')
+ raise NotImplementedError('Waiting on interp level method')
def put(a, ind, v, mode='raise'):
@@ -366,7 +367,7 @@
array([ 0, 1, 2, 3, -5])
"""
- raise NotImplemented('Waiting on interp level method')
+ raise NotImplementedError('Waiting on interp level method')
def swapaxes(a, axis1, axis2):
@@ -410,7 +411,7 @@
[3, 7]]])
"""
- raise NotImplemented('Waiting on interp level method')
+ raise NotImplementedError('Waiting on interp level method')
def transpose(a, axes=None):
@@ -451,8 +452,11 @@
(2, 1, 3)
"""
- raise NotImplemented('Waiting on interp level method')
-
+ if axes is not None:
+ raise NotImplementedError('No "axes" arg yet.')
+ if not hasattr(a, 'T'):
+ a = numpypy.array(a)
+ return a.T
def sort(a, axis=-1, kind='quicksort', order=None):
"""
@@ -553,7 +557,7 @@
dtype=[('name', '|S10'), ('height', '<f8'), ('age', '<i4')])
"""
- raise NotImplemented('Waiting on interp level method')
+ raise NotImplementedError('Waiting on interp level method')
def argsort(a, axis=-1, kind='quicksort', order=None):
@@ -634,7 +638,7 @@
array([0, 1])
"""
- raise NotImplemented('Waiting on interp level method')
+ raise NotImplementedError('Waiting on interp level method')
def argmax(a, axis=None):
@@ -687,6 +691,7 @@
1
"""
+ assert axis is None
if not hasattr(a, 'argmax'):
a = numpypy.array(a)
return a.argmax()
@@ -702,6 +707,7 @@
documentation.
"""
+ assert axis is None
if not hasattr(a, 'argmin'):
a = numpypy.array(a)
return a.argmin()
@@ -753,7 +759,7 @@
array([0, 5, 1, 2])
"""
- raise NotImplemented('Waiting on interp level method')
+ raise NotImplementedError('Waiting on interp level method')
def resize(a, new_shape):
@@ -794,7 +800,7 @@
[0, 1, 2, 3]])
"""
- raise NotImplemented('Waiting on interp level method')
+ raise NotImplementedError('Waiting on interp level method')
def squeeze(a):
@@ -821,7 +827,7 @@
(3,)
"""
- raise NotImplemented('Waiting on interp level method')
+ raise NotImplementedError('Waiting on interp level method')
def diagonal(a, offset=0, axis1=0, axis2=1):
@@ -905,7 +911,7 @@
[5, 7]])
"""
- raise NotImplemented('Waiting on interp level method')
+ raise NotImplementedError('Waiting on interp level method')
def trace(a, offset=0, axis1=0, axis2=1, dtype=None, out=None):
@@ -964,7 +970,7 @@
(2, 3)
"""
- raise NotImplemented('Waiting on interp level method')
+ raise NotImplementedError('Waiting on interp level method')
def ravel(a, order='C'):
"""
@@ -1048,8 +1054,9 @@
array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
"""
- raise NotImplemented('Waiting on interp level method')
-
+ if not hasattr(a, 'ravel'):
+ a = numpypy.array(a)
+ return a.ravel(order=order)
def nonzero(a):
"""
@@ -1124,7 +1131,7 @@
(array([1, 1, 1, 2, 2, 2]), array([0, 1, 2, 0, 1, 2]))
"""
- raise NotImplemented('Waiting on interp level method')
+ raise NotImplementedError('Waiting on interp level method')
def shape(a):
@@ -1229,7 +1236,7 @@
array([2])
"""
- raise NotImplemented('Waiting on interp level method')
+ raise NotImplementedError('Waiting on interp level method')
def clip(a, a_min, a_max, out=None):
@@ -1282,7 +1289,7 @@
array([3, 4, 2, 3, 4, 5, 6, 7, 8, 8])
"""
- raise NotImplemented('Waiting on interp level method')
+ raise NotImplementedError('Waiting on interp level method')
def sum(a, axis=None, dtype=None, out=None):
@@ -1351,9 +1358,11 @@
-128
"""
+ assert dtype is None
+ assert out is None
if not hasattr(a, "sum"):
a = numpypy.array(a)
- return a.sum()
+ return a.sum(axis=axis)
def product (a, axis=None, dtype=None, out=None):
@@ -1365,7 +1374,7 @@
prod : equivalent function; see for details.
"""
- raise NotImplemented('Waiting on interp level method')
+ raise NotImplementedError('Waiting on interp level method')
def sometrue(a, axis=None, out=None):
@@ -1379,6 +1388,8 @@
any : equivalent function
"""
+ assert axis is None
+ assert out is None
if not hasattr(a, 'any'):
a = numpypy.array(a)
return a.any()
@@ -1393,6 +1404,8 @@
numpy.all : Equivalent function; see for details.
"""
+ assert axis is None
+ assert out is None
if not hasattr(a, 'all'):
a = numpypy.array(a)
return a.all()
@@ -1461,6 +1474,8 @@
(191614240, 191614240)
"""
+ assert axis is None
+ assert out is None
if not hasattr(a, 'any'):
a = numpypy.array(a)
return a.any()
@@ -1523,6 +1538,8 @@
(28293632, 28293632, array([ True], dtype=bool))
"""
+ assert axis is None
+ assert out is None
if not hasattr(a, 'all'):
a = numpypy.array(a)
return a.all()
@@ -1590,7 +1607,7 @@
[ 4, 9, 15]])
"""
- raise NotImplemented('Waiting on interp level method')
+ raise NotImplementedError('Waiting on interp level method')
def cumproduct(a, axis=None, dtype=None, out=None):
@@ -1603,7 +1620,7 @@
cumprod : equivalent function; see for details.
"""
- raise NotImplemented('Waiting on interp level method')
+ raise NotImplementedError('Waiting on interp level method')
def ptp(a, axis=None, out=None):
@@ -1644,7 +1661,7 @@
array([1, 1])
"""
- raise NotImplemented('Waiting on interp level method')
+ raise NotImplementedError('Waiting on interp level method')
def amax(a, axis=None, out=None):
@@ -1702,6 +1719,8 @@
4.0
"""
+ assert axis is None
+ assert out is None
if not hasattr(a, "max"):
a = numpypy.array(a)
return a.max()
@@ -1763,6 +1782,8 @@
"""
# amin() is equivalent to min()
+ assert axis is None
+ assert out is None
if not hasattr(a, 'min'):
a = numpypy.array(a)
return a.min()
@@ -1873,7 +1894,7 @@
True
"""
- raise NotImplemented('Waiting on interp level method')
+ raise NotImplementedError('Waiting on interp level method')
def cumprod(a, axis=None, dtype=None, out=None):
@@ -1936,7 +1957,7 @@
[ 4, 20, 120]])
"""
- raise NotImplemented('Waiting on interp level method')
+ raise NotImplementedError('Waiting on interp level method')
def ndim(a):
@@ -2054,7 +2075,7 @@
2
"""
- raise NotImplemented('Waiting on interp level method')
+ raise NotImplementedError('Waiting on interp level method')
def around(a, decimals=0, out=None):
@@ -2123,7 +2144,7 @@
array([ 0, 0, 0, 10])
"""
- raise NotImplemented('Waiting on interp level method')
+ raise NotImplementedError('Waiting on interp level method')
def round_(a, decimals=0, out=None):
@@ -2137,7 +2158,7 @@
around : equivalent function
"""
- raise NotImplemented('Waiting on interp level method')
+ raise NotImplementedError('Waiting on interp level method')
def mean(a, axis=None, dtype=None, out=None):
@@ -2211,9 +2232,11 @@
0.55000000074505806
"""
+ assert dtype is None
+ assert out is None
if not hasattr(a, "mean"):
a = numpypy.array(a)
- return a.mean()
+ return a.mean(axis=axis)
def std(a, axis=None, dtype=None, out=None, ddof=0):
@@ -2302,9 +2325,12 @@
0.44999999925552653
"""
+ assert dtype is None
+ assert out is None
+ assert ddof == 0
if not hasattr(a, "std"):
a = numpypy.array(a)
- return a.std()
+ return a.std(axis=axis)
def var(a, axis=None, dtype=None, out=None, ddof=0):
@@ -2395,6 +2421,9 @@
0.20250000000000001
"""
+ assert dtype is None
+ assert out is None
+ assert ddof == 0
if not hasattr(a, "var"):
a = numpypy.array(a)
- return a.var()
+ return a.var(axis=axis)
diff --git a/lib_pypy/numpypy/core/numeric.py b/lib_pypy/numpypy/core/numeric.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/numpypy/core/numeric.py
@@ -0,0 +1,311 @@
+
+from _numpypy import array, ndarray, int_, float_ #, complex_# , longlong
+from _numpypy import concatenate
+import sys
+import _numpypy as multiarray # ARGH
+from numpypy.core.arrayprint import array2string
+
+
+
+def asanyarray(a, dtype=None, order=None, maskna=None, ownmaskna=False):
+ """
+ Convert the input to an ndarray, but pass ndarray subclasses through.
+
+ Parameters
+ ----------
+ a : array_like
+ Input data, in any form that can be converted to an array. This
+ includes scalars, lists, lists of tuples, tuples, tuples of tuples,
+ tuples of lists, and ndarrays.
+ dtype : data-type, optional
+ By default, the data-type is inferred from the input data.
+ order : {'C', 'F'}, optional
+ Whether to use row-major ('C') or column-major ('F') memory
+ representation. Defaults to 'C'.
+ maskna : bool or None, optional
+ If this is set to True, it forces the array to have an NA mask.
+ If this is set to False, it forces the array to not have an NA
+ mask.
+ ownmaskna : bool, optional
+ If this is set to True, forces the array to have a mask which
+ it owns.
+
+ Returns
+ -------
+ out : ndarray or an ndarray subclass
+ Array interpretation of `a`. If `a` is an ndarray or a subclass
+ of ndarray, it is returned as-is and no copy is performed.
+
+ See Also
+ --------
+ asarray : Similar function which always returns ndarrays.
+ ascontiguousarray : Convert input to a contiguous array.
+ asfarray : Convert input to a floating point ndarray.
+ asfortranarray : Convert input to an ndarray with column-major
+ memory order.
+ asarray_chkfinite : Similar function which checks input for NaNs and
+ Infs.
+ fromiter : Create an array from an iterator.
+ fromfunction : Construct an array by executing a function on grid
+ positions.
+
+ Examples
+ --------
+ Convert a list into an array:
+
+ >>> a = [1, 2]
+ >>> np.asanyarray(a)
+ array([1, 2])
+
+ Instances of `ndarray` subclasses are passed through as-is:
+
+ >>> a = np.matrix([1, 2])
+ >>> np.asanyarray(a) is a
+ True
+
+ """
+ return array(a, dtype, copy=False, order=order, subok=True,
+ maskna=maskna, ownmaskna=ownmaskna)
+
+def base_repr(number, base=2, padding=0):
+ """
+ Return a string representation of a number in the given base system.
+
+ Parameters
+ ----------
+ number : int
+ The value to convert. Only positive values are handled.
+ base : int, optional
+ Convert `number` to the `base` number system. The valid range is 2-36,
+ the default value is 2.
+ padding : int, optional
+ Number of zeros padded on the left. Default is 0 (no padding).
+
+ Returns
+ -------
+ out : str
+ String representation of `number` in `base` system.
+
+ See Also
+ --------
+ binary_repr : Faster version of `base_repr` for base 2.
+
+ Examples
+ --------
+ >>> np.base_repr(5)
+ '101'
+ >>> np.base_repr(6, 5)
+ '11'
+ >>> np.base_repr(7, base=5, padding=3)
+ '00012'
+
+ >>> np.base_repr(10, base=16)
+ 'A'
+ >>> np.base_repr(32, base=16)
+ '20'
+
+ """
+ digits = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+ if base > len(digits):
+ raise ValueError("Bases greater than 36 not handled in base_repr.")
+
+ num = abs(number)
+ res = []
+ while num:
+ res.append(digits[num % base])
+ num //= base
+ if padding:
+ res.append('0' * padding)
+ if number < 0:
+ res.append('-')
+ return ''.join(reversed(res or '0'))
+
+_typelessdata = [int_, float_]#, complex_]
+# XXX
+#if issubclass(intc, int):
+# _typelessdata.append(intc)
+
+#if issubclass(longlong, int):
+# _typelessdata.append(longlong)
+
+def array_repr(arr, max_line_width=None, precision=None, suppress_small=None):
+ """
+ Return the string representation of an array.
+
+ Parameters
+ ----------
+ arr : ndarray
+ Input array.
+ max_line_width : int, optional
+ The maximum number of columns the string should span. Newline
+ characters split the string appropriately after array elements.
+ precision : int, optional
+ Floating point precision. Default is the current printing precision
+ (usually 8), which can be altered using `set_printoptions`.
+ suppress_small : bool, optional
+ Represent very small numbers as zero, default is False. Very small
+ is defined by `precision`, if the precision is 8 then
+ numbers smaller than 5e-9 are represented as zero.
+
+ Returns
+ -------
+ string : str
+ The string representation of an array.
+
+ See Also
+ --------
+ array_str, array2string, set_printoptions
+
+ Examples
+ --------
+ >>> np.array_repr(np.array([1,2]))
+ 'array([1, 2])'
+ >>> np.array_repr(np.ma.array([0.]))
+ 'MaskedArray([ 0.])'
+ >>> np.array_repr(np.array([], np.int32))
+ 'array([], dtype=int32)'
+
+ >>> x = np.array([1e-6, 4e-7, 2, 3])
+ >>> np.array_repr(x, precision=6, suppress_small=True)
+ 'array([ 0.000001, 0. , 2. , 3. ])'
+
+ """
+ if arr.size > 0 or arr.shape==(0,):
+ lst = array2string(arr, max_line_width, precision, suppress_small,
+ ', ', "array(")
+ else: # show zero-length shape unless it is (0,)
+ lst = "[], shape=%s" % (repr(arr.shape),)
+
+ if arr.__class__ is not ndarray:
+ cName= arr.__class__.__name__
+ else:
+ cName = "array"
+
+ skipdtype = (arr.dtype.type in _typelessdata) and arr.size > 0
+
+ # XXX pypy lacks support
+ if 0 and arr.flags.maskna:
+ whichna = isna(arr)
+ # If nothing is NA, explicitly signal the NA-mask
+ if not any(whichna):
+ lst += ", maskna=True"
+ # If everything is NA, can't skip the dtype
+ if skipdtype and all(whichna):
+ skipdtype = False
+
+ if skipdtype:
+ return "%s(%s)" % (cName, lst)
+ else:
+ typename = arr.dtype.name
+ # Quote typename in the output if it is "complex".
+ if typename and not (typename[0].isalpha() and typename.isalnum()):
+ typename = "'%s'" % typename
+
+ lf = ''
+ if 0: # or issubclass(arr.dtype.type, flexible):
+ if arr.dtype.names:
+ typename = "%s" % str(arr.dtype)
+ else:
+ typename = "'%s'" % str(arr.dtype)
+ lf = '\n'+' '*len("array(")
+ return cName + "(%s, %sdtype=%s)" % (lst, lf, typename)
+
+def array_str(a, max_line_width=None, precision=None, suppress_small=None):
+ """
+ Return a string representation of the data in an array.
+
+ The data in the array is returned as a single string. This function is
+ similar to `array_repr`, the difference being that `array_repr` also
+ returns information on the kind of array and its data type.
+
+ Parameters
+ ----------
+ a : ndarray
+ Input array.
+ max_line_width : int, optional
+ Inserts newlines if text is longer than `max_line_width`. The
+ default is, indirectly, 75.
+ precision : int, optional
+ Floating point precision. Default is the current printing precision
+ (usually 8), which can be altered using `set_printoptions`.
+ suppress_small : bool, optional
+ Represent numbers "very close" to zero as zero; default is False.
+ Very close is defined by precision: if the precision is 8, e.g.,
+ numbers smaller (in absolute value) than 5e-9 are represented as
+ zero.
+
+ See Also
+ --------
+ array2string, array_repr, set_printoptions
+
+ Examples
+ --------
+ >>> np.array_str(np.arange(3))
+ '[0 1 2]'
+
+ """
+ return array2string(a, max_line_width, precision, suppress_small, ' ', "", str)
+
+def set_string_function(f, repr=True):
+ """
+ Set a Python function to be used when pretty printing arrays.
+
+ Parameters
+ ----------
+ f : function or None
+ Function to be used to pretty print arrays. The function should expect
+ a single array argument and return a string of the representation of
+ the array. If None, the function is reset to the default NumPy function
+ to print arrays.
+ repr : bool, optional
+ If True (default), the function for pretty printing (``__repr__``)
+ is set, if False the function that returns the default string
+ representation (``__str__``) is set.
+
+ See Also
+ --------
+ set_printoptions, get_printoptions
+
+ Examples
+ --------
+ >>> def pprint(arr):
+ ... return 'HA! - What are you going to do now?'
+ ...
+ >>> np.set_string_function(pprint)
+ >>> a = np.arange(10)
+ >>> a
+ HA! - What are you going to do now?
+ >>> print a
+ [0 1 2 3 4 5 6 7 8 9]
+
+ We can reset the function to the default:
+
+ >>> np.set_string_function(None)
+ >>> a
+ array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
+
+ `repr` affects either pretty printing or normal string representation.
+ Note that ``__repr__`` is still affected by setting ``__str__``
+ because the width of each array element in the returned string becomes
+ equal to the length of the result of ``__str__()``.
+
+ >>> x = np.arange(4)
+ >>> np.set_string_function(lambda x:'random', repr=False)
+ >>> x.__str__()
+ 'random'
+ >>> x.__repr__()
+ 'array([ 0, 1, 2, 3])'
+
+ """
+ if f is None:
+ if repr:
+ return multiarray.set_string_function(array_repr, 1)
+ else:
+ return multiarray.set_string_function(array_str, 0)
+ else:
+ return multiarray.set_string_function(f, repr)
+
+set_string_function(array_str, 0)
+set_string_function(array_repr, 1)
+
+little_endian = (sys.byteorder == 'little')
diff --git a/pypy/annotation/unaryop.py b/pypy/annotation/unaryop.py
--- a/pypy/annotation/unaryop.py
+++ b/pypy/annotation/unaryop.py
@@ -440,6 +440,12 @@
def method_popitem(dct):
return dct.getanyitem('items')
+ def method_pop(dct, s_key, s_dfl=None):
+ dct.dictdef.generalize_key(s_key)
+ if s_dfl is not None:
+ dct.dictdef.generalize_value(s_dfl)
+ return dct.dictdef.read_value()
+
def _can_only_throw(dic, *ignore):
if dic1.dictdef.dictkey.custom_eq_hash:
return None # r_dict: can throw anything
diff --git a/pypy/bin/py.py b/pypy/bin/py.py
--- a/pypy/bin/py.py
+++ b/pypy/bin/py.py
@@ -76,6 +76,8 @@
config.objspace.suggest(allworkingmodules=False)
if config.objspace.allworkingmodules:
pypyoption.enable_allworkingmodules(config)
+ if config.objspace.usemodules._continuation:
+ config.translation.continuation = True
if config.objspace.usemodules.thread:
config.translation.thread = True
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -340,7 +340,7 @@
requires=[("objspace.std.builtinshortcut", True)]),
BoolOption("withidentitydict",
"track types that override __hash__, __eq__ or __cmp__ and use a special dict strategy for those which do not",
- default=True),
+ default=False),
]),
])
@@ -370,6 +370,7 @@
config.objspace.std.suggest(getattributeshortcut=True)
config.objspace.std.suggest(newshortcut=True)
config.objspace.std.suggest(withspecialisedtuple=True)
+ config.objspace.std.suggest(withidentitydict=True)
#if not IS_64_BITS:
# config.objspace.std.suggest(withsmalllong=True)
diff --git a/pypy/doc/getting-started.rst b/pypy/doc/getting-started.rst
--- a/pypy/doc/getting-started.rst
+++ b/pypy/doc/getting-started.rst
@@ -53,11 +53,11 @@
PyPy is ready to be executed as soon as you unpack the tarball or the zip
file, with no need to install it in any specific location::
- $ tar xf pypy-1.6-linux.tar.bz2
+ $ tar xf pypy-1.7-linux.tar.bz2
- $ ./pypy-1.6/bin/pypy
+ $ ./pypy-1.7/bin/pypy
Python 2.7.1 (?, Apr 27 2011, 12:44:21)
- [PyPy 1.6.0 with GCC 4.4.3] on linux2
+ [PyPy 1.7.0 with GCC 4.4.3] on linux2
Type "help", "copyright", "credits" or "license" for more information.
And now for something completely different: ``implementing LOGO in LOGO:
"turtles all the way down"''
@@ -75,14 +75,14 @@
$ curl -O https://raw.github.com/pypa/pip/master/contrib/get-pip.py
- $ ./pypy-1.6/bin/pypy distribute_setup.py
+ $ ./pypy-1.7/bin/pypy distribute_setup.py
- $ ./pypy-1.6/bin/pypy get-pip.py
+ $ ./pypy-1.7/bin/pypy get-pip.py
- $ ./pypy-1.6/bin/pip install pygments # for example
+ $ ./pypy-1.7/bin/pip install pygments # for example
-3rd party libraries will be installed in ``pypy-1.6/site-packages``, and
-the scripts in ``pypy-1.6/bin``.
+3rd party libraries will be installed in ``pypy-1.7/site-packages``, and
+the scripts in ``pypy-1.7/bin``.
Installing using virtualenv
---------------------------
diff --git a/pypy/doc/translation.rst b/pypy/doc/translation.rst
--- a/pypy/doc/translation.rst
+++ b/pypy/doc/translation.rst
@@ -155,7 +155,7 @@
function. The two input variables are the exception class
and the exception value, respectively. (No other block will
actually link to the exceptblock if the function does not
- explicitely raise exceptions.)
+ explicitly raise exceptions.)
``Block``
@@ -325,7 +325,7 @@
Mutable objects need special treatment during annotation, because
the annotation of contained values needs to be possibly updated to account
for mutation operations, and consequently the annotation information
-reflown through the relevant parts of the flow the graphs.
+reflown through the relevant parts of the flow graphs.
* ``SomeList`` stands for a list of homogeneous type (i.e. all the
elements of the list are represented by a single common ``SomeXxx``
@@ -503,8 +503,8 @@
Since RPython is a garbage collected language there is a lot of heap memory
allocation going on all the time, which would either not occur at all in a more
-traditional explicitely managed language or results in an object which dies at
-a time known in advance and can thus be explicitely deallocated. For example a
+traditional explicitly managed language or results in an object which dies at
+a time known in advance and can thus be explicitly deallocated. For example a
loop of the following form::
for i in range(n):
@@ -696,7 +696,7 @@
So far it is the second most mature high level backend after GenCLI:
it still can't translate the full Standard Interpreter, but after the
-Leysin sprint we were able to compile and run the rpytstone and
+Leysin sprint we were able to compile and run the rpystone and
richards benchmarks.
GenJVM is almost entirely the work of Niko Matsakis, who worked on it
diff --git a/pypy/interpreter/eval.py b/pypy/interpreter/eval.py
--- a/pypy/interpreter/eval.py
+++ b/pypy/interpreter/eval.py
@@ -2,7 +2,6 @@
This module defines the abstract base classes that support execution:
Code and Frame.
"""
-from pypy.rlib import jit
from pypy.interpreter.error import OperationError
from pypy.interpreter.baseobjspace import Wrappable
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -445,6 +445,7 @@
AsyncAction.__init__(self, space)
self.dying_objects = []
self.finalizers_lock_count = 0
+ self.enabled_at_app_level = True
def register_callback(self, w_obj, callback, descrname):
self.dying_objects.append((w_obj, callback, descrname))
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -162,7 +162,8 @@
# generate 2 versions of the function and 2 jit drivers.
def _create_unpack_into():
jitdriver = jit.JitDriver(greens=['pycode'],
- reds=['self', 'frame', 'results'])
+ reds=['self', 'frame', 'results'],
+ name='unpack_into')
def unpack_into(self, results):
"""This is a hack for performance: runs the generator and collects
all produced items in a list."""
@@ -196,4 +197,4 @@
self.frame = None
return unpack_into
unpack_into = _create_unpack_into()
- unpack_into_w = _create_unpack_into()
\ No newline at end of file
+ unpack_into_w = _create_unpack_into()
diff --git a/pypy/jit/backend/llsupport/test/test_runner.py b/pypy/jit/backend/llsupport/test/test_runner.py
--- a/pypy/jit/backend/llsupport/test/test_runner.py
+++ b/pypy/jit/backend/llsupport/test/test_runner.py
@@ -8,6 +8,12 @@
class MyLLCPU(AbstractLLCPU):
supports_floats = True
+
+ class assembler(object):
+ @staticmethod
+ def set_debug(flag):
+ pass
+
def compile_loop(self, inputargs, operations, looptoken):
py.test.skip("llsupport test: cannot compile operations")
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -17,6 +17,7 @@
from pypy.rpython.llinterp import LLException
from pypy.jit.codewriter import heaptracker, longlong
from pypy.rlib.rarithmetic import intmask
+from pypy.jit.backend.detect_cpu import autodetect_main_model_and_size
def boxfloat(x):
return BoxFloat(longlong.getfloatstorage(x))
@@ -27,6 +28,9 @@
class Runner(object):
+ add_loop_instruction = ['overload for a specific cpu']
+ bridge_loop_instruction = ['overload for a specific cpu']
+
def execute_operation(self, opname, valueboxes, result_type, descr=None):
inputargs, operations = self._get_single_operation_list(opname,
result_type,
@@ -547,6 +551,28 @@
res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr)
assert res.value == func(*args)
+ def test_call_box_func(self):
+ def a(a1, a2):
+ return a1 + a2
+ def b(b1, b2):
+ return b1 * b2
+
+ arg1 = 40
+ arg2 = 2
+ for f in [a, b]:
+ TP = lltype.Signed
+ FPTR = self.Ptr(self.FuncType([TP, TP], TP))
+ func_ptr = llhelper(FPTR, f)
+ FUNC = deref(FPTR)
+ funcconst = self.get_funcbox(self.cpu, func_ptr)
+ funcbox = funcconst.clonebox()
+ calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
+ EffectInfo.MOST_GENERAL)
+ res = self.execute_operation(rop.CALL,
+ [funcbox, BoxInt(arg1), BoxInt(arg2)],
+ 'int', descr=calldescr)
+ assert res.getint() == f(arg1, arg2)
+
def test_call_stack_alignment(self):
# test stack alignment issues, notably for Mac OS/X.
# also test the ordering of the arguments.
@@ -1868,6 +1894,7 @@
values.append(descr)
values.append(self.cpu.get_latest_value_int(0))
values.append(self.cpu.get_latest_value_int(1))
+ values.append(token)
FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Void)
func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force)
@@ -1898,7 +1925,8 @@
assert fail.identifier == 1
assert self.cpu.get_latest_value_int(0) == 1
assert self.cpu.get_latest_value_int(1) == 10
- assert values == [faildescr, 1, 10]
+ token = self.cpu.get_latest_force_token()
+ assert values == [faildescr, 1, 10, token]
def test_force_operations_returning_int(self):
values = []
@@ -1907,6 +1935,7 @@
self.cpu.force(token)
values.append(self.cpu.get_latest_value_int(0))
values.append(self.cpu.get_latest_value_int(2))
+ values.append(token)
return 42
FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)
@@ -1940,7 +1969,8 @@
assert self.cpu.get_latest_value_int(0) == 1
assert self.cpu.get_latest_value_int(1) == 42
assert self.cpu.get_latest_value_int(2) == 10
- assert values == [1, 10]
+ token = self.cpu.get_latest_force_token()
+ assert values == [1, 10, token]
def test_force_operations_returning_float(self):
values = []
@@ -1949,6 +1979,7 @@
self.cpu.force(token)
values.append(self.cpu.get_latest_value_int(0))
values.append(self.cpu.get_latest_value_int(2))
+ values.append(token)
return 42.5
FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Float)
@@ -1984,7 +2015,8 @@
x = self.cpu.get_latest_value_float(1)
assert longlong.getrealfloat(x) == 42.5
assert self.cpu.get_latest_value_int(2) == 10
- assert values == [1, 10]
+ token = self.cpu.get_latest_force_token()
+ assert values == [1, 10, token]
def test_call_to_c_function(self):
from pypy.rlib.libffi import CDLL, types, ArgChain, FUNCFLAG_CDECL
@@ -2974,6 +3006,56 @@
res = self.cpu.get_latest_value_int(0)
assert res == -10
+ def test_compile_asmlen(self):
+ from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU
+ if not isinstance(self.cpu, AbstractLLCPU):
+ py.test.skip("pointless test on non-asm")
+ from pypy.jit.backend.x86.tool.viewcode import machine_code_dump
+ import ctypes
+ ops = """
+ [i2]
+ i0 = same_as(i2) # but forced to be in a register
+ label(i0, descr=1)
+ i1 = int_add(i0, i0)
+ guard_true(i1, descr=faildesr) [i1]
+ jump(i1, descr=1)
+ """
+ faildescr = BasicFailDescr(2)
+ loop = parse(ops, self.cpu, namespace=locals())
+ faildescr = loop.operations[-2].getdescr()
+ jumpdescr = loop.operations[-1].getdescr()
+ bridge_ops = """
+ [i0]
+ jump(i0, descr=jumpdescr)
+ """
+ bridge = parse(bridge_ops, self.cpu, namespace=locals())
+ looptoken = JitCellToken()
+ self.cpu.assembler.set_debug(False)
+ info = self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
+ bridge_info = self.cpu.compile_bridge(faildescr, bridge.inputargs,
+ bridge.operations,
+ looptoken)
+ self.cpu.assembler.set_debug(True) # always on untranslated
+ assert info.asmlen != 0
+ cpuname = autodetect_main_model_and_size()
+ # XXX we have to check the precise assembler, otherwise
+ # we don't quite know if borders are correct
+
+ def checkops(mc, ops):
+ assert len(mc) == len(ops)
+ for i in range(len(mc)):
+ assert mc[i].split("\t")[-1].startswith(ops[i])
+
+ data = ctypes.string_at(info.asmaddr, info.asmlen)
+ mc = list(machine_code_dump(data, info.asmaddr, cpuname))
+ lines = [line for line in mc if line.count('\t') == 2]
+ checkops(lines, self.add_loop_instructions)
+ data = ctypes.string_at(bridge_info.asmaddr, bridge_info.asmlen)
+ mc = list(machine_code_dump(data, bridge_info.asmaddr, cpuname))
+ lines = [line for line in mc if line.count('\t') == 2]
+ checkops(lines, self.bridge_loop_instructions)
+
+
def test_compile_bridge_with_target(self):
# This test creates a loopy piece of code in a bridge, and builds another
# unrelated loop that ends in a jump directly to this loopy bit of code.
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -7,6 +7,7 @@
from pypy.rpython.lltypesystem import lltype, rffi, rstr, llmemory
from pypy.rpython.lltypesystem.lloperation import llop
from pypy.rpython.annlowlevel import llhelper
+from pypy.rlib.jit import AsmInfo
from pypy.jit.backend.model import CompiledLoopToken
from pypy.jit.backend.x86.regalloc import (RegAlloc, get_ebp_ofs, _get_scale,
gpr_reg_mgr_cls, _valid_addressing_size)
@@ -411,6 +412,7 @@
'''adds the following attributes to looptoken:
_x86_function_addr (address of the generated func, as an int)
_x86_loop_code (debug: addr of the start of the ResOps)
+ _x86_fullsize (debug: full size including failure)
_x86_debug_checksum
'''
# XXX this function is too longish and contains some code
@@ -476,7 +478,8 @@
name = "Loop # %s: %s" % (looptoken.number, loopname)
self.cpu.profile_agent.native_code_written(name,
rawstart, full_size)
- return ops_offset
+ return AsmInfo(ops_offset, rawstart + looppos,
+ size_excluding_failure_stuff - looppos)
def assemble_bridge(self, faildescr, inputargs, operations,
original_loop_token, log):
@@ -485,12 +488,7 @@
assert len(set(inputargs)) == len(inputargs)
descr_number = self.cpu.get_fail_descr_number(faildescr)
- try:
- failure_recovery = self._find_failure_recovery_bytecode(faildescr)
- except ValueError:
- debug_print("Bridge out of guard", descr_number,
- "was already compiled!")
- return
+ failure_recovery = self._find_failure_recovery_bytecode(faildescr)
self.setup(original_loop_token)
if log:
@@ -503,6 +501,7 @@
[loc.assembler() for loc in faildescr._x86_debug_faillocs])
regalloc = RegAlloc(self, self.cpu.translate_support_code)
fail_depths = faildescr._x86_current_depths
+ startpos = self.mc.get_relative_pos()
operations = regalloc.prepare_bridge(fail_depths, inputargs, arglocs,
operations,
self.current_clt.allgcrefs)
@@ -537,7 +536,7 @@
name = "Bridge # %s" % (descr_number,)
self.cpu.profile_agent.native_code_written(name,
rawstart, fullsize)
- return ops_offset
+ return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos)
def write_pending_failure_recoveries(self):
# for each pending guard, generate the code of the recovery stub
@@ -621,7 +620,10 @@
def _find_failure_recovery_bytecode(self, faildescr):
adr_jump_offset = faildescr._x86_adr_jump_offset
if adr_jump_offset == 0:
- raise ValueError
+ # This case should be prevented by the logic in compile.py:
+ # look for CNT_BUSY_FLAG, which disables tracing from a guard
+ # when another tracing from the same guard is already in progress.
+ raise BridgeAlreadyCompiled
# follow the JMP/Jcond
p = rffi.cast(rffi.INTP, adr_jump_offset)
adr_target = adr_jump_offset + 4 + rffi.cast(lltype.Signed, p[0])
@@ -810,7 +812,10 @@
target = newlooptoken._x86_function_addr
mc = codebuf.MachineCodeBlockWrapper()
mc.JMP(imm(target))
- assert mc.get_relative_pos() <= 13 # keep in sync with prepare_loop()
+ if WORD == 4: # keep in sync with prepare_loop()
+ assert mc.get_relative_pos() == 5
+ else:
+ assert mc.get_relative_pos() <= 13
mc.copy_to_raw_memory(oldadr)
def dump(self, text):
@@ -1113,6 +1118,12 @@
for src, dst in singlefloats:
self.mc.MOVD(dst, src)
# Finally remap the arguments in the main regs
+ # If x is a register and is in dst_locs, then oups, it needs to
+ # be moved away:
+ if x in dst_locs:
+ src_locs.append(x)
+ dst_locs.append(r10)
+ x = r10
remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG)
self._regalloc.reserve_param(len(pass_on_stack))
@@ -2037,10 +2048,7 @@
size = sizeloc.value
signloc = arglocs[1]
- if isinstance(op.getarg(0), Const):
- x = imm(op.getarg(0).getint())
- else:
- x = arglocs[2]
+ x = arglocs[2] # the function address
if x is eax:
tmp = ecx
else:
@@ -2550,3 +2558,6 @@
def not_implemented(msg):
os.write(2, '[x86/asm] %s\n' % msg)
raise NotImplementedError(msg)
+
+class BridgeAlreadyCompiled(Exception):
+ pass
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -188,7 +188,10 @@
# note: we need to make a copy of inputargs because possibly_free_vars
# is also used on op args, which is a non-resizable list
self.possibly_free_vars(list(inputargs))
- self.min_bytes_before_label = 13
+ if WORD == 4: # see redirect_call_assembler()
+ self.min_bytes_before_label = 5
+ else:
+ self.min_bytes_before_label = 13
return operations
def prepare_bridge(self, prev_depths, inputargs, arglocs, operations,
diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py
--- a/pypy/jit/backend/x86/runner.py
+++ b/pypy/jit/backend/x86/runner.py
@@ -6,7 +6,7 @@
from pypy.jit.codewriter import longlong
from pypy.jit.metainterp import history, compile
from pypy.jit.backend.x86.assembler import Assembler386
-from pypy.jit.backend.x86.arch import FORCE_INDEX_OFS
+from pypy.jit.backend.x86.arch import FORCE_INDEX_OFS, IS_X86_32
from pypy.jit.backend.x86.profagent import ProfileAgent
from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU
from pypy.jit.backend.x86 import regloc
@@ -142,7 +142,9 @@
cast_ptr_to_int._annspecialcase_ = 'specialize:arglltype(0)'
cast_ptr_to_int = staticmethod(cast_ptr_to_int)
- all_null_registers = lltype.malloc(rffi.LONGP.TO, 24,
+ all_null_registers = lltype.malloc(rffi.LONGP.TO,
+ IS_X86_32 and (16+8) # 16 + 8 regs
+ or (16+16), # 16 + 16 regs
flavor='raw', zero=True,
immortal=True)
diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py
--- a/pypy/jit/backend/x86/test/test_runner.py
+++ b/pypy/jit/backend/x86/test/test_runner.py
@@ -33,6 +33,13 @@
# for the individual tests see
# ====> ../../test/runner_test.py
+ add_loop_instructions = ['mov', 'add', 'test', 'je', 'jmp']
+ if WORD == 4:
+ bridge_loop_instructions = ['lea', 'jmp']
+ else:
+ # the 'mov' is part of the 'jmp' so far
+ bridge_loop_instructions = ['lea', 'mov', 'jmp']
+
def setup_method(self, meth):
self.cpu = CPU(rtyper=None, stats=FakeStats())
self.cpu.setup_once()
@@ -416,7 +423,8 @@
]
inputargs = [i0]
debug._log = dlog = debug.DebugLog()
- ops_offset = self.cpu.compile_loop(inputargs, operations, looptoken)
+ info = self.cpu.compile_loop(inputargs, operations, looptoken)
+ ops_offset = info.ops_offset
debug._log = None
#
assert ops_offset is looptoken._x86_ops_offset
diff --git a/pypy/jit/backend/x86/tool/viewcode.py b/pypy/jit/backend/x86/tool/viewcode.py
--- a/pypy/jit/backend/x86/tool/viewcode.py
+++ b/pypy/jit/backend/x86/tool/viewcode.py
@@ -39,6 +39,7 @@
def machine_code_dump(data, originaddr, backend_name, label_list=None):
objdump_backend_option = {
'x86': 'i386',
+ 'x86_32': 'i386',
'x86_64': 'x86-64',
'i386': 'i386',
}
diff --git a/pypy/jit/codewriter/assembler.py b/pypy/jit/codewriter/assembler.py
--- a/pypy/jit/codewriter/assembler.py
+++ b/pypy/jit/codewriter/assembler.py
@@ -81,10 +81,15 @@
if not isinstance(value, (llmemory.AddressAsInt,
ComputedIntSymbolic)):
value = lltype.cast_primitive(lltype.Signed, value)
- if allow_short and -128 <= value <= 127:
- # emit the constant as a small integer
- self.code.append(chr(value & 0xFF))
- return True
+ if allow_short:
+ try:
+ short_num = -128 <= value <= 127
+ except TypeError: # "Symbolics cannot be compared!"
+ short_num = False
+ if short_num:
+ # emit the constant as a small integer
+ self.code.append(chr(value & 0xFF))
+ return True
constants = self.constants_i
elif kind == 'ref':
value = lltype.cast_opaque_ptr(llmemory.GCREF, value)
diff --git a/pypy/jit/codewriter/policy.py b/pypy/jit/codewriter/policy.py
--- a/pypy/jit/codewriter/policy.py
+++ b/pypy/jit/codewriter/policy.py
@@ -8,11 +8,15 @@
class JitPolicy(object):
- def __init__(self):
+ def __init__(self, jithookiface=None):
self.unsafe_loopy_graphs = set()
self.supports_floats = False
self.supports_longlong = False
self.supports_singlefloats = False
+ if jithookiface is None:
+ from pypy.rlib.jit import JitHookInterface
+ jithookiface = JitHookInterface()
+ self.jithookiface = jithookiface
def set_supports_floats(self, flag):
self.supports_floats = flag
diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py
--- a/pypy/jit/metainterp/compile.py
+++ b/pypy/jit/metainterp/compile.py
@@ -5,6 +5,7 @@
from pypy.rlib.objectmodel import we_are_translated
from pypy.rlib.debug import debug_start, debug_stop, debug_print
from pypy.rlib import rstack
+from pypy.rlib.jit import JitDebugInfo
from pypy.conftest import option
from pypy.tool.sourcetools import func_with_new_name
@@ -75,7 +76,7 @@
if descr is not original_jitcell_token:
original_jitcell_token.record_jump_to(descr)
descr.exported_state = None
- op._descr = None # clear reference, mostly for tests
+ op.cleardescr() # clear reference, mostly for tests
elif isinstance(descr, TargetToken):
# for a JUMP: record it as a potential jump.
# (the following test is not enough to prevent more complicated
@@ -90,8 +91,8 @@
assert descr.exported_state is None
if not we_are_translated():
op._descr_wref = weakref.ref(op._descr)
- op._descr = None # clear reference to prevent the history.Stats
- # from keeping the loop alive during tests
+ op.cleardescr() # clear reference to prevent the history.Stats
+ # from keeping the loop alive during tests
# record this looptoken on the QuasiImmut used in the code
if loop.quasi_immutable_deps is not None:
for qmut in loop.quasi_immutable_deps:
@@ -296,8 +297,6 @@
patch_new_loop_to_load_virtualizable_fields(loop, jitdriver_sd)
original_jitcell_token = loop.original_jitcell_token
- jitdriver_sd.on_compile(metainterp_sd.logger_ops, original_jitcell_token,
- loop.operations, type, greenkey)
loopname = jitdriver_sd.warmstate.get_location_str(greenkey)
globaldata = metainterp_sd.globaldata
original_jitcell_token.number = n = globaldata.loopnumbering
@@ -307,21 +306,38 @@
show_procedures(metainterp_sd, loop)
loop.check_consistency()
+ if metainterp_sd.warmrunnerdesc is not None:
+ hooks = metainterp_sd.warmrunnerdesc.hooks
+ debug_info = JitDebugInfo(jitdriver_sd, metainterp_sd.logger_ops,
+ original_jitcell_token, loop.operations,
+ type, greenkey)
+ hooks.before_compile(debug_info)
+ else:
+ debug_info = None
+ hooks = None
operations = get_deep_immutable_oplist(loop.operations)
metainterp_sd.profiler.start_backend()
debug_start("jit-backend")
try:
- ops_offset = metainterp_sd.cpu.compile_loop(loop.inputargs, operations,
- original_jitcell_token, name=loopname)
+ asminfo = metainterp_sd.cpu.compile_loop(loop.inputargs, operations,
+ original_jitcell_token,
+ name=loopname)
finally:
debug_stop("jit-backend")
metainterp_sd.profiler.end_backend()
+ if hooks is not None:
+ debug_info.asminfo = asminfo
+ hooks.after_compile(debug_info)
metainterp_sd.stats.add_new_loop(loop)
if not we_are_translated():
metainterp_sd.stats.compiled()
metainterp_sd.log("compiled new " + type)
#
loopname = jitdriver_sd.warmstate.get_location_str(greenkey)
+ if asminfo is not None:
+ ops_offset = asminfo.ops_offset
+ else:
+ ops_offset = None
metainterp_sd.logger_ops.log_loop(loop.inputargs, loop.operations, n,
type, ops_offset,
name=loopname)
@@ -332,25 +348,40 @@
def send_bridge_to_backend(jitdriver_sd, metainterp_sd, faildescr, inputargs,
operations, original_loop_token):
n = metainterp_sd.cpu.get_fail_descr_number(faildescr)
- jitdriver_sd.on_compile_bridge(metainterp_sd.logger_ops,
- original_loop_token, operations, n)
if not we_are_translated():
show_procedures(metainterp_sd)
seen = dict.fromkeys(inputargs)
TreeLoop.check_consistency_of_branch(operations, seen)
+ if metainterp_sd.warmrunnerdesc is not None:
+ hooks = metainterp_sd.warmrunnerdesc.hooks
+ debug_info = JitDebugInfo(jitdriver_sd, metainterp_sd.logger_ops,
+ original_loop_token, operations, 'bridge',
+ fail_descr_no=n)
+ hooks.before_compile_bridge(debug_info)
+ else:
+ hooks = None
+ debug_info = None
+ operations = get_deep_immutable_oplist(operations)
metainterp_sd.profiler.start_backend()
- operations = get_deep_immutable_oplist(operations)
debug_start("jit-backend")
try:
- ops_offset = metainterp_sd.cpu.compile_bridge(faildescr, inputargs, operations,
- original_loop_token)
+ asminfo = metainterp_sd.cpu.compile_bridge(faildescr, inputargs,
+ operations,
+ original_loop_token)
finally:
debug_stop("jit-backend")
metainterp_sd.profiler.end_backend()
+ if hooks is not None:
+ debug_info.asminfo = asminfo
+ hooks.after_compile_bridge(debug_info)
if not we_are_translated():
metainterp_sd.stats.compiled()
metainterp_sd.log("compiled new bridge")
#
+ if asminfo is not None:
+ ops_offset = asminfo.ops_offset
+ else:
+ ops_offset = None
metainterp_sd.logger_ops.log_bridge(inputargs, operations, n, ops_offset)
#
#if metainterp_sd.warmrunnerdesc is not None: # for tests
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -1003,16 +1003,16 @@
return insns
def check_simple_loop(self, expected=None, **check):
- # Usefull in the simplest case when we have only one trace ending with
- # a jump back to itself and possibly a few bridges ending with finnish.
- # Only the operations within the loop formed by that single jump will
- # be counted.
+ """ Usefull in the simplest case when we have only one trace ending with
+ a jump back to itself and possibly a few bridges.
+ Only the operations within the loop formed by that single jump will
+ be counted.
+ """
loops = self.get_all_loops()
assert len(loops) == 1
loop = loops[0]
jumpop = loop.operations[-1]
assert jumpop.getopnum() == rop.JUMP
- assert self.check_resops(jump=1)
labels = [op for op in loop.operations if op.getopnum() == rop.LABEL]
targets = [op._descr_wref() for op in labels]
assert None not in targets # TargetToken was freed, give up
diff --git a/pypy/jit/metainterp/jitdriver.py b/pypy/jit/metainterp/jitdriver.py
--- a/pypy/jit/metainterp/jitdriver.py
+++ b/pypy/jit/metainterp/jitdriver.py
@@ -21,7 +21,6 @@
# self.portal_finishtoken... pypy.jit.metainterp.pyjitpl
# self.index ... pypy.jit.codewriter.call
# self.mainjitcode ... pypy.jit.codewriter.call
- # self.on_compile ... pypy.jit.metainterp.warmstate
# These attributes are read by the backend in CALL_ASSEMBLER:
# self.assembler_helper_adr
diff --git a/pypy/jit/metainterp/jitprof.py b/pypy/jit/metainterp/jitprof.py
--- a/pypy/jit/metainterp/jitprof.py
+++ b/pypy/jit/metainterp/jitprof.py
@@ -18,8 +18,8 @@
OPT_FORCINGS
ABORT_TOO_LONG
ABORT_BRIDGE
+ABORT_BAD_LOOP
ABORT_ESCAPE
-ABORT_BAD_LOOP
ABORT_FORCE_QUASIIMMUT
NVIRTUALS
NVHOLES
@@ -30,10 +30,13 @@
TOTAL_FREED_BRIDGES
"""
+counter_names = []
+
def _setup():
names = counters.split()
for i, name in enumerate(names):
globals()[name] = i
+ counter_names.append(name)
global ncounters
ncounters = len(names)
_setup()
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -117,7 +117,7 @@
def optimize_loop(self, ops, optops, call_pure_results=None):
loop = self.parse(ops)
- token = JitCellToken()
+ token = JitCellToken()
loop.operations = [ResOperation(rop.LABEL, loop.inputargs, None, descr=TargetToken(token))] + \
loop.operations
if loop.operations[-1].getopnum() == rop.JUMP:
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -1553,6 +1553,7 @@
class MetaInterp(object):
in_recursion = 0
+ cancel_count = 0
def __init__(self, staticdata, jitdriver_sd):
self.staticdata = staticdata
@@ -1793,6 +1794,15 @@
def aborted_tracing(self, reason):
self.staticdata.profiler.count(reason)
debug_print('~~~ ABORTING TRACING')
+ jd_sd = self.jitdriver_sd
+ if not self.current_merge_points:
+ greenkey = None # we're in the bridge
+ else:
+ greenkey = self.current_merge_points[0][0][:jd_sd.num_green_args]
+ self.staticdata.warmrunnerdesc.hooks.on_abort(reason,
+ jd_sd.jitdriver,
+ greenkey,
+ jd_sd.warmstate.get_location_str(greenkey))
self.staticdata.stats.aborted()
def blackhole_if_trace_too_long(self):
@@ -1966,9 +1976,14 @@
raise SwitchToBlackhole(ABORT_BAD_LOOP) # For now
self.compile_loop(original_boxes, live_arg_boxes, start, resumedescr)
# creation of the loop was cancelled!
+ self.cancel_count += 1
+ if self.staticdata.warmrunnerdesc:
+ memmgr = self.staticdata.warmrunnerdesc.memory_manager
+ if memmgr:
+ if self.cancel_count > memmgr.max_unroll_loops:
+ self.staticdata.log('cancelled too many times!')
+ raise SwitchToBlackhole(ABORT_BAD_LOOP)
self.staticdata.log('cancelled, tracing more...')
- #self.staticdata.log('cancelled, stopping tracing')
- #raise SwitchToBlackhole(ABORT_BAD_LOOP)
# Otherwise, no loop found so far, so continue tracing.
start = len(self.history.operations)
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -18,6 +18,8 @@
pc = 0
opnum = 0
+ _attrs_ = ('result',)
+
def __init__(self, result):
self.result = result
@@ -62,6 +64,9 @@
def setdescr(self, descr):
raise NotImplementedError
+ def cleardescr(self):
+ pass
+
# common methods
# --------------
@@ -194,6 +199,9 @@
self._check_descr(descr)
self._descr = descr
+ def cleardescr(self):
+ self._descr = None
+
def _check_descr(self, descr):
if not we_are_translated() and getattr(descr, 'I_am_a_descr', False):
return # needed for the mock case in oparser_model
diff --git a/pypy/jit/metainterp/test/support.py b/pypy/jit/metainterp/test/support.py
--- a/pypy/jit/metainterp/test/support.py
+++ b/pypy/jit/metainterp/test/support.py
@@ -56,8 +56,6 @@
greenfield_info = None
result_type = result_kind
portal_runner_ptr = "???"
- on_compile = lambda *args: None
- on_compile_bridge = lambda *args: None
stats = history.Stats()
cpu = CPUClass(rtyper, stats, None, False)
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -2629,6 +2629,38 @@
self.check_jitcell_token_count(1)
self.check_target_token_count(5)
+ def test_max_unroll_loops(self):
+ from pypy.jit.metainterp.optimize import InvalidLoop
+ from pypy.jit.metainterp import optimizeopt
+ myjitdriver = JitDriver(greens = [], reds = ['n', 'i'])
+ #
+ def f(n, limit):
+ set_param(myjitdriver, 'threshold', 5)
+ set_param(myjitdriver, 'max_unroll_loops', limit)
+ i = 0
+ while i < n:
+ myjitdriver.jit_merge_point(n=n, i=i)
+ print i
+ i += 1
+ return i
+ #
+ def my_optimize_trace(*args, **kwds):
+ raise InvalidLoop
+ old_optimize_trace = optimizeopt.optimize_trace
+ optimizeopt.optimize_trace = my_optimize_trace
+ try:
+ res = self.meta_interp(f, [23, 4])
+ assert res == 23
+ self.check_trace_count(0)
+ self.check_aborted_count(3)
+ #
+ res = self.meta_interp(f, [23, 20])
+ assert res == 23
+ self.check_trace_count(0)
+ self.check_aborted_count(2)
+ finally:
+ optimizeopt.optimize_trace = old_optimize_trace
+
def test_retrace_limit_with_extra_guards(self):
myjitdriver = JitDriver(greens = [], reds = ['n', 'i', 'sa', 'a',
'node'])
diff --git a/pypy/jit/metainterp/test/test_compile.py b/pypy/jit/metainterp/test/test_compile.py
--- a/pypy/jit/metainterp/test/test_compile.py
+++ b/pypy/jit/metainterp/test/test_compile.py
@@ -53,8 +53,6 @@
call_pure_results = {}
class jitdriver_sd:
warmstate = FakeState()
- on_compile = staticmethod(lambda *args: None)
- on_compile_bridge = staticmethod(lambda *args: None)
virtualizable_info = None
def test_compile_loop():
diff --git a/pypy/jit/metainterp/test/test_jitdriver.py b/pypy/jit/metainterp/test/test_jitdriver.py
--- a/pypy/jit/metainterp/test/test_jitdriver.py
+++ b/pypy/jit/metainterp/test/test_jitdriver.py
@@ -10,57 +10,6 @@
def getloc2(g):
return "in jitdriver2, with g=%d" % g
-class JitDriverTests(object):
- def test_on_compile(self):
- called = {}
-
- class MyJitDriver(JitDriver):
- def on_compile(self, logger, looptoken, operations, type, n, m):
- called[(m, n, type)] = looptoken
-
- driver = MyJitDriver(greens = ['n', 'm'], reds = ['i'])
-
- def loop(n, m):
- i = 0
- while i < n + m:
- driver.can_enter_jit(n=n, m=m, i=i)
- driver.jit_merge_point(n=n, m=m, i=i)
- i += 1
-
- self.meta_interp(loop, [1, 4])
- assert sorted(called.keys()) == [(4, 1, "loop")]
- self.meta_interp(loop, [2, 4])
- assert sorted(called.keys()) == [(4, 1, "loop"),
- (4, 2, "loop")]
-
- def test_on_compile_bridge(self):
- called = {}
-
- class MyJitDriver(JitDriver):
- def on_compile(self, logger, looptoken, operations, type, n, m):
- called[(m, n, type)] = loop
- def on_compile_bridge(self, logger, orig_token, operations, n):
- assert 'bridge' not in called
- called['bridge'] = orig_token
-
- driver = MyJitDriver(greens = ['n', 'm'], reds = ['i'])
-
- def loop(n, m):
- i = 0
- while i < n + m:
- driver.can_enter_jit(n=n, m=m, i=i)
- driver.jit_merge_point(n=n, m=m, i=i)
- if i >= 4:
- i += 2
- i += 1
-
- self.meta_interp(loop, [1, 10])
- assert sorted(called.keys()) == ['bridge', (10, 1, "loop")]
-
-
-class TestLLtypeSingle(JitDriverTests, LLJitMixin):
- pass
-
class MultipleJitDriversTests(object):
def test_simple(self):
diff --git a/pypy/jit/metainterp/test/test_jitiface.py b/pypy/jit/metainterp/test/test_jitiface.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/test/test_jitiface.py
@@ -0,0 +1,148 @@
+
+from pypy.rlib.jit import JitDriver, JitHookInterface
+from pypy.rlib import jit_hooks
+from pypy.jit.metainterp.test.support import LLJitMixin
+from pypy.jit.codewriter.policy import JitPolicy
+from pypy.jit.metainterp.jitprof import ABORT_FORCE_QUASIIMMUT
+from pypy.jit.metainterp.resoperation import rop
+from pypy.rpython.annlowlevel import hlstr
+
+class TestJitHookInterface(LLJitMixin):
+ def test_abort_quasi_immut(self):
+ reasons = []
+
+ class MyJitIface(JitHookInterface):
+ def on_abort(self, reason, jitdriver, greenkey, greenkey_repr):
+ assert jitdriver is myjitdriver
+ assert len(greenkey) == 1
+ reasons.append(reason)
+ assert greenkey_repr == 'blah'
+
+ iface = MyJitIface()
+
+ myjitdriver = JitDriver(greens=['foo'], reds=['x', 'total'],
+ get_printable_location=lambda *args: 'blah')
+
+ class Foo:
+ _immutable_fields_ = ['a?']
+ def __init__(self, a):
+ self.a = a
+ def f(a, x):
+ foo = Foo(a)
+ total = 0
+ while x > 0:
+ myjitdriver.jit_merge_point(foo=foo, x=x, total=total)
+ # read a quasi-immutable field out of a Constant
+ total += foo.a
+ foo.a += 1
+ x -= 1
+ return total
+ #
+ assert f(100, 7) == 721
+ res = self.meta_interp(f, [100, 7], policy=JitPolicy(iface))
+ assert res == 721
+ assert reasons == [ABORT_FORCE_QUASIIMMUT] * 2
+
+ def test_on_compile(self):
+ called = []
+
+ class MyJitIface(JitHookInterface):
+ def after_compile(self, di):
+ called.append(("compile", di.greenkey[1].getint(),
+ di.greenkey[0].getint(), di.type))
+
+ def before_compile(self, di):
+ called.append(("optimize", di.greenkey[1].getint(),
+ di.greenkey[0].getint(), di.type))
+
+ #def before_optimize(self, jitdriver, logger, looptoken, oeprations,
+ # type, greenkey):
+ # called.append(("trace", greenkey[1].getint(),
+ # greenkey[0].getint(), type))
+
+ iface = MyJitIface()
+
+ driver = JitDriver(greens = ['n', 'm'], reds = ['i'])
+
+ def loop(n, m):
+ i = 0
+ while i < n + m:
+ driver.can_enter_jit(n=n, m=m, i=i)
+ driver.jit_merge_point(n=n, m=m, i=i)
+ i += 1
+
+ self.meta_interp(loop, [1, 4], policy=JitPolicy(iface))
+ assert called == [#("trace", 4, 1, "loop"),
+ ("optimize", 4, 1, "loop"),
+ ("compile", 4, 1, "loop")]
+ self.meta_interp(loop, [2, 4], policy=JitPolicy(iface))
+ assert called == [#("trace", 4, 1, "loop"),
+ ("optimize", 4, 1, "loop"),
+ ("compile", 4, 1, "loop"),
+ #("trace", 4, 2, "loop"),
+ ("optimize", 4, 2, "loop"),
+ ("compile", 4, 2, "loop")]
+
+ def test_on_compile_bridge(self):
+ called = []
+
+ class MyJitIface(JitHookInterface):
+ def after_compile(self, di):
+ called.append("compile")
+
+ def after_compile_bridge(self, di):
+ called.append("compile_bridge")
+
+ def before_compile_bridge(self, di):
+ called.append("before_compile_bridge")
+
+ driver = JitDriver(greens = ['n', 'm'], reds = ['i'])
+
+ def loop(n, m):
+ i = 0
+ while i < n + m:
+ driver.can_enter_jit(n=n, m=m, i=i)
+ driver.jit_merge_point(n=n, m=m, i=i)
+ if i >= 4:
+ i += 2
+ i += 1
+
+ self.meta_interp(loop, [1, 10], policy=JitPolicy(MyJitIface()))
+ assert called == ["compile", "before_compile_bridge", "compile_bridge"]
+
+ def test_resop_interface(self):
+ driver = JitDriver(greens = [], reds = ['i'])
+
+ def loop(i):
+ while i > 0:
+ driver.jit_merge_point(i=i)
+ i -= 1
+
+ def main():
+ loop(1)
+ op = jit_hooks.resop_new(rop.INT_ADD,
+ [jit_hooks.boxint_new(3),
+ jit_hooks.boxint_new(4)],
+ jit_hooks.boxint_new(1))
+ assert hlstr(jit_hooks.resop_getopname(op)) == 'int_add'
+ assert jit_hooks.resop_getopnum(op) == rop.INT_ADD
+ box = jit_hooks.resop_getarg(op, 0)
+ assert jit_hooks.box_getint(box) == 3
+ box2 = jit_hooks.box_clone(box)
+ assert box2 != box
+ assert jit_hooks.box_getint(box2) == 3
+ assert not jit_hooks.box_isconst(box2)
+ box3 = jit_hooks.box_constbox(box)
+ assert jit_hooks.box_getint(box) == 3
+ assert jit_hooks.box_isconst(box3)
+ box4 = jit_hooks.box_nonconstbox(box)
+ assert not jit_hooks.box_isconst(box4)
+ box5 = jit_hooks.boxint_new(18)
+ jit_hooks.resop_setarg(op, 0, box5)
+ assert jit_hooks.resop_getarg(op, 0) == box5
+ box6 = jit_hooks.resop_getresult(op)
+ assert jit_hooks.box_getint(box6) == 1
+ jit_hooks.resop_setresult(op, box5)
+ assert jit_hooks.resop_getresult(op) == box5
+
+ self.meta_interp(main, [])
diff --git a/pypy/jit/metainterp/test/test_virtualstate.py b/pypy/jit/metainterp/test/test_virtualstate.py
--- a/pypy/jit/metainterp/test/test_virtualstate.py
+++ b/pypy/jit/metainterp/test/test_virtualstate.py
@@ -5,7 +5,7 @@
VArrayStateInfo, NotVirtualStateInfo, VirtualState, ShortBoxes
from pypy.jit.metainterp.optimizeopt.optimizer import OptValue
from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr, ConstInt, ConstPtr
-from pypy.rpython.lltypesystem import lltype
+from pypy.rpython.lltypesystem import lltype, llmemory
from pypy.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin, BaseTest, \
equaloplists, FakeDescrWithSnapshot
from pypy.jit.metainterp.optimizeopt.intutils import IntBound
@@ -82,6 +82,13 @@
assert isgeneral(value1, value2)
assert not isgeneral(value2, value1)
+ assert isgeneral(OptValue(ConstInt(7)), OptValue(ConstInt(7)))
+ S = lltype.GcStruct('S')
+ foo = lltype.malloc(S)
+ fooref = lltype.cast_opaque_ptr(llmemory.GCREF, foo)
+ assert isgeneral(OptValue(ConstPtr(fooref)),
+ OptValue(ConstPtr(fooref)))
+
def test_field_matching_generalization(self):
const1 = NotVirtualStateInfo(OptValue(ConstInt(1)))
const2 = NotVirtualStateInfo(OptValue(ConstInt(2)))
diff --git a/pypy/jit/metainterp/test/test_ztranslation.py b/pypy/jit/metainterp/test/test_ztranslation.py
--- a/pypy/jit/metainterp/test/test_ztranslation.py
+++ b/pypy/jit/metainterp/test/test_ztranslation.py
@@ -3,7 +3,9 @@
from pypy.jit.backend.llgraph import runner
from pypy.rlib.jit import JitDriver, unroll_parameters, set_param
from pypy.rlib.jit import PARAMETERS, dont_look_inside, hint
+from pypy.rlib.jit_hooks import boxint_new, resop_new, resop_getopnum
from pypy.jit.metainterp.jitprof import Profiler
+from pypy.jit.metainterp.resoperation import rop
from pypy.rpython.lltypesystem import lltype, llmemory
class TranslationTest:
@@ -22,6 +24,7 @@
# - jitdriver hooks
# - two JITs
# - string concatenation, slicing and comparison
+ # - jit hooks interface
class Frame(object):
_virtualizable2_ = ['l[*]']
@@ -91,7 +94,9 @@
return f.i
#
def main(i, j):
- return f(i) - f2(i+j, i, j)
+ op = resop_new(rop.INT_ADD, [boxint_new(3), boxint_new(5)],
+ boxint_new(8))
+ return f(i) - f2(i+j, i, j) + resop_getopnum(op)
res = ll_meta_interp(main, [40, 5], CPUClass=self.CPUClass,
type_system=self.type_system,
listops=True)
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -1,4 +1,5 @@
import sys, py
+from pypy.tool.sourcetools import func_with_new_name
from pypy.rpython.lltypesystem import lltype, llmemory
from pypy.rpython.annlowlevel import llhelper, MixLevelHelperAnnotator,\
cast_base_ptr_to_instance, hlstr
@@ -112,7 +113,7 @@
return ll_meta_interp(function, args, backendopt=backendopt,
translate_support_code=True, **kwds)
-def _find_jit_marker(graphs, marker_name):
+def _find_jit_marker(graphs, marker_name, check_driver=True):
results = []
for graph in graphs:
for block in graph.iterblocks():
@@ -120,8 +121,8 @@
op = block.operations[i]
if (op.opname == 'jit_marker' and
op.args[0].value == marker_name and
- (op.args[1].value is None or
- op.args[1].value.active)): # the jitdriver
+ (not check_driver or op.args[1].value is None or
+ op.args[1].value.active)): # the jitdriver
results.append((graph, block, i))
return results
@@ -140,6 +141,9 @@
"found several jit_merge_points in the same graph")
return results
+def find_access_helpers(graphs):
+ return _find_jit_marker(graphs, 'access_helper', False)
+
def locate_jit_merge_point(graph):
[(graph, block, pos)] = find_jit_merge_points([graph])
return block, pos, block.operations[pos]
@@ -206,6 +210,7 @@
vrefinfo = VirtualRefInfo(self)
self.codewriter.setup_vrefinfo(vrefinfo)
#
+ self.hooks = policy.jithookiface
self.make_virtualizable_infos()
self.make_exception_classes()
self.make_driverhook_graphs()
@@ -213,6 +218,7 @@
self.rewrite_jit_merge_points(policy)
verbose = False # not self.cpu.translate_support_code
+ self.rewrite_access_helpers()
self.codewriter.make_jitcodes(verbose=verbose)
self.rewrite_can_enter_jits()
self.rewrite_set_param()
@@ -619,6 +625,24 @@
graph = self.annhelper.getgraph(func, args_s, s_result)
return self.annhelper.graph2delayed(graph, FUNC)
+ def rewrite_access_helpers(self):
+ ah = find_access_helpers(self.translator.graphs)
+ for graph, block, index in ah:
+ op = block.operations[index]
+ self.rewrite_access_helper(op)
+
+ def rewrite_access_helper(self, op):
+ ARGS = [arg.concretetype for arg in op.args[2:]]
+ RESULT = op.result.concretetype
+ FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, RESULT))
+ # make sure we make a copy of function so it no longer belongs
+ # to extregistry
+ func = op.args[1].value
+ func = func_with_new_name(func, func.func_name + '_compiled')
+ ptr = self.helper_func(FUNCPTR, func)
+ op.opname = 'direct_call'
+ op.args = [Constant(ptr, FUNCPTR)] + op.args[2:]
+
def rewrite_jit_merge_points(self, policy):
for jd in self.jitdrivers_sd:
self.rewrite_jit_merge_point(jd, policy)
diff --git a/pypy/jit/metainterp/warmstate.py b/pypy/jit/metainterp/warmstate.py
--- a/pypy/jit/metainterp/warmstate.py
+++ b/pypy/jit/metainterp/warmstate.py
@@ -244,6 +244,11 @@
if self.warmrunnerdesc.memory_manager:
self.warmrunnerdesc.memory_manager.max_retrace_guards = value
+ def set_param_max_unroll_loops(self, value):
+ if self.warmrunnerdesc:
+ if self.warmrunnerdesc.memory_manager:
+ self.warmrunnerdesc.memory_manager.max_unroll_loops = value
+
def disable_noninlinable_function(self, greenkey):
cell = self.jit_cell_at_key(greenkey)
cell.dont_trace_here = True
@@ -596,20 +601,6 @@
return fn(*greenargs)
self.should_unroll_one_iteration = should_unroll_one_iteration
- if hasattr(jd.jitdriver, 'on_compile'):
- def on_compile(logger, token, operations, type, greenkey):
- greenargs = unwrap_greenkey(greenkey)
- return jd.jitdriver.on_compile(logger, token, operations, type,
- *greenargs)
- def on_compile_bridge(logger, orig_token, operations, n):
- return jd.jitdriver.on_compile_bridge(logger, orig_token,
- operations, n)
- jd.on_compile = on_compile
- jd.on_compile_bridge = on_compile_bridge
- else:
- jd.on_compile = lambda *args: None
- jd.on_compile_bridge = lambda *args: None
-
redargtypes = ''.join([kind[0] for kind in jd.red_args_types])
def get_assembler_token(greenkey):
diff --git a/pypy/jit/tool/oparser.py b/pypy/jit/tool/oparser.py
--- a/pypy/jit/tool/oparser.py
+++ b/pypy/jit/tool/oparser.py
@@ -89,11 +89,18 @@
assert typ == 'class'
return self.model.ConstObj(ootype.cast_to_object(obj))
- def get_descr(self, poss_descr):
+ def get_descr(self, poss_descr, allow_invent):
if poss_descr.startswith('<'):
return None
- else:
+ try:
return self._consts[poss_descr]
+ except KeyError:
+ if allow_invent:
+ int(poss_descr)
+ token = self.model.JitCellToken()
+ tt = self.model.TargetToken(token)
+ self._consts[poss_descr] = tt
+ return tt
def box_for_var(self, elem):
try:
@@ -186,7 +193,8 @@
poss_descr = allargs[-1].strip()
if poss_descr.startswith('descr='):
- descr = self.get_descr(poss_descr[len('descr='):])
+ descr = self.get_descr(poss_descr[len('descr='):],
+ opname == 'label')
allargs = allargs[:-1]
for arg in allargs:
arg = arg.strip()
diff --git a/pypy/jit/tool/oparser_model.py b/pypy/jit/tool/oparser_model.py
--- a/pypy/jit/tool/oparser_model.py
+++ b/pypy/jit/tool/oparser_model.py
@@ -6,7 +6,7 @@
from pypy.jit.metainterp.history import TreeLoop, JitCellToken
from pypy.jit.metainterp.history import Box, BoxInt, BoxFloat
from pypy.jit.metainterp.history import ConstInt, ConstObj, ConstPtr, ConstFloat
- from pypy.jit.metainterp.history import BasicFailDescr
+ from pypy.jit.metainterp.history import BasicFailDescr, TargetToken
from pypy.jit.metainterp.typesystem import llhelper
from pypy.jit.metainterp.history import get_const_ptr_for_string
@@ -42,6 +42,10 @@
class JitCellToken(object):
I_am_a_descr = True
+ class TargetToken(object):
+ def __init__(self, jct):
+ pass
+
class BasicFailDescr(object):
I_am_a_descr = True
diff --git a/pypy/jit/tool/pypytrace.vim b/pypy/jit/tool/pypytrace.vim
--- a/pypy/jit/tool/pypytrace.vim
+++ b/pypy/jit/tool/pypytrace.vim
@@ -19,6 +19,7 @@
syn match pypyLoopArgs '^[[].*'
syn match pypyLoopStart '^#.*'
syn match pypyDebugMergePoint '^debug_merge_point(.\+)'
+syn match pypyLogBoundary '[[][0-9a-f]\+[]] \([{].\+\|.\+[}]\)$'
hi def link pypyLoopStart Structure
"hi def link pypyLoopArgs PreProc
@@ -29,3 +30,4 @@
hi def link pypyNumber Number
hi def link pypyDescr PreProc
hi def link pypyDescrField Label
+hi def link pypyLogBoundary Statement
diff --git a/pypy/jit/tool/test/test_oparser.py b/pypy/jit/tool/test/test_oparser.py
--- a/pypy/jit/tool/test/test_oparser.py
+++ b/pypy/jit/tool/test/test_oparser.py
@@ -4,7 +4,8 @@
from pypy.jit.tool.oparser import parse, OpParser
from pypy.jit.metainterp.resoperation import rop
-from pypy.jit.metainterp.history import AbstractDescr, BoxInt, JitCellToken
+from pypy.jit.metainterp.history import AbstractDescr, BoxInt, JitCellToken,\
+ TargetToken
class BaseTestOparser(object):
@@ -243,6 +244,16 @@
b = loop.getboxes()
assert isinstance(b.sum0, BoxInt)
+ def test_label(self):
+ x = """
+ [i0]
+ label(i0, descr=1)
+ jump(i0, descr=1)
+ """
+ loop = self.parse(x)
+ assert loop.operations[0].getdescr() is loop.operations[1].getdescr()
+ assert isinstance(loop.operations[0].getdescr(), TargetToken)
+
class ForbiddenModule(object):
def __init__(self, name, old_mod):
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -108,6 +108,10 @@
w_result = state.codec_search_cache.get(normalized_encoding, None)
if w_result is not None:
return w_result
+ return _lookup_codec_loop(space, encoding, normalized_encoding)
+
+def _lookup_codec_loop(space, encoding, normalized_encoding):
+ state = space.fromcache(CodecState)
if state.codec_need_encodings:
w_import = space.getattr(space.builtin, space.wrap("__import__"))
# registers new codecs
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -588,10 +588,18 @@
raises(UnicodeDecodeError, '+3ADYAA-'.decode, 'utf-7')
def test_utf_16_encode_decode(self):
- import codecs
+ import codecs, sys
x = u'123abc'
- assert codecs.getencoder('utf-16')(x) == ('\xff\xfe1\x002\x003\x00a\x00b\x00c\x00', 6)
- assert codecs.getdecoder('utf-16')('\xff\xfe1\x002\x003\x00a\x00b\x00c\x00') == (x, 14)
+ if sys.byteorder == 'big':
+ assert codecs.getencoder('utf-16')(x) == (
+ '\xfe\xff\x001\x002\x003\x00a\x00b\x00c', 6)
+ assert codecs.getdecoder('utf-16')(
+ '\xfe\xff\x001\x002\x003\x00a\x00b\x00c') == (x, 14)
+ else:
+ assert codecs.getencoder('utf-16')(x) == (
+ '\xff\xfe1\x002\x003\x00a\x00b\x00c\x00', 6)
+ assert codecs.getdecoder('utf-16')(
+ '\xff\xfe1\x002\x003\x00a\x00b\x00c\x00') == (x, 14)
def test_unicode_escape(self):
assert u'\\'.encode('unicode-escape') == '\\\\'
diff --git a/pypy/module/_codecs/test/test_ztranslation.py b/pypy/module/_codecs/test/test_ztranslation.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_codecs/test/test_ztranslation.py
@@ -0,0 +1,5 @@
+from pypy.objspace.fake.checkmodule import checkmodule
+
+
+def test__codecs_translates():
+ checkmodule('_codecs')
diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -34,8 +34,12 @@
ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO, flavor='raw')
rgc.add_memory_pressure(HASH_MALLOC_SIZE + self.digest_size)
- ropenssl.EVP_DigestInit(ctx, digest_type)
- self.ctx = ctx
+ try:
+ ropenssl.EVP_DigestInit(ctx, digest_type)
+ self.ctx = ctx
+ except:
+ lltype.free(ctx, flavor='raw')
+ raise
def __del__(self):
# self.lock.free()
diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py
--- a/pypy/module/_io/interp_fileio.py
+++ b/pypy/module/_io/interp_fileio.py
@@ -349,6 +349,8 @@
try:
s = os.read(self.fd, size)
except OSError, e:
+ if e.errno == errno.EAGAIN:
+ return space.w_None
raise wrap_oserror(space, e,
exception_name='w_IOError')
@@ -362,6 +364,8 @@
try:
buf = os.read(self.fd, length)
except OSError, e:
+ if e.errno == errno.EAGAIN:
+ return space.w_None
raise wrap_oserror(space, e,
exception_name='w_IOError')
rwbuffer.setslice(0, buf)
diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py
--- a/pypy/module/_io/test/test_fileio.py
+++ b/pypy/module/_io/test/test_fileio.py
@@ -133,6 +133,19 @@
f.close()
assert a == 'a\nbxxxxxxx'
+ def test_nonblocking_read(self):
+ import os, fcntl
+ r_fd, w_fd = os.pipe()
+ # set nonblocking
+ fcntl.fcntl(r_fd, fcntl.F_SETFL, os.O_NONBLOCK)
+ import _io
+ f = _io.FileIO(r_fd, 'r')
+ # Read from stream sould return None
+ assert f.read() is None
+ assert f.read(10) is None
+ a = bytearray('x' * 10)
+ assert f.readinto(a) is None
+
def test_repr(self):
import _io
f = _io.FileIO(self.tmpfile, 'r')
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -67,9 +67,6 @@
self.connect(self.addr_from_object(space, w_addr))
except SocketError, e:
raise converted_error(space, e)
- except TypeError, e:
- raise OperationError(space.w_TypeError,
- space.wrap(str(e)))
def connect_ex_w(self, space, w_addr):
"""connect_ex(address) -> errno
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -529,26 +529,31 @@
import _socket, os
if not hasattr(_socket, 'AF_UNIX'):
skip('AF_UNIX not supported.')
- sockpath = os.path.join(self.udir, 'app_test_unix_socket_connect')
+ oldcwd = os.getcwd()
+ os.chdir(self.udir)
+ try:
+ sockpath = 'app_test_unix_socket_connect'
- serversock = _socket.socket(_socket.AF_UNIX)
- serversock.bind(sockpath)
- serversock.listen(1)
+ serversock = _socket.socket(_socket.AF_UNIX)
+ serversock.bind(sockpath)
+ serversock.listen(1)
- clientsock = _socket.socket(_socket.AF_UNIX)
- clientsock.connect(sockpath)
- s, addr = serversock.accept()
- assert not addr
+ clientsock = _socket.socket(_socket.AF_UNIX)
+ clientsock.connect(sockpath)
+ s, addr = serversock.accept()
+ assert not addr
- s.send('X')
- data = clientsock.recv(100)
- assert data == 'X'
- clientsock.send('Y')
- data = s.recv(100)
- assert data == 'Y'
+ s.send('X')
+ data = clientsock.recv(100)
+ assert data == 'X'
+ clientsock.send('Y')
+ data = s.recv(100)
+ assert data == 'Y'
- clientsock.close()
- s.close()
+ clientsock.close()
+ s.close()
+ finally:
+ os.chdir(oldcwd)
class AppTestSocketTCP:
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -64,8 +64,8 @@
def test_sslwrap(self):
import _ssl, _socket, sys, gc
- if sys.platform == 'darwin':
- skip("hangs indefinitely on OSX (also on CPython)")
+ if sys.platform == 'darwin' or 'freebsd' in sys.platform:
+ skip("hangs indefinitely on OSX & FreeBSD (also on CPython)")
s = _socket.socket()
ss = _ssl.sslwrap(s, 0)
exc = raises(_socket.error, ss.do_handshake)
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,7 +29,7 @@
#define PY_VERSION "2.7.1"
/* PyPy version as a string */
-#define PYPY_VERSION "1.7.1"
+#define PYPY_VERSION "1.8.1"
/* Subversion Revision number of this file (not of the repository).
* Empty since Mercurial migration. */
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -32,11 +32,10 @@
Py_DecRef(space, w_item)
if not isinstance(w_list, W_ListObject):
PyErr_BadInternalCall(space)
- wrappeditems = w_list.getitems()
- if index < 0 or index >= len(wrappeditems):
+ if index < 0 or index >= w_list.length():
raise OperationError(space.w_IndexError, space.wrap(
"list assignment index out of range"))
- wrappeditems[index] = w_item
+ w_list.setitem(index, w_item)
return 0
@cpython_api([PyObject, Py_ssize_t], PyObject)
@@ -74,7 +73,7 @@
"""Macro form of PyList_Size() without error checking.
"""
assert isinstance(w_list, W_ListObject)
- return len(w_list.getitems())
+ return w_list.length()
@cpython_api([PyObject], Py_ssize_t, error=-1)
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -128,3 +128,6 @@
module.setslice(l, None)
assert l == [0, 4, 5]
+ l = [1, 2, 3]
+ module.setlistitem(l,0)
+ assert l == [None, 2, 3]
diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py
--- a/pypy/module/fcntl/test/test_fcntl.py
+++ b/pypy/module/fcntl/test/test_fcntl.py
@@ -42,13 +42,9 @@
else:
start_len = "qq"
- if sys.platform in ('netbsd1', 'netbsd2', 'netbsd3',
- 'Darwin1.2', 'darwin',
- 'freebsd2', 'freebsd3', 'freebsd4', 'freebsd5',
- 'freebsd6', 'freebsd7', 'freebsd8', 'freebsd9',
- 'bsdos2', 'bsdos3', 'bsdos4',
- 'openbsd', 'openbsd2', 'openbsd3', 'openbsd4',
- 'openbsd5'):
+ if any(substring in sys.platform.lower()
+ for substring in ('netbsd', 'darwin', 'freebsd', 'bsdos',
+ 'openbsd')):
if struct.calcsize('l') == 8:
off_t = 'l'
pid_t = 'i'
@@ -182,7 +178,8 @@
def test_large_flag(self):
import sys
- if sys.platform == "darwin" or sys.platform.startswith("openbsd"):
+ if any(plat in sys.platform
+ for plat in ('darwin', 'openbsd', 'freebsd')):
skip("Mac OS doesn't have any large flag in fcntl.h")
import fcntl, sys
if sys.maxint == 2147483647:
diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py
--- a/pypy/module/gc/__init__.py
+++ b/pypy/module/gc/__init__.py
@@ -1,18 +1,18 @@
from pypy.interpreter.mixedmodule import MixedModule
class Module(MixedModule):
- appleveldefs = {
- 'enable': 'app_gc.enable',
- 'disable': 'app_gc.disable',
- 'isenabled': 'app_gc.isenabled',
- }
interpleveldefs = {
'collect': 'interp_gc.collect',
+ 'enable': 'interp_gc.enable',
+ 'disable': 'interp_gc.disable',
+ 'isenabled': 'interp_gc.isenabled',
'enable_finalizers': 'interp_gc.enable_finalizers',
'disable_finalizers': 'interp_gc.disable_finalizers',
'garbage' : 'space.newlist([])',
#'dump_heap_stats': 'interp_gc.dump_heap_stats',
}
+ appleveldefs = {
+ }
def __init__(self, space, w_name):
if (not space.config.translating or
diff --git a/pypy/module/gc/app_gc.py b/pypy/module/gc/app_gc.py
deleted file mode 100644
--- a/pypy/module/gc/app_gc.py
+++ /dev/null
@@ -1,21 +0,0 @@
-# NOT_RPYTHON
-
-enabled = True
-
-def isenabled():
- global enabled
- return enabled
-
-def enable():
- global enabled
- import gc
- if not enabled:
- gc.enable_finalizers()
- enabled = True
-
-def disable():
- global enabled
- import gc
- if enabled:
- gc.disable_finalizers()
- enabled = False
diff --git a/pypy/module/gc/interp_gc.py b/pypy/module/gc/interp_gc.py
--- a/pypy/module/gc/interp_gc.py
+++ b/pypy/module/gc/interp_gc.py
@@ -17,6 +17,26 @@
rgc.collect()
return space.wrap(0)
+def enable(space):
+ """Non-recursive version. Enable finalizers now.
+ If they were already enabled, no-op.
+ If they were disabled even several times, enable them anyway.
+ """
+ if not space.user_del_action.enabled_at_app_level:
+ space.user_del_action.enabled_at_app_level = True
+ enable_finalizers(space)
+
+def disable(space):
+ """Non-recursive version. Disable finalizers now. Several calls
+ to this function are ignored.
+ """
+ if space.user_del_action.enabled_at_app_level:
+ space.user_del_action.enabled_at_app_level = False
+ disable_finalizers(space)
+
+def isenabled(space):
+ return space.newbool(space.user_del_action.enabled_at_app_level)
+
def enable_finalizers(space):
if space.user_del_action.finalizers_lock_count == 0:
raise OperationError(space.w_ValueError,
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -27,6 +27,12 @@
'dot': 'interp_numarray.dot',
'fromstring': 'interp_support.fromstring',
'flatiter': 'interp_numarray.W_FlatIterator',
+ 'isna': 'interp_numarray.isna',
+ 'concatenate': 'interp_numarray.concatenate',
+
+ 'set_string_function': 'appbridge.set_string_function',
+
+ 'count_reduce_items': 'interp_numarray.count_reduce_items',
'True_': 'types.Bool.True',
'False_': 'types.Bool.False',
@@ -66,10 +72,12 @@
("copysign", "copysign"),
("cos", "cos"),
("divide", "divide"),
+ ("true_divide", "true_divide"),
("equal", "equal"),
("exp", "exp"),
("fabs", "fabs"),
("floor", "floor"),
+ ("ceil", "ceil"),
("greater", "greater"),
("greater_equal", "greater_equal"),
("less", "less"),
@@ -85,12 +93,16 @@
("subtract", "subtract"),
('sqrt', 'sqrt'),
("tan", "tan"),
+ ('bitwise_and', 'bitwise_and'),
+ ('bitwise_or', 'bitwise_or'),
+ ('bitwise_not', 'invert'),
+ ('isnan', 'isnan'),
+ ('isinf', 'isinf'),
]:
interpleveldefs[exposed] = "interp_ufuncs.get(space).%s" % impl
appleveldefs = {
'average': 'app_numpy.average',
- 'mean': 'app_numpy.mean',
'sum': 'app_numpy.sum',
'min': 'app_numpy.min',
'identity': 'app_numpy.identity',
@@ -99,5 +111,4 @@
'e': 'app_numpy.e',
'pi': 'app_numpy.pi',
'arange': 'app_numpy.arange',
- 'reshape': 'app_numpy.reshape',
}
diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py
--- a/pypy/module/micronumpy/app_numpy.py
+++ b/pypy/module/micronumpy/app_numpy.py
@@ -11,34 +11,55 @@
def average(a):
# This implements a weighted average, for now we don't implement the
# weighting, just the average part!
- return mean(a)
+ if not hasattr(a, "mean"):
+ a = _numpypy.array(a)
+ return a.mean()
def identity(n, dtype=None):
- a = _numpypy.zeros((n,n), dtype=dtype)
+ a = _numpypy.zeros((n, n), dtype=dtype)
for i in range(n):
a[i][i] = 1
return a
-def mean(a):
- if not hasattr(a, "mean"):
- a = _numpypy.array(a)
- return a.mean()
+def sum(a,axis=None):
+ '''sum(a, axis=None)
+ Sum of array elements over a given axis.
-def sum(a):
+ Parameters
+ ----------
+ a : array_like
+ Elements to sum.
+ axis : integer, optional
+ Axis over which the sum is taken. By default `axis` is None,
+ and all elements are summed.
+
+ Returns
+ -------
+ sum_along_axis : ndarray
+ An array with the same shape as `a`, with the specified
+ axis removed. If `a` is a 0-d array, or if `axis` is None, a scalar
+ is returned. If an output array is specified, a reference to
+ `out` is returned.
+
+ See Also
+ --------
+ ndarray.sum : Equivalent method.
+ '''
+ # TODO: add to doc (once it's implemented): cumsum : Cumulative sum of array elements.
if not hasattr(a, "sum"):
a = _numpypy.array(a)
- return a.sum()
+ return a.sum(axis)
-def min(a):
+def min(a, axis=None):
if not hasattr(a, "min"):
a = _numpypy.array(a)
- return a.min()
+ return a.min(axis)
-def max(a):
+def max(a, axis=None):
if not hasattr(a, "max"):
a = _numpypy.array(a)
- return a.max()
-
+ return a.max(axis)
+
def arange(start, stop=None, step=1, dtype=None):
'''arange([start], stop[, step], dtype=None)
Generate values in the half-interval [start, stop).
@@ -55,40 +76,3 @@
arr[j] = i
i += step
return arr
-
-
-def reshape(a, shape):
- '''reshape(a, newshape)
- Gives a new shape to an array without changing its data.
-
- Parameters
- ----------
- a : array_like
- Array to be reshaped.
- newshape : int or tuple of ints
- The new shape should be compatible with the original shape. If
- an integer, then the result will be a 1-D array of that length.
- One shape dimension can be -1. In this case, the value is inferred
- from the length of the array and remaining dimensions.
-
- Returns
- -------
- reshaped_array : ndarray
- This will be a new view object if possible; otherwise, it will
- be a copy.
-
-
- See Also
- --------
- ndarray.reshape : Equivalent method.
-
- Notes
- -----
-
- It is not always possible to change the shape of an array without
- copying the data. If you want an error to be raise if the data is copied,
- you should assign the new shape to the shape attribute of the array
-'''
- if not hasattr(a, 'reshape'):
- a = _numpypy.array(a)
- return a.reshape(shape)
diff --git a/pypy/module/micronumpy/appbridge.py b/pypy/module/micronumpy/appbridge.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/appbridge.py
@@ -0,0 +1,38 @@
+
+from pypy.rlib.objectmodel import specialize
+
+class AppBridgeCache(object):
+ w__var = None
+ w__std = None
+ w_module = None
+ w_array_repr = None
+ w_array_str = None
+
+ def __init__(self, space):
+ self.w_import = space.appexec([], """():
+ def f():
+ import sys
+ __import__('numpypy.core._methods')
+ return sys.modules['numpypy.core._methods']
+ return f
+ """)
+
+ @specialize.arg(2)
+ def call_method(self, space, name, *args):
+ w_meth = getattr(self, 'w_' + name)
+ if w_meth is None:
+ if self.w_module is None:
+ self.w_module = space.call_function(self.w_import)
+ w_meth = space.getattr(self.w_module, space.wrap(name))
+ setattr(self, 'w_' + name, w_meth)
+ return space.call_function(w_meth, *args)
+
+def set_string_function(space, w_f, w_repr):
+ cache = get_appbridge_cache(space)
+ if space.is_true(w_repr):
+ cache.w_array_repr = w_f
+ else:
+ cache.w_array_str = w_f
+
+def get_appbridge_cache(space):
+ return space.fromcache(AppBridgeCache)
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -32,13 +32,16 @@
class BadToken(Exception):
pass
-SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any", "unegative"]
+SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any",
+ "unegative", "flat"]
+TWO_ARG_FUNCTIONS = ['take']
class FakeSpace(object):
w_ValueError = None
w_TypeError = None
w_IndexError = None
w_OverflowError = None
+ w_NotImplementedError = None
w_None = None
w_bool = "bool"
@@ -53,6 +56,9 @@
"""NOT_RPYTHON"""
self.fromcache = InternalSpaceCache(self).getorbuild
+ def _freeze_(self):
+ return True
+
def issequence_w(self, w_obj):
return isinstance(w_obj, ListObject) or isinstance(w_obj, W_NDimArray)
@@ -144,6 +150,9 @@
def allocate_instance(self, klass, w_subtype):
return instantiate(klass)
+ def newtuple(self, list_w):
+ raise ValueError
+
def len_w(self, w_obj):
if isinstance(w_obj, ListObject):
return len(w_obj.items)
@@ -371,14 +380,18 @@
for arg in self.args]))
def execute(self, interp):
+ arr = self.args[0].execute(interp)
+ if not isinstance(arr, BaseArray):
+ raise ArgumentNotAnArray
if self.name in SINGLE_ARG_FUNCTIONS:
- if len(self.args) != 1:
+ if len(self.args) != 1 and self.name != 'sum':
raise ArgumentMismatch
- arr = self.args[0].execute(interp)
- if not isinstance(arr, BaseArray):
- raise ArgumentNotAnArray
if self.name == "sum":
- w_res = arr.descr_sum(interp.space)
+ if len(self.args)>1:
+ w_res = arr.descr_sum(interp.space,
+ self.args[1].execute(interp))
+ else:
+ w_res = arr.descr_sum(interp.space)
elif self.name == "prod":
w_res = arr.descr_prod(interp.space)
elif self.name == "max":
@@ -392,21 +405,31 @@
elif self.name == "unegative":
neg = interp_ufuncs.get(interp.space).negative
w_res = neg.call(interp.space, [arr])
+ elif self.name == "flat":
+ w_res = arr.descr_get_flatiter(interp.space)
else:
assert False # unreachable code
- if isinstance(w_res, BaseArray):
- return w_res
- if isinstance(w_res, FloatObject):
- dtype = get_dtype_cache(interp.space).w_float64dtype
- elif isinstance(w_res, BoolObject):
- dtype = get_dtype_cache(interp.space).w_booldtype
- elif isinstance(w_res, interp_boxes.W_GenericBox):
- dtype = w_res.get_dtype(interp.space)
+ elif self.name in TWO_ARG_FUNCTIONS:
+ arg = self.args[1].execute(interp)
+ if not isinstance(arg, BaseArray):
+ raise ArgumentNotAnArray
+ if self.name == 'take':
+ w_res = arr.descr_take(interp.space, arg)
else:
- dtype = None
- return scalar_w(interp.space, dtype, w_res)
+ assert False # unreachable
else:
raise WrongFunctionName
+ if isinstance(w_res, BaseArray):
+ return w_res
+ if isinstance(w_res, FloatObject):
+ dtype = get_dtype_cache(interp.space).w_float64dtype
+ elif isinstance(w_res, BoolObject):
+ dtype = get_dtype_cache(interp.space).w_booldtype
+ elif isinstance(w_res, interp_boxes.W_GenericBox):
+ dtype = w_res.get_dtype(interp.space)
+ else:
+ dtype = None
+ return scalar_w(interp.space, dtype, w_res)
_REGEXES = [
('-?[\d\.]+', 'number'),
@@ -416,7 +439,7 @@
('\]', 'array_right'),
('(->)|[\+\-\*\/]', 'operator'),
('=', 'assign'),
- (',', 'coma'),
+ (',', 'comma'),
('\|', 'pipe'),
('\(', 'paren_left'),
('\)', 'paren_right'),
@@ -504,7 +527,7 @@
return SliceConstant(start, stop, step)
- def parse_expression(self, tokens):
+ def parse_expression(self, tokens, accept_comma=False):
stack = []
while tokens.remaining():
token = tokens.pop()
@@ -524,9 +547,13 @@
stack.append(RangeConstant(tokens.pop().v))
end = tokens.pop()
assert end.name == 'pipe'
+ elif accept_comma and token.name == 'comma':
+ continue
else:
tokens.push()
break
+ if accept_comma:
+ return stack
stack.reverse()
lhs = stack.pop()
while stack:
@@ -540,7 +567,7 @@
args = []
tokens.pop() # lparen
while tokens.get(0).name != 'paren_right':
- args.append(self.parse_expression(tokens))
+ args += self.parse_expression(tokens, accept_comma=True)
return FunctionCall(name, args)
def parse_array_const(self, tokens):
@@ -556,7 +583,7 @@
token = tokens.pop()
if token.name == 'array_right':
return elems
- assert token.name == 'coma'
+ assert token.name == 'comma'
def parse_statement(self, tokens):
if (tokens.get(0).name == 'identifier' and
diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -8,6 +8,7 @@
from pypy.tool.sourcetools import func_with_new_name
+MIXIN_32 = (int_typedef,) if LONG_BIT == 32 else ()
MIXIN_64 = (int_typedef,) if LONG_BIT == 64 else ()
def new_dtype_getter(name):
@@ -28,6 +29,7 @@
def convert_to(self, dtype):
return dtype.box(self.value)
+
class W_GenericBox(Wrappable):
_attrs_ = ()
@@ -93,7 +95,7 @@
descr_neg = _unaryop_impl("negative")
descr_abs = _unaryop_impl("absolute")
- def descr_tolist(self, space):
+ def item(self, space):
return self.get_dtype(space).itemtype.to_builtin_type(space, self)
@@ -104,7 +106,8 @@
_attrs_ = ()
class W_IntegerBox(W_NumberBox):
- pass
+ def int_w(self, space):
+ return space.int_w(self.descr_int(space))
class W_SignedIntegerBox(W_IntegerBox):
pass
@@ -187,7 +190,7 @@
__neg__ = interp2app(W_GenericBox.descr_neg),
__abs__ = interp2app(W_GenericBox.descr_abs),
- tolist = interp2app(W_GenericBox.descr_tolist),
+ tolist = interp2app(W_GenericBox.item),
)
W_BoolBox.typedef = TypeDef("bool_", W_GenericBox.typedef,
@@ -231,7 +234,7 @@
__new__ = interp2app(W_UInt16Box.descr__new__.im_func),
)
-W_Int32Box.typedef = TypeDef("int32", W_SignedIntegerBox.typedef,
+W_Int32Box.typedef = TypeDef("int32", (W_SignedIntegerBox.typedef,) + MIXIN_32,
__module__ = "numpypy",
__new__ = interp2app(W_Int32Box.descr__new__.im_func),
)
@@ -241,24 +244,18 @@
__new__ = interp2app(W_UInt32Box.descr__new__.im_func),
)
-if LONG_BIT == 32:
- long_name = "int32"
-elif LONG_BIT == 64:
- long_name = "int64"
-W_LongBox.typedef = TypeDef(long_name, (W_SignedIntegerBox.typedef, int_typedef,),
- __module__ = "numpypy",
- __new__ = interp2app(W_LongBox.descr__new__.im_func),
-)
-
-W_ULongBox.typedef = TypeDef("u" + long_name, W_UnsignedIntegerBox.typedef,
- __module__ = "numpypy",
-)
-
W_Int64Box.typedef = TypeDef("int64", (W_SignedIntegerBox.typedef,) + MIXIN_64,
__module__ = "numpypy",
__new__ = interp2app(W_Int64Box.descr__new__.im_func),
)
+if LONG_BIT == 32:
+ W_LongBox = W_Int32Box
+ W_ULongBox = W_UInt32Box
+elif LONG_BIT == 64:
+ W_LongBox = W_Int64Box
+ W_ULongBox = W_UInt64Box
+
W_UInt64Box.typedef = TypeDef("uint64", W_UnsignedIntegerBox.typedef,
__module__ = "numpypy",
__new__ = interp2app(W_UInt64Box.descr__new__.im_func),
@@ -283,3 +280,4 @@
__new__ = interp2app(W_Float64Box.descr__new__.im_func),
)
+
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -20,7 +20,7 @@
class W_Dtype(Wrappable):
_immutable_fields_ = ["itemtype", "num", "kind"]
- def __init__(self, itemtype, num, kind, name, char, w_box_type, alternate_constructors=[]):
+ def __init__(self, itemtype, num, kind, name, char, w_box_type, alternate_constructors=[], aliases=[]):
self.itemtype = itemtype
self.num = num
self.kind = kind
@@ -28,6 +28,7 @@
self.char = char
self.w_box_type = w_box_type
self.alternate_constructors = alternate_constructors
+ self.aliases = aliases
def malloc(self, length):
# XXX find out why test_zjit explodes with tracking of allocations
@@ -46,6 +47,10 @@
def getitem(self, storage, i):
return self.itemtype.read(storage, self.itemtype.get_element_size(), i, 0)
+ def getitem_bool(self, storage, i):
+ isize = self.itemtype.get_element_size()
+ return self.itemtype.read_bool(storage, isize, i, 0)
+
def setitem(self, storage, i, box):
self.itemtype.store(storage, self.itemtype.get_element_size(), i, 0, box)
@@ -62,7 +67,7 @@
elif space.isinstance_w(w_dtype, space.w_str):
name = space.str_w(w_dtype)
for dtype in cache.builtin_dtypes:
- if dtype.name == name or dtype.char == name:
+ if dtype.name == name or dtype.char == name or name in dtype.aliases:
return dtype
else:
for dtype in cache.builtin_dtypes:
@@ -84,18 +89,38 @@
def descr_get_shape(self, space):
return space.newtuple([])
+ def eq(self, space, w_other):
+ w_other = space.call_function(space.gettypefor(W_Dtype), w_other)
+ return space.is_w(self, w_other)
+
+ def descr_eq(self, space, w_other):
+ return space.wrap(self.eq(space, w_other))
+
+ def descr_ne(self, space, w_other):
+ return space.wrap(not self.eq(space, w_other))
+
+ def is_int_type(self):
+ return (self.kind == SIGNEDLTR or self.kind == UNSIGNEDLTR or
+ self.kind == BOOLLTR)
+
+ def is_bool_type(self):
+ return self.kind == BOOLLTR
+
W_Dtype.typedef = TypeDef("dtype",
__module__ = "numpypy",
__new__ = interp2app(W_Dtype.descr__new__.im_func),
__str__= interp2app(W_Dtype.descr_str),
__repr__ = interp2app(W_Dtype.descr_repr),
+ __eq__ = interp2app(W_Dtype.descr_eq),
+ __ne__ = interp2app(W_Dtype.descr_ne),
num = interp_attrproperty("num", cls=W_Dtype),
kind = interp_attrproperty("kind", cls=W_Dtype),
type = interp_attrproperty_w("w_box_type", cls=W_Dtype),
itemsize = GetSetProperty(W_Dtype.descr_get_itemsize),
shape = GetSetProperty(W_Dtype.descr_get_shape),
+ name = interp_attrproperty('name', cls=W_Dtype),
)
W_Dtype.typedef.acceptable_as_base_class = False
@@ -107,7 +132,7 @@
kind=BOOLLTR,
name="bool",
char="?",
- w_box_type = space.gettypefor(interp_boxes.W_BoolBox),
+ w_box_type=space.gettypefor(interp_boxes.W_BoolBox),
alternate_constructors=[space.w_bool],
)
self.w_int8dtype = W_Dtype(
@@ -116,7 +141,7 @@
kind=SIGNEDLTR,
name="int8",
char="b",
- w_box_type = space.gettypefor(interp_boxes.W_Int8Box)
+ w_box_type=space.gettypefor(interp_boxes.W_Int8Box)
)
self.w_uint8dtype = W_Dtype(
types.UInt8(),
@@ -124,7 +149,7 @@
kind=UNSIGNEDLTR,
name="uint8",
char="B",
- w_box_type = space.gettypefor(interp_boxes.W_UInt8Box),
+ w_box_type=space.gettypefor(interp_boxes.W_UInt8Box),
)
self.w_int16dtype = W_Dtype(
types.Int16(),
@@ -132,7 +157,7 @@
kind=SIGNEDLTR,
name="int16",
char="h",
- w_box_type = space.gettypefor(interp_boxes.W_Int16Box),
+ w_box_type=space.gettypefor(interp_boxes.W_Int16Box),
)
self.w_uint16dtype = W_Dtype(
types.UInt16(),
@@ -140,7 +165,7 @@
kind=UNSIGNEDLTR,
name="uint16",
char="H",
- w_box_type = space.gettypefor(interp_boxes.W_UInt16Box),
+ w_box_type=space.gettypefor(interp_boxes.W_UInt16Box),
)
self.w_int32dtype = W_Dtype(
types.Int32(),
@@ -148,7 +173,7 @@
kind=SIGNEDLTR,
name="int32",
char="i",
- w_box_type = space.gettypefor(interp_boxes.W_Int32Box),
+ w_box_type=space.gettypefor(interp_boxes.W_Int32Box),
)
self.w_uint32dtype = W_Dtype(
types.UInt32(),
@@ -156,7 +181,7 @@
kind=UNSIGNEDLTR,
name="uint32",
char="I",
- w_box_type = space.gettypefor(interp_boxes.W_UInt32Box),
+ w_box_type=space.gettypefor(interp_boxes.W_UInt32Box),
)
if LONG_BIT == 32:
name = "int32"
@@ -168,7 +193,7 @@
kind=SIGNEDLTR,
name=name,
char="l",
- w_box_type = space.gettypefor(interp_boxes.W_LongBox),
+ w_box_type=space.gettypefor(interp_boxes.W_LongBox),
alternate_constructors=[space.w_int],
)
self.w_ulongdtype = W_Dtype(
@@ -177,7 +202,7 @@
kind=UNSIGNEDLTR,
name="u" + name,
char="L",
- w_box_type = space.gettypefor(interp_boxes.W_ULongBox),
+ w_box_type=space.gettypefor(interp_boxes.W_ULongBox),
)
self.w_int64dtype = W_Dtype(
types.Int64(),
@@ -185,7 +210,7 @@
kind=SIGNEDLTR,
name="int64",
char="q",
- w_box_type = space.gettypefor(interp_boxes.W_Int64Box),
+ w_box_type=space.gettypefor(interp_boxes.W_Int64Box),
alternate_constructors=[space.w_long],
)
self.w_uint64dtype = W_Dtype(
@@ -194,7 +219,7 @@
kind=UNSIGNEDLTR,
name="uint64",
char="Q",
- w_box_type = space.gettypefor(interp_boxes.W_UInt64Box),
+ w_box_type=space.gettypefor(interp_boxes.W_UInt64Box),
)
self.w_float32dtype = W_Dtype(
types.Float32(),
@@ -202,7 +227,7 @@
kind=FLOATINGLTR,
name="float32",
char="f",
- w_box_type = space.gettypefor(interp_boxes.W_Float32Box),
+ w_box_type=space.gettypefor(interp_boxes.W_Float32Box),
)
self.w_float64dtype = W_Dtype(
types.Float64(),
@@ -212,6 +237,7 @@
char="d",
w_box_type = space.gettypefor(interp_boxes.W_Float64Box),
alternate_constructors=[space.w_float],
+ aliases=["float"],
)
self.builtin_dtypes = [
diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py
--- a/pypy/module/micronumpy/interp_iter.py
+++ b/pypy/module/micronumpy/interp_iter.py
@@ -1,19 +1,80 @@
from pypy.rlib import jit
from pypy.rlib.objectmodel import instantiate
-from pypy.module.micronumpy.strides import calculate_broadcast_strides
+from pypy.module.micronumpy.strides import calculate_broadcast_strides,\
+ calculate_slice_strides
-# Iterators for arrays
-# --------------------
-# all those iterators with the exception of BroadcastIterator iterate over the
-# entire array in C order (the last index changes the fastest). This will
-# yield all elements. Views iterate over indices and look towards strides and
-# backstrides to find the correct position. Notably the offset between
-# x[..., i + 1] and x[..., i] will be strides[-1]. Offset between
-# x[..., k + 1, 0] and x[..., k, i_max] will be backstrides[-2] etc.
+""" This is a mini-tutorial on iterators, strides, and
+memory layout. It assumes you are familiar with the terms, see
+http://docs.scipy.org/doc/numpy/reference/arrays.ndarray.html
+for a more gentle introduction.
-# BroadcastIterator works like that, but for indexes that don't change source
-# in the original array, strides[i] == backstrides[i] == 0
+Given an array x: x.shape == [5,6],
+
+At which byte in x.data does the item x[3,4] begin?
+if x.strides==[1,5]:
+ pData = x.pData + (x.start + 3*1 + 4*5)*sizeof(x.pData[0])
+ pData = x.pData + (x.start + 24) * sizeof(x.pData[0])
+so the offset of the element is 24 elements after the first
+
+What is the next element in x after coordinates [3,4]?
+if x.order =='C':
+ next == [3,5] => offset is 28
+if x.order =='F':
+ next == [4,4] => offset is 24
+so for the strides [1,5] x is 'F' contiguous
+likewise, for the strides [6,1] x would be 'C' contiguous.
+
+Iterators have an internal representation of the current coordinates
+(indices), the array, strides, and backstrides. A short digression to
+explain backstrides: what is the coordinate and offset after [3,5] in
+the example above?
+if x.order == 'C':
+ next == [4,0] => offset is 4
+if x.order == 'F':
+ next == [4,5] => offset is 25
+Note that in 'C' order we stepped BACKWARDS 24 while 'overflowing' a
+shape dimension
+ which is back 25 and forward 1,
+ which is x.strides[1] * (x.shape[1] - 1) + x.strides[0]
+so if we precalculate the overflow backstride as
+[x.strides[i] * (x.shape[i] - 1) for i in range(len(x.shape))]
+we can go faster.
+All the calculations happen in next()
+
+next_step_x() tries to do the iteration for a number of steps at once,
+but then we cannot gaurentee that we only overflow one single shape
+dimension, perhaps we could overflow times in one big step.
+"""
+
+# structures to describe slicing
+
+class Chunk(object):
+ def __init__(self, start, stop, step, lgt):
+ self.start = start
+ self.stop = stop
+ self.step = step
+ self.lgt = lgt
+
+ def extend_shape(self, shape):
+ if self.step != 0:
+ shape.append(self.lgt)
+
+ def __repr__(self):
+ return 'Chunk(%d, %d, %d, %d)' % (self.start, self.stop, self.step,
+ self.lgt)
+
+class BaseTransform(object):
+ pass
+
+class ViewTransform(BaseTransform):
+ def __init__(self, chunks):
+ # 4-tuple specifying slicing
+ self.chunks = chunks
+
+class BroadcastTransform(BaseTransform):
+ def __init__(self, res_shape):
+ self.res_shape = res_shape
class BaseIterator(object):
def next(self, shapelen):
@@ -22,20 +83,40 @@
def done(self):
raise NotImplementedError
+ def apply_transformations(self, arr, transformations):
+ v = self
+ for transform in transformations:
+ v = v.transform(arr, transform)
+ return v
+
+ def transform(self, arr, t):
+ raise NotImplementedError
+
class ArrayIterator(BaseIterator):
def __init__(self, size):
self.offset = 0
self.size = size
def next(self, shapelen):
+ return self.next_skip_x(1)
+
+ def next_skip_x(self, ofs):
arr = instantiate(ArrayIterator)
arr.size = self.size
- arr.offset = self.offset + 1
+ arr.offset = self.offset + ofs
return arr
+ def next_no_increase(self, shapelen):
+ # a hack to make JIT believe this is always virtual
+ return self.next_skip_x(0)
+
def done(self):
return self.offset >= self.size
+ def transform(self, arr, t):
+ return ViewIterator(arr.start, arr.strides, arr.backstrides,
+ arr.shape).transform(arr, t)
+
class OneDimIterator(BaseIterator):
def __init__(self, start, step, stop):
self.offset = start
@@ -52,26 +133,29 @@
def done(self):
return self.offset == self.size
-def view_iter_from_arr(arr):
- return ViewIterator(arr.start, arr.strides, arr.backstrides, arr.shape)
-
class ViewIterator(BaseIterator):
- def __init__(self, start, strides, backstrides, shape, res_shape=None):
+ def __init__(self, start, strides, backstrides, shape):
self.offset = start
self._done = False
- if res_shape is not None and res_shape != shape:
- r = calculate_broadcast_strides(strides, backstrides,
- shape, res_shape)
- self.strides, self.backstrides = r
- self.res_shape = res_shape
- else:
- self.strides = strides
- self.backstrides = backstrides
- self.res_shape = shape
+ self.strides = strides
+ self.backstrides = backstrides
+ self.res_shape = shape
self.indices = [0] * len(self.res_shape)
+ def transform(self, arr, t):
+ if isinstance(t, BroadcastTransform):
+ r = calculate_broadcast_strides(self.strides, self.backstrides,
+ self.res_shape, t.res_shape)
+ return ViewIterator(self.offset, r[0], r[1], t.res_shape)
+ elif isinstance(t, ViewTransform):
+ r = calculate_slice_strides(self.res_shape, self.offset,
+ self.strides,
+ self.backstrides, t.chunks)
+ return ViewIterator(r[1], r[2], r[3], r[0])
+
@jit.unroll_safe
def next(self, shapelen):
+ shapelen = jit.promote(len(self.res_shape))
offset = self.offset
indices = [0] * shapelen
for i in range(shapelen):
@@ -96,6 +180,43 @@
res._done = done
return res
+ @jit.unroll_safe
+ def next_skip_x(self, shapelen, step):
+ shapelen = jit.promote(len(self.res_shape))
+ offset = self.offset
+ indices = [0] * shapelen
+ for i in range(shapelen):
+ indices[i] = self.indices[i]
+ done = False
+ for i in range(shapelen - 1, -1, -1):
+ if indices[i] < self.res_shape[i] - step:
+ indices[i] += step
+ offset += self.strides[i] * step
+ break
+ else:
+ remaining_step = (indices[i] + step) // self.res_shape[i]
+ this_i_step = step - remaining_step * self.res_shape[i]
+ offset += self.strides[i] * this_i_step
+ indices[i] = indices[i] + this_i_step
+ step = remaining_step
+ else:
+ done = True
+ res = instantiate(ViewIterator)
+ res.offset = offset
+ res.indices = indices
+ res.strides = self.strides
+ res.backstrides = self.backstrides
+ res.res_shape = self.res_shape
+ res._done = done
+ return res
+
+ def apply_transformations(self, arr, transformations):
+ v = BaseIterator.apply_transformations(self, arr, transformations)
+ if len(arr.shape) == 1:
+ return OneDimIterator(self.offset, self.strides[0],
+ self.res_shape[0])
+ return v
+
def done(self):
return self._done
@@ -103,11 +224,64 @@
def next(self, shapelen):
return self
+ def transform(self, arr, t):
+ pass
+
+class AxisIterator(BaseIterator):
+ def __init__(self, start, dim, shape, strides, backstrides):
+ self.res_shape = shape[:]
+ if len(shape) == len(strides):
+ # keepdims = True
+ self.strides = strides[:dim] + [0] + strides[dim + 1:]
+ self.backstrides = backstrides[:dim] + [0] + backstrides[dim + 1:]
+ else:
+ self.strides = strides[:dim] + [0] + strides[dim:]
+ self.backstrides = backstrides[:dim] + [0] + backstrides[dim:]
+ self.first_line = True
+ self.indices = [0] * len(shape)
+ self._done = False
+ self.offset = start
+ self.dim = dim
+
+ @jit.unroll_safe
+ def next(self, shapelen):
+ offset = self.offset
+ first_line = self.first_line
+ indices = [0] * shapelen
+ for i in range(shapelen):
+ indices[i] = self.indices[i]
+ done = False
+ for i in range(shapelen - 1, -1, -1):
+ if indices[i] < self.res_shape[i] - 1:
+ if i == self.dim:
+ first_line = False
+ indices[i] += 1
+ offset += self.strides[i]
+ break
+ else:
+ if i == self.dim:
+ first_line = True
+ indices[i] = 0
+ offset -= self.backstrides[i]
+ else:
+ done = True
+ res = instantiate(AxisIterator)
+ res.offset = offset
+ res.indices = indices
+ res.strides = self.strides
+ res.backstrides = self.backstrides
+ res.res_shape = self.res_shape
+ res._done = done
+ res.first_line = first_line
+ res.dim = self.dim
+ return res
+
+ def done(self):
+ return self._done
+
# ------ other iterators that are not part of the computation frame ----------
-
-class AxisIterator(object):
- """ This object will return offsets of each start of the last stride
- """
+
+class SkipLastAxisIterator(object):
def __init__(self, arr):
self.arr = arr
self.indices = [0] * (len(arr.shape) - 1)
@@ -125,4 +299,3 @@
self.offset -= self.arr.backstrides[i]
else:
self.done = True
-
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1,211 +1,82 @@
from pypy.interpreter.baseobjspace import Wrappable
from pypy.interpreter.error import OperationError, operationerrfmt
-from pypy.interpreter.gateway import interp2app, NoneNotWrapped
+from pypy.interpreter.gateway import interp2app, unwrap_spec
from pypy.interpreter.typedef import TypeDef, GetSetProperty
-from pypy.module.micronumpy import interp_ufuncs, interp_dtype, signature
-from pypy.module.micronumpy.strides import calculate_slice_strides
+from pypy.module.micronumpy import (interp_ufuncs, interp_dtype, interp_boxes,
+ signature, support)
+from pypy.module.micronumpy.strides import (calculate_slice_strides,
+ shape_agreement, find_shape_and_elems, get_shape_from_iterable,
+ calc_new_strides, to_coords)
from pypy.rlib import jit
from pypy.rpython.lltypesystem import lltype, rffi
from pypy.tool.sourcetools import func_with_new_name
from pypy.rlib.rstring import StringBuilder
-from pypy.module.micronumpy.interp_iter import ArrayIterator,\
- view_iter_from_arr, OneDimIterator, AxisIterator
+from pypy.module.micronumpy.interp_iter import (ArrayIterator, OneDimIterator,
+ SkipLastAxisIterator, Chunk, ViewIterator)
+from pypy.module.micronumpy.appbridge import get_appbridge_cache
+
numpy_driver = jit.JitDriver(
greens=['shapelen', 'sig'],
virtualizables=['frame'],
reds=['result_size', 'frame', 'ri', 'self', 'result'],
get_printable_location=signature.new_printable_location('numpy'),
+ name='numpy',
)
all_driver = jit.JitDriver(
greens=['shapelen', 'sig'],
virtualizables=['frame'],
reds=['frame', 'self', 'dtype'],
get_printable_location=signature.new_printable_location('all'),
+ name='numpy_all',
)
any_driver = jit.JitDriver(
greens=['shapelen', 'sig'],
virtualizables=['frame'],
reds=['frame', 'self', 'dtype'],
get_printable_location=signature.new_printable_location('any'),
+ name='numpy_any',
)
slice_driver = jit.JitDriver(
greens=['shapelen', 'sig'],
virtualizables=['frame'],
- reds=['self', 'frame', 'source', 'res_iter'],
+ reds=['self', 'frame', 'arr'],
get_printable_location=signature.new_printable_location('slice'),
+ name='numpy_slice',
)
-
-def _find_shape_and_elems(space, w_iterable):
- shape = [space.len_w(w_iterable)]
- batch = space.listview(w_iterable)
- while True:
- new_batch = []
- if not batch:
- return shape, []
- if not space.issequence_w(batch[0]):
- for elem in batch:
- if space.issequence_w(elem):
- raise OperationError(space.w_ValueError, space.wrap(
- "setting an array element with a sequence"))
- return shape, batch
- size = space.len_w(batch[0])
- for w_elem in batch:
- if not space.issequence_w(w_elem) or space.len_w(w_elem) != size:
- raise OperationError(space.w_ValueError, space.wrap(
- "setting an array element with a sequence"))
- new_batch += space.listview(w_elem)
- shape.append(size)
- batch = new_batch
-
-def shape_agreement(space, shape1, shape2):
- ret = _shape_agreement(shape1, shape2)
- if len(ret) < max(len(shape1), len(shape2)):
- raise OperationError(space.w_ValueError,
- space.wrap("operands could not be broadcast together with shapes (%s) (%s)" % (
- ",".join([str(x) for x in shape1]),
- ",".join([str(x) for x in shape2]),
- ))
- )
- return ret
-
-def _shape_agreement(shape1, shape2):
- """ Checks agreement about two shapes with respect to broadcasting. Returns
- the resulting shape.
- """
- lshift = 0
- rshift = 0
- if len(shape1) > len(shape2):
- m = len(shape1)
- n = len(shape2)
- rshift = len(shape2) - len(shape1)
- remainder = shape1
- else:
- m = len(shape2)
- n = len(shape1)
- lshift = len(shape1) - len(shape2)
- remainder = shape2
- endshape = [0] * m
- indices1 = [True] * m
- indices2 = [True] * m
- for i in range(m - 1, m - n - 1, -1):
- left = shape1[i + lshift]
- right = shape2[i + rshift]
- if left == right:
- endshape[i] = left
- elif left == 1:
- endshape[i] = right
- indices1[i + lshift] = False
- elif right == 1:
- endshape[i] = left
- indices2[i + rshift] = False
- else:
- return []
- #raise OperationError(space.w_ValueError, space.wrap(
- # "frames are not aligned"))
- for i in range(m - n):
- endshape[i] = remainder[i]
- return endshape
-
-def get_shape_from_iterable(space, old_size, w_iterable):
- new_size = 0
- new_shape = []
- if space.isinstance_w(w_iterable, space.w_int):
- new_size = space.int_w(w_iterable)
- if new_size < 0:
- new_size = old_size
- new_shape = [new_size]
- else:
- neg_dim = -1
- batch = space.listview(w_iterable)
- new_size = 1
- if len(batch) < 1:
- if old_size == 1:
- # Scalars can have an empty size.
- new_size = 1
- else:
- new_size = 0
- new_shape = []
- i = 0
- for elem in batch:
- s = space.int_w(elem)
- if s < 0:
- if neg_dim >= 0:
- raise OperationError(space.w_ValueError, space.wrap(
- "can only specify one unknown dimension"))
- s = 1
- neg_dim = i
- new_size *= s
- new_shape.append(s)
- i += 1
- if neg_dim >= 0:
- new_shape[neg_dim] = old_size / new_size
- new_size *= new_shape[neg_dim]
- if new_size != old_size:
- raise OperationError(space.w_ValueError,
- space.wrap("total size of new array must be unchanged"))
- return new_shape
-
-# Recalculating strides. Find the steps that the iteration does for each
-# dimension, given the stride and shape. Then try to create a new stride that
-# fits the new shape, using those steps. If there is a shape/step mismatch
-# (meaning that the realignment of elements crosses from one step into another)
-# return None so that the caller can raise an exception.
-def calc_new_strides(new_shape, old_shape, old_strides):
- # Return the proper strides for new_shape, or None if the mapping crosses
- # stepping boundaries
-
- # Assumes that prod(old_shape) == prod(new_shape), len(old_shape) > 1, and
- # len(new_shape) > 0
- steps = []
- last_step = 1
- oldI = 0
- new_strides = []
- if old_strides[0] < old_strides[-1]:
- for i in range(len(old_shape)):
- steps.append(old_strides[i] / last_step)
- last_step *= old_shape[i]
- cur_step = steps[0]
- n_new_elems_used = 1
- n_old_elems_to_use = old_shape[0]
- for s in new_shape:
- new_strides.append(cur_step * n_new_elems_used)
- n_new_elems_used *= s
- while n_new_elems_used > n_old_elems_to_use:
- oldI += 1
- if steps[oldI] != steps[oldI - 1]:
- return None
- n_old_elems_to_use *= old_shape[oldI]
- if n_new_elems_used == n_old_elems_to_use:
- oldI += 1
- if oldI >= len(old_shape):
- break
- cur_step = steps[oldI]
- n_old_elems_to_use *= old_shape[oldI]
- else:
- for i in range(len(old_shape) - 1, -1, -1):
- steps.insert(0, old_strides[i] / last_step)
- last_step *= old_shape[i]
- cur_step = steps[-1]
- n_new_elems_used = 1
- oldI = -1
- n_old_elems_to_use = old_shape[-1]
- for i in range(len(new_shape) - 1, -1, -1):
- s = new_shape[i]
- new_strides.insert(0, cur_step * n_new_elems_used)
- n_new_elems_used *= s
- while n_new_elems_used > n_old_elems_to_use:
- oldI -= 1
- if steps[oldI] != steps[oldI + 1]:
- return None
- n_old_elems_to_use *= old_shape[oldI]
- if n_new_elems_used == n_old_elems_to_use:
- oldI -= 1
- if oldI < -len(old_shape):
- break
- cur_step = steps[oldI]
- n_old_elems_to_use *= old_shape[oldI]
- return new_strides
+count_driver = jit.JitDriver(
+ greens=['shapelen'],
+ virtualizables=['frame'],
+ reds=['s', 'frame', 'iter', 'arr'],
+ name='numpy_count'
+)
+filter_driver = jit.JitDriver(
+ greens=['shapelen', 'sig'],
+ virtualizables=['frame'],
+ reds=['concr', 'argi', 'ri', 'frame', 'v', 'res', 'self'],
+ name='numpy_filter',
+)
+filter_set_driver = jit.JitDriver(
+ greens=['shapelen', 'sig'],
+ virtualizables=['frame'],
+ reds=['idx', 'idxi', 'frame', 'arr'],
+ name='numpy_filterset',
+)
+take_driver = jit.JitDriver(
+ greens=['shapelen', 'sig'],
+ reds=['index_i', 'res_i', 'concr', 'index', 'res'],
+ name='numpy_take',
+)
+flat_get_driver = jit.JitDriver(
+ greens=['shapelen', 'base'],
+ reds=['step', 'ri', 'basei', 'res'],
+ name='numpy_flatget',
+)
+flat_set_driver = jit.JitDriver(
+ greens=['shapelen', 'base'],
+ reds=['step', 'ai', 'lngth', 'arr', 'basei'],
+ name='numpy_flatset',
+)
class BaseArray(Wrappable):
_attrs_ = ["invalidates", "shape", 'size']
@@ -246,6 +117,7 @@
descr_pos = _unaryop_impl("positive")
descr_neg = _unaryop_impl("negative")
descr_abs = _unaryop_impl("absolute")
+ descr_invert = _unaryop_impl("invert")
def _binop_impl(ufunc_name):
def impl(self, space, w_other):
@@ -266,6 +138,9 @@
descr_gt = _binop_impl("greater")
descr_ge = _binop_impl("greater_equal")
+ descr_and = _binop_impl("bitwise_and")
+ descr_or = _binop_impl("bitwise_or")
+
def _binop_right_impl(ufunc_name):
def impl(self, space, w_other):
w_other = scalar_w(space,
@@ -282,13 +157,19 @@
descr_rpow = _binop_right_impl("power")
descr_rmod = _binop_right_impl("mod")
- def _reduce_ufunc_impl(ufunc_name):
- def impl(self, space):
- return getattr(interp_ufuncs.get(space), ufunc_name).reduce(space, self, multidim=True)
+ def _reduce_ufunc_impl(ufunc_name, promote_to_largest=False):
+ def impl(self, space, w_axis=None):
+ if space.is_w(w_axis, space.w_None):
+ axis = -1
+ else:
+ axis = space.int_w(w_axis)
+ return getattr(interp_ufuncs.get(space), ufunc_name).reduce(space,
+ self, True, promote_to_largest, axis)
return func_with_new_name(impl, "reduce_%s_impl" % ufunc_name)
descr_sum = _reduce_ufunc_impl("add")
- descr_prod = _reduce_ufunc_impl("multiply")
+ descr_sum_promote = _reduce_ufunc_impl("add", True)
+ descr_prod = _reduce_ufunc_impl("multiply", True)
descr_max = _reduce_ufunc_impl("maximum")
descr_min = _reduce_ufunc_impl("minimum")
@@ -297,6 +178,7 @@
greens=['shapelen', 'sig'],
reds=['result', 'idx', 'frame', 'self', 'cur_best', 'dtype'],
get_printable_location=signature.new_printable_location(op_name),
+ name='numpy_' + op_name,
)
def loop(self):
sig = self.find_sig()
@@ -372,7 +254,7 @@
else:
w_res = self.descr_mul(space, w_other)
assert isinstance(w_res, BaseArray)
- return w_res.descr_sum(space)
+ return w_res.descr_sum(space, space.wrap(-1))
def get_concrete(self):
raise NotImplementedError
@@ -400,9 +282,23 @@
def descr_copy(self, space):
return self.copy(space)
+ def descr_flatten(self, space, w_order=None):
+ if isinstance(self, Scalar):
+ # scalars have no storage
+ return self.descr_reshape(space, [space.wrap(1)])
+ concr = self.get_concrete()
+ w_res = concr.descr_ravel(space, w_order)
+ if w_res.storage == concr.storage:
+ return w_res.copy(space)
+ return w_res
+
def copy(self, space):
return self.get_concrete().copy(space)
+ def empty_copy(self, space, dtype):
+ shape = self.shape
+ return W_NDimArray(support.product(shape), shape[:], dtype, 'C')
+
def descr_len(self, space):
if len(self.shape):
return space.wrap(self.shape[0])
@@ -410,33 +306,32 @@
"len() of unsized object"))
def descr_repr(self, space):
- res = StringBuilder()
- res.append("array(")
- concrete = self.get_concrete_or_scalar()
- dtype = concrete.find_dtype()
- if not concrete.size:
- res.append('[]')
- if len(self.shape) > 1:
- # An empty slice reports its shape
- res.append(", shape=(")
- self_shape = str(self.shape)
- res.append_slice(str(self_shape), 1, len(self_shape) - 1)
- res.append(')')
- else:
- concrete.to_str(space, 1, res, indent=' ')
- if (dtype is not interp_dtype.get_dtype_cache(space).w_float64dtype and
- not (dtype.kind == interp_dtype.SIGNEDLTR and
- dtype.itemtype.get_element_size() == rffi.sizeof(lltype.Signed)) or
- not self.size):
- res.append(", dtype=" + dtype.name)
- res.append(")")
- return space.wrap(res.build())
+ cache = get_appbridge_cache(space)
+ if cache.w_array_repr is None:
+ return space.wrap(self.dump_data())
+ return space.call_function(cache.w_array_repr, self)
+
+ def dump_data(self):
+ concr = self.get_concrete()
+ i = concr.create_iter()
+ first = True
+ s = StringBuilder()
+ s.append('array([')
+ while not i.done():
+ if first:
+ first = False
+ else:
+ s.append(', ')
+ s.append(concr.dtype.itemtype.str_format(concr.getitem(i.offset)))
+ i = i.next(len(concr.shape))
+ s.append('])')
+ return s.build()
def descr_str(self, space):
- ret = StringBuilder()
- concrete = self.get_concrete_or_scalar()
- concrete.to_str(space, 0, ret, ' ')
- return space.wrap(ret.build())
+ cache = get_appbridge_cache(space)
+ if cache.w_array_str is None:
+ return space.wrap(self.dump_data())
+ return space.call_function(cache.w_array_str, self)
@jit.unroll_safe
def _single_item_result(self, space, w_idx):
@@ -470,20 +365,86 @@
def _prepare_slice_args(self, space, w_idx):
if (space.isinstance_w(w_idx, space.w_int) or
space.isinstance_w(w_idx, space.w_slice)):
- return [space.decode_index4(w_idx, self.shape[0])]
- return [space.decode_index4(w_item, self.shape[i]) for i, w_item in
+ return [Chunk(*space.decode_index4(w_idx, self.shape[0]))]
+ return [Chunk(*space.decode_index4(w_item, self.shape[i])) for i, w_item in
enumerate(space.fixedview(w_idx))]
+ def count_all_true(self, arr):
+ sig = arr.find_sig()
+ frame = sig.create_frame(arr)
+ shapelen = len(arr.shape)
+ s = 0
+ iter = None
+ while not frame.done():
+ count_driver.jit_merge_point(arr=arr, frame=frame, iter=iter, s=s,
+ shapelen=shapelen)
+ iter = frame.get_final_iter()
+ s += arr.dtype.getitem_bool(arr.storage, iter.offset)
+ frame.next(shapelen)
+ return s
+
+ def getitem_filter(self, space, arr):
+ concr = arr.get_concrete()
+ if concr.size > self.size:
+ raise OperationError(space.w_IndexError,
+ space.wrap("index out of range for array"))
+ size = self.count_all_true(concr)
+ res = W_NDimArray(size, [size], self.find_dtype())
+ ri = ArrayIterator(size)
+ shapelen = len(self.shape)
+ argi = concr.create_iter()
+ sig = self.find_sig()
+ frame = sig.create_frame(self)
+ v = None
+ while not ri.done():
+ filter_driver.jit_merge_point(concr=concr, argi=argi, ri=ri,
+ frame=frame, v=v, res=res, sig=sig,
+ shapelen=shapelen, self=self)
+ if concr.dtype.getitem_bool(concr.storage, argi.offset):
+ v = sig.eval(frame, self)
+ res.setitem(ri.offset, v)
+ ri = ri.next(1)
+ else:
+ ri = ri.next_no_increase(1)
+ argi = argi.next(shapelen)
+ frame.next(shapelen)
+ return res
+
+ def setitem_filter(self, space, idx, val):
+ size = self.count_all_true(idx)
+ arr = SliceArray([size], self.dtype, self, val)
+ sig = arr.find_sig()
+ shapelen = len(self.shape)
+ frame = sig.create_frame(arr)
+ idxi = idx.create_iter()
+ while not frame.done():
+ filter_set_driver.jit_merge_point(idx=idx, idxi=idxi, sig=sig,
+ frame=frame, arr=arr,
+ shapelen=shapelen)
+ if idx.dtype.getitem_bool(idx.storage, idxi.offset):
+ sig.eval(frame, arr)
+ frame.next_from_second(1)
+ frame.next_first(shapelen)
+ idxi = idxi.next(shapelen)
+
def descr_getitem(self, space, w_idx):
+ if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and
+ w_idx.find_dtype().is_bool_type()):
+ return self.getitem_filter(space, w_idx)
if self._single_item_result(space, w_idx):
concrete = self.get_concrete()
item = concrete._index_of_single_item(space, w_idx)
return concrete.getitem(item)
chunks = self._prepare_slice_args(space, w_idx)
- return space.wrap(self.create_slice(chunks))
+ return self.create_slice(chunks)
def descr_setitem(self, space, w_idx, w_value):
self.invalidated()
+ if (isinstance(w_idx, BaseArray) and w_idx.shape == self.shape and
+ w_idx.find_dtype().is_bool_type()):
+ return self.get_concrete().setitem_filter(space,
+ w_idx.get_concrete(),
+ convert_to_array(space, w_value))
if self._single_item_result(space, w_idx):
concrete = self.get_concrete()
item = concrete._index_of_single_item(space, w_idx)
@@ -500,9 +461,8 @@
def create_slice(self, chunks):
shape = []
i = -1
- for i, (start_, stop, step, lgt) in enumerate(chunks):
- if step != 0:
- shape.append(lgt)
+ for i, chunk in enumerate(chunks):
+ chunk.extend_shape(shape)
s = i + 1
assert s >= 0
shape += self.shape[s:]
@@ -530,11 +490,14 @@
w_shape = args_w[0]
else:
w_shape = space.newtuple(args_w)
+ new_shape = get_shape_from_iterable(space, self.size, w_shape)
+ return self.reshape(space, new_shape)
+
+ def reshape(self, space, new_shape):
concrete = self.get_concrete()
- new_shape = get_shape_from_iterable(space, concrete.size, w_shape)
# Since we got to here, prod(new_shape) == self.size
- new_strides = calc_new_strides(new_shape,
- concrete.shape, concrete.strides)
+ new_strides = calc_new_strides(new_shape, concrete.shape,
+ concrete.strides, concrete.order)
if new_strides:
# We can create a view, strides somehow match up.
ndims = len(new_shape)
@@ -542,7 +505,7 @@
for nd in range(ndims):
new_backstrides[nd] = (new_shape[nd] - 1) * new_strides[nd]
arr = W_NDimSlice(concrete.start, new_strides, new_backstrides,
- new_shape, self)
+ new_shape, concrete)
else:
# Create copy with contiguous data
arr = concrete.copy(space)
@@ -552,7 +515,7 @@
def descr_tolist(self, space):
if len(self.shape) == 0:
assert isinstance(self, Scalar)
- return self.value.descr_tolist(space)
+ return self.value.item(space)
w_result = space.newlist([])
for i in range(self.shape[0]):
space.call_method(w_result, "append",
@@ -560,20 +523,26 @@
)
return w_result
- def descr_mean(self, space):
- return space.div(self.descr_sum(space), space.wrap(self.size))
+ def descr_mean(self, space, w_axis=None):
+ if space.is_w(w_axis, space.w_None):
+ w_axis = space.wrap(-1)
+ w_denom = space.wrap(self.size)
+ else:
+ dim = space.int_w(w_axis)
+ w_denom = space.wrap(self.shape[dim])
+ return space.div(self.descr_sum_promote(space, w_axis), w_denom)
- def descr_var(self, space):
- # var = mean((values - mean(values)) ** 2)
- w_res = self.descr_sub(space, self.descr_mean(space))
- assert isinstance(w_res, BaseArray)
- w_res = w_res.descr_pow(space, space.wrap(2))
- assert isinstance(w_res, BaseArray)
- return w_res.descr_mean(space)
+ def descr_var(self, space, w_axis=None):
+ return get_appbridge_cache(space).call_method(space, '_var', self,
+ w_axis)
- def descr_std(self, space):
- # std(v) = sqrt(var(v))
- return interp_ufuncs.get(space).sqrt.call(space, [self.descr_var(space)])
+ def descr_std(self, space, w_axis=None):
+ return get_appbridge_cache(space).call_method(space, '_std', self,
+ w_axis)
+
+ def descr_fill(self, space, w_value):
+ concr = self.get_concrete_or_scalar()
+ concr.fill(space, w_value)
def descr_nonzero(self, space):
if self.size > 1:
@@ -602,17 +571,28 @@
return space.wrap(W_NDimSlice(concrete.start, strides,
backstrides, shape, concrete))
+ def descr_ravel(self, space, w_order=None):
+ if w_order is None or space.is_w(w_order, space.w_None):
+ order = 'C'
+ else:
+ order = space.str_w(w_order)
+ if order != 'C':
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ "order not implemented"))
+ return self.descr_reshape(space, [space.wrap(-1)])
+
def descr_get_flatiter(self, space):
return space.wrap(W_FlatIterator(self))
def getitem(self, item):
raise NotImplementedError
- def find_sig(self, res_shape=None):
+ def find_sig(self, res_shape=None, arr=None):
""" find a correct signature for the array
"""
res_shape = res_shape or self.shape
- return signature.find_sig(self.create_sig(res_shape), self)
+ arr = arr or self
+ return signature.find_sig(self.create_sig(), arr)
def descr_array_iface(self, space):
if not self.shape:
@@ -630,6 +610,60 @@
def supports_fast_slicing(self):
return False
+ def descr_compress(self, space, w_obj, w_axis=None):
+ index = convert_to_array(space, w_obj)
+ return self.getitem_filter(space, index)
+
+ def descr_take(self, space, w_obj, w_axis=None):
+ index = convert_to_array(space, w_obj).get_concrete()
+ concr = self.get_concrete()
+ if space.is_w(w_axis, space.w_None):
+ concr = concr.descr_ravel(space)
+ else:
+ raise OperationError(space.w_NotImplementedError,
+ space.wrap("axis unsupported for take"))
+ index_i = index.create_iter()
+ res_shape = index.shape
+ size = support.product(res_shape)
+ res = W_NDimArray(size, res_shape[:], concr.dtype, concr.order)
+ res_i = res.create_iter()
+ shapelen = len(index.shape)
+ sig = concr.find_sig()
+ while not index_i.done():
+ take_driver.jit_merge_point(index_i=index_i, index=index,
+ res_i=res_i, concr=concr,
+ res=res,
+ shapelen=shapelen, sig=sig)
+ w_item = index._getitem_long(space, index_i.offset)
+ res.setitem(res_i.offset, concr.descr_getitem(space, w_item))
+ index_i = index_i.next(shapelen)
+ res_i = res_i.next(shapelen)
+ return res
+
+ def _getitem_long(self, space, offset):
+ # an obscure hack to not have longdtype inside a jitted loop
+ longdtype = interp_dtype.get_dtype_cache(space).w_longdtype
+ return self.getitem(offset).convert_to(longdtype).item(
+ space)
+
+ def descr_item(self, space, w_arg=None):
+ if space.is_w(w_arg, space.w_None):
+ if not isinstance(self, Scalar):
+ raise OperationError(space.w_ValueError, space.wrap("index out of bounds"))
+ return self.value.item(space)
+ if space.isinstance_w(w_arg, space.w_int):
+ if isinstance(self, Scalar):
+ raise OperationError(space.w_ValueError, space.wrap("index out of bounds"))
+ concr = self.get_concrete()
+ i = to_coords(space, self.shape, concr.size, concr.order, w_arg)[0]
+ # XXX a bit around
+ item = self.descr_getitem(space, space.newtuple([space.wrap(x)
+ for x in i]))
+ assert isinstance(item, interp_boxes.W_GenericBox)
+ return item.item(space)
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ "non-int arg not supported"))
+
def convert_to_array(space, w_obj):
if isinstance(w_obj, BaseArray):
return w_obj
@@ -655,23 +689,29 @@
self.shape = []
BaseArray.__init__(self, [])
self.dtype = dtype
+ assert isinstance(value, interp_boxes.W_GenericBox)
self.value = value
def find_dtype(self):
return self.dtype
- def to_str(self, space, comma, builder, indent=' ', use_ellipsis=False):
- builder.append(self.dtype.itemtype.str_format(self.value))
-
def copy(self, space):
return Scalar(self.dtype, self.value)
- def create_sig(self, res_shape):
+ def fill(self, space, w_value):
+ self.value = self.dtype.coerce(space, w_value)
+
+ def create_sig(self):
return signature.ScalarSignature(self.dtype)
def get_concrete_or_scalar(self):
return self
+ def reshape(self, space, new_shape):
+ size = support.product(new_shape)
+ res = W_NDimArray(size, new_shape, self.dtype, 'C')
+ res.setitem(0, self.value)
+ return res
class VirtualArray(BaseArray):
"""
@@ -684,7 +724,8 @@
self.name = name
def _del_sources(self):
- # Function for deleting references to source arrays, to allow garbage-collecting them
+ # Function for deleting references to source arrays,
+ # to allow garbage-collecting them
raise NotImplementedError
def compute(self):
@@ -700,8 +741,7 @@
frame=frame,
ri=ri,
self=self, result=result)
- result.dtype.setitem(result.storage, ri.offset,
- sig.eval(frame, self))
+ result.setitem(ri.offset, sig.eval(frame, self))
frame.next(shapelen)
ri = ri.next(shapelen)
return result
@@ -728,19 +768,16 @@
class VirtualSlice(VirtualArray):
def __init__(self, child, chunks, shape):
- size = 1
- for sh in shape:
- size *= sh
self.child = child
self.chunks = chunks
- self.size = size
+ self.size = support.product(shape)
VirtualArray.__init__(self, 'slice', shape, child.find_dtype())
- def create_sig(self, res_shape):
+ def create_sig(self):
if self.forced_result is not None:
- return self.forced_result.create_sig(res_shape)
+ return self.forced_result.create_sig()
return signature.VirtualSliceSignature(
- self.child.create_sig(res_shape))
+ self.child.create_sig())
def force_if_needed(self):
if self.forced_result is None:
@@ -750,46 +787,91 @@
def _del_sources(self):
self.child = None
+
class Call1(VirtualArray):
- def __init__(self, ufunc, name, shape, res_dtype, values):
+ def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, values):
VirtualArray.__init__(self, name, shape, res_dtype)
self.values = values
self.size = values.size
self.ufunc = ufunc
+ self.calc_dtype = calc_dtype
def _del_sources(self):
self.values = None
- def create_sig(self, res_shape):
+ def create_sig(self):
if self.forced_result is not None:
- return self.forced_result.create_sig(res_shape)
- return signature.Call1(self.ufunc, self.name,
- self.values.create_sig(res_shape))
+ return self.forced_result.create_sig()
+ return signature.Call1(self.ufunc, self.name, self.values.create_sig())
class Call2(VirtualArray):
"""
Intermediate class for performing binary operations.
"""
+ _immutable_fields_ = ['left', 'right']
+
def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, left, right):
VirtualArray.__init__(self, name, shape, res_dtype)
self.ufunc = ufunc
self.left = left
self.right = right
self.calc_dtype = calc_dtype
- self.size = 1
- for s in self.shape:
- self.size *= s
+ self.size = support.product(self.shape)
def _del_sources(self):
self.left = None
self.right = None
- def create_sig(self, res_shape):
+ def create_sig(self):
if self.forced_result is not None:
- return self.forced_result.create_sig(res_shape)
+ return self.forced_result.create_sig()
+ if self.shape != self.left.shape and self.shape != self.right.shape:
+ return signature.BroadcastBoth(self.ufunc, self.name,
+ self.calc_dtype,
+ self.left.create_sig(),
+ self.right.create_sig())
+ elif self.shape != self.left.shape:
+ return signature.BroadcastLeft(self.ufunc, self.name,
+ self.calc_dtype,
+ self.left.create_sig(),
+ self.right.create_sig())
+ elif self.shape != self.right.shape:
+ return signature.BroadcastRight(self.ufunc, self.name,
+ self.calc_dtype,
+ self.left.create_sig(),
+ self.right.create_sig())
return signature.Call2(self.ufunc, self.name, self.calc_dtype,
- self.left.create_sig(res_shape),
- self.right.create_sig(res_shape))
+ self.left.create_sig(), self.right.create_sig())
+
+class SliceArray(Call2):
+ def __init__(self, shape, dtype, left, right, no_broadcast=False):
+ self.no_broadcast = no_broadcast
+ Call2.__init__(self, None, 'sliceloop', shape, dtype, dtype, left,
+ right)
+
+ def create_sig(self):
+ lsig = self.left.create_sig()
+ rsig = self.right.create_sig()
+ if not self.no_broadcast and self.shape != self.right.shape:
+ return signature.SliceloopBroadcastSignature(self.ufunc,
+ self.name,
+ self.calc_dtype,
+ lsig, rsig)
+ return signature.SliceloopSignature(self.ufunc, self.name,
+ self.calc_dtype,
+ lsig, rsig)
+
+class AxisReduce(Call2):
+ """ NOTE: this is only used as a container, you should never
+ encounter such things in the wild. Remove this comment
+ when we'll make AxisReduce lazy
+ """
+ _immutable_fields_ = ['left', 'right']
+
+ def __init__(self, ufunc, name, shape, dtype, left, right, dim):
+ Call2.__init__(self, ufunc, name, shape, dtype, dtype,
+ left, right)
+ self.dim = dim
class ConcreteArray(BaseArray):
""" An array that have actual storage, whether owned or not
@@ -844,94 +926,6 @@
self.strides = strides
self.backstrides = backstrides
- def array_sig(self, res_shape):
- if res_shape is not None and self.shape != res_shape:
- return signature.ViewSignature(self.dtype)
- return signature.ArraySignature(self.dtype)
-
- def to_str(self, space, comma, builder, indent=' ', use_ellipsis=False):
- '''Modifies builder with a representation of the array/slice
- The items will be seperated by a comma if comma is 1
- Multidimensional arrays/slices will span a number of lines,
- each line will begin with indent.
- '''
- size = self.size
- ccomma = ',' * comma
- ncomma = ',' * (1 - comma)
- dtype = self.find_dtype()
- if size < 1:
- builder.append('[]')
- return
- elif size == 1:
- builder.append(dtype.itemtype.str_format(self.getitem(0)))
- return
- if size > 1000:
- # Once this goes True it does not go back to False for recursive
- # calls
- use_ellipsis = True
- ndims = len(self.shape)
- i = 0
- builder.append('[')
- if ndims > 1:
- if use_ellipsis:
- for i in range(min(3, self.shape[0])):
- if i > 0:
- builder.append(ccomma + '\n')
- if ndims >= 3:
- builder.append('\n' + indent)
- else:
- builder.append(indent)
- view = self.create_slice([(i, 0, 0, 1)]).get_concrete()
- view.to_str(space, comma, builder, indent=indent + ' ',
- use_ellipsis=use_ellipsis)
- if i < self.shape[0] - 1:
- builder.append(ccomma +'\n' + indent + '...' + ncomma)
- i = self.shape[0] - 3
- else:
- i += 1
- while i < self.shape[0]:
- if i > 0:
- builder.append(ccomma + '\n')
- if ndims >= 3:
- builder.append('\n' + indent)
- else:
- builder.append(indent)
- # create_slice requires len(chunks) > 1 in order to reduce
- # shape
- view = self.create_slice([(i, 0, 0, 1)]).get_concrete()
- view.to_str(space, comma, builder, indent=indent + ' ',
- use_ellipsis=use_ellipsis)
- i += 1
- elif ndims == 1:
- spacer = ccomma + ' '
- item = self.start
- # An iterator would be a nicer way to walk along the 1d array, but
- # how do I reset it if printing ellipsis? iterators have no
- # "set_offset()"
- i = 0
- if use_ellipsis:
- for i in range(min(3, self.shape[0])):
- if i > 0:
- builder.append(spacer)
- builder.append(dtype.itemtype.str_format(self.getitem(item)))
- item += self.strides[0]
- if i < self.shape[0] - 1:
- # Add a comma only if comma is False - this prevents adding
- # two commas
- builder.append(spacer + '...' + ncomma)
- # Ugly, but can this be done with an iterator?
- item = self.start + self.backstrides[0] - 2 * self.strides[0]
- i = self.shape[0] - 3
- else:
- i += 1
- while i < self.shape[0]:
- if i > 0:
- builder.append(spacer)
- builder.append(dtype.itemtype.str_format(self.getitem(item)))
- item += self.strides[0]
- i += 1
- builder.append(']')
-
@jit.unroll_safe
def _index_of_single_item(self, space, w_idx):
if space.isinstance_w(w_idx, space.w_int):
@@ -963,20 +957,22 @@
self.dtype is w_value.find_dtype()):
self._fast_setslice(space, w_value)
else:
- self._sliceloop(w_value, res_shape)
+ arr = SliceArray(self.shape, self.dtype, self, w_value)
+ self._sliceloop(arr)
def _fast_setslice(self, space, w_value):
assert isinstance(w_value, ConcreteArray)
itemsize = self.dtype.itemtype.get_element_size()
- if len(self.shape) == 1:
+ shapelen = len(self.shape)
+ if shapelen == 1:
rffi.c_memcpy(
rffi.ptradd(self.storage, self.start * itemsize),
rffi.ptradd(w_value.storage, w_value.start * itemsize),
self.size * itemsize
)
else:
- dest = AxisIterator(self)
- source = AxisIterator(w_value)
+ dest = SkipLastAxisIterator(self)
+ source = SkipLastAxisIterator(w_value)
while not dest.done:
rffi.c_memcpy(
rffi.ptradd(self.storage, dest.offset * itemsize),
@@ -986,30 +982,28 @@
source.next()
dest.next()
- def _sliceloop(self, source, res_shape):
- sig = source.find_sig(res_shape)
- frame = sig.create_frame(source, res_shape)
- res_iter = view_iter_from_arr(self)
- shapelen = len(res_shape)
- while not res_iter.done():
- slice_driver.jit_merge_point(sig=sig,
- frame=frame,
- shapelen=shapelen,
- self=self, source=source,
- res_iter=res_iter)
- self.setitem(res_iter.offset, sig.eval(frame, source).convert_to(
- self.find_dtype()))
+ def _sliceloop(self, arr):
+ sig = arr.find_sig()
+ frame = sig.create_frame(arr)
+ shapelen = len(self.shape)
+ while not frame.done():
+ slice_driver.jit_merge_point(sig=sig, frame=frame, self=self,
+ arr=arr,
+ shapelen=shapelen)
+ sig.eval(frame, arr)
frame.next(shapelen)
- res_iter = res_iter.next(shapelen)
def copy(self, space):
array = W_NDimArray(self.size, self.shape[:], self.dtype, self.order)
array.setslice(space, self)
return array
+ def fill(self, space, w_value):
+ self.setslice(space, scalar_w(space, self.dtype, w_value))
+
class ViewArray(ConcreteArray):
- def create_sig(self, res_shape):
+ def create_sig(self):
return signature.ViewSignature(self.dtype)
@@ -1018,15 +1012,16 @@
assert isinstance(parent, ConcreteArray)
if isinstance(parent, W_NDimSlice):
parent = parent.parent
- size = 1
- for sh in shape:
- size *= sh
self.strides = strides
self.backstrides = backstrides
- ViewArray.__init__(self, size, shape, parent.dtype, parent.order,
- parent)
+ ViewArray.__init__(self, support.product(shape), shape, parent.dtype,
+ parent.order, parent)
self.start = start
+ def create_iter(self):
+ return ViewIterator(self.start, self.strides, self.backstrides,
+ self.shape)
+
def setshape(self, space, new_shape):
if len(self.shape) < 1:
return
@@ -1050,7 +1045,8 @@
self.backstrides = backstrides
self.shape = new_shape
return
- new_strides = calc_new_strides(new_shape, self.shape, self.strides)
+ new_strides = calc_new_strides(new_shape, self.shape, self.strides,
+ self.order)
if new_strides is None:
raise OperationError(space.w_AttributeError, space.wrap(
"incompatible shape for a non-contiguous array"))
@@ -1073,8 +1069,11 @@
self.shape = new_shape
self.calc_strides(new_shape)
- def create_sig(self, res_shape):
- return self.array_sig(res_shape)
+ def create_iter(self):
+ return ArrayIterator(self.size)
+
+ def create_sig(self):
+ return signature.ArraySignature(self.dtype)
def __del__(self):
lltype.free(self.storage, flavor='raw', track_allocation=False)
@@ -1092,27 +1091,42 @@
shape.append(item)
return size, shape
-def array(space, w_item_or_iterable, w_dtype=None, w_order=NoneNotWrapped):
+ at unwrap_spec(subok=bool, copy=bool, ownmaskna=bool)
+def array(space, w_item_or_iterable, w_dtype=None, w_order=None,
+ subok=True, copy=True, w_maskna=None, ownmaskna=False):
# find scalar
+ if w_maskna is None:
+ w_maskna = space.w_None
+ if (not subok or not space.is_w(w_maskna, space.w_None) or
+ ownmaskna):
+ raise OperationError(space.w_NotImplementedError, space.wrap("Unsupported args"))
if not space.issequence_w(w_item_or_iterable):
- if space.is_w(w_dtype, space.w_None):
+ if w_dtype is None or space.is_w(w_dtype, space.w_None):
w_dtype = interp_ufuncs.find_dtype_for_scalar(space,
w_item_or_iterable)
dtype = space.interp_w(interp_dtype.W_Dtype,
space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
)
return scalar_w(space, dtype, w_item_or_iterable)
- if w_order is None:
+ if space.is_w(w_order, space.w_None) or w_order is None:
order = 'C'
else:
order = space.str_w(w_order)
if order != 'C': # or order != 'F':
raise operationerrfmt(space.w_ValueError, "Unknown order: %s",
order)
- shape, elems_w = _find_shape_and_elems(space, w_item_or_iterable)
+ if isinstance(w_item_or_iterable, BaseArray):
+ if (not space.is_w(w_dtype, space.w_None) and
+ w_item_or_iterable.find_dtype() is not w_dtype):
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ "copying over different dtypes unsupported"))
+ if copy:
+ return w_item_or_iterable.copy(space)
+ return w_item_or_iterable
+ shape, elems_w = find_shape_and_elems(space, w_item_or_iterable)
# they come back in C order
size = len(elems_w)
- if space.is_w(w_dtype, space.w_None):
+ if w_dtype is None or space.is_w(w_dtype, space.w_None):
w_dtype = None
for w_elem in elems_w:
w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_elem,
@@ -1127,6 +1141,7 @@
arr = W_NDimArray(size, shape[:], dtype=dtype, order=order)
shapelen = len(shape)
arr_iter = ArrayIterator(arr.size)
+ # XXX we might want to have a jitdriver here
for i in range(len(elems_w)):
w_elem = elems_w[i]
dtype.setitem(arr.storage, arr_iter.offset,
@@ -1139,6 +1154,8 @@
space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
)
size, shape = _find_size_and_shape(space, w_size)
+ if not shape:
+ return scalar_w(space, dtype, space.wrap(0))
return space.wrap(W_NDimArray(size, shape[:], dtype=dtype))
def ones(space, w_size, w_dtype=None):
@@ -1147,17 +1164,66 @@
)
size, shape = _find_size_and_shape(space, w_size)
+ if not shape:
+ return scalar_w(space, dtype, space.wrap(1))
arr = W_NDimArray(size, shape[:], dtype=dtype)
one = dtype.box(1)
arr.dtype.fill(arr.storage, one, 0, size)
return space.wrap(arr)
+ at unwrap_spec(arr=BaseArray, skipna=bool, keepdims=bool)
+def count_reduce_items(space, arr, w_axis=None, skipna=False, keepdims=True):
+ if not keepdims:
+ raise OperationError(space.w_NotImplementedError, space.wrap("unsupported"))
+ if space.is_w(w_axis, space.w_None):
+ return space.wrap(support.product(arr.shape))
+ if space.isinstance_w(w_axis, space.w_int):
+ return space.wrap(arr.shape[space.int_w(w_axis)])
+ s = 1
+ elems = space.fixedview(w_axis)
+ for w_elem in elems:
+ s *= arr.shape[space.int_w(w_elem)]
+ return space.wrap(s)
+
def dot(space, w_obj, w_obj2):
w_arr = convert_to_array(space, w_obj)
if isinstance(w_arr, Scalar):
return convert_to_array(space, w_obj2).descr_dot(space, w_arr)
return w_arr.descr_dot(space, w_obj2)
+ at unwrap_spec(axis=int)
+def concatenate(space, w_args, axis=0):
+ args_w = space.listview(w_args)
+ if len(args_w) == 0:
+ raise OperationError(space.w_ValueError, space.wrap("concatenation of zero-length sequences is impossible"))
+ args_w = [convert_to_array(space, w_arg) for w_arg in args_w]
+ dtype = args_w[0].find_dtype()
+ shape = args_w[0].shape[:]
+ if len(shape) <= axis:
+ raise OperationError(space.w_ValueError,
+ space.wrap("bad axis argument"))
+ for arr in args_w[1:]:
+ dtype = interp_ufuncs.find_binop_result_dtype(space, dtype,
+ arr.find_dtype())
+ if len(arr.shape) <= axis:
+ raise OperationError(space.w_ValueError,
+ space.wrap("bad axis argument"))
+ for i, axis_size in enumerate(arr.shape):
+ if len(arr.shape) != len(shape) or (i != axis and axis_size != shape[i]):
+ raise OperationError(space.w_ValueError, space.wrap(
+ "array dimensions must agree except for axis being concatenated"))
+ elif i == axis:
+ shape[i] += axis_size
+ res = W_NDimArray(support.product(shape), shape, dtype, 'C')
+ chunks = [Chunk(0, i, 1, i) for i in shape]
+ axis_start = 0
+ for arr in args_w:
+ chunks[axis] = Chunk(axis_start, axis_start + arr.shape[axis], 1,
+ arr.shape[axis])
+ res.create_slice(chunks).setslice(space, arr)
+ axis_start += arr.shape[axis]
+ return res
+
BaseArray.typedef = TypeDef(
'ndarray',
__module__ = "numpypy",
@@ -1193,6 +1259,10 @@
__gt__ = interp2app(BaseArray.descr_gt),
__ge__ = interp2app(BaseArray.descr_ge),
+ __and__ = interp2app(BaseArray.descr_and),
+ __or__ = interp2app(BaseArray.descr_or),
+ __invert__ = interp2app(BaseArray.descr_invert),
+
__repr__ = interp2app(BaseArray.descr_repr),
__str__ = interp2app(BaseArray.descr_str),
__array_interface__ = GetSetProperty(BaseArray.descr_array_iface),
@@ -1202,9 +1272,11 @@
BaseArray.descr_set_shape),
size = GetSetProperty(BaseArray.descr_get_size),
ndim = GetSetProperty(BaseArray.descr_get_ndim),
+ item = interp2app(BaseArray.descr_item),
T = GetSetProperty(BaseArray.descr_get_transpose),
flat = GetSetProperty(BaseArray.descr_get_flatiter),
+ ravel = interp2app(BaseArray.descr_ravel),
mean = interp2app(BaseArray.descr_mean),
sum = interp2app(BaseArray.descr_sum),
@@ -1219,9 +1291,14 @@
var = interp2app(BaseArray.descr_var),
std = interp2app(BaseArray.descr_std),
+ fill = interp2app(BaseArray.descr_fill),
+
copy = interp2app(BaseArray.descr_copy),
+ flatten = interp2app(BaseArray.descr_flatten),
reshape = interp2app(BaseArray.descr_reshape),
tolist = interp2app(BaseArray.descr_tolist),
+ take = interp2app(BaseArray.descr_take),
+ compress = interp2app(BaseArray.descr_compress),
)
@@ -1230,30 +1307,129 @@
@jit.unroll_safe
def __init__(self, arr):
arr = arr.get_concrete()
- size = 1
- for sh in arr.shape:
- size *= sh
self.strides = [arr.strides[-1]]
self.backstrides = [arr.backstrides[-1]]
- ViewArray.__init__(self, size, [size], arr.dtype, arr.order,
- arr)
self.shapelen = len(arr.shape)
- self.iter = OneDimIterator(arr.start, self.strides[0],
- self.shape[0])
+ sig = arr.find_sig()
+ self.iter = sig.create_frame(arr).get_final_iter()
+ self.base = arr
+ self.index = 0
+ ViewArray.__init__(self, arr.size, [arr.size], arr.dtype, arr.order,
+ arr)
def descr_next(self, space):
if self.iter.done():
raise OperationError(space.w_StopIteration, space.w_None)
- result = self.getitem(self.iter.offset)
+ result = self.base.getitem(self.iter.offset)
self.iter = self.iter.next(self.shapelen)
+ self.index += 1
return result
def descr_iter(self):
return self
+ def descr_index(self, space):
+ return space.wrap(self.index)
+
+ def descr_coords(self, space):
+ coords, step, lngth = to_coords(space, self.base.shape,
+ self.base.size, self.base.order,
+ space.wrap(self.index))
+ return space.newtuple([space.wrap(c) for c in coords])
+
+ @jit.unroll_safe
+ def descr_getitem(self, space, w_idx):
+ if not (space.isinstance_w(w_idx, space.w_int) or
+ space.isinstance_w(w_idx, space.w_slice)):
+ raise OperationError(space.w_IndexError,
+ space.wrap('unsupported iterator index'))
+ base = self.base
+ start, stop, step, lngth = space.decode_index4(w_idx, base.size)
+ # setslice would have been better, but flat[u:v] for arbitrary
+ # shapes of array a cannot be represented as a[x1:x2, y1:y2]
+ basei = ViewIterator(base.start, base.strides,
+ base.backstrides,base.shape)
+ shapelen = len(base.shape)
+ basei = basei.next_skip_x(shapelen, start)
+ if lngth <2:
+ return base.getitem(basei.offset)
+ ri = ArrayIterator(lngth)
+ res = W_NDimArray(lngth, [lngth], base.dtype,
+ base.order)
+ while not ri.done():
+ flat_get_driver.jit_merge_point(shapelen=shapelen,
+ base=base,
+ basei=basei,
+ step=step,
+ res=res,
+ ri=ri,
+ )
+ w_val = base.getitem(basei.offset)
+ res.setitem(ri.offset,w_val)
+ basei = basei.next_skip_x(shapelen, step)
+ ri = ri.next(shapelen)
+ return res
+
+ def descr_setitem(self, space, w_idx, w_value):
+ if not (space.isinstance_w(w_idx, space.w_int) or
+ space.isinstance_w(w_idx, space.w_slice)):
+ raise OperationError(space.w_IndexError,
+ space.wrap('unsupported iterator index'))
+ base = self.base
+ start, stop, step, lngth = space.decode_index4(w_idx, base.size)
+ arr = convert_to_array(space, w_value)
+ ai = 0
+ basei = ViewIterator(base.start, base.strides,
+ base.backstrides,base.shape)
+ shapelen = len(base.shape)
+ basei = basei.next_skip_x(shapelen, start)
+ while lngth > 0:
+ flat_set_driver.jit_merge_point(shapelen=shapelen,
+ basei=basei,
+ base=base,
+ step=step,
+ arr=arr,
+ ai=ai,
+ lngth=lngth,
+ )
+ v = arr.getitem(ai).convert_to(base.dtype)
+ base.setitem(basei.offset, v)
+ # need to repeat input values until all assignments are done
+ ai = (ai + 1) % arr.size
+ basei = basei.next_skip_x(shapelen, step)
+ lngth -= 1
+
+ def create_sig(self):
+ return signature.FlatSignature(self.base.dtype)
+
+ def descr_base(self, space):
+ return space.wrap(self.base)
+
W_FlatIterator.typedef = TypeDef(
'flatiter',
+ #__array__ = #MISSING
+ __iter__ = interp2app(W_FlatIterator.descr_iter),
+ __getitem__ = interp2app(W_FlatIterator.descr_getitem),
+ __setitem__ = interp2app(W_FlatIterator.descr_setitem),
+ __eq__ = interp2app(BaseArray.descr_eq),
+ __ne__ = interp2app(BaseArray.descr_ne),
+ __lt__ = interp2app(BaseArray.descr_lt),
+ __le__ = interp2app(BaseArray.descr_le),
+ __gt__ = interp2app(BaseArray.descr_gt),
+ __ge__ = interp2app(BaseArray.descr_ge),
+ #__sizeof__ #MISSING
+ base = GetSetProperty(W_FlatIterator.descr_base),
+ index = GetSetProperty(W_FlatIterator.descr_index),
+ coords = GetSetProperty(W_FlatIterator.descr_coords),
next = interp2app(W_FlatIterator.descr_next),
- __iter__ = interp2app(W_FlatIterator.descr_iter),
+
)
W_FlatIterator.acceptable_as_base_class = False
+
+def isna(space, w_obj):
+ if isinstance(w_obj, BaseArray):
+ arr = w_obj.empty_copy(space,
+ interp_dtype.get_dtype_cache(space).w_booldtype)
+ arr.fill(space, space.wrap(False))
+ return arr
+ return space.wrap(False)
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -1,31 +1,44 @@
from pypy.interpreter.baseobjspace import Wrappable
from pypy.interpreter.error import OperationError, operationerrfmt
-from pypy.interpreter.gateway import interp2app
+from pypy.interpreter.gateway import interp2app, unwrap_spec, NoneNotWrapped
from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
-from pypy.module.micronumpy import interp_boxes, interp_dtype
-from pypy.module.micronumpy.signature import ReduceSignature, ScalarSignature,\
- find_sig, new_printable_location
+from pypy.module.micronumpy import interp_boxes, interp_dtype, support
+from pypy.module.micronumpy.signature import (ReduceSignature, find_sig,
+ new_printable_location, AxisReduceSignature, ScalarSignature)
from pypy.rlib import jit
from pypy.rlib.rarithmetic import LONG_BIT
from pypy.tool.sourcetools import func_with_new_name
+
reduce_driver = jit.JitDriver(
- greens = ['shapelen', "sig"],
- virtualizables = ["frame"],
- reds = ["frame", "self", "dtype", "value", "obj"],
+ greens=['shapelen', "sig"],
+ virtualizables=["frame"],
+ reds=["frame", "self", "dtype", "value", "obj"],
get_printable_location=new_printable_location('reduce'),
+ name='numpy_reduce',
)
+axisreduce_driver = jit.JitDriver(
+ greens=['shapelen', 'sig'],
+ virtualizables=['frame'],
+ reds=['self','arr', 'identity', 'frame'],
+ name='numpy_axisreduce',
+ get_printable_location=new_printable_location('axisreduce'),
+)
+
+
class W_Ufunc(Wrappable):
_attrs_ = ["name", "promote_to_float", "promote_bools", "identity"]
_immutable_fields_ = ["promote_to_float", "promote_bools", "name"]
- def __init__(self, name, promote_to_float, promote_bools, identity):
+ def __init__(self, name, promote_to_float, promote_bools, identity,
+ int_only):
self.name = name
self.promote_to_float = promote_to_float
self.promote_bools = promote_bools
self.identity = identity
+ self.int_only = int_only
def descr_repr(self, space):
return space.wrap("<ufunc '%s'>" % self.name)
@@ -36,30 +49,105 @@
return self.identity
def descr_call(self, space, __args__):
- if __args__.keywords or len(__args__.arguments_w) < self.argcount:
+ args_w, kwds_w = __args__.unpack()
+ # it occurs to me that we don't support any datatypes that
+ # require casting, change it later when we do
+ kwds_w.pop('casting', None)
+ w_subok = kwds_w.pop('subok', None)
+ w_out = kwds_w.pop('out', space.w_None)
+ if ((w_subok is not None and space.is_true(w_subok)) or
+ not space.is_w(w_out, space.w_None)):
+ raise OperationError(space.w_NotImplementedError,
+ space.wrap("parameters unsupported"))
+ if kwds_w or len(args_w) < self.argcount:
raise OperationError(space.w_ValueError,
space.wrap("invalid number of arguments")
)
- elif len(__args__.arguments_w) > self.argcount:
+ elif len(args_w) > self.argcount:
# The extra arguments should actually be the output array, but we
# don't support that yet.
raise OperationError(space.w_TypeError,
space.wrap("invalid number of arguments")
)
- return self.call(space, __args__.arguments_w)
+ return self.call(space, args_w)
- def descr_reduce(self, space, w_obj):
- return self.reduce(space, w_obj, multidim=False)
+ @unwrap_spec(skipna=bool, keepdims=bool)
+ def descr_reduce(self, space, w_obj, w_axis=NoneNotWrapped, w_dtype=None,
+ skipna=False, keepdims=False, w_out=None):
+ """reduce(...)
+ reduce(a, axis=0)
- def reduce(self, space, w_obj, multidim):
- from pypy.module.micronumpy.interp_numarray import convert_to_array, Scalar
-
+ Reduces `a`'s dimension by one, by applying ufunc along one axis.
+
+ Let :math:`a.shape = (N_0, ..., N_i, ..., N_{M-1})`. Then
+ :math:`ufunc.reduce(a, axis=i)[k_0, ..,k_{i-1}, k_{i+1}, .., k_{M-1}]` =
+ the result of iterating `j` over :math:`range(N_i)`, cumulatively applying
+ ufunc to each :math:`a[k_0, ..,k_{i-1}, j, k_{i+1}, .., k_{M-1}]`.
+ For a one-dimensional array, reduce produces results equivalent to:
+ ::
+
+ r = op.identity # op = ufunc
+ for i in xrange(len(A)):
+ r = op(r, A[i])
+ return r
+
+ For example, add.reduce() is equivalent to sum().
+
+ Parameters
+ ----------
+ a : array_like
+ The array to act on.
+ axis : int, optional
+ The axis along which to apply the reduction.
+
+ Examples
+ --------
+ >>> np.multiply.reduce([2,3,5])
+ 30
+
+ A multi-dimensional array example:
+
+ >>> X = np.arange(8).reshape((2,2,2))
+ >>> X
+ array([[[0, 1],
+ [2, 3]],
+ [[4, 5],
+ [6, 7]]])
+ >>> np.add.reduce(X, 0)
+ array([[ 4, 6],
+ [ 8, 10]])
+ >>> np.add.reduce(X) # confirm: default axis value is 0
+ array([[ 4, 6],
+ [ 8, 10]])
+ >>> np.add.reduce(X, 1)
+ array([[ 2, 4],
+ [10, 12]])
+ >>> np.add.reduce(X, 2)
+ array([[ 1, 5],
+ [ 9, 13]])
+ """
+ if not space.is_w(w_out, space.w_None):
+ raise OperationError(space.w_NotImplementedError, space.wrap(
+ "out not supported"))
+ if w_axis is None:
+ axis = 0
+ elif space.is_w(w_axis, space.w_None):
+ axis = -1
+ else:
+ axis = space.int_w(w_axis)
+ return self.reduce(space, w_obj, False, False, axis, keepdims)
+
+ def reduce(self, space, w_obj, multidim, promote_to_largest, dim,
+ keepdims=False):
+ from pypy.module.micronumpy.interp_numarray import convert_to_array, \
+ Scalar
if self.argcount != 2:
raise OperationError(space.w_ValueError, space.wrap("reduce only "
"supported for binary functions"))
-
assert isinstance(self, W_Ufunc2)
obj = convert_to_array(space, w_obj)
+ if dim >= len(obj.shape):
+ raise OperationError(space.w_ValueError, space.wrap("axis(=%d) out of bounds" % dim))
if isinstance(obj, Scalar):
raise OperationError(space.w_TypeError, space.wrap("cannot reduce "
"on a scalar"))
@@ -67,26 +155,80 @@
size = obj.size
dtype = find_unaryop_result_dtype(
space, obj.find_dtype(),
- promote_to_largest=True
+ promote_to_float=self.promote_to_float,
+ promote_to_largest=promote_to_largest,
+ promote_bools=True
)
shapelen = len(obj.shape)
+ if self.identity is None and size == 0:
+ raise operationerrfmt(space.w_ValueError, "zero-size array to "
+ "%s.reduce without identity", self.name)
+ if shapelen > 1 and dim >= 0:
+ res = self.do_axis_reduce(obj, dtype, dim, keepdims)
+ return space.wrap(res)
+ scalarsig = ScalarSignature(dtype)
sig = find_sig(ReduceSignature(self.func, self.name, dtype,
- ScalarSignature(dtype),
- obj.create_sig(obj.shape)), obj)
+ scalarsig,
+ obj.create_sig()), obj)
frame = sig.create_frame(obj)
- if shapelen > 1 and not multidim:
- raise OperationError(space.w_NotImplementedError,
- space.wrap("not implemented yet"))
if self.identity is None:
- if size == 0:
- raise operationerrfmt(space.w_ValueError, "zero-size array to "
- "%s.reduce without identity", self.name)
value = sig.eval(frame, obj).convert_to(dtype)
frame.next(shapelen)
else:
value = self.identity.convert_to(dtype)
return self.reduce_loop(shapelen, sig, frame, value, obj, dtype)
+ def do_axis_reduce(self, obj, dtype, dim, keepdims):
+ from pypy.module.micronumpy.interp_numarray import AxisReduce,\
+ W_NDimArray
+
+ if keepdims:
+ shape = obj.shape[:dim] + [1] + obj.shape[dim + 1:]
+ else:
+ shape = obj.shape[:dim] + obj.shape[dim + 1:]
+ result = W_NDimArray(support.product(shape), shape, dtype)
+ rightsig = obj.create_sig()
+ # note - this is just a wrapper so signature can fetch
+ # both left and right, nothing more, especially
+ # this is not a true virtual array, because shapes
+ # don't quite match
+ arr = AxisReduce(self.func, self.name, obj.shape, dtype,
+ result, obj, dim)
+ scalarsig = ScalarSignature(dtype)
+ sig = find_sig(AxisReduceSignature(self.func, self.name, dtype,
+ scalarsig, rightsig), arr)
+ assert isinstance(sig, AxisReduceSignature)
+ frame = sig.create_frame(arr)
+ shapelen = len(obj.shape)
+ if self.identity is not None:
+ identity = self.identity.convert_to(dtype)
+ else:
+ identity = None
+ self.reduce_axis_loop(frame, sig, shapelen, arr, identity)
+ return result
+
+ def reduce_axis_loop(self, frame, sig, shapelen, arr, identity):
+ # note - we can be advanterous here, depending on the exact field
+ # layout. For now let's say we iterate the original way and
+ # simply follow the original iteration order
+ while not frame.done():
+ axisreduce_driver.jit_merge_point(frame=frame, self=self,
+ sig=sig,
+ identity=identity,
+ shapelen=shapelen, arr=arr)
+ iter = frame.get_final_iter()
+ v = sig.eval(frame, arr).convert_to(sig.calc_dtype)
+ if iter.first_line:
+ if identity is not None:
+ value = self.func(sig.calc_dtype, identity, v)
+ else:
+ value = v
+ else:
+ cur = arr.left.getitem(iter.offset)
+ value = self.func(sig.calc_dtype, cur, v)
+ arr.left.setitem(iter.offset, value)
+ frame.next(shapelen)
+
def reduce_loop(self, shapelen, sig, frame, value, obj, dtype):
while not frame.done():
reduce_driver.jit_merge_point(sig=sig,
@@ -94,20 +236,24 @@
value=value, obj=obj, frame=frame,
dtype=dtype)
assert isinstance(sig, ReduceSignature)
- value = sig.binfunc(dtype, value, sig.eval(frame, obj).convert_to(dtype))
+ value = sig.binfunc(dtype, value,
+ sig.eval(frame, obj).convert_to(dtype))
frame.next(shapelen)
return value
+
class W_Ufunc1(W_Ufunc):
argcount = 1
_immutable_fields_ = ["func", "name"]
def __init__(self, func, name, promote_to_float=False, promote_bools=False,
- identity=None):
+ identity=None, bool_result=False, int_only=False):
- W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity)
+ W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity,
+ int_only)
self.func = func
+ self.bool_result = bool_result
def call(self, space, args_w):
from pypy.module.micronumpy.interp_numarray import (Call1,
@@ -115,27 +261,32 @@
[w_obj] = args_w
w_obj = convert_to_array(space, w_obj)
- res_dtype = find_unaryop_result_dtype(space,
- w_obj.find_dtype(),
- promote_to_float=self.promote_to_float,
- promote_bools=self.promote_bools,
- )
+ calc_dtype = find_unaryop_result_dtype(space,
+ w_obj.find_dtype(),
+ promote_to_float=self.promote_to_float,
+ promote_bools=self.promote_bools)
+ if self.bool_result:
+ res_dtype = interp_dtype.get_dtype_cache(space).w_booldtype
+ else:
+ res_dtype = calc_dtype
if isinstance(w_obj, Scalar):
- return self.func(res_dtype, w_obj.value.convert_to(res_dtype))
+ return space.wrap(self.func(calc_dtype, w_obj.value.convert_to(calc_dtype)))
- w_res = Call1(self.func, self.name, w_obj.shape, res_dtype, w_obj)
+ w_res = Call1(self.func, self.name, w_obj.shape, calc_dtype, res_dtype,
+ w_obj)
w_obj.add_invalidates(w_res)
return w_res
class W_Ufunc2(W_Ufunc):
- _immutable_fields_ = ["comparison_func", "func", "name"]
+ _immutable_fields_ = ["comparison_func", "func", "name", "int_only"]
argcount = 2
def __init__(self, func, name, promote_to_float=False, promote_bools=False,
- identity=None, comparison_func=False):
+ identity=None, comparison_func=False, int_only=False):
- W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity)
+ W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity,
+ int_only)
self.func = func
self.comparison_func = comparison_func
@@ -148,6 +299,7 @@
w_rhs = convert_to_array(space, w_rhs)
calc_dtype = find_binop_result_dtype(space,
w_lhs.find_dtype(), w_rhs.find_dtype(),
+ int_only=self.int_only,
promote_to_float=self.promote_to_float,
promote_bools=self.promote_bools,
)
@@ -156,10 +308,10 @@
else:
res_dtype = calc_dtype
if isinstance(w_lhs, Scalar) and isinstance(w_rhs, Scalar):
- return self.func(calc_dtype,
+ return space.wrap(self.func(calc_dtype,
w_lhs.value.convert_to(calc_dtype),
w_rhs.value.convert_to(calc_dtype)
- )
+ ))
new_shape = shape_agreement(space, w_lhs.shape, w_rhs.shape)
w_res = Call2(self.func, self.name,
@@ -182,11 +334,14 @@
reduce = interp2app(W_Ufunc.descr_reduce),
)
+
def find_binop_result_dtype(space, dt1, dt2, promote_to_float=False,
- promote_bools=False):
+ promote_bools=False, int_only=False):
# dt1.num should be <= dt2.num
if dt1.num > dt2.num:
dt1, dt2 = dt2, dt1
+ if int_only and (not dt1.is_int_type() or not dt2.is_int_type()):
+ raise OperationError(space.w_TypeError, space.wrap("Unsupported types"))
# Some operations promote op(bool, bool) to return int8, rather than bool
if promote_bools and (dt1.kind == dt2.kind == interp_dtype.BOOLLTR):
return interp_dtype.get_dtype_cache(space).w_int8dtype
@@ -230,6 +385,7 @@
dtypenum += 3
return interp_dtype.get_dtype_cache(space).builtin_dtypes[dtypenum]
+
def find_unaryop_result_dtype(space, dt, promote_to_float=False,
promote_bools=False, promote_to_largest=False):
if promote_bools and (dt.kind == interp_dtype.BOOLLTR):
@@ -254,6 +410,7 @@
assert False
return dt
+
def find_dtype_for_scalar(space, w_obj, current_guess=None):
bool_dtype = interp_dtype.get_dtype_cache(space).w_booldtype
long_dtype = interp_dtype.get_dtype_cache(space).w_longdtype
@@ -282,12 +439,16 @@
return interp_dtype.get_dtype_cache(space).w_float64dtype
-def ufunc_dtype_caller(space, ufunc_name, op_name, argcount, comparison_func):
+def ufunc_dtype_caller(space, ufunc_name, op_name, argcount, comparison_func,
+ bool_result):
+ dtype_cache = interp_dtype.get_dtype_cache(space)
if argcount == 1:
def impl(res_dtype, value):
- return getattr(res_dtype.itemtype, op_name)(value)
+ res = getattr(res_dtype.itemtype, op_name)(value)
+ if bool_result:
+ return dtype_cache.w_booldtype.box(res)
+ return res
elif argcount == 2:
- dtype_cache = interp_dtype.get_dtype_cache(space)
def impl(res_dtype, lvalue, rvalue):
res = getattr(res_dtype.itemtype, op_name)(lvalue, rvalue)
if comparison_func:
@@ -302,7 +463,13 @@
("add", "add", 2, {"identity": 0}),
("subtract", "sub", 2),
("multiply", "mul", 2, {"identity": 1}),
+ ("bitwise_and", "bitwise_and", 2, {"identity": 1,
+ 'int_only': True}),
+ ("bitwise_or", "bitwise_or", 2, {"identity": 0,
+ 'int_only': True}),
+ ("invert", "invert", 1, {"int_only": True}),
("divide", "div", 2, {"promote_bools": True}),
+ ("true_divide", "div", 2, {"promote_to_float": True}),
("mod", "mod", 2, {"promote_bools": True}),
("power", "pow", 2, {"promote_bools": True}),
@@ -312,6 +479,8 @@
("less_equal", "le", 2, {"comparison_func": True}),
("greater", "gt", 2, {"comparison_func": True}),
("greater_equal", "ge", 2, {"comparison_func": True}),
+ ("isnan", "isnan", 1, {"bool_result": True}),
+ ("isinf", "isinf", 1, {"bool_result": True}),
("maximum", "max", 2),
("minimum", "min", 2),
@@ -326,6 +495,7 @@
("fabs", "fabs", 1, {"promote_to_float": True}),
("floor", "floor", 1, {"promote_to_float": True}),
+ ("ceil", "ceil", 1, {"promote_to_float": True}),
("exp", "exp", 1, {"promote_to_float": True}),
('sqrt', 'sqrt', 1, {'promote_to_float': True}),
@@ -347,11 +517,13 @@
identity = extra_kwargs.get("identity")
if identity is not None:
- identity = interp_dtype.get_dtype_cache(space).w_longdtype.box(identity)
+ identity = \
+ interp_dtype.get_dtype_cache(space).w_longdtype.box(identity)
extra_kwargs["identity"] = identity
func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount,
- comparison_func=extra_kwargs.get("comparison_func", False)
+ comparison_func=extra_kwargs.get("comparison_func", False),
+ bool_result=extra_kwargs.get("bool_result", False),
)
if argcount == 1:
ufunc = W_Ufunc1(func, ufunc_name, **extra_kwargs)
@@ -361,3 +533,4 @@
def get(space):
return space.fromcache(UfuncState)
+
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -1,10 +1,32 @@
from pypy.rlib.objectmodel import r_dict, compute_identity_hash, compute_hash
from pypy.rlib.rarithmetic import intmask
from pypy.module.micronumpy.interp_iter import ViewIterator, ArrayIterator, \
- OneDimIterator, ConstantIterator
-from pypy.module.micronumpy.strides import calculate_slice_strides
+ ConstantIterator, AxisIterator, ViewTransform,\
+ BroadcastTransform
from pypy.rlib.jit import hint, unroll_safe, promote
+""" Signature specifies both the numpy expression that has been constructed
+and the assembler to be compiled. This is a very important observation -
+Two expressions will be using the same assembler if and only if they are
+compiled to the same signature.
+
+This is also a very convinient tool for specializations. For example
+a + a and a + b (where a != b) will compile to different assembler because
+we specialize on the same array access.
+
+When evaluating, signatures will create iterators per signature node,
+potentially sharing some of them. Iterators depend also on the actual
+expression, they're not only dependant on the array itself. For example
+a + b where a is dim 2 and b is dim 1 would create a broadcasted iterator for
+the array b.
+
+Such iterator changes are called Transformations. An actual iterator would
+be a combination of array and various transformation, like view, broadcast,
+dimension swapping etc.
+
+See interp_iter for transformations
+"""
+
def new_printable_location(driver_name):
def get_printable_location(shapelen, sig):
return 'numpy ' + sig.debug_repr() + ' [%d dims,%s]' % (shapelen, driver_name)
@@ -33,7 +55,8 @@
return sig
class NumpyEvalFrame(object):
- _virtualizable2_ = ['iterators[*]', 'final_iter', 'arraylist[*]']
+ _virtualizable2_ = ['iterators[*]', 'final_iter', 'arraylist[*]',
+ 'value', 'identity']
@unroll_safe
def __init__(self, iterators, arrays):
@@ -51,7 +74,7 @@
def done(self):
final_iter = promote(self.final_iter)
if final_iter < 0:
- return False
+ assert False
return self.iterators[final_iter].done()
@unroll_safe
@@ -59,6 +82,22 @@
for i in range(len(self.iterators)):
self.iterators[i] = self.iterators[i].next(shapelen)
+ @unroll_safe
+ def next_from_second(self, shapelen):
+ """ Don't increase the first iterator
+ """
+ for i in range(1, len(self.iterators)):
+ self.iterators[i] = self.iterators[i].next(shapelen)
+
+ def next_first(self, shapelen):
+ self.iterators[0] = self.iterators[0].next(shapelen)
+
+ def get_final_iter(self):
+ final_iter = promote(self.final_iter)
+ if final_iter < 0:
+ assert False
+ return self.iterators[final_iter]
+
def _add_ptr_to_cache(ptr, cache):
i = 0
for p in cache:
@@ -70,6 +109,9 @@
cache.append(ptr)
return res
+def new_cache():
+ return r_dict(sigeq_no_numbering, sighash)
+
class Signature(object):
_attrs_ = ['iter_no', 'array_no']
_immutable_fields_ = ['iter_no', 'array_no']
@@ -78,7 +120,7 @@
iter_no = 0
def invent_numbering(self):
- cache = r_dict(sigeq_no_numbering, sighash)
+ cache = new_cache()
allnumbers = []
self._invent_numbering(cache, allnumbers)
@@ -95,13 +137,13 @@
allnumbers.append(no)
self.iter_no = no
- def create_frame(self, arr, res_shape=None):
- res_shape = res_shape or arr.shape
+ def create_frame(self, arr):
iterlist = []
arraylist = []
- self._create_iter(iterlist, arraylist, arr, res_shape, [])
+ self._create_iter(iterlist, arraylist, arr, [])
return NumpyEvalFrame(iterlist, arraylist)
+
class ConcreteSignature(Signature):
_immutable_fields_ = ['dtype']
@@ -120,16 +162,6 @@
def hash(self):
return compute_identity_hash(self.dtype)
- def allocate_view_iter(self, arr, res_shape, chunklist):
- r = arr.shape, arr.start, arr.strides, arr.backstrides
- if chunklist:
- for chunkelem in chunklist:
- r = calculate_slice_strides(r[0], r[1], r[2], r[3], chunkelem)
- shape, start, strides, backstrides = r
- if len(res_shape) == 1:
- return OneDimIterator(start, strides[0], res_shape[0])
- return ViewIterator(start, strides, backstrides, shape, res_shape)
-
class ArraySignature(ConcreteSignature):
def debug_repr(self):
return 'Array'
@@ -137,23 +169,25 @@
def _invent_array_numbering(self, arr, cache):
from pypy.module.micronumpy.interp_numarray import ConcreteArray
concr = arr.get_concrete()
+ # this get_concrete never forces assembler. If we're here and array
+ # is not of a concrete class it means that we have a _forced_result,
+ # otherwise the signature would not match
assert isinstance(concr, ConcreteArray)
+ assert concr.dtype is self.dtype
self.array_no = _add_ptr_to_cache(concr.storage, cache)
- def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist):
+ def _create_iter(self, iterlist, arraylist, arr, transforms):
from pypy.module.micronumpy.interp_numarray import ConcreteArray
concr = arr.get_concrete()
assert isinstance(concr, ConcreteArray)
storage = concr.storage
if self.iter_no >= len(iterlist):
- iterlist.append(self.allocate_iter(concr, res_shape, chunklist))
+ iterlist.append(self.allocate_iter(concr, transforms))
if self.array_no >= len(arraylist):
arraylist.append(storage)
- def allocate_iter(self, arr, res_shape, chunklist):
- if chunklist:
- return self.allocate_view_iter(arr, res_shape, chunklist)
- return ArrayIterator(arr.size)
+ def allocate_iter(self, arr, transforms):
+ return ArrayIterator(arr.size).apply_transformations(arr, transforms)
def eval(self, frame, arr):
iter = frame.iterators[self.iter_no]
@@ -166,7 +200,7 @@
def _invent_array_numbering(self, arr, cache):
pass
- def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist):
+ def _create_iter(self, iterlist, arraylist, arr, transforms):
if self.iter_no >= len(iterlist):
iter = ConstantIterator()
iterlist.append(iter)
@@ -186,8 +220,21 @@
allnumbers.append(no)
self.iter_no = no
- def allocate_iter(self, arr, res_shape, chunklist):
- return self.allocate_view_iter(arr, res_shape, chunklist)
+ def allocate_iter(self, arr, transforms):
+ return ViewIterator(arr.start, arr.strides, arr.backstrides,
+ arr.shape).apply_transformations(arr, transforms)
+
+class FlatSignature(ViewSignature):
+ def debug_repr(self):
+ return 'Flat'
+
+ def allocate_iter(self, arr, transforms):
+ from pypy.module.micronumpy.interp_numarray import W_FlatIterator
+ assert isinstance(arr, W_FlatIterator)
+ return ViewIterator(arr.base.start, arr.base.strides,
+ arr.base.backstrides,
+ arr.base.shape).apply_transformations(arr.base,
+ transforms)
class VirtualSliceSignature(Signature):
def __init__(self, child):
@@ -198,6 +245,9 @@
assert isinstance(arr, VirtualSlice)
self.child._invent_array_numbering(arr.child, cache)
+ def _invent_numbering(self, cache, allnumbers):
+ self.child._invent_numbering(new_cache(), allnumbers)
+
def hash(self):
return intmask(self.child.hash() ^ 1234)
@@ -207,12 +257,11 @@
assert isinstance(other, VirtualSliceSignature)
return self.child.eq(other.child, compare_array_no)
- def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist):
+ def _create_iter(self, iterlist, arraylist, arr, transforms):
from pypy.module.micronumpy.interp_numarray import VirtualSlice
assert isinstance(arr, VirtualSlice)
- chunklist.append(arr.chunks)
- self.child._create_iter(iterlist, arraylist, arr.child, res_shape,
- chunklist)
+ transforms = transforms + [ViewTransform(arr.chunks)]
+ self.child._create_iter(iterlist, arraylist, arr.child, transforms)
def eval(self, frame, arr):
from pypy.module.micronumpy.interp_numarray import VirtualSlice
@@ -248,17 +297,16 @@
assert isinstance(arr, Call1)
self.child._invent_array_numbering(arr.values, cache)
- def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist):
+ def _create_iter(self, iterlist, arraylist, arr, transforms):
from pypy.module.micronumpy.interp_numarray import Call1
assert isinstance(arr, Call1)
- self.child._create_iter(iterlist, arraylist, arr.values, res_shape,
- chunklist)
+ self.child._create_iter(iterlist, arraylist, arr.values, transforms)
def eval(self, frame, arr):
from pypy.module.micronumpy.interp_numarray import Call1
assert isinstance(arr, Call1)
- v = self.child.eval(frame, arr.values).convert_to(arr.res_dtype)
- return self.unfunc(arr.res_dtype, v)
+ v = self.child.eval(frame, arr.values).convert_to(arr.calc_dtype)
+ return self.unfunc(arr.calc_dtype, v)
class Call2(Signature):
_immutable_fields_ = ['binfunc', 'name', 'calc_dtype', 'left', 'right']
@@ -293,29 +341,68 @@
self.left._invent_numbering(cache, allnumbers)
self.right._invent_numbering(cache, allnumbers)
- def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist):
+ def _create_iter(self, iterlist, arraylist, arr, transforms):
from pypy.module.micronumpy.interp_numarray import Call2
assert isinstance(arr, Call2)
- self.left._create_iter(iterlist, arraylist, arr.left, res_shape,
- chunklist)
- self.right._create_iter(iterlist, arraylist, arr.right, res_shape,
- chunklist)
+ self.left._create_iter(iterlist, arraylist, arr.left, transforms)
+ self.right._create_iter(iterlist, arraylist, arr.right, transforms)
def eval(self, frame, arr):
from pypy.module.micronumpy.interp_numarray import Call2
assert isinstance(arr, Call2)
lhs = self.left.eval(frame, arr.left).convert_to(self.calc_dtype)
rhs = self.right.eval(frame, arr.right).convert_to(self.calc_dtype)
+
return self.binfunc(self.calc_dtype, lhs, rhs)
def debug_repr(self):
return 'Call2(%s, %s, %s)' % (self.name, self.left.debug_repr(),
self.right.debug_repr())
+class BroadcastLeft(Call2):
+ def _invent_numbering(self, cache, allnumbers):
+ self.left._invent_numbering(new_cache(), allnumbers)
+ self.right._invent_numbering(cache, allnumbers)
+
+ def _create_iter(self, iterlist, arraylist, arr, transforms):
+ from pypy.module.micronumpy.interp_numarray import Call2
+
+ assert isinstance(arr, Call2)
+ ltransforms = transforms + [BroadcastTransform(arr.shape)]
+ self.left._create_iter(iterlist, arraylist, arr.left, ltransforms)
+ self.right._create_iter(iterlist, arraylist, arr.right, transforms)
+
+class BroadcastRight(Call2):
+ def _invent_numbering(self, cache, allnumbers):
+ self.left._invent_numbering(cache, allnumbers)
+ self.right._invent_numbering(new_cache(), allnumbers)
+
+ def _create_iter(self, iterlist, arraylist, arr, transforms):
+ from pypy.module.micronumpy.interp_numarray import Call2
+
+ assert isinstance(arr, Call2)
+ rtransforms = transforms + [BroadcastTransform(arr.shape)]
+ self.left._create_iter(iterlist, arraylist, arr.left, transforms)
+ self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
+
+class BroadcastBoth(Call2):
+ def _invent_numbering(self, cache, allnumbers):
+ self.left._invent_numbering(new_cache(), allnumbers)
+ self.right._invent_numbering(new_cache(), allnumbers)
+
+ def _create_iter(self, iterlist, arraylist, arr, transforms):
+ from pypy.module.micronumpy.interp_numarray import Call2
+
+ assert isinstance(arr, Call2)
+ rtransforms = transforms + [BroadcastTransform(arr.shape)]
+ ltransforms = transforms + [BroadcastTransform(arr.shape)]
+ self.left._create_iter(iterlist, arraylist, arr.left, ltransforms)
+ self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
+
class ReduceSignature(Call2):
- def _create_iter(self, iterlist, arraylist, arr, res_shape, chunklist):
- self.right._create_iter(iterlist, arraylist, arr, res_shape, chunklist)
+ def _create_iter(self, iterlist, arraylist, arr, transforms):
+ self.right._create_iter(iterlist, arraylist, arr, transforms)
def _invent_numbering(self, cache, allnumbers):
self.right._invent_numbering(cache, allnumbers)
@@ -325,3 +412,63 @@
def eval(self, frame, arr):
return self.right.eval(frame, arr)
+
+ def debug_repr(self):
+ return 'ReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
+
+class SliceloopSignature(Call2):
+ def eval(self, frame, arr):
+ from pypy.module.micronumpy.interp_numarray import Call2
+
+ assert isinstance(arr, Call2)
+ ofs = frame.iterators[0].offset
+ arr.left.setitem(ofs, self.right.eval(frame, arr.right).convert_to(
+ self.calc_dtype))
+
+ def debug_repr(self):
+ return 'SliceLoop(%s, %s, %s)' % (self.name, self.left.debug_repr(),
+ self.right.debug_repr())
+
+class SliceloopBroadcastSignature(SliceloopSignature):
+ def _invent_numbering(self, cache, allnumbers):
+ self.left._invent_numbering(new_cache(), allnumbers)
+ self.right._invent_numbering(cache, allnumbers)
+
+ def _create_iter(self, iterlist, arraylist, arr, transforms):
+ from pypy.module.micronumpy.interp_numarray import SliceArray
+
+ assert isinstance(arr, SliceArray)
+ rtransforms = transforms + [BroadcastTransform(arr.shape)]
+ self.left._create_iter(iterlist, arraylist, arr.left, transforms)
+ self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
+
+class AxisReduceSignature(Call2):
+ def _create_iter(self, iterlist, arraylist, arr, transforms):
+ from pypy.module.micronumpy.interp_numarray import AxisReduce,\
+ ConcreteArray
+
+ assert isinstance(arr, AxisReduce)
+ left = arr.left
+ assert isinstance(left, ConcreteArray)
+ iterlist.append(AxisIterator(left.start, arr.dim, arr.shape,
+ left.strides, left.backstrides))
+ self.right._create_iter(iterlist, arraylist, arr.right, transforms)
+
+ def _invent_numbering(self, cache, allnumbers):
+ allnumbers.append(0)
+ self.right._invent_numbering(cache, allnumbers)
+
+ def _invent_array_numbering(self, arr, cache):
+ from pypy.module.micronumpy.interp_numarray import AxisReduce
+
+ assert isinstance(arr, AxisReduce)
+ self.right._invent_array_numbering(arr.right, cache)
+
+ def eval(self, frame, arr):
+ from pypy.module.micronumpy.interp_numarray import AxisReduce
+
+ assert isinstance(arr, AxisReduce)
+ return self.right.eval(frame, arr.right).convert_to(self.calc_dtype)
+
+ def debug_repr(self):
+ return 'AxisReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -1,5 +1,5 @@
from pypy.rlib import jit
-
+from pypy.interpreter.error import OperationError
@jit.look_inside_iff(lambda shape, start, strides, backstrides, chunks:
jit.isconstant(len(chunks))
@@ -10,12 +10,12 @@
rstart = start
rshape = []
i = -1
- for i, (start_, stop, step, lgt) in enumerate(chunks):
- if step != 0:
- rstrides.append(strides[i] * step)
- rbackstrides.append(strides[i] * (lgt - 1) * step)
- rshape.append(lgt)
- rstart += strides[i] * start_
+ for i, chunk in enumerate(chunks):
+ if chunk.step != 0:
+ rstrides.append(strides[i] * chunk.step)
+ rbackstrides.append(strides[i] * (chunk.lgt - 1) * chunk.step)
+ rshape.append(chunk.lgt)
+ rstart += strides[i] * chunk.start
# add a reminder
s = i + 1
assert s >= 0
@@ -37,3 +37,196 @@
rstrides = [0] * (len(res_shape) - len(orig_shape)) + rstrides
rbackstrides = [0] * (len(res_shape) - len(orig_shape)) + rbackstrides
return rstrides, rbackstrides
+
+def find_shape_and_elems(space, w_iterable):
+ shape = [space.len_w(w_iterable)]
+ batch = space.listview(w_iterable)
+ while True:
+ new_batch = []
+ if not batch:
+ return shape, []
+ if not space.issequence_w(batch[0]):
+ for elem in batch:
+ if space.issequence_w(elem):
+ raise OperationError(space.w_ValueError, space.wrap(
+ "setting an array element with a sequence"))
+ return shape, batch
+ size = space.len_w(batch[0])
+ for w_elem in batch:
+ if not space.issequence_w(w_elem) or space.len_w(w_elem) != size:
+ raise OperationError(space.w_ValueError, space.wrap(
+ "setting an array element with a sequence"))
+ new_batch += space.listview(w_elem)
+ shape.append(size)
+ batch = new_batch
+
+def to_coords(space, shape, size, order, w_item_or_slice):
+ '''Returns a start coord, step, and length.
+ '''
+ start = lngth = step = 0
+ if not (space.isinstance_w(w_item_or_slice, space.w_int) or
+ space.isinstance_w(w_item_or_slice, space.w_slice)):
+ raise OperationError(space.w_IndexError,
+ space.wrap('unsupported iterator index'))
+
+ start, stop, step, lngth = space.decode_index4(w_item_or_slice, size)
+
+ coords = [0] * len(shape)
+ i = start
+ if order == 'C':
+ for s in range(len(shape) -1, -1, -1):
+ coords[s] = i % shape[s]
+ i //= shape[s]
+ else:
+ for s in range(len(shape)):
+ coords[s] = i % shape[s]
+ i //= shape[s]
+ return coords, step, lngth
+
+def shape_agreement(space, shape1, shape2):
+ ret = _shape_agreement(shape1, shape2)
+ if len(ret) < max(len(shape1), len(shape2)):
+ raise OperationError(space.w_ValueError,
+ space.wrap("operands could not be broadcast together with shapes (%s) (%s)" % (
+ ",".join([str(x) for x in shape1]),
+ ",".join([str(x) for x in shape2]),
+ ))
+ )
+ return ret
+
+def _shape_agreement(shape1, shape2):
+ """ Checks agreement about two shapes with respect to broadcasting. Returns
+ the resulting shape.
+ """
+ lshift = 0
+ rshift = 0
+ if len(shape1) > len(shape2):
+ m = len(shape1)
+ n = len(shape2)
+ rshift = len(shape2) - len(shape1)
+ remainder = shape1
+ else:
+ m = len(shape2)
+ n = len(shape1)
+ lshift = len(shape1) - len(shape2)
+ remainder = shape2
+ endshape = [0] * m
+ indices1 = [True] * m
+ indices2 = [True] * m
+ for i in range(m - 1, m - n - 1, -1):
+ left = shape1[i + lshift]
+ right = shape2[i + rshift]
+ if left == right:
+ endshape[i] = left
+ elif left == 1:
+ endshape[i] = right
+ indices1[i + lshift] = False
+ elif right == 1:
+ endshape[i] = left
+ indices2[i + rshift] = False
+ else:
+ return []
+ #raise OperationError(space.w_ValueError, space.wrap(
+ # "frames are not aligned"))
+ for i in range(m - n):
+ endshape[i] = remainder[i]
+ return endshape
+
+def get_shape_from_iterable(space, old_size, w_iterable):
+ new_size = 0
+ new_shape = []
+ if space.isinstance_w(w_iterable, space.w_int):
+ new_size = space.int_w(w_iterable)
+ if new_size < 0:
+ new_size = old_size
+ new_shape = [new_size]
+ else:
+ neg_dim = -1
+ batch = space.listview(w_iterable)
+ new_size = 1
+ if len(batch) < 1:
+ if old_size == 1:
+ # Scalars can have an empty size.
+ new_size = 1
+ else:
+ new_size = 0
+ new_shape = []
+ i = 0
+ for elem in batch:
+ s = space.int_w(elem)
+ if s < 0:
+ if neg_dim >= 0:
+ raise OperationError(space.w_ValueError, space.wrap(
+ "can only specify one unknown dimension"))
+ s = 1
+ neg_dim = i
+ new_size *= s
+ new_shape.append(s)
+ i += 1
+ if neg_dim >= 0:
+ new_shape[neg_dim] = old_size / new_size
+ new_size *= new_shape[neg_dim]
+ if new_size != old_size:
+ raise OperationError(space.w_ValueError,
+ space.wrap("total size of new array must be unchanged"))
+ return new_shape
+
+# Recalculating strides. Find the steps that the iteration does for each
+# dimension, given the stride and shape. Then try to create a new stride that
+# fits the new shape, using those steps. If there is a shape/step mismatch
+# (meaning that the realignment of elements crosses from one step into another)
+# return None so that the caller can raise an exception.
+def calc_new_strides(new_shape, old_shape, old_strides, order):
+ # Return the proper strides for new_shape, or None if the mapping crosses
+ # stepping boundaries
+
+ # Assumes that prod(old_shape) == prod(new_shape), len(old_shape) > 1, and
+ # len(new_shape) > 0
+ steps = []
+ last_step = 1
+ oldI = 0
+ new_strides = []
+ if order == 'F':
+ for i in range(len(old_shape)):
+ steps.append(old_strides[i] / last_step)
+ last_step *= old_shape[i]
+ cur_step = steps[0]
+ n_new_elems_used = 1
+ n_old_elems_to_use = old_shape[0]
+ for s in new_shape:
+ new_strides.append(cur_step * n_new_elems_used)
+ n_new_elems_used *= s
+ while n_new_elems_used > n_old_elems_to_use:
+ oldI += 1
+ if steps[oldI] != steps[oldI - 1]:
+ return None
+ n_old_elems_to_use *= old_shape[oldI]
+ if n_new_elems_used == n_old_elems_to_use:
+ oldI += 1
+ if oldI < len(old_shape):
+ cur_step = steps[oldI]
+ n_old_elems_to_use *= old_shape[oldI]
+ elif order == 'C':
+ for i in range(len(old_shape) - 1, -1, -1):
+ steps.insert(0, old_strides[i] / last_step)
+ last_step *= old_shape[i]
+ cur_step = steps[-1]
+ n_new_elems_used = 1
+ oldI = -1
+ n_old_elems_to_use = old_shape[-1]
+ for i in range(len(new_shape) - 1, -1, -1):
+ s = new_shape[i]
+ new_strides.insert(0, cur_step * n_new_elems_used)
+ n_new_elems_used *= s
+ while n_new_elems_used > n_old_elems_to_use:
+ oldI -= 1
+ if steps[oldI] != steps[oldI + 1]:
+ return None
+ n_old_elems_to_use *= old_shape[oldI]
+ if n_new_elems_used == n_old_elems_to_use:
+ oldI -= 1
+ if oldI >= -len(old_shape):
+ cur_step = steps[oldI]
+ n_old_elems_to_use *= old_shape[oldI]
+ assert len(new_strides) == len(new_shape)
+ return new_strides
diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/support.py
@@ -0,0 +1,5 @@
+def product(s):
+ i = 1
+ for x in s:
+ i *= x
+ return i
\ No newline at end of file
diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -3,9 +3,17 @@
from pypy.module.micronumpy.interp_numarray import W_NDimArray, Scalar
from pypy.module.micronumpy.interp_ufuncs import (find_binop_result_dtype,
find_unaryop_result_dtype)
+from pypy.module.micronumpy.interp_boxes import W_Float64Box
+from pypy.conftest import option
+import sys
class BaseNumpyAppTest(object):
def setup_class(cls):
+ if option.runappdirect:
+ if '__pypy__' not in sys.builtin_module_names:
+ import numpy
+ sys.modules['numpypy'] = numpy
+ sys.modules['_numpypy'] = numpy
cls.space = gettestobjspace(usemodules=['micronumpy'])
class TestSignature(object):
@@ -16,7 +24,7 @@
ar = W_NDimArray(10, [10], dtype=float64_dtype)
ar2 = W_NDimArray(10, [10], dtype=float64_dtype)
v1 = ar.descr_add(space, ar)
- v2 = ar.descr_add(space, Scalar(float64_dtype, 2.0))
+ v2 = ar.descr_add(space, Scalar(float64_dtype, W_Float64Box(2.0)))
sig1 = v1.find_sig()
sig2 = v2.find_sig()
assert v1 is not v2
@@ -26,7 +34,7 @@
sig1b = ar2.descr_add(space, ar).find_sig()
assert sig1b.left.array_no != sig1b.right.array_no
assert sig1b is not sig1
- v3 = ar.descr_add(space, Scalar(float64_dtype, 1.0))
+ v3 = ar.descr_add(space, Scalar(float64_dtype, W_Float64Box(1.0)))
sig3 = v3.find_sig()
assert sig2 is sig3
v4 = ar.descr_add(space, ar)
diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py
--- a/pypy/module/micronumpy/test/test_compile.py
+++ b/pypy/module/micronumpy/test/test_compile.py
@@ -245,3 +245,19 @@
a -> 3
""")
assert interp.results[0].value == 11
+
+ def test_flat_iter(self):
+ interp = self.run('''
+ a = |30|
+ b = flat(a)
+ b -> 3
+ ''')
+ assert interp.results[0].value == 3
+
+ def test_take(self):
+ interp = self.run("""
+ a = |10|
+ b = take(a, [1, 1, 3, 2])
+ b -> 2
+ """)
+ assert interp.results[0].value == 3
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -11,8 +11,17 @@
assert dtype('int8').num == 1
assert dtype(d) is d
assert dtype(None) is dtype(float)
+ assert dtype('int8').name == 'int8'
raises(TypeError, dtype, 1042)
+ def test_dtype_eq(self):
+ from _numpypy import dtype
+
+ assert dtype("int8") == "int8"
+ assert "int8" == dtype("int8")
+ raises(TypeError, lambda: dtype("int8") == 3)
+ assert dtype(bool) == bool
+
def test_dtype_with_types(self):
from _numpypy import dtype
@@ -30,7 +39,7 @@
def test_repr_str(self):
from _numpypy import dtype
- assert repr(dtype) == "<type 'numpypy.dtype'>"
+ assert '.dtype' in repr(dtype)
d = dtype('?')
assert repr(d) == "dtype('bool')"
assert str(d) == "bool"
@@ -166,14 +175,11 @@
# You can't subclass dtype
raises(TypeError, type, "Foo", (dtype,), {})
- def test_new(self):
- import _numpypy as np
- assert np.int_(4) == 4
- assert np.float_(3.4) == 3.4
+ def test_aliases(self):
+ from _numpypy import dtype
- def test_pow(self):
- from _numpypy import int_
- assert int_(4) ** 2 == 16
+ assert dtype("float") is dtype(float)
+
class AppTestTypes(BaseNumpyAppTest):
def test_abstract_types(self):
@@ -189,6 +195,15 @@
raises(TypeError, numpy.floating, 0)
raises(TypeError, numpy.inexact, 0)
+ def test_new(self):
+ import _numpypy as np
+ assert np.int_(4) == 4
+ assert np.float_(3.4) == 3.4
+
+ def test_pow(self):
+ from _numpypy import int_
+ assert int_(4) ** 2 == 16
+
def test_bool(self):
import _numpypy as numpy
@@ -318,7 +333,7 @@
else:
raises(OverflowError, numpy.int64, 9223372036854775807)
raises(OverflowError, numpy.int64, '9223372036854775807')
-
+
raises(OverflowError, numpy.int64, 9223372036854775808)
raises(OverflowError, numpy.int64, '9223372036854775808')
@@ -370,3 +385,19 @@
b = X(10)
assert type(b) is X
assert b.m() == 12
+
+ def test_long_as_index(self):
+ skip("waiting for removal of multimethods of __index__")
+ from _numpypy import int_
+ assert (1, 2, 3)[int_(1)] == 2
+
+ def test_int(self):
+ import sys
+ from _numpypy import int32, int64, int_
+ assert issubclass(int_, int)
+ if sys.maxint == (1<<31) - 1:
+ assert issubclass(int32, int)
+ assert int_ is int32
+ else:
+ assert issubclass(int64, int)
+ assert int_ is int64
diff --git a/pypy/module/micronumpy/test/test_iter.py b/pypy/module/micronumpy/test/test_iter.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_iter.py
@@ -0,0 +1,88 @@
+from pypy.module.micronumpy.interp_iter import ViewIterator
+
+class TestIterDirect(object):
+ def test_C_viewiterator(self):
+ #Let's get started, simple iteration in C order with
+ #contiguous layout => strides[-1] is 1
+ start = 0
+ shape = [3, 5]
+ strides = [5, 1]
+ backstrides = [x * (y - 1) for x,y in zip(strides, shape)]
+ assert backstrides == [10, 4]
+ i = ViewIterator(start, strides, backstrides, shape)
+ i = i.next(2)
+ i = i.next(2)
+ i = i.next(2)
+ assert i.offset == 3
+ assert not i.done()
+ assert i.indices == [0,3]
+ #cause a dimension overflow
+ i = i.next(2)
+ i = i.next(2)
+ assert i.offset == 5
+ assert i.indices == [1,0]
+
+ #Now what happens if the array is transposed? strides[-1] != 1
+ # therefore layout is non-contiguous
+ strides = [1, 3]
+ backstrides = [x * (y - 1) for x,y in zip(strides, shape)]
+ assert backstrides == [2, 12]
+ i = ViewIterator(start, strides, backstrides, shape)
+ i = i.next(2)
+ i = i.next(2)
+ i = i.next(2)
+ assert i.offset == 9
+ assert not i.done()
+ assert i.indices == [0,3]
+ #cause a dimension overflow
+ i = i.next(2)
+ i = i.next(2)
+ assert i.offset == 1
+ assert i.indices == [1,0]
+
+ def test_C_viewiterator_step(self):
+ #iteration in C order with #contiguous layout => strides[-1] is 1
+ #skip less than the shape
+ start = 0
+ shape = [3, 5]
+ strides = [5, 1]
+ backstrides = [x * (y - 1) for x,y in zip(strides, shape)]
+ assert backstrides == [10, 4]
+ i = ViewIterator(start, strides, backstrides, shape)
+ i = i.next_skip_x(2,2)
+ i = i.next_skip_x(2,2)
+ i = i.next_skip_x(2,2)
+ assert i.offset == 6
+ assert not i.done()
+ assert i.indices == [1,1]
+ #And for some big skips
+ i = i.next_skip_x(2,5)
+ assert i.offset == 11
+ assert i.indices == [2,1]
+ i = i.next_skip_x(2,5)
+ # Note: the offset does not overflow but recycles,
+ # this is good for broadcast
+ assert i.offset == 1
+ assert i.indices == [0,1]
+ assert i.done()
+
+ #Now what happens if the array is transposed? strides[-1] != 1
+ # therefore layout is non-contiguous
+ strides = [1, 3]
+ backstrides = [x * (y - 1) for x,y in zip(strides, shape)]
+ assert backstrides == [2, 12]
+ i = ViewIterator(start, strides, backstrides, shape)
+ i = i.next_skip_x(2,2)
+ i = i.next_skip_x(2,2)
+ i = i.next_skip_x(2,2)
+ assert i.offset == 4
+ assert i.indices == [1,1]
+ assert not i.done()
+ i = i.next_skip_x(2,5)
+ assert i.offset == 5
+ assert i.indices == [2,1]
+ assert not i.done()
+ i = i.next_skip_x(2,5)
+ assert i.indices == [0,1]
+ assert i.offset == 3
+ assert i.done()
diff --git a/pypy/module/micronumpy/test/test_module.py b/pypy/module/micronumpy/test/test_module.py
--- a/pypy/module/micronumpy/test/test_module.py
+++ b/pypy/module/micronumpy/test/test_module.py
@@ -2,16 +2,11 @@
class AppTestNumPyModule(BaseNumpyAppTest):
- def test_mean(self):
- from _numpypy import array, mean
- assert mean(array(range(5))) == 2.0
- assert mean(range(5)) == 2.0
-
def test_average(self):
from _numpypy import array, average
assert average(range(10)) == 4.5
assert average(array(range(10))) == 4.5
-
+
def test_sum(self):
from _numpypy import array, sum
assert sum(range(10)) == 45
@@ -21,7 +16,7 @@
from _numpypy import array, min
assert min(range(10)) == 0
assert min(array(range(10))) == 0
-
+
def test_max(self):
from _numpypy import array, max
assert max(range(10)) == 9
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -2,6 +2,7 @@
import py
from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement
+from pypy.module.micronumpy.interp_iter import Chunk
from pypy.module.micronumpy import signature
from pypy.interpreter.error import OperationError
from pypy.conftest import gettestobjspace
@@ -37,53 +38,54 @@
def test_create_slice_f(self):
a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
- s = a.create_slice([(3, 0, 0, 1)])
+ s = a.create_slice([Chunk(3, 0, 0, 1)])
assert s.start == 3
assert s.strides == [10, 50]
assert s.backstrides == [40, 100]
- s = a.create_slice([(1, 9, 2, 4)])
+ s = a.create_slice([Chunk(1, 9, 2, 4)])
assert s.start == 1
assert s.strides == [2, 10, 50]
assert s.backstrides == [6, 40, 100]
- s = a.create_slice([(1, 5, 3, 2), (1, 2, 1, 1), (1, 0, 0, 1)])
+ s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1), Chunk(1, 0, 0, 1)])
assert s.shape == [2, 1]
assert s.strides == [3, 10]
assert s.backstrides == [3, 0]
- s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+ s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
assert s.start == 20
assert s.shape == [10, 3]
def test_create_slice_c(self):
a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C')
- s = a.create_slice([(3, 0, 0, 1)])
+ s = a.create_slice([Chunk(3, 0, 0, 1)])
assert s.start == 45
assert s.strides == [3, 1]
assert s.backstrides == [12, 2]
- s = a.create_slice([(1, 9, 2, 4)])
+ s = a.create_slice([Chunk(1, 9, 2, 4)])
assert s.start == 15
assert s.strides == [30, 3, 1]
assert s.backstrides == [90, 12, 2]
- s = a.create_slice([(1, 5, 3, 2), (1, 2, 1, 1), (1, 0, 0, 1)])
+ s = a.create_slice([Chunk(1, 5, 3, 2), Chunk(1, 2, 1, 1),
+ Chunk(1, 0, 0, 1)])
assert s.start == 19
assert s.shape == [2, 1]
assert s.strides == [45, 3]
assert s.backstrides == [45, 0]
- s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+ s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
assert s.start == 6
assert s.shape == [10, 3]
def test_slice_of_slice_f(self):
a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
- s = a.create_slice([(5, 0, 0, 1)])
+ s = a.create_slice([Chunk(5, 0, 0, 1)])
assert s.start == 5
- s2 = s.create_slice([(3, 0, 0, 1)])
+ s2 = s.create_slice([Chunk(3, 0, 0, 1)])
assert s2.shape == [3]
assert s2.strides == [50]
assert s2.parent is a
assert s2.backstrides == [100]
assert s2.start == 35
- s = a.create_slice([(1, 5, 3, 2)])
- s2 = s.create_slice([(0, 2, 1, 2), (2, 0, 0, 1)])
+ s = a.create_slice([Chunk(1, 5, 3, 2)])
+ s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)])
assert s2.shape == [2, 3]
assert s2.strides == [3, 50]
assert s2.backstrides == [3, 100]
@@ -91,16 +93,16 @@
def test_slice_of_slice_c(self):
a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C')
- s = a.create_slice([(5, 0, 0, 1)])
+ s = a.create_slice([Chunk(5, 0, 0, 1)])
assert s.start == 15 * 5
- s2 = s.create_slice([(3, 0, 0, 1)])
+ s2 = s.create_slice([Chunk(3, 0, 0, 1)])
assert s2.shape == [3]
assert s2.strides == [1]
assert s2.parent is a
assert s2.backstrides == [2]
assert s2.start == 5 * 15 + 3 * 3
- s = a.create_slice([(1, 5, 3, 2)])
- s2 = s.create_slice([(0, 2, 1, 2), (2, 0, 0, 1)])
+ s = a.create_slice([Chunk(1, 5, 3, 2)])
+ s2 = s.create_slice([Chunk(0, 2, 1, 2), Chunk(2, 0, 0, 1)])
assert s2.shape == [2, 3]
assert s2.strides == [45, 1]
assert s2.backstrides == [45, 2]
@@ -108,14 +110,14 @@
def test_negative_step_f(self):
a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
- s = a.create_slice([(9, -1, -2, 5)])
+ s = a.create_slice([Chunk(9, -1, -2, 5)])
assert s.start == 9
assert s.strides == [-2, 10, 50]
assert s.backstrides == [-8, 40, 100]
def test_negative_step_c(self):
a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), order='C')
- s = a.create_slice([(9, -1, -2, 5)])
+ s = a.create_slice([Chunk(9, -1, -2, 5)])
assert s.start == 135
assert s.strides == [-30, 3, 1]
assert s.backstrides == [-120, 12, 2]
@@ -124,7 +126,7 @@
a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'F')
r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2))
assert r == 1 + 2 * 10 + 2 * 50
- s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+ s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
r = s._index_of_single_item(self.space, self.newtuple(1, 0))
assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0))
r = s._index_of_single_item(self.space, self.newtuple(1, 1))
@@ -134,7 +136,7 @@
a = W_NDimArray(10 * 5 * 3, [10, 5, 3], MockDtype(), 'C')
r = a._index_of_single_item(self.space, self.newtuple(1, 2, 2))
assert r == 1 * 3 * 5 + 2 * 3 + 2
- s = a.create_slice([(0, 10, 1, 10), (2, 0, 0, 1)])
+ s = a.create_slice([Chunk(0, 10, 1, 10), Chunk(2, 0, 0, 1)])
r = s._index_of_single_item(self.space, self.newtuple(1, 0))
assert r == a._index_of_single_item(self.space, self.newtuple(1, 2, 0))
r = s._index_of_single_item(self.space, self.newtuple(1, 1))
@@ -152,12 +154,36 @@
def test_calc_new_strides(self):
from pypy.module.micronumpy.interp_numarray import calc_new_strides
- assert calc_new_strides([2, 4], [4, 2], [4, 2]) == [8, 2]
- assert calc_new_strides([2, 4, 3], [8, 3], [1, 16]) == [1, 2, 16]
- assert calc_new_strides([2, 3, 4], [8, 3], [1, 16]) is None
- assert calc_new_strides([24], [2, 4, 3], [48, 6, 1]) is None
- assert calc_new_strides([24], [2, 4, 3], [24, 6, 2]) == [2]
+ assert calc_new_strides([2, 4], [4, 2], [4, 2], "C") == [8, 2]
+ assert calc_new_strides([2, 4, 3], [8, 3], [1, 16], 'F') == [1, 2, 16]
+ assert calc_new_strides([2, 3, 4], [8, 3], [1, 16], 'F') is None
+ assert calc_new_strides([24], [2, 4, 3], [48, 6, 1], 'C') is None
+ assert calc_new_strides([24], [2, 4, 3], [24, 6, 2], 'C') == [2]
+ assert calc_new_strides([105, 1], [3, 5, 7], [35, 7, 1],'C') == [1, 1]
+ assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1],'C') == [105, 1]
+ assert calc_new_strides([1, 105], [3, 5, 7], [35, 7, 1],'F') is None
+ assert calc_new_strides([1, 1, 1, 105, 1], [15, 7], [7, 1],'C') == \
+ [105, 105, 105, 1, 1]
+ assert calc_new_strides([1, 1, 105, 1, 1], [7, 15], [1, 7],'F') == \
+ [1, 1, 1, 105, 105]
+ def test_to_coords(self):
+ from pypy.module.micronumpy.strides import to_coords
+
+ def _to_coords(index, order):
+ return to_coords(self.space, [2, 3, 4], 24, order,
+ self.space.wrap(index))[0]
+
+ assert _to_coords(0, 'C') == [0, 0, 0]
+ assert _to_coords(1, 'C') == [0, 0, 1]
+ assert _to_coords(-1, 'C') == [1, 2, 3]
+ assert _to_coords(5, 'C') == [0, 1, 1]
+ assert _to_coords(13, 'C') == [1, 0, 1]
+ assert _to_coords(0, 'F') == [0, 0, 0]
+ assert _to_coords(1, 'F') == [1, 0, 0]
+ assert _to_coords(-1, 'F') == [1, 2, 3]
+ assert _to_coords(5, 'F') == [1, 2, 0]
+ assert _to_coords(13, 'F') == [1, 0, 2]
class AppTestNumArray(BaseNumpyAppTest):
def test_ndarray(self):
@@ -202,6 +228,7 @@
# And check that changes stick.
a[13] = 5.3
assert a[13] == 5.3
+ assert zeros(()).shape == ()
def test_size(self):
from _numpypy import array
@@ -245,6 +272,11 @@
b = a[::2]
c = b.copy()
assert (c == b).all()
+ assert ((a + a).copy() == (a + a)).all()
+
+ a = arange(15).reshape(5,3)
+ b = a.copy()
+ assert (b == a).all()
def test_iterator_init(self):
from _numpypy import array
@@ -270,6 +302,12 @@
for i in xrange(5):
assert a[i] == b[i]
+ def test_getitem_nd(self):
+ from _numpypy import arange
+ a = arange(15).reshape(3, 5)
+ assert a[1, 3] == 8
+ assert a.T[1, 2] == 11
+
def test_setitem(self):
from _numpypy import array
a = array(range(5))
@@ -357,6 +395,7 @@
assert b.shape == (5,)
c = a[:3]
assert c.shape == (3,)
+ assert array([]).shape == (0,)
def test_set_shape(self):
from _numpypy import array, zeros
@@ -377,6 +416,8 @@
a.shape = ()
#numpy allows this
a.shape = (1,)
+ a = array(range(6)).reshape(2,3).T
+ raises(AttributeError, 'a.shape = 6')
def test_reshape(self):
from _numpypy import array, zeros
@@ -390,6 +431,7 @@
assert (a == [1000, 1, 2, 3, 1000, 5, 6, 7, 1000, 9, 10, 11]).all()
a = zeros((4, 2, 3))
a.shape = (12, 2)
+ (a + a).reshape(2, 12) # assert did not explode
def test_slice_reshape(self):
from _numpypy import zeros, arange
@@ -433,6 +475,13 @@
y = z.reshape(4, 3, 8)
assert y.shape == (4, 3, 8)
+ def test_scalar_reshape(self):
+ from numpypy import array
+ a = array(3)
+ assert a.reshape([1, 1]).shape == (1, 1)
+ assert a.reshape([1]).shape == (1,)
+ raises(ValueError, "a.reshape(3)")
+
def test_add(self):
from _numpypy import array
a = array(range(5))
@@ -720,10 +769,17 @@
assert d[1] == 12
def test_mean(self):
- from _numpypy import array
+ from _numpypy import array, arange
a = array(range(5))
assert a.mean() == 2.0
assert a[:4].mean() == 1.5
+ a = array(range(105)).reshape(3, 5, 7)
+ b = a.mean(axis=0)
+ b[0, 0]==35.
+ assert a.mean(axis=0)[0, 0] == 35
+ assert (b == array(range(35, 70), dtype=float).reshape(5, 7)).all()
+ assert (a.mean(2) == array(range(0, 15), dtype=float).reshape(3, 5) * 7 + 3).all()
+ assert (arange(10).reshape(5, 2).mean(axis=1) == [0.5, 2.5, 4.5, 6.5, 8.5]).all()
def test_sum(self):
from _numpypy import array
@@ -734,6 +790,34 @@
a = array([True] * 5, bool)
assert a.sum() == 5
+ raises(TypeError, 'a.sum(2, 3)')
+
+ def test_reduce_nd(self):
+ from numpypy import arange, array, multiply
+ a = arange(15).reshape(5, 3)
+ assert a.sum() == 105
+ assert a.max() == 14
+ assert array([]).sum() == 0.0
+ raises(ValueError, 'array([]).max()')
+ assert (a.sum(0) == [30, 35, 40]).all()
+ assert (a.sum(axis=0) == [30, 35, 40]).all()
+ assert (a.sum(1) == [3, 12, 21, 30, 39]).all()
+ assert (a.max(0) == [12, 13, 14]).all()
+ assert (a.max(1) == [2, 5, 8, 11, 14]).all()
+ assert ((a + a).max() == 28)
+ assert ((a + a).max(0) == [24, 26, 28]).all()
+ assert ((a + a).sum(1) == [6, 24, 42, 60, 78]).all()
+ assert (multiply.reduce(a) == array([0, 3640, 12320])).all()
+ a = array(range(105)).reshape(3, 5, 7)
+ assert (a[:, 1, :].sum(0) == [126, 129, 132, 135, 138, 141, 144]).all()
+ assert (a[:, 1, :].sum(1) == [70, 315, 560]).all()
+ raises (ValueError, 'a[:, 1, :].sum(2)')
+ assert ((a + a).T.sum(2).T == (a + a).sum(0)).all()
+ assert (a.reshape(1,-1).sum(0) == range(105)).all()
+ assert (a.reshape(1,-1).sum(1) == 5460)
+ assert (array([[1,2],[3,4]]).prod(0) == [3, 8]).all()
+ assert (array([[1,2],[3,4]]).prod(1) == [2, 12]).all()
+
def test_identity(self):
from _numpypy import identity, array
from _numpypy import int32, float64, dtype
@@ -906,7 +990,7 @@
assert debug_repr(a + a) == 'Call2(add, Array, Array)'
assert debug_repr(a[::2]) == 'Slice'
assert debug_repr(a + 2) == 'Call2(add, Array, Scalar)'
- assert debug_repr(a + a.flat) == 'Call2(add, Array, Slice)'
+ assert debug_repr(a + a.flat) == 'Call2(add, Array, Flat)'
assert debug_repr(sin(a)) == 'Call1(sin, Array)'
b = a + a
@@ -979,11 +1063,49 @@
assert a[0].tolist() == [17.1, 27.2]
def test_var(self):
- from _numpypy import array
+ from _numpypy import array, arange
a = array(range(10))
assert a.var() == 8.25
a = array([5.0])
assert a.var() == 0.0
+ a = arange(10).reshape(5, 2)
+ assert a.var() == 8.25
+ assert (a.var(0) == [8, 8]).all()
+ assert (a.var(1) == [.25] * 5).all()
+
+ def test_concatenate(self):
+ from numpypy import array, concatenate, dtype
+ a1 = array([0,1,2])
+ a2 = array([3,4,5])
+ a = concatenate((a1, a2))
+ assert len(a) == 6
+ assert (a == [0,1,2,3,4,5]).all()
+ assert a.dtype is dtype(int)
+ b1 = array([[1, 2], [3, 4]])
+ b2 = array([[5, 6]])
+ b = concatenate((b1, b2), axis=0)
+ assert (b == [[1, 2],[3, 4],[5, 6]]).all()
+ c = concatenate((b1, b2.T), axis=1)
+ assert (c == [[1, 2, 5],[3, 4, 6]]).all()
+ d = concatenate(([0],[1]))
+ assert (d == [0,1]).all()
+ e1 = array([[0,1],[2,3]])
+ e = concatenate(e1)
+ assert (e == [0,1,2,3]).all()
+ f1 = array([0,1])
+ f = concatenate((f1, [2], f1, [7]))
+ assert (f == [0,1,2,0,1,7]).all()
+
+ bad_axis = raises(ValueError, concatenate, (a1,a2), axis=1)
+ assert str(bad_axis.value) == "bad axis argument"
+
+ concat_zero = raises(ValueError, concatenate, ())
+ assert str(concat_zero.value) == \
+ "concatenation of zero-length sequences is impossible"
+
+ dims_disagree = raises(ValueError, concatenate, (a1, b1), axis=0)
+ assert str(dims_disagree.value) == \
+ "array dimensions must agree except for axis being concatenated"
def test_std(self):
from _numpypy import array
@@ -992,6 +1114,28 @@
a = array([5.0])
assert a.std() == 0.0
+ def test_flatten(self):
+ from _numpypy import array
+
+ assert array(3).flatten().shape == (1,)
+ a = array([[1, 2], [3, 4]])
+ b = a.flatten()
+ c = a.ravel()
+ a[0, 0] = 15
+ assert b[0] == 1
+ assert c[0] == 15
+ a = array([[1, 2, 3], [4, 5, 6]])
+ assert (a.flatten() == [1, 2, 3, 4, 5, 6]).all()
+ a = array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])
+ assert (a.flatten() == [1, 2, 3, 4, 5, 6, 7, 8]).all()
+ a = array([1, 2, 3, 4, 5, 6, 7, 8])
+ assert (a[::2].flatten() == [1, 3, 5, 7]).all()
+ a = array([1, 2, 3])
+ assert ((a + a).flatten() == [2, 4, 6]).all()
+ a = array(2)
+ assert (a.flatten() == [2]).all()
+ a = array([[1, 2], [3, 4]])
+ assert (a.T.flatten() == [1, 3, 2, 4]).all()
class AppTestMultiDim(BaseNumpyAppTest):
def test_init(self):
@@ -1007,7 +1151,7 @@
assert _numpypy.array([[1], [2], [3]]).shape == (3, 1)
assert len(_numpypy.zeros((3, 1, 2))) == 3
raises(TypeError, len, _numpypy.zeros(()))
- raises(ValueError, _numpypy.array, [[1, 2], 3])
+ raises(ValueError, _numpypy.array, [[1, 2], 3], dtype=float)
def test_getsetitem(self):
import _numpypy
@@ -1222,7 +1366,7 @@
assert(b[:, 0] == a[0, :]).all()
def test_flatiter(self):
- from _numpypy import array, flatiter
+ from _numpypy import array, flatiter, arange
a = array([[10, 30], [40, 60]])
f_iter = a.flat
assert f_iter.next() == 10
@@ -1235,6 +1379,9 @@
for k in a.flat:
s += k
assert s == 140
+ a = arange(10).reshape(5, 2)
+ raises(IndexError, 'a.flat[(1, 2)]')
+ assert a.flat.base is a
def test_flatiter_array_conv(self):
from _numpypy import array, dot
@@ -1246,6 +1393,75 @@
a = ones((2, 2))
assert list(((a + a).flat)) == [2, 2, 2, 2]
+ def test_flatiter_getitem(self):
+ from _numpypy import arange
+ a = arange(10)
+ assert a.flat[3] == 3
+ assert a[2:].flat[3] == 5
+ assert (a + a).flat[3] == 6
+ assert a[::2].flat[3] == 6
+ assert a.reshape(2,5).flat[3] == 3
+ b = a.reshape(2,5).flat
+ b.next()
+ b.next()
+ b.next()
+ assert b[3] == 3
+ assert (b[::3] == [0, 3, 6, 9]).all()
+ assert (b[2::5] == [2, 7]).all()
+ assert b[-2] == 8
+ raises(IndexError, "b[11]")
+ raises(IndexError, "b[-11]")
+ raises(IndexError, 'b[0, 1]')
+ assert b.index == 3
+ assert b.coords == (0,3)
+
+ def test_flatiter_setitem(self):
+ from _numpypy import arange, array
+ a = arange(12).reshape(3,4)
+ b = a.T.flat
+ b[6::2] = [-1, -2]
+ assert (a == [[0, 1, -1, 3], [4, 5, 6, -1], [8, 9, -2, 11]]).all()
+ b[0:2] = [[[100]]]
+ assert(a[0,0] == 100)
+ assert(a[1,0] == 100)
+ raises(IndexError, 'b[array([10, 11])] == [-20, -40]')
+
+ def test_flatiter_ops(self):
+ from _numpypy import arange, array
+ a = arange(12).reshape(3,4)
+ b = a.T.flat
+ assert (b == [0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11]).all()
+ assert not (b != [0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11]).any()
+ assert ((b >= range(12)) == [True, True, True,False, True, True,
+ False, False, True, False, False, True]).all()
+ assert ((b < range(12)) != [True, True, True,False, True, True,
+ False, False, True, False, False, True]).all()
+ assert ((b <= range(12)) != [False, True, True,False, True, True,
+ False, False, True, False, False, False]).all()
+ assert ((b > range(12)) == [False, True, True,False, True, True,
+ False, False, True, False, False, False]).all()
+ def test_flatiter_view(self):
+ from _numpypy import arange
+ a = arange(10).reshape(5, 2)
+ #no == yet.
+ # a[::2].flat == [0, 1, 4, 5, 8, 9]
+ isequal = True
+ for y,z in zip(a[::2].flat, [0, 1, 4, 5, 8, 9]):
+ if y != z:
+ isequal = False
+ assert isequal == True
+
+ def test_flatiter_transpose(self):
+ from _numpypy import arange
+ a = arange(10).reshape(2,5).T
+ b = a.flat
+ assert (b[:5] == [0, 5, 1, 6, 2]).all()
+ b.next()
+ b.next()
+ b.next()
+ assert b.index == 3
+ assert b.coords == (1,1)
+
def test_slice_copy(self):
from _numpypy import zeros
a = zeros((10, 10))
@@ -1262,6 +1478,110 @@
assert isinstance(i['data'][0], int)
raises(TypeError, getattr, array(3), '__array_interface__')
+ def test_array_indexing_one_elem(self):
+ skip("not yet")
+ from _numpypy import array, arange
+ raises(IndexError, 'arange(3)[array([3.5])]')
+ a = arange(3)[array([1])]
+ assert a == 1
+ assert a[0] == 1
+ raises(IndexError,'arange(3)[array([15])]')
+ assert arange(3)[array([-3])] == 0
+ raises(IndexError,'arange(3)[array([-15])]')
+ assert arange(3)[array(1)] == 1
+
+ def test_fill(self):
+ from _numpypy import array
+ a = array([1, 2, 3])
+ a.fill(10)
+ assert (a == [10, 10, 10]).all()
+ a.fill(False)
+ assert (a == [0, 0, 0]).all()
+ b = a[:1]
+ b.fill(4)
+ assert (b == [4]).all()
+ assert (a == [4, 0, 0]).all()
+
+ c = b + b
+ c.fill(27)
+ assert (c == [27]).all()
+
+ d = array(10)
+ d.fill(100)
+ assert d == 100
+
+ def test_array_indexing_bool(self):
+ from _numpypy import arange
+ a = arange(10)
+ assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all()
+ a = arange(10).reshape(5, 2)
+ assert (a[a > 3] == [4, 5, 6, 7, 8, 9]).all()
+ assert (a[a & 1 == 1] == [1, 3, 5, 7, 9]).all()
+
+ def test_array_indexing_bool_setitem(self):
+ from _numpypy import arange, array
+ a = arange(6)
+ a[a > 3] = 15
+ assert (a == [0, 1, 2, 3, 15, 15]).all()
+ a = arange(6).reshape(3, 2)
+ a[a & 1 == 1] = array([8, 9, 10])
+ assert (a == [[0, 8], [2, 9], [4, 10]]).all()
+
+ def test_copy_kwarg(self):
+ from _numpypy import array
+ x = array([1, 2, 3])
+ assert (array(x) == x).all()
+ assert array(x) is not x
+ assert array(x, copy=False) is x
+ assert array(x, copy=True) is not x
+
+ def test_isna(self):
+ from _numpypy import isna, array
+ # XXX for now
+ assert not isna(3)
+ assert (isna(array([1, 2, 3, 4])) == [False, False, False, False]).all()
+
+ def test_ravel(self):
+ from _numpypy import arange
+ assert (arange(3).ravel() == arange(3)).all()
+ assert (arange(6).reshape(2, 3).ravel() == arange(6)).all()
+ assert (arange(6).reshape(2, 3).T.ravel() == [0, 3, 1, 4, 2, 5]).all()
+
+ def test_take(self):
+ from _numpypy import arange
+ assert (arange(10).take([1, 2, 1, 1]) == [1, 2, 1, 1]).all()
+ raises(IndexError, "arange(3).take([15])")
+ a = arange(6).reshape(2, 3)
+ assert (a.take([1, 0, 3]) == [1, 0, 3]).all()
+ assert ((a + a).take([3]) == [6]).all()
+ a = arange(12).reshape(2, 6)
+ assert (a[:,::2].take([3, 2, 1]) == [6, 4, 2]).all()
+
+ def test_compress(self):
+ from _numpypy import arange
+ a = arange(10)
+ assert (a.compress([True, False, True]) == [0, 2]).all()
+ assert (a.compress([1, 0, 13]) == [0, 2]).all()
+ assert (a.compress([1, 0, 13.5]) == [0, 2]).all()
+ a = arange(10).reshape(2, 5)
+ assert (a.compress([True, False, True]) == [0, 2]).all()
+ raises(IndexError, "a.compress([1] * 100)")
+
+ def test_item(self):
+ from _numpypy import array
+ assert array(3).item() == 3
+ assert type(array(3).item()) is int
+ assert type(array(True).item()) is bool
+ assert type(array(3.5).item()) is float
+ raises((ValueError, IndexError), "array(3).item(15)")
+ raises(ValueError, "array([1, 2, 3]).item()")
+ assert array([3]).item(0) == 3
+ assert type(array([3]).item(0)) is int
+ assert array([1, 2, 3]).item(-1) == 3
+ a = array([1, 2, 3])
+ assert a[::2].item(1) == 3
+ assert (a + a).item(1) == 4
+ raises(ValueError, "array(5).item(1)")
class AppTestSupport(BaseNumpyAppTest):
def setup_class(cls):
@@ -1373,123 +1693,6 @@
raises(ValueError, fromstring, "\x01\x02\x03", count=5, dtype=uint8)
-class AppTestRepr(BaseNumpyAppTest):
- def test_repr(self):
- from _numpypy import array, zeros
- int_size = array(5).dtype.itemsize
- a = array(range(5), float)
- assert repr(a) == "array([0.0, 1.0, 2.0, 3.0, 4.0])"
- a = array([], float)
- assert repr(a) == "array([], dtype=float64)"
- a = zeros(1001)
- assert repr(a) == "array([0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0])"
- a = array(range(5), long)
- if a.dtype.itemsize == int_size:
- assert repr(a) == "array([0, 1, 2, 3, 4])"
- else:
- assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int64)"
- a = array(range(5), 'int32')
- if a.dtype.itemsize == int_size:
- assert repr(a) == "array([0, 1, 2, 3, 4])"
- else:
- assert repr(a) == "array([0, 1, 2, 3, 4], dtype=int32)"
- a = array([], long)
- assert repr(a) == "array([], dtype=int64)"
- a = array([True, False, True, False], "?")
- assert repr(a) == "array([True, False, True, False], dtype=bool)"
- a = zeros([])
- assert repr(a) == "array(0.0)"
- a = array(0.2)
- assert repr(a) == "array(0.2)"
-
- def test_repr_multi(self):
- from _numpypy import arange, zeros
- a = zeros((3, 4))
- assert repr(a) == '''array([[0.0, 0.0, 0.0, 0.0],
- [0.0, 0.0, 0.0, 0.0],
- [0.0, 0.0, 0.0, 0.0]])'''
- a = zeros((2, 3, 4))
- assert repr(a) == '''array([[[0.0, 0.0, 0.0, 0.0],
- [0.0, 0.0, 0.0, 0.0],
- [0.0, 0.0, 0.0, 0.0]],
-
- [[0.0, 0.0, 0.0, 0.0],
- [0.0, 0.0, 0.0, 0.0],
- [0.0, 0.0, 0.0, 0.0]]])'''
- a = arange(1002).reshape((2, 501))
- assert repr(a) == '''array([[0, 1, 2, ..., 498, 499, 500],
- [501, 502, 503, ..., 999, 1000, 1001]])'''
- assert repr(a.T) == '''array([[0, 501],
- [1, 502],
- [2, 503],
- ...,
- [498, 999],
- [499, 1000],
- [500, 1001]])'''
-
- def test_repr_slice(self):
- from _numpypy import array, zeros
- a = array(range(5), float)
- b = a[1::2]
- assert repr(b) == "array([1.0, 3.0])"
- a = zeros(2002)
- b = a[::2]
- assert repr(b) == "array([0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0])"
- a = array((range(5), range(5, 10)), dtype="int16")
- b = a[1, 2:]
- assert repr(b) == "array([7, 8, 9], dtype=int16)"
- # an empty slice prints its shape
- b = a[2:1, ]
- assert repr(b) == "array([], shape=(0, 5), dtype=int16)"
-
- def test_str(self):
- from _numpypy import array, zeros
- a = array(range(5), float)
- assert str(a) == "[0.0 1.0 2.0 3.0 4.0]"
- assert str((2 * a)[:]) == "[0.0 2.0 4.0 6.0 8.0]"
- a = zeros(1001)
- assert str(a) == "[0.0 0.0 0.0 ..., 0.0 0.0 0.0]"
-
- a = array(range(5), dtype=long)
- assert str(a) == "[0 1 2 3 4]"
- a = array([True, False, True, False], dtype="?")
- assert str(a) == "[True False True False]"
-
- a = array(range(5), dtype="int8")
- assert str(a) == "[0 1 2 3 4]"
-
- a = array(range(5), dtype="int16")
- assert str(a) == "[0 1 2 3 4]"
-
- a = array((range(5), range(5, 10)), dtype="int16")
- assert str(a) == "[[0 1 2 3 4]\n [5 6 7 8 9]]"
-
- a = array(3, dtype=int)
- assert str(a) == "3"
-
- a = zeros((400, 400), dtype=int)
- assert str(a) == "[[0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]\n" \
- " [0 0 0 ..., 0 0 0]\n ...,\n [0 0 0 ..., 0 0 0]\n" \
- " [0 0 0 ..., 0 0 0]\n [0 0 0 ..., 0 0 0]]"
- a = zeros((2, 2, 2))
- r = str(a)
- assert r == '[[[0.0 0.0]\n [0.0 0.0]]\n\n [[0.0 0.0]\n [0.0 0.0]]]'
-
- def test_str_slice(self):
- from _numpypy import array, zeros
- a = array(range(5), float)
- b = a[1::2]
- assert str(b) == "[1.0 3.0]"
- a = zeros(2002)
- b = a[::2]
- assert str(b) == "[0.0 0.0 0.0 ..., 0.0 0.0 0.0]"
- a = array((range(5), range(5, 10)), dtype="int16")
- b = a[1, 2:]
- assert str(b) == "[7 8 9]"
- b = a[2:1, ]
- assert str(b) == "[]"
-
-
class AppTestRanges(BaseNumpyAppTest):
def test_arange(self):
from _numpypy import arange, array, dtype
@@ -1511,13 +1714,23 @@
assert len(a) == 8
assert arange(False, True, True).dtype is dtype(int)
+from pypy.module.micronumpy.appbridge import get_appbridge_cache
-class AppTestRanges(BaseNumpyAppTest):
- def test_app_reshape(self):
- from _numpypy import arange, array, dtype, reshape
- a = arange(12)
- b = reshape(a, (3, 4))
- assert b.shape == (3, 4)
- a = range(12)
- b = reshape(a, (3, 4))
- assert b.shape == (3, 4)
+class AppTestRepr(BaseNumpyAppTest):
+ def setup_class(cls):
+ BaseNumpyAppTest.setup_class.im_func(cls)
+ cache = get_appbridge_cache(cls.space)
+ cls.old_array_repr = cache.w_array_repr
+ cls.old_array_str = cache.w_array_str
+ cache.w_array_str = None
+ cache.w_array_repr = None
+
+ def test_repr_str(self):
+ from _numpypy import array
+ assert repr(array([1, 2, 3])) == 'array([1, 2, 3])'
+ assert str(array([1, 2, 3])) == 'array([1, 2, 3])'
+
+ def teardown_class(cls):
+ cache = get_appbridge_cache(cls.space)
+ cache.w_array_repr = cls.old_array_repr
+ cache.w_array_str = cls.old_array_str
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -190,14 +190,24 @@
for i in range(3):
assert c[i] == a[i] - b[i]
- def test_floor(self):
- from _numpypy import array, floor
-
- reference = [-2.0, -1.0, 0.0, 1.0, 1.0]
- a = array([-1.4, -1.0, 0.0, 1.0, 1.4])
+ def test_floorceil(self):
+ from _numpypy import array, floor, ceil
+ import math
+ reference = [-2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0]
+ a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
b = floor(a)
for i in range(5):
assert b[i] == reference[i]
+ reference = [-1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0]
+ a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
+ b = ceil(a)
+ assert (reference == b).all()
+ inf = float("inf")
+ data = [1.5, 2.9999, -1.999, inf]
+ results = [math.floor(x) for x in data]
+ assert (floor(data) == results).all()
+ results = [math.ceil(x) for x in data]
+ assert (ceil(data) == results).all()
def test_copysign(self):
from _numpypy import array, copysign
@@ -238,7 +248,7 @@
assert b[i] == math.sin(a[i])
a = sin(array([True, False], dtype=bool))
- assert abs(a[0] - sin(1)) < 1e-7 # a[0] will be less precise
+ assert abs(a[0] - sin(1)) < 1e-7 # a[0] will be less precise
assert a[1] == 0.0
def test_cos(self):
@@ -259,7 +269,6 @@
for i in range(len(a)):
assert b[i] == math.tan(a[i])
-
def test_arcsin(self):
import math
from _numpypy import array, arcsin
@@ -283,7 +292,6 @@
for i in range(len(a)):
assert b[i] == math.acos(a[i])
-
a = array([-10, -1.5, -1.01, 1.01, 1.5, 10, float('nan'), float('inf'), float('-inf')])
b = arccos(a)
for f in b:
@@ -298,7 +306,7 @@
for i in range(len(a)):
assert b[i] == math.atan(a[i])
- a = array([float('nan')])
+ a = array([float('nan')])
b = arctan(a)
assert math.isnan(b[0])
@@ -336,9 +344,9 @@
from _numpypy import sin, add
raises(ValueError, sin.reduce, [1, 2, 3])
- raises(TypeError, add.reduce, 1)
+ raises((ValueError, TypeError), add.reduce, 1)
- def test_reduce(self):
+ def test_reduce_1d(self):
from _numpypy import add, maximum
assert add.reduce([1, 2, 3]) == 6
@@ -346,6 +354,35 @@
assert maximum.reduce([1, 2, 3]) == 3
raises(ValueError, maximum.reduce, [])
+ def test_reduceND(self):
+ from _numpypy import add, arange
+ a = arange(12).reshape(3, 4)
+ assert (add.reduce(a, 0) == [12, 15, 18, 21]).all()
+ assert (add.reduce(a, 1) == [6.0, 22.0, 38.0]).all()
+
+ def test_reduce_keepdims(self):
+ from _numpypy import add, arange
+ a = arange(12).reshape(3, 4)
+ b = add.reduce(a, 0, keepdims=True)
+ assert b.shape == (1, 4)
+ assert (add.reduce(a, 0, keepdims=True) == [12, 15, 18, 21]).all()
+
+
+ def test_bitwise(self):
+ from _numpypy import bitwise_and, bitwise_or, arange, array
+ a = arange(6).reshape(2, 3)
+ assert (a & 1 == [[0, 1, 0], [1, 0, 1]]).all()
+ assert (a & 1 == bitwise_and(a, 1)).all()
+ assert (a | 1 == [[1, 1, 3], [3, 5, 5]]).all()
+ assert (a | 1 == bitwise_or(a, 1)).all()
+ raises(TypeError, 'array([1.0]) & 1')
+
+ def test_unary_bitops(self):
+ from _numpypy import bitwise_not, array
+ a = array([1, 2, 3, 4])
+ assert (~a == [-2, -3, -4, -5]).all()
+ assert (bitwise_not(a) == ~a).all()
+
def test_comparisons(self):
import operator
from _numpypy import equal, not_equal, less, less_equal, greater, greater_equal
@@ -371,3 +408,28 @@
(3, 3.5),
]:
assert ufunc(a, b) == func(a, b)
+
+ def test_count_reduce_items(self):
+ from _numpypy import count_reduce_items, arange
+ a = arange(24).reshape(2, 3, 4)
+ assert count_reduce_items(a) == 24
+ assert count_reduce_items(a, 1) == 3
+ assert count_reduce_items(a, (1, 2)) == 3 * 4
+
+ def test_true_divide(self):
+ from _numpypy import arange, array, true_divide
+ assert (true_divide(arange(3), array([2, 2, 2])) == array([0, 0.5, 1])).all()
+
+ def test_isnan_isinf(self):
+ from _numpypy import isnan, isinf, float64, array
+ assert isnan(float('nan'))
+ assert isnan(float64(float('nan')))
+ assert not isnan(3)
+ assert isinf(float('inf'))
+ assert not isnan(3.5)
+ assert not isinf(3.5)
+ assert not isnan(float('inf'))
+ assert not isinf(float('nan'))
+ assert (isnan(array([0.2, float('inf'), float('nan')])) == [False, False, True]).all()
+ assert (isinf(array([0.2, float('inf'), float('nan')])) == [False, True, False]).all()
+ assert isinf(array([0.2])).dtype.kind == 'b'
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -12,7 +12,7 @@
from pypy.module.micronumpy.compile import (FakeSpace,
IntObject, Parser, InterpreterState)
from pypy.module.micronumpy.interp_numarray import (W_NDimArray,
- BaseArray)
+ BaseArray, W_FlatIterator)
from pypy.rlib.nonconst import NonConstant
@@ -47,6 +47,8 @@
def f(i):
interp = InterpreterState(codes[i])
interp.run(space)
+ if not len(interp.results):
+ raise Exception("need results")
w_res = interp.results[-1]
if isinstance(w_res, BaseArray):
concr = w_res.get_concrete_or_scalar()
@@ -115,6 +117,28 @@
"int_add": 1, "int_ge": 1, "guard_false": 1,
"jump": 1, 'arraylen_gc': 1})
+ def define_axissum():
+ return """
+ a = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]]
+ b = sum(a,0)
+ b -> 1
+ """
+
+ def test_axissum(self):
+ result = self.run("axissum")
+ assert result == 30
+ # XXX note - the bridge here is fairly crucial and yet it's pretty
+ # bogus. We need to improve the situation somehow.
+ self.check_simple_loop({'getinteriorfield_raw': 2,
+ 'setinteriorfield_raw': 1,
+ 'arraylen_gc': 1,
+ 'guard_true': 1,
+ 'int_lt': 1,
+ 'jump': 1,
+ 'float_add': 1,
+ 'int_add': 3,
+ })
+
def define_prod():
return """
a = |30|
@@ -193,9 +217,10 @@
# This is the sum of the ops for both loops, however if you remove the
# optimization then you end up with 2 float_adds, so we can still be
# sure it was optimized correctly.
- self.check_resops({'setinteriorfield_raw': 4, 'getfield_gc': 26,
+ py.test.skip("too fragile")
+ self.check_resops({'setinteriorfield_raw': 4, 'getfield_gc': 22,
'getarrayitem_gc': 4, 'getarrayitem_gc_pure': 2,
- 'getfield_gc_pure': 4,
+ 'getfield_gc_pure': 8,
'guard_class': 8, 'int_add': 8, 'float_mul': 2,
'jump': 2, 'int_ge': 4,
'getinteriorfield_raw': 4, 'float_add': 2,
@@ -212,7 +237,8 @@
def test_ufunc(self):
result = self.run("ufunc")
assert result == -6
- self.check_simple_loop({"getinteriorfield_raw": 2, "float_add": 1, "float_neg": 1,
+ self.check_simple_loop({"getinteriorfield_raw": 2, "float_add": 1,
+ "float_neg": 1,
"setinteriorfield_raw": 1, "int_add": 2,
"int_ge": 1, "guard_false": 1, "jump": 1,
'arraylen_gc': 1})
@@ -261,6 +287,27 @@
'jump': 1,
'arraylen_gc': 1})
+ def define_take():
+ return """
+ a = |10|
+ b = take(a, [1, 1, 3, 2])
+ b -> 2
+ """
+
+ def test_take(self):
+ result = self.run("take")
+ assert result == 3
+ self.check_simple_loop({'getinteriorfield_raw': 2,
+ 'cast_float_to_int': 1,
+ 'int_lt': 1,
+ 'int_ge': 2,
+ 'guard_false': 3,
+ 'setinteriorfield_raw': 1,
+ 'int_mul': 1,
+ 'int_add': 3,
+ 'jump': 1,
+ 'arraylen_gc': 2})
+
def define_multidim():
return """
a = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]]
@@ -322,10 +369,10 @@
result = self.run("setslice")
assert result == 11.0
self.check_trace_count(1)
- self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add' : 1,
- 'setinteriorfield_raw': 1, 'int_add': 3,
- 'int_lt': 1, 'guard_true': 1, 'jump': 1,
- 'arraylen_gc': 3})
+ self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1,
+ 'setinteriorfield_raw': 1, 'int_add': 2,
+ 'int_eq': 1, 'guard_false': 1, 'jump': 1,
+ 'arraylen_gc': 1})
def define_virtual_slice():
return """
@@ -339,10 +386,75 @@
result = self.run("virtual_slice")
assert result == 4
self.check_trace_count(1)
- self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add' : 1,
+ self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1,
'setinteriorfield_raw': 1, 'int_add': 2,
'int_ge': 1, 'guard_false': 1, 'jump': 1,
'arraylen_gc': 1})
+ def define_flat_iter():
+ return '''
+ a = |30|
+ b = flat(a)
+ c = b + a
+ c -> 3
+ '''
+
+ def test_flat_iter(self):
+ result = self.run("flat_iter")
+ assert result == 6
+ self.check_trace_count(1)
+ self.check_simple_loop({'getinteriorfield_raw': 2, 'float_add': 1,
+ 'setinteriorfield_raw': 1, 'int_add': 3,
+ 'int_ge': 1, 'guard_false': 1,
+ 'arraylen_gc': 1, 'jump': 1})
+
+ def define_flat_getitem():
+ return '''
+ a = |30|
+ b = flat(a)
+ b -> 4: -> 6
+ '''
+
+ def test_flat_getitem(self):
+ result = self.run("flat_getitem")
+ assert result == 10.0
+ self.check_trace_count(1)
+ self.check_simple_loop({'getinteriorfield_raw': 1,
+ 'setinteriorfield_raw': 1,
+ 'int_lt': 1,
+ 'int_ge': 1,
+ 'int_add': 3,
+ 'guard_true': 1,
+ 'guard_false': 1,
+ 'arraylen_gc': 2,
+ 'jump': 1})
+
+ def define_flat_setitem():
+ return '''
+ a = |30|
+ b = flat(a)
+ b[4:] = a->:26
+ a -> 5
+ '''
+
+ def test_flat_setitem(self):
+ result = self.run("flat_setitem")
+ assert result == 1.0
+ self.check_trace_count(1)
+ # XXX not ideal, but hey, let's ignore it for now
+ self.check_simple_loop({'getinteriorfield_raw': 1,
+ 'setinteriorfield_raw': 1,
+ 'int_lt': 1,
+ 'int_gt': 1,
+ 'int_add': 4,
+ 'guard_true': 2,
+ 'arraylen_gc': 2,
+ 'jump': 1,
+ 'int_sub': 1,
+ # XXX bad part
+ 'int_and': 1,
+ 'int_mod': 1,
+ 'int_rshift': 1,
+ })
class TestNumpyOld(LLJitMixin):
def setup_class(cls):
@@ -377,4 +489,3 @@
result = self.meta_interp(f, [5], listops=True, backendopt=True)
assert result == f(5)
-
diff --git a/pypy/module/micronumpy/tool/numready/__init__.py b/pypy/module/micronumpy/tool/numready/__init__.py
new file mode 100644
diff --git a/pypy/module/micronumpy/tool/numready/__main__.py b/pypy/module/micronumpy/tool/numready/__main__.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/tool/numready/__main__.py
@@ -0,0 +1,6 @@
+import sys
+
+from .main import main
+
+
+main(sys.argv)
diff --git a/pypy/module/micronumpy/tool/numready/kinds.py b/pypy/module/micronumpy/tool/numready/kinds.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/tool/numready/kinds.py
@@ -0,0 +1,4 @@
+KINDS = {
+ "UNKNOWN": "U",
+ "TYPE": "T",
+}
diff --git a/pypy/module/micronumpy/tool/numready/main.py b/pypy/module/micronumpy/tool/numready/main.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/tool/numready/main.py
@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+This should be run under PyPy.
+"""
+
+import os
+import platform
+import subprocess
+import tempfile
+import webbrowser
+from collections import OrderedDict
+
+import jinja2
+
+from .kinds import KINDS
+
+
+class SearchableSet(object):
+ def __init__(self, items=()):
+ self._items = {}
+ for item in items:
+ self.add(item)
+
+ def __iter__(self):
+ return iter(self._items)
+
+ def __contains__(self, other):
+ return other in self._items
+
+ def __getitem__(self, idx):
+ return self._items[idx]
+
+ def add(self, item):
+ self._items[item] = item
+
+ def __len__(self):
+ return len(self._items)
+
+class Item(object):
+ def __init__(self, name, kind, subitems=None):
+ self.name = name
+ self.kind = kind
+ self.subitems = subitems
+
+ def __hash__(self):
+ return hash(self.name)
+
+ def __eq__(self, other):
+ if isinstance(other, str):
+ return self.name == other
+ return self.name == other.name
+
+
+class ItemStatus(object):
+ def __init__(self, name, pypy_exists):
+ self.name = name
+ self.cls = 'exists' if pypy_exists else ''
+ self.symbol = u"✔" if pypy_exists else u'✖'
+
+ def __lt__(self, other):
+ return self.name < other.name
+
+def find_numpy_items(python, modname="numpy", attr=None):
+ args = [
+ python, os.path.join(os.path.dirname(__file__), "search.py"), modname
+ ]
+ if attr is not None:
+ args.append(attr)
+ lines = subprocess.check_output(args).splitlines()
+ items = SearchableSet()
+ for line in lines:
+ kind, name = line.split(" : ", 1)
+ subitems = None
+ if kind == KINDS["TYPE"] and name in SPECIAL_NAMES and attr is None:
+ subitems = find_numpy_items(python, modname, name)
+ items.add(Item(name, kind, subitems))
+ return items
+
+def split(lst):
+ SPLIT = 5
+ lgt = len(lst) // SPLIT + 1
+ l = [[] for i in range(lgt)]
+ for i in range(lgt):
+ for k in range(SPLIT):
+ if k * lgt + i < len(lst):
+ l[i].append(lst[k * lgt + i])
+ return l
+
+SPECIAL_NAMES = ["ndarray", "dtype", "generic"]
+
+def main(argv):
+ cpy_items = find_numpy_items("/usr/bin/python")
+ pypy_items = find_numpy_items(argv[1], "numpypy")
+ all_items = []
+
+ msg = "{:d}/{:d} names".format(len(pypy_items), len(cpy_items)) + " "
+ msg += ", ".join(
+ "{:d}/{:d} {} attributes".format(
+ len(pypy_items[name].subitems), len(cpy_items[name].subitems), name
+ )
+ for name in SPECIAL_NAMES
+ )
+ for item in cpy_items:
+ pypy_exists = item in pypy_items
+ if item.subitems:
+ for sub in item.subitems:
+ all_items.append(
+ ItemStatus(item.name + "." + sub.name, pypy_exists=pypy_exists and pypy_items[item].subitems and sub in pypy_items[item].subitems)
+ )
+ all_items.append(ItemStatus(item.name, pypy_exists=item in pypy_items))
+ env = jinja2.Environment(
+ loader=jinja2.FileSystemLoader(os.path.dirname(__file__))
+ )
+ html = env.get_template("page.html").render(all_items=split(sorted(all_items)), msg=msg)
+ if len(argv) > 2:
+ with open(argv[2], 'w') as f:
+ f.write(html.encode("utf-8"))
+ else:
+ with tempfile.NamedTemporaryFile(delete=False, suffix=".html") as f:
+ f.write(html.encode("utf-8"))
+ print "Saved in: %s" % f.name
diff --git a/pypy/module/micronumpy/tool/numready/page.html b/pypy/module/micronumpy/tool/numready/page.html
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/tool/numready/page.html
@@ -0,0 +1,65 @@
+<!DOCTYPE html>
+<html lang="en">
+ <head>
+ <title>NumPyPy Status</title>
+ <meta http-equiv="content-type" content="text/html; charset=utf-8">
+ <style type="text/css">
+ body {
+ font-family: 'Consolas', 'Bitstream Vera Sans Mono', monospace;
+ }
+ h1 {
+ text-align: center;
+ }
+ h3 {
+ text-align: center;
+ }
+ table {
+ border: 8px solid #DFDECB;
+ margin: 30px auto;
+ font-size: 12px;
+ }
+ table th {
+ text-align: left;
+ }
+ table td {
+ padding: 4px 10px;
+ text-align: center;
+ }
+ .exists {
+ background-color: #337792;
+ color: white;
+ border: 1px solid #234F61;
+ }
+ </style>
+ </head>
+ <body>
+ <h1>NumPyPy Status</h1>
+ <h3>Overall: {{ msg }}</h3>
+ <table>
+ <thead>
+ <tr>
+ <th></th>
+ <th>PyPy</th>
+ <th></th>
+ <th>PyPy</th>
+ <th></th>
+ <th>PyPy</th>
+ <th></th>
+ <th>PyPy</th>
+ <th></th>
+ <th>PyPy</th>
+ </tr>
+ </thead>
+ <tbody>
+ {% for chunk in all_items %}
+ <tr>
+ {% for item in chunk %}
+ <th class='{{ item.cls }}'>{{ item.name }}</th>
+ <td class='{{ item.cls }}'>{{ item.symbol }}</td>
+ {% endfor %}
+ </tr>
+ {% endfor %}
+ </tbody>
+ </table>
+ </body>
+</html>
diff --git a/pypy/module/micronumpy/tool/numready/search.py b/pypy/module/micronumpy/tool/numready/search.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/tool/numready/search.py
@@ -0,0 +1,33 @@
+import sys
+import types
+
+# Evil implicit relative import.
+from kinds import KINDS
+
+
+def main(argv):
+ if len(argv) == 2:
+ [_, modname] = argv
+ attr = None
+ elif len(argv) == 3:
+ [_, modname, attr] = argv
+ else:
+ sys.exit("Wrong number of args")
+ __import__(modname)
+ obj = sys.modules[modname]
+
+ if attr is not None:
+ obj = getattr(obj, attr)
+
+ for name in dir(obj):
+ if attr is None and name.startswith("_"):
+ continue
+ subobj = getattr(obj, name)
+ if isinstance(subobj, types.TypeType):
+ kind = KINDS["TYPE"]
+ else:
+ kind = KINDS["UNKNOWN"]
+ print kind, ":", name
+
+if __name__ == "__main__":
+ main(sys.argv)
\ No newline at end of file
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -23,6 +23,16 @@
)
return dispatcher
+def raw_unary_op(func):
+ specialize.argtype(1)(func)
+ @functools.wraps(func)
+ def dispatcher(self, v):
+ return func(
+ self,
+ self.for_computation(self.unbox(v))
+ )
+ return dispatcher
+
def simple_binary_op(func):
specialize.argtype(1, 2)(func)
@functools.wraps(func)
@@ -94,6 +104,11 @@
width, storage, i, offset
))
+ def read_bool(self, storage, width, i, offset):
+ return bool(self.for_computation(
+ libffi.array_getitem(clibffi.cast_type_to_ffitype(self.T),
+ width, storage, i, offset)))
+
def store(self, storage, width, i, offset, box):
value = self.unbox(box)
libffi.array_setitem(clibffi.cast_type_to_ffitype(self.T),
@@ -134,6 +149,14 @@
def abs(self, v):
return abs(v)
+ @raw_unary_op
+ def isnan(self, v):
+ return False
+
+ @raw_unary_op
+ def isinf(self, v):
+ return False
+
@raw_binary_op
def eq(self, v1, v2):
return v1 == v2
@@ -169,6 +192,7 @@
def min(self, v1, v2):
return min(v1, v2)
+
class Bool(BaseType, Primitive):
T = lltype.Bool
BoxType = interp_boxes.W_BoolBox
@@ -205,6 +229,18 @@
def default_fromstring(self, space):
return self.box(False)
+ @simple_binary_op
+ def bitwise_and(self, v1, v2):
+ return v1 & v2
+
+ @simple_binary_op
+ def bitwise_or(self, v1, v2):
+ return v1 | v2
+
+ @simple_unary_op
+ def invert(self, v):
+ return ~v
+
class Integer(Primitive):
_mixin_ = True
@@ -253,6 +289,18 @@
assert v == 0
return 0
+ @simple_binary_op
+ def bitwise_and(self, v1, v2):
+ return v1 & v2
+
+ @simple_binary_op
+ def bitwise_or(self, v1, v2):
+ return v1 | v2
+
+ @simple_unary_op
+ def invert(self, v):
+ return ~v
+
class Int8(BaseType, Integer):
T = rffi.SIGNEDCHAR
BoxType = interp_boxes.W_Int8Box
@@ -374,6 +422,10 @@
return math.floor(v)
@simple_unary_op
+ def ceil(self, v):
+ return math.ceil(v)
+
+ @simple_unary_op
def exp(self, v):
try:
return math.exp(v)
@@ -427,6 +479,14 @@
except ValueError:
return rfloat.NAN
+ @raw_unary_op
+ def isnan(self, v):
+ return rfloat.isnan(v)
+
+ @raw_unary_op
+ def isinf(self, v):
+ return rfloat.isinf(v)
+
class Float32(BaseType, Float):
T = rffi.FLOAT
@@ -436,4 +496,4 @@
class Float64(BaseType, Float):
T = rffi.DOUBLE
BoxType = interp_boxes.W_Float64Box
- format_code = "d"
\ No newline at end of file
+ format_code = "d"
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -371,6 +371,8 @@
if hasattr(__import__(os.name), "forkpty"):
def test_forkpty(self):
import sys
+ if 'freebsd' in sys.platform:
+ skip("hangs indifinitly on FreeBSD (also on CPython).")
os = self.posix
childpid, master_fd = os.forkpty()
assert isinstance(childpid, int)
diff --git a/pypy/module/pypyjit/__init__.py b/pypy/module/pypyjit/__init__.py
--- a/pypy/module/pypyjit/__init__.py
+++ b/pypy/module/pypyjit/__init__.py
@@ -7,16 +7,22 @@
interpleveldefs = {
'set_param': 'interp_jit.set_param',
'residual_call': 'interp_jit.residual_call',
- 'set_compile_hook': 'interp_jit.set_compile_hook',
- 'DebugMergePoint': 'interp_resop.W_DebugMergePoint',
+ 'set_compile_hook': 'interp_resop.set_compile_hook',
+ 'set_optimize_hook': 'interp_resop.set_optimize_hook',
+ 'set_abort_hook': 'interp_resop.set_abort_hook',
+ 'ResOperation': 'interp_resop.WrappedOp',
+ 'DebugMergePoint': 'interp_resop.DebugMergePoint',
+ 'Box': 'interp_resop.WrappedBox',
}
def setup_after_space_initialization(self):
# force the __extend__ hacks to occur early
from pypy.module.pypyjit.interp_jit import pypyjitdriver
+ from pypy.module.pypyjit.policy import pypy_hooks
# add the 'defaults' attribute
from pypy.rlib.jit import PARAMETERS
space = self.space
pypyjitdriver.space = space
w_obj = space.wrap(PARAMETERS)
space.setattr(space.wrap(self), space.wrap('defaults'), w_obj)
+ pypy_hooks.space = space
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -13,11 +13,7 @@
from pypy.interpreter.pycode import PyCode, CO_GENERATOR
from pypy.interpreter.pyframe import PyFrame
from pypy.interpreter.pyopcode import ExitFrame
-from pypy.interpreter.gateway import unwrap_spec
from opcode import opmap
-from pypy.rlib.nonconst import NonConstant
-from pypy.jit.metainterp.resoperation import rop
-from pypy.module.pypyjit.interp_resop import debug_merge_point_from_boxes
PyFrame._virtualizable2_ = ['last_instr', 'pycode',
'valuestackdepth', 'locals_stack_w[*]',
@@ -51,72 +47,19 @@
def should_unroll_one_iteration(next_instr, is_being_profiled, bytecode):
return (bytecode.co_flags & CO_GENERATOR) != 0
-def wrap_oplist(space, logops, operations):
- list_w = []
- for op in operations:
- if op.getopnum() == rop.DEBUG_MERGE_POINT:
- list_w.append(space.wrap(debug_merge_point_from_boxes(
- op.getarglist())))
- else:
- list_w.append(space.wrap(logops.repr_of_resop(op)))
- return list_w
-
class PyPyJitDriver(JitDriver):
reds = ['frame', 'ec']
greens = ['next_instr', 'is_being_profiled', 'pycode']
virtualizables = ['frame']
- def on_compile(self, logger, looptoken, operations, type, next_instr,
- is_being_profiled, ll_pycode):
- from pypy.rpython.annlowlevel import cast_base_ptr_to_instance
-
- space = self.space
- cache = space.fromcache(Cache)
- if cache.in_recursion:
- return
- if space.is_true(cache.w_compile_hook):
- logops = logger._make_log_operations()
- list_w = wrap_oplist(space, logops, operations)
- pycode = cast_base_ptr_to_instance(PyCode, ll_pycode)
- cache.in_recursion = True
- try:
- space.call_function(cache.w_compile_hook,
- space.wrap('main'),
- space.wrap(type),
- space.newtuple([pycode,
- space.wrap(next_instr),
- space.wrap(is_being_profiled)]),
- space.newlist(list_w))
- except OperationError, e:
- e.write_unraisable(space, "jit hook ", cache.w_compile_hook)
- cache.in_recursion = False
-
- def on_compile_bridge(self, logger, orig_looptoken, operations, n):
- space = self.space
- cache = space.fromcache(Cache)
- if cache.in_recursion:
- return
- if space.is_true(cache.w_compile_hook):
- logops = logger._make_log_operations()
- list_w = wrap_oplist(space, logops, operations)
- cache.in_recursion = True
- try:
- space.call_function(cache.w_compile_hook,
- space.wrap('main'),
- space.wrap('bridge'),
- space.wrap(n),
- space.newlist(list_w))
- except OperationError, e:
- e.write_unraisable(space, "jit hook ", cache.w_compile_hook)
- cache.in_recursion = False
-
pypyjitdriver = PyPyJitDriver(get_printable_location = get_printable_location,
get_jitcell_at = get_jitcell_at,
set_jitcell_at = set_jitcell_at,
confirm_enter_jit = confirm_enter_jit,
can_never_inline = can_never_inline,
should_unroll_one_iteration =
- should_unroll_one_iteration)
+ should_unroll_one_iteration,
+ name='pypyjit')
class __extend__(PyFrame):
@@ -223,34 +166,3 @@
'''For testing. Invokes callable(...), but without letting
the JIT follow the call.'''
return space.call_args(w_callable, __args__)
-
-class Cache(object):
- in_recursion = False
-
- def __init__(self, space):
- self.w_compile_hook = space.w_None
-
-def set_compile_hook(space, w_hook):
- """ set_compile_hook(hook)
-
- Set a compiling hook that will be called each time a loop is compiled.
- The hook will be called with the following signature:
- hook(merge_point_type, loop_type, greenkey or guard_number, operations)
-
- for now merge point type is always `main`
-
- loop_type can be either `loop` `entry_bridge` or `bridge`
- in case loop is not `bridge`, greenkey will be a set of constants
- for jit merge point. in case it's `main` it'll be a tuple
- (code, offset, is_being_profiled)
-
More information about the pypy-commit
mailing list